This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.9 by this push:
new 460e55300 [KYUUBI #6458] Remove commons-logging from binary release
460e55300 is described below
commit 460e553002eb773c596a286bf378b0ddbea1e9d8
Author: Cheng Pan <[email protected]>
AuthorDate: Tue Jun 11 11:32:30 2024 +0800
[KYUUBI #6458] Remove commons-logging from binary release
# :mag: Description
[`jcl-over-slf4j`](https://www.slf4j.org/legacy.html#jcl-over-slf4j) is a
drop-in replacement of `commons-logging`, the latter one should not be present
in the final classpath, otherwise, there are potential class conflict issues.
The current dep check is problematic, this PR also changes it to always
perform "install" to fix the false negative report.
## Types of changes :bookmark:
- [x] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
Simply delete `commons-logging-1.1.3.jar` from
`apache-kyuubi-1.9.1-bin.tgz` and everything goes well.
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6458 from pan3793/commons-logging.
Closes #6458
114ec766a [Cheng Pan] fix
79d4121a1 [Cheng Pan] fix
6633e83ee [Cheng Pan] fix
21127ed0b [Cheng Pan] always perform install on dep check
98b13dfcf [Cheng Pan] Remove commons-logging from binary release
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 017d8ccd7ecb2f819363aa75bc7f047834c7fec1)
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/dep.yml | 28 +++++++---------------
LICENSE-binary | 2 --
NOTICE-binary | 11 ---------
dev/dependencyList | 1 -
.../engine/flink/WithFlinkSQLEngineOnYarn.scala | 9 ++++++-
.../flink/WithKyuubiServerAndYarnMiniCluster.scala | 9 ++++++-
.../deployment/KyuubiOnKubernetesTestsSuite.scala | 5 ++++
.../test/spark/SparkOnKubernetesTestsSuite.scala | 5 ++++
.../WithKyuubiServerAndHadoopMiniCluster.scala | 9 ++++++-
.../org/apache/kyuubi/WithSecuredDFSService.scala | 6 +++++
.../org/apache/kyuubi/WithSimpleDFSService.scala | 10 +++++++-
pom.xml | 4 ++++
12 files changed, 61 insertions(+), 38 deletions(-)
diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml
index 96f49d8d9..c9cfdbf06 100644
--- a/.github/workflows/dep.yml
+++ b/.github/workflows/dep.yml
@@ -48,27 +48,15 @@ jobs:
check-latest: false
- name: Setup Maven
uses: ./.github/actions/setup-maven
- - name: Check kyuubi modules available
- id: modules-check
- run: >-
- build/mvn dependency:resolve validate
- -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile"
- -Pfast -Denforcer.skip=false
- -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
- continue-on-error: true
- - name: build
+ - name: Install modules
env:
- MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
- if: steps.modules-check.conclusion == 'success' &&
steps.modules-check.outcome == 'failure'
- run: >-
- build/mvn clean install
- -Pflink-provided,spark-provided,hive-provided
- -Dmaven.javadoc.skip=true
- -Drat.skip=true
- -Dscalastyle.skip=true
- -Dspotless.check.skip
- -DskipTests
- -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
+ MAVEN_ARGS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
+ run: |
+ build/mvn clean install \
+ -Pflink-provided,spark-provided,hive-provided \
+ -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true
-Dspotless.check.skip \
+ -DskipTests \
+ -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
- name: Check dependency list
run: build/dependency.sh
- name: Dependency Review
diff --git a/LICENSE-binary b/LICENSE-binary
index c4d8b185a..e78047f54 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -209,8 +209,6 @@ Apache License Version 2.0
--------------------------
com.zaxxer:HikariCP
com.google.android:annotations
-commons-lang:commons-lang
-commons-logging:commons-logging
org.apache.commons:commons-lang3
com.google.errorprone:error_prone_annotations
dev.failsafe:failsafe
diff --git a/NOTICE-binary b/NOTICE-binary
index 237fdda56..b9bcdb214 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -115,9 +115,6 @@ Copyright 2002-2020 The Apache Software Foundation
Apache Commons Lang
Copyright 2001-2020 The Apache Software Foundation
-Apache Commons Logging
-Copyright 2003-2013 The Apache Software Foundation
-
Apache Hadoop
Copyright 2006 and onwards The Apache Software Foundation.
@@ -986,14 +983,6 @@ benchmarking framework, which can be obtained at:
* HOMEPAGE:
* https://github.com/google/caliper
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
- * LICENSE:
- * license/LICENSE.commons-logging.txt (Apache License 2.0)
- * HOMEPAGE:
- * https://commons.apache.org/logging/
-
This product optionally depends on 'Apache Log4J', a logging framework, which
can be obtained at:
diff --git a/dev/dependencyList b/dev/dependencyList
index 348bf08b6..69bfd2de5 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -30,7 +30,6 @@ checker-qual/3.42.0//checker-qual-3.42.0.jar
classgraph/4.8.138//classgraph-4.8.138.jar
commons-codec/1.15//commons-codec-1.15.jar
commons-lang3/3.13.0//commons-lang3-3.13.0.jar
-commons-logging/1.1.3//commons-logging-1.1.3.jar
error_prone_annotations/2.20.0//error_prone_annotations-2.20.0.jar
failsafe/3.3.2//failsafe-3.3.2.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
diff --git
a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
index 001768946..d85d93334 100644
---
a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
+++
b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala
@@ -105,7 +105,14 @@ trait WithFlinkSQLEngineOnYarn extends KyuubiFunSuite with
WithFlinkTestResource
zkServer.start()
conf.set(HA_ADDRESSES, zkServer.getConnectString)
- hdfsCluster = new MiniDFSCluster.Builder(new Configuration)
+ val hdfsConf = new Configuration()
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConf)
.numDataNodes(1)
.checkDataNodeAddrConfig(true)
.checkDataNodeHostConfig(true)
diff --git
a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala
b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala
index b672bfb38..fc58a7833 100644
---
a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala
+++
b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala
@@ -20,6 +20,7 @@ package org.apache.kyuubi.it.flink
import java.io.{File, FileWriter}
import java.nio.file.Paths
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.kyuubi.{KyuubiFunSuite, Utils, WithKyuubiServer}
@@ -84,7 +85,13 @@ trait WithKyuubiServerAndYarnMiniCluster extends
KyuubiFunSuite with WithKyuubiS
}
override def beforeAll(): Unit = {
- miniHdfsService = new MiniDFSService()
+ val hdfsConf = new Configuration()
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+ miniHdfsService = new MiniDFSService(hdfsConf)
miniHdfsService.initialize(conf)
miniHdfsService.start()
diff --git
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
index 1256687ae..6ffd157d6 100644
---
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@@ -130,6 +130,11 @@ class KyuubiOnKubernetesWithClusterSparkTestsSuite
hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0")
hdfsConf.set("dfs.datanode.hostname", localhostAddress)
hdfsConf.set("dfs.datanode.address",
s"0.0.0.0:${NetUtils.getFreeSocketPort}")
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
// spark use 185 as userid in docker
hdfsConf.set("hadoop.proxyuser.185.groups", "*")
hdfsConf.set("hadoop.proxyuser.185.hosts", "*")
diff --git
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
index cf4b3ff3b..be07dd557 100644
---
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
+++
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@@ -106,6 +106,11 @@ class SparkClusterModeOnKubernetesSuiteBase
hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0")
hdfsConf.set("dfs.datanode.hostname", localhostAddress)
hdfsConf.set("dfs.datanode.address",
s"0.0.0.0:${NetUtils.getFreeSocketPort}")
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
// spark use 185 as userid in docker
hdfsConf.set("hadoop.proxyuser.185.groups", "*")
hdfsConf.set("hadoop.proxyuser.185.hosts", "*")
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala
index 90046ca37..64a521413 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServerAndHadoopMiniCluster.scala
@@ -18,6 +18,7 @@ package org.apache.kyuubi
import java.io.File
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.kyuubi.config.KyuubiConf
@@ -38,7 +39,13 @@ trait WithKyuubiServerAndHadoopMiniCluster extends
KyuubiFunSuite with WithKyuub
protected var miniYarnService: MiniYarnService = _
override def beforeAll(): Unit = {
- miniHdfsService = new MiniDFSService()
+ val hdfsConf = new Configuration()
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+ miniHdfsService = new MiniDFSService(hdfsConf)
miniHdfsService.initialize(conf)
miniHdfsService.start()
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala
index 43bea7c6a..f5f3eedc7 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala
@@ -37,6 +37,12 @@ trait WithSecuredDFSService extends KerberizedTestHelper {
hdfsConf.set("dfs.namenode.kerberos.internal.spnego.principal",
testPrincipal)
hdfsConf.set("dfs.web.authentication.kerberos.principal", testPrincipal)
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+
hdfsConf.set("dfs.datanode.address", "0.0.0.0:1025")
hdfsConf.set("dfs.datanode.kerberos.principal", testPrincipal)
hdfsConf.set("dfs.datanode.keytab.file", testKeytab)
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
index 0e29f4411..d3307c269 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
@@ -26,7 +26,15 @@ trait WithSimpleDFSService extends KyuubiFunSuite {
private var miniDFSService: MiniDFSService = _
- def hadoopConf: Configuration = new Configuration()
+ def hadoopConf: Configuration = {
+ val hdfsConf = new Configuration()
+ // before HADOOP-18206 (3.4.0), HDFS MetricsLogger strongly depends on
+ // commons-logging, we should disable it explicitly, otherwise, it throws
+ // ClassNotFound: org.apache.commons.logging.impl.Log4JLogger
+ hdfsConf.set("dfs.namenode.metrics.logger.period.seconds", "0")
+ hdfsConf.set("dfs.datanode.metrics.logger.period.seconds", "0")
+ hdfsConf
+ }
override def beforeAll(): Unit = {
miniDFSService = new MiniDFSService(hadoopConf)
diff --git a/pom.xml b/pom.xml
index 0642f39b9..c900617c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -355,6 +355,10 @@
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
</exclusions>
</dependency>
