This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 89b6cac5a [CELEBORN-1113] Bump Hadoop client version from 3.2.4 to
3.3.6
89b6cac5a is described below
commit 89b6cac5abd43c1a2bd89d5fda137222006e2896
Author: sychen <[email protected]>
AuthorDate: Fri Dec 1 15:41:04 2023 +0800
[CELEBORN-1113] Bump Hadoop client version from 3.2.4 to 3.3.6
### What changes were proposed in this pull request?
### Why are the changes needed?
[[HADOOP-17098](https://issues.apache.org/jira/browse/HADOOP-17098)] Reduce
Guava dependency in Hadoop source code
The higher version of hadoop client removes many guava-related methods,
which avoids some conflicts on guava.
`hadoop-client-api` 3.3.6
`hadoop-client-runtime` 3.3.6
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
Closes #2077 from cxzl25/CELEBORN-1113.
Authored-by: sychen <[email protected]>
Signed-off-by: mingji <[email protected]>
---
client-flink/common/pom.xml | 6 ++
client-flink/flink-1.14/pom.xml | 6 ++
client-flink/flink-1.15/pom.xml | 6 ++
client-flink/flink-1.17/pom.xml | 6 ++
client-flink/flink-1.18/pom.xml | 6 ++
client-mr/mr/pom.xml | 11 +++
client-spark/common/pom.xml | 12 +++
client-spark/spark-3/pom.xml | 12 +++
dev/deps/dependencies-client-flink-1.14 | 6 +-
dev/deps/dependencies-client-flink-1.15 | 6 +-
dev/deps/dependencies-client-flink-1.17 | 6 +-
dev/deps/dependencies-client-flink-1.18 | 6 +-
dev/deps/dependencies-client-mr | 133 ++++++++++++++++++--------------
dev/deps/dependencies-client-spark-2.4 | 6 +-
dev/deps/dependencies-client-spark-3.0 | 6 +-
dev/deps/dependencies-client-spark-3.1 | 6 +-
dev/deps/dependencies-client-spark-3.2 | 6 +-
dev/deps/dependencies-client-spark-3.3 | 6 +-
dev/deps/dependencies-client-spark-3.4 | 6 +-
dev/deps/dependencies-client-spark-3.5 | 6 +-
dev/deps/dependencies-server | 6 +-
pom.xml | 2 +-
project/CelebornBuild.scala | 3 +-
23 files changed, 178 insertions(+), 97 deletions(-)
diff --git a/client-flink/common/pom.xml b/client-flink/common/pom.xml
index 9cf960a2a..499e086e0 100644
--- a/client-flink/common/pom.xml
+++ b/client-flink/common/pom.xml
@@ -43,6 +43,12 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
diff --git a/client-flink/flink-1.14/pom.xml b/client-flink/flink-1.14/pom.xml
index 903774e07..183cf6a65 100644
--- a/client-flink/flink-1.14/pom.xml
+++ b/client-flink/flink-1.14/pom.xml
@@ -48,6 +48,12 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Test dependencies -->
<dependency>
diff --git a/client-flink/flink-1.15/pom.xml b/client-flink/flink-1.15/pom.xml
index ec9f8ac70..f11e34be0 100644
--- a/client-flink/flink-1.15/pom.xml
+++ b/client-flink/flink-1.15/pom.xml
@@ -48,6 +48,12 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Test dependencies -->
<dependency>
diff --git a/client-flink/flink-1.17/pom.xml b/client-flink/flink-1.17/pom.xml
index 96d8ff863..8c8602987 100644
--- a/client-flink/flink-1.17/pom.xml
+++ b/client-flink/flink-1.17/pom.xml
@@ -48,6 +48,12 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Test dependencies -->
<dependency>
diff --git a/client-flink/flink-1.18/pom.xml b/client-flink/flink-1.18/pom.xml
index 1d51f8dc8..339b0b832 100644
--- a/client-flink/flink-1.18/pom.xml
+++ b/client-flink/flink-1.18/pom.xml
@@ -48,6 +48,12 @@
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- Test dependencies -->
<dependency>
diff --git a/client-mr/mr/pom.xml b/client-mr/mr/pom.xml
index 8b16614cb..de9fe1e85 100644
--- a/client-mr/mr/pom.xml
+++ b/client-mr/mr/pom.xml
@@ -44,6 +44,17 @@
<artifactId>hadoop-mapreduce-client-app</artifactId>
<version>${hadoop.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ <version>1.7.36</version>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<profiles>
diff --git a/client-spark/common/pom.xml b/client-spark/common/pom.xml
index 07e7699f9..c1165d621 100644
--- a/client-spark/common/pom.xml
+++ b/client-spark/common/pom.xml
@@ -43,6 +43,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -57,6 +63,12 @@
<artifactId>spark-core_${scala.binary.version}</artifactId>
<type>test-jar</type>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/client-spark/spark-3/pom.xml b/client-spark/spark-3/pom.xml
index 14bd07e01..084e6e731 100644
--- a/client-spark/spark-3/pom.xml
+++ b/client-spark/spark-3/pom.xml
@@ -54,6 +54,12 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -73,6 +79,12 @@
<artifactId>spark-core_${scala.binary.version}</artifactId>
<type>test-jar</type>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
diff --git a/dev/deps/dependencies-client-flink-1.14
b/dev/deps/dependencies-client-flink-1.14
index be6b7d17f..111bf56cf 100644
--- a/dev/deps/dependencies-client-flink-1.14
+++ b/dev/deps/dependencies-client-flink-1.14
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/dev/deps/dependencies-client-flink-1.15
b/dev/deps/dependencies-client-flink-1.15
index be6b7d17f..111bf56cf 100644
--- a/dev/deps/dependencies-client-flink-1.15
+++ b/dev/deps/dependencies-client-flink-1.15
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/dev/deps/dependencies-client-flink-1.17
b/dev/deps/dependencies-client-flink-1.17
index be6b7d17f..111bf56cf 100644
--- a/dev/deps/dependencies-client-flink-1.17
+++ b/dev/deps/dependencies-client-flink-1.17
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/dev/deps/dependencies-client-flink-1.18
b/dev/deps/dependencies-client-flink-1.18
index be6b7d17f..111bf56cf 100644
--- a/dev/deps/dependencies-client-flink-1.18
+++ b/dev/deps/dependencies-client-flink-1.18
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
index b3764b2ea..d1e8720c4 100644
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@@ -17,28 +17,28 @@
HikariCP-java7/2.4.12//HikariCP-java7-2.4.12.jar
RoaringBitmap/0.9.32//RoaringBitmap-0.9.32.jar
-accessors-smart/2.4.7//accessors-smart-2.4.7.jar
aopalliance/1.0//aopalliance-1.0.jar
-asm/9.1//asm-9.1.jar
+asm-commons/9.4//asm-commons-9.4.jar
+asm-tree/9.4//asm-tree-9.4.jar
audience-annotations/0.5.0//audience-annotations-0.5.0.jar
avro/1.7.7//avro-1.7.7.jar
commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.11//commons-codec-1.11.jar
+commons-codec/1.15//commons-codec-1.15.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-compress/1.4.1//commons-compress-1.4.1.jar
-commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
+commons-configuration2/2.8.0//commons-configuration2-2.8.0.jar
commons-crypto/1.0.0//commons-crypto-1.0.0.jar
commons-daemon/1.0.13//commons-daemon-1.0.13.jar
commons-io/2.13.0//commons-io-2.13.0.jar
commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
commons-math3/3.1.1//commons-math3-3.1.1.jar
-commons-net/3.6//commons-net-3.6.jar
-commons-text/1.4//commons-text-1.4.jar
-curator-client/2.13.0//curator-client-2.13.0.jar
-curator-framework/2.13.0//curator-framework-2.13.0.jar
-curator-recipes/2.13.0//curator-recipes-2.13.0.jar
+commons-net/3.9.0//commons-net-3.9.0.jar
+commons-text/1.10.0//commons-text-1.10.0.jar
+curator-client/5.2.0//curator-client-5.2.0.jar
+curator-framework/5.2.0//curator-framework-5.2.0.jar
+curator-recipes/5.2.0//curator-recipes-5.2.0.jar
dnsjava/2.1.7//dnsjava-2.1.7.jar
ehcache/3.3.1//ehcache-3.3.1.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
@@ -47,61 +47,69 @@ gson/2.9.0//gson-2.9.0.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
guice-servlet/4.0//guice-servlet-4.0.jar
guice/4.0//guice-4.0.jar
-hadoop-annotations/3.2.4//hadoop-annotations-3.2.4.jar
-hadoop-auth/3.2.4//hadoop-auth-3.2.4.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-hadoop-common/3.2.4//hadoop-common-3.2.4.jar
-hadoop-hdfs-client/3.2.4//hadoop-hdfs-client-3.2.4.jar
-hadoop-mapreduce-client-app/3.2.4//hadoop-mapreduce-client-app-3.2.4.jar
-hadoop-mapreduce-client-common/3.2.4//hadoop-mapreduce-client-common-3.2.4.jar
-hadoop-mapreduce-client-core/3.2.4//hadoop-mapreduce-client-core-3.2.4.jar
-hadoop-mapreduce-client-shuffle/3.2.4//hadoop-mapreduce-client-shuffle-3.2.4.jar
-hadoop-yarn-api/3.2.4//hadoop-yarn-api-3.2.4.jar
-hadoop-yarn-client/3.2.4//hadoop-yarn-client-3.2.4.jar
-hadoop-yarn-common/3.2.4//hadoop-yarn-common-3.2.4.jar
-hadoop-yarn-registry/3.2.4//hadoop-yarn-registry-3.2.4.jar
-hadoop-yarn-server-common/3.2.4//hadoop-yarn-server-common-3.2.4.jar
-hadoop-yarn-server-nodemanager/3.2.4//hadoop-yarn-server-nodemanager-3.2.4.jar
-hadoop-yarn-server-web-proxy/3.2.4//hadoop-yarn-server-web-proxy-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-annotations/3.3.6//hadoop-annotations-3.3.6.jar
+hadoop-auth/3.3.6//hadoop-auth-3.3.6.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
+hadoop-common/3.3.6//hadoop-common-3.3.6.jar
+hadoop-hdfs-client/3.3.6//hadoop-hdfs-client-3.3.6.jar
+hadoop-mapreduce-client-app/3.3.6//hadoop-mapreduce-client-app-3.3.6.jar
+hadoop-mapreduce-client-common/3.3.6//hadoop-mapreduce-client-common-3.3.6.jar
+hadoop-mapreduce-client-core/3.3.6//hadoop-mapreduce-client-core-3.3.6.jar
+hadoop-mapreduce-client-shuffle/3.3.6//hadoop-mapreduce-client-shuffle-3.3.6.jar
+hadoop-registry/3.3.6//hadoop-registry-3.3.6.jar
+hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar
+hadoop-shaded-protobuf_3_7/1.1.1//hadoop-shaded-protobuf_3_7-1.1.1.jar
+hadoop-yarn-api/3.3.6//hadoop-yarn-api-3.3.6.jar
+hadoop-yarn-client/3.3.6//hadoop-yarn-client-3.3.6.jar
+hadoop-yarn-common/3.3.6//hadoop-yarn-common-3.3.6.jar
+hadoop-yarn-server-common/3.3.6//hadoop-yarn-server-common-3.3.6.jar
+hadoop-yarn-server-nodemanager/3.3.6//hadoop-yarn-server-nodemanager-3.3.6.jar
+hadoop-yarn-server-web-proxy/3.3.6//hadoop-yarn-server-web-proxy-3.3.6.jar
httpclient/4.5.13//httpclient-4.5.13.jar
httpcore/4.4.13//httpcore-4.4.13.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.10.5//jackson-core-2.10.5.jar
-jackson-jaxrs-base/2.10.5//jackson-jaxrs-base-2.10.5.jar
-jackson-jaxrs-json-provider/2.10.5//jackson-jaxrs-json-provider-2.10.5.jar
-jackson-jaxrs/1.9.2//jackson-jaxrs-1.9.2.jar
+jackson-core/2.12.7//jackson-core-2.12.7.jar
+jackson-jaxrs-base/2.12.7//jackson-jaxrs-base-2.12.7.jar
+jackson-jaxrs-json-provider/2.12.7//jackson-jaxrs-json-provider-2.12.7.jar
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.10.5//jackson-module-jaxb-annotations-2.10.5.jar
-jackson-xc/1.9.2//jackson-xc-1.9.2.jar
+jackson-module-jaxb-annotations/2.12.7//jackson-module-jaxb-annotations-2.12.7.jar
jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-javax.activation-api/1.2.0//javax.activation-api-1.2.0.jar
+javax-websocket-client-impl/9.4.51.v20230217//javax-websocket-client-impl-9.4.51.v20230217.jar
+javax-websocket-server-impl/9.4.51.v20230217//javax-websocket-server-impl-9.4.51.v20230217.jar
+javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
javax.inject/1//javax.inject-1.jar
javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
+javax.websocket-api/1.0//javax.websocket-api-1.0.jar
+javax.websocket-client-api/1.0//javax.websocket-client-api-1.0.jar
jaxb-api/2.2.11//jaxb-api-2.2.11.jar
jaxb-impl/2.2.3-1//jaxb-impl-2.2.3-1.jar
jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
-jersey-client/1.19//jersey-client-1.19.jar
-jersey-core/1.19//jersey-core-1.19.jar
-jersey-guice/1.19//jersey-guice-1.19.jar
-jersey-json/1.19//jersey-json-1.19.jar
-jersey-server/1.19//jersey-server-1.19.jar
-jersey-servlet/1.19//jersey-servlet-1.19.jar
-jettison/1.1//jettison-1.1.jar
-jetty-http/9.4.43.v20210629//jetty-http-9.4.43.v20210629.jar
-jetty-io/9.4.43.v20210629//jetty-io-9.4.43.v20210629.jar
-jetty-security/9.4.43.v20210629//jetty-security-9.4.43.v20210629.jar
-jetty-server/9.4.43.v20210629//jetty-server-9.4.43.v20210629.jar
-jetty-servlet/9.4.43.v20210629//jetty-servlet-9.4.43.v20210629.jar
-jetty-util-ajax/9.4.43.v20210629//jetty-util-ajax-9.4.43.v20210629.jar
-jetty-util/9.4.43.v20210629//jetty-util-9.4.43.v20210629.jar
-jetty-webapp/9.4.43.v20210629//jetty-webapp-9.4.43.v20210629.jar
-jetty-xml/9.4.43.v20210629//jetty-xml-9.4.43.v20210629.jar
+jersey-client/1.19.4//jersey-client-1.19.4.jar
+jersey-core/1.19.4//jersey-core-1.19.4.jar
+jersey-guice/1.19.4//jersey-guice-1.19.4.jar
+jersey-json/1.20//jersey-json-1.20.jar
+jersey-server/1.19.4//jersey-server-1.19.4.jar
+jersey-servlet/1.19.4//jersey-servlet-1.19.4.jar
+jettison/1.5.4//jettison-1.5.4.jar
+jetty-annotations/9.4.51.v20230217//jetty-annotations-9.4.51.v20230217.jar
+jetty-client/9.4.51.v20230217//jetty-client-9.4.51.v20230217.jar
+jetty-http/9.4.51.v20230217//jetty-http-9.4.51.v20230217.jar
+jetty-io/9.4.51.v20230217//jetty-io-9.4.51.v20230217.jar
+jetty-jndi/9.4.51.v20230217//jetty-jndi-9.4.51.v20230217.jar
+jetty-plus/9.4.51.v20230217//jetty-plus-9.4.51.v20230217.jar
+jetty-security/9.4.51.v20230217//jetty-security-9.4.51.v20230217.jar
+jetty-server/9.4.51.v20230217//jetty-server-9.4.51.v20230217.jar
+jetty-servlet/9.4.51.v20230217//jetty-servlet-9.4.51.v20230217.jar
+jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar
+jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar
+jetty-webapp/9.4.51.v20230217//jetty-webapp-9.4.51.v20230217.jar
+jetty-xml/9.4.51.v20230217//jetty-xml-9.4.51.v20230217.jar
+jline/3.9.0//jline-3.9.0.jar
+jna/5.2.0//jna-5.2.0.jar
jsch/0.1.55//jsch-0.1.55.jar
-json-smart/2.4.7//json-smart-2.4.7.jar
jsp-api/2.1//jsp-api-2.1.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jsr311-api/1.1.1//jsr311-api-1.1.1.jar
@@ -120,6 +128,8 @@ kerby-config/1.0.1//kerby-config-1.0.1.jar
kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar
kerby-util/1.0.1//kerby-util-1.0.1.jar
kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
+kotlin-stdlib-common/1.4.10//kotlin-stdlib-common-1.4.10.jar
+kotlin-stdlib/1.4.10//kotlin-stdlib-1.4.10.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
lz4-java/1.8.0//lz4-java-1.8.0.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
@@ -162,8 +172,8 @@
netty-transport-udt/4.1.93.Final//netty-transport-udt-4.1.93.Final.jar
netty-transport/4.1.93.Final//netty-transport-4.1.93.Final.jar
netty/3.10.6.Final//netty-3.10.6.Final.jar
nimbus-jose-jwt/9.8.1//nimbus-jose-jwt-9.8.1.jar
-okhttp/2.7.5//okhttp-2.7.5.jar
-okio/1.6.0//okio-1.6.0.jar
+okhttp/4.9.3//okhttp-4.9.3.jar
+okio/2.8.0//okio-2.8.0.jar
paranamer/2.3//paranamer-2.3.jar
protobuf-java/3.19.2//protobuf-java-3.19.2.jar
ratis-client/2.5.1//ratis-client-2.5.1.jar
@@ -171,18 +181,23 @@ ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
re2j/1.1//re2j-1.1.jar
-reload4j/1.2.18.3//reload4j-1.2.18.3.jar
+reload4j/1.2.22//reload4j-1.2.22.jar
scala-library/2.12.15//scala-library-2.12.15.jar
scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
-slf4j-reload4j/1.7.35//slf4j-reload4j-1.7.35.jar
+slf4j-reload4j/1.7.36//slf4j-reload4j-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
-snappy-java/1.0.5//snappy-java-1.0.5.jar
-spotbugs-annotations/3.1.9//spotbugs-annotations-3.1.9.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
stax2-api/4.2.1//stax2-api-4.2.1.jar
token-provider/1.0.1//token-provider-1.0.1.jar
-woodstox-core/5.3.0//woodstox-core-5.3.0.jar
+websocket-api/9.4.51.v20230217//websocket-api-9.4.51.v20230217.jar
+websocket-client/9.4.51.v20230217//websocket-client-9.4.51.v20230217.jar
+websocket-common/9.4.51.v20230217//websocket-common-9.4.51.v20230217.jar
+websocket-server/9.4.51.v20230217//websocket-server-9.4.51.v20230217.jar
+websocket-servlet/9.4.51.v20230217//websocket-servlet-9.4.51.v20230217.jar
+woodstox-core/5.4.0//woodstox-core-5.4.0.jar
xz/1.0//xz-1.0.jar
-zookeeper/3.4.14//zookeeper-3.4.14.jar
+zookeeper-jute/3.6.3//zookeeper-jute-3.6.3.jar
+zookeeper/3.6.3//zookeeper-3.6.3.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/dev/deps/dependencies-client-spark-2.4
b/dev/deps/dependencies-client-spark-2.4
index 9beaa1f8b..a002ea22b 100644
--- a/dev/deps/dependencies-client-spark-2.4
+++ b/dev/deps/dependencies-client-spark-2.4
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.11.12//scala-reflect-2.11.12.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar
diff --git a/dev/deps/dependencies-client-spark-3.0
b/dev/deps/dependencies-client-spark-3.0
index 03618f785..00adfa190 100644
--- a/dev/deps/dependencies-client-spark-3.0
+++ b/dev/deps/dependencies-client-spark-3.0
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.4.4-3//zstd-jni-1.4.4-3.jar
diff --git a/dev/deps/dependencies-client-spark-3.1
b/dev/deps/dependencies-client-spark-3.1
index da8331b10..b9c2c81fd 100644
--- a/dev/deps/dependencies-client-spark-3.1
+++ b/dev/deps/dependencies-client-spark-3.1
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.4.8-1//zstd-jni-1.4.8-1.jar
diff --git a/dev/deps/dependencies-client-spark-3.2
b/dev/deps/dependencies-client-spark-3.2
index 1feb81a8e..2c3cf1672 100644
--- a/dev/deps/dependencies-client-spark-3.2
+++ b/dev/deps/dependencies-client-spark-3.2
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.0-4//zstd-jni-1.5.0-4.jar
diff --git a/dev/deps/dependencies-client-spark-3.3
b/dev/deps/dependencies-client-spark-3.3
index be6b7d17f..111bf56cf 100644
--- a/dev/deps/dependencies-client-spark-3.3
+++ b/dev/deps/dependencies-client-spark-3.3
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/dev/deps/dependencies-client-spark-3.4
b/dev/deps/dependencies-client-spark-3.4
index eabdee5a8..dd2ad1a3e 100644
--- a/dev/deps/dependencies-client-spark-3.4
+++ b/dev/deps/dependencies-client-spark-3.4
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.17//scala-reflect-2.12.17.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-5//zstd-jni-1.5.2-5.jar
diff --git a/dev/deps/dependencies-client-spark-3.5
b/dev/deps/dependencies-client-spark-3.5
index 4aa62aef5..d3a66fca3 100644
--- a/dev/deps/dependencies-client-spark-3.5
+++ b/dev/deps/dependencies-client-spark-3.5
@@ -22,9 +22,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
@@ -77,4 +76,5 @@ scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.5-4//zstd-jni-1.5.5-4.jar
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index 1be1e05dd..efc6c68c4 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -23,9 +23,8 @@ commons-lang3/3.12.0//commons-lang3-3.12.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.1//failureaccess-1.0.1.jar
guava/32.1.3-jre//guava-32.1.3-jre.jar
-hadoop-client-api/3.2.4//hadoop-client-api-3.2.4.jar
-hadoop-client-runtime/3.2.4//hadoop-client-runtime-3.2.4.jar
-htrace-core4/4.1.0-incubating//htrace-core4-4.1.0-incubating.jar
+hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
+hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
javassist/3.28.0-GA//javassist-3.28.0-GA.jar
javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
@@ -92,4 +91,5 @@ scala-reflect/2.12.15//scala-reflect-2.12.15.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/1.33//snakeyaml-1.33.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
zstd-jni/1.5.2-1//zstd-jni-1.5.2-1.jar
diff --git a/pom.xml b/pom.xml
index e1c96c704..cbf1f3c6b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,7 +67,7 @@
<spark.version>3.3.2</spark.version>
<!-- use hadoop-3 as default -->
- <hadoop.version>3.2.4</hadoop.version>
+ <hadoop.version>3.3.6</hadoop.version>
<codahale.metrics.version>3.2.6</codahale.metrics.version>
<commons-lang3.version>3.12.0</commons-lang3.version>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index b4f56782a..ebe8d1c52 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -43,7 +43,7 @@ object Dependencies {
val commonsLang3Version = "3.12.0"
val findbugsVersion = "1.3.9"
val guavaVersion = "32.1.3-jre"
- val hadoopVersion = "3.2.4"
+ val hadoopVersion = "3.3.6"
val javaxServletVersion = "3.1.0"
val junitInterfaceVersion = "0.13.3"
// don't forget update `junitInterfaceVersion` when we upgrade junit
@@ -83,6 +83,7 @@ object Dependencies {
val hadoopClientApi = "org.apache.hadoop" % "hadoop-client-api" %
hadoopVersion
val hadoopClientRuntime = "org.apache.hadoop" % "hadoop-client-runtime" %
hadoopVersion
val hadoopMapreduceClientApp = "org.apache.hadoop" %
"hadoop-mapreduce-client-app" % hadoopVersion excludeAll(
+ ExclusionRule("io.netty", "netty-transport-native-epoll"),
ExclusionRule("com.google.guava", "guava"),
ExclusionRule("com.fasterxml.jackson.core", "jackson-annotations"),
ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"),
