This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new e5f09ce4e [CELEBORN-1443] Remove ratis dependencies from common module
e5f09ce4e is described below
commit e5f09ce4e06154c3cadd1ff13a9f304b672e9cdb
Author: SteNicholas <[email protected]>
AuthorDate: Mon Jun 3 10:15:51 2024 +0800
[CELEBORN-1443] Remove ratis dependencies from common module
### What changes were proposed in this pull request?
Remove ratis dependencies from common module.
### Why are the changes needed?
Ratis is only depended on by the master module. Removing ratis dependencies
from the common module reduces the size of the Celeborn client package.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
GA.
Closes #2538 from SteNicholas/CELEBORN-1443.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: Shuang <[email protected]>
---
common/pom.xml | 8 --------
dev/deps/dependencies-client-flink-1.14 | 4 ----
dev/deps/dependencies-client-flink-1.15 | 4 ----
dev/deps/dependencies-client-flink-1.17 | 4 ----
dev/deps/dependencies-client-flink-1.18 | 4 ----
dev/deps/dependencies-client-flink-1.19 | 4 ----
dev/deps/dependencies-client-mr | 4 ----
dev/deps/dependencies-client-spark-2.4 | 4 ----
dev/deps/dependencies-client-spark-3.0 | 4 ----
dev/deps/dependencies-client-spark-3.1 | 4 ----
dev/deps/dependencies-client-spark-3.2 | 4 ----
dev/deps/dependencies-client-spark-3.3 | 4 ----
dev/deps/dependencies-client-spark-3.4 | 4 ----
dev/deps/dependencies-client-spark-3.5 | 4 ----
project/CelebornBuild.scala | 7 -------
worker/pom.xml | 10 ----------
16 files changed, 77 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index 3273303dd..29baba769 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -30,14 +30,6 @@
<name>Celeborn Common</name>
<dependencies>
- <dependency>
- <groupId>org.apache.ratis</groupId>
- <artifactId>ratis-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.ratis</groupId>
- <artifactId>ratis-client</artifactId>
- </dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
diff --git a/dev/deps/dependencies-client-flink-1.14
b/dev/deps/dependencies-client-flink-1.14
index f3ce797ac..0380d321b 100644
--- a/dev/deps/dependencies-client-flink-1.14
+++ b/dev/deps/dependencies-client-flink-1.14
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-flink-1.15
b/dev/deps/dependencies-client-flink-1.15
index f3ce797ac..0380d321b 100644
--- a/dev/deps/dependencies-client-flink-1.15
+++ b/dev/deps/dependencies-client-flink-1.15
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-flink-1.17
b/dev/deps/dependencies-client-flink-1.17
index f3ce797ac..0380d321b 100644
--- a/dev/deps/dependencies-client-flink-1.17
+++ b/dev/deps/dependencies-client-flink-1.17
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-flink-1.18
b/dev/deps/dependencies-client-flink-1.18
index f3ce797ac..0380d321b 100644
--- a/dev/deps/dependencies-client-flink-1.18
+++ b/dev/deps/dependencies-client-flink-1.18
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-flink-1.19
b/dev/deps/dependencies-client-flink-1.19
index f3ce797ac..0380d321b 100644
--- a/dev/deps/dependencies-client-flink-1.19
+++ b/dev/deps/dependencies-client-flink-1.19
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
index e952be60b..4a6aa7d7a 100644
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@@ -180,10 +180,6 @@ okhttp/4.9.3//okhttp-4.9.3.jar
okio/2.8.0//okio-2.8.0.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
re2j/1.1//re2j-1.1.jar
reload4j/1.2.22//reload4j-1.2.22.jar
scala-library/2.12.18//scala-library-2.12.18.jar
diff --git a/dev/deps/dependencies-client-spark-2.4
b/dev/deps/dependencies-client-spark-2.4
index dc0d1a718..bb5e0f9f2 100644
--- a/dev/deps/dependencies-client-spark-2.4
+++ b/dev/deps/dependencies-client-spark-2.4
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.11.12//scala-library-2.11.12.jar
scala-reflect/2.11.12//scala-reflect-2.11.12.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.0
b/dev/deps/dependencies-client-spark-3.0
index ffb7b0599..fffc6e468 100644
--- a/dev/deps/dependencies-client-spark-3.0
+++ b/dev/deps/dependencies-client-spark-3.0
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.10//scala-library-2.12.10.jar
scala-reflect/2.12.10//scala-reflect-2.12.10.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.1
b/dev/deps/dependencies-client-spark-3.1
index 381b67ff4..da095542c 100644
--- a/dev/deps/dependencies-client-spark-3.1
+++ b/dev/deps/dependencies-client-spark-3.1
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.10//scala-library-2.12.10.jar
scala-reflect/2.12.10//scala-reflect-2.12.10.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.2
b/dev/deps/dependencies-client-spark-3.2
index ba87c7a60..76f5b0211 100644
--- a/dev/deps/dependencies-client-spark-3.2
+++ b/dev/deps/dependencies-client-spark-3.2
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.15//scala-library-2.12.15.jar
scala-reflect/2.12.15//scala-reflect-2.12.15.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.3
b/dev/deps/dependencies-client-spark-3.3
index a5da10cf3..8102a540f 100644
--- a/dev/deps/dependencies-client-spark-3.3
+++ b/dev/deps/dependencies-client-spark-3.3
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.15//scala-library-2.12.15.jar
scala-reflect/2.12.15//scala-reflect-2.12.15.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.4
b/dev/deps/dependencies-client-spark-3.4
index 6f0005936..96996db77 100644
--- a/dev/deps/dependencies-client-spark-3.4
+++ b/dev/deps/dependencies-client-spark-3.4
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.17//scala-library-2.12.17.jar
scala-reflect/2.12.17//scala-reflect-2.12.17.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/dev/deps/dependencies-client-spark-3.5
b/dev/deps/dependencies-client-spark-3.5
index bce13ca7c..371e6479a 100644
--- a/dev/deps/dependencies-client-spark-3.5
+++ b/dev/deps/dependencies-client-spark-3.5
@@ -73,10 +73,6 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
paranamer/2.8//paranamer-2.8.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
-ratis-client/3.0.1//ratis-client-3.0.1.jar
-ratis-common/3.0.1//ratis-common-3.0.1.jar
-ratis-proto/3.0.1//ratis-proto-3.0.1.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index c616237b0..0ce59ddf6 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -438,8 +438,6 @@ object CelebornCommon {
Dependencies.hadoopClientApi,
Dependencies.hadoopClientRuntime,
Dependencies.jdkTools,
- Dependencies.ratisClient,
- Dependencies.ratisCommon,
Dependencies.leveldbJniAll,
Dependencies.roaringBitmap,
Dependencies.scalaReflect,
@@ -569,11 +567,6 @@ object CelebornWorker {
.dependsOn(CelebornMaster.master % "test->compile")
.settings (
commonSettings,
- excludeDependencies ++= Seq(
- // ratis-common/ratis-client are the transitive dependencies from
celeborn-common
- ExclusionRule("org.apache.ratis", "ratis-common"),
- ExclusionRule("org.apache.ratis", "ratis-client")
- ),
libraryDependencies ++= Seq(
Dependencies.apLoader,
Dependencies.guava,
diff --git a/worker/pom.xml b/worker/pom.xml
index e0be5e904..31c624e3b 100644
--- a/worker/pom.xml
+++ b/worker/pom.xml
@@ -32,16 +32,6 @@
<groupId>org.apache.celeborn</groupId>
<artifactId>celeborn-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.ratis</groupId>
- <artifactId>ratis-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ratis</groupId>
- <artifactId>ratis-client</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.celeborn</groupId>
