This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-0.4
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/branch-0.4 by this push:
new c598aa8fa [CELEBORN-1380] leveldbjni uses
org.openlabtesting.leveldbjni to support linux aarch64 platform for leveldb
c598aa8fa is described below
commit c598aa8fa8435770f3089771796e38a052ed4a85
Author: SteNicholas <[email protected]>
AuthorDate: Wed Apr 24 11:52:56 2024 +0800
[CELEBORN-1380] leveldbjni uses org.openlabtesting.leveldbjni to support
linux aarch64 platform for leveldb
Dependency leveldbjni uses `org.openlabtesting.leveldbjni` to support linux
aarch64 platform for leveldb.
Celeborn worker could not start on arm arch devices if db backend is
`LevelDB`, which should support leveldbjni on the aarch64 platform.
aarch64 uses `org.openlabtesting.leveldbjni:leveldbjni-all.1.8`, and other
platforms use `org.fusesource.leveldbjni:leveldbjni-all.1.8`. Meanwhile,
because some hadoop dependencies packages are also depend on
`org.fusesource.leveldbjni:leveldbjni-all`, but hadoop merge the similar change
on trunk, details see
[HADOOP-16614](https://issues.apache.org/jira/browse/HADOOP-16614),
therefore it should exclude the dependency of `org.fusesource.leveldbjni` for
these hadoop packages related.
Backport:
- https://github.com/apache/spark/pull/26636
- https://github.com/apache/spark/pull/31036
No.
No.
Closes #2476 from SteNicholas/CELEBORN-1380.
Authored-by: SteNicholas <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 9110eab9968be0f07c73e6e90d90a1b5b6bcf00e)
Signed-off-by: Cheng Pan <[email protected]>
---
common/pom.xml | 2 +-
pom.xml | 6 +++++-
project/CelebornBuild.scala | 2 +-
worker/pom.xml | 2 +-
4 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/common/pom.xml b/common/pom.xml
index ac5d8e5fc..936fe4053 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -87,7 +87,7 @@
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
+ <groupId>org.openlabtesting.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
<dependency>
diff --git a/pom.xml b/pom.xml
index 586bf7adb..ae7f25cb1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -385,7 +385,7 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
+ <groupId>org.openlabtesting.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
<version>${leveldb.version}</version>
</dependency>
@@ -1053,6 +1053,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 5b050b067..aed4eb947 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -100,7 +100,7 @@ object Dependencies {
val ioNetty = "io.netty" % "netty-all" % nettyVersion excludeAll(
ExclusionRule("io.netty", "netty-handler-ssl-ocsp"))
val javaxServletApi = "javax.servlet" % "javax.servlet-api" %
javaxServletVersion
- val leveldbJniAll = "org.fusesource.leveldbjni" % "leveldbjni-all" %
leveldbJniVersion
+ val leveldbJniAll = "org.openlabtesting.leveldbjni" % "leveldbjni-all" %
leveldbJniVersion
val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version
val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" %
log4j2Version
val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion
diff --git a/worker/pom.xml b/worker/pom.xml
index 5a8914e3f..8d3620de6 100644
--- a/worker/pom.xml
+++ b/worker/pom.xml
@@ -49,7 +49,7 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
+ <groupId>org.openlabtesting.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
<dependency>
