This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch branch-0.6
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/branch-0.6 by this push:
new e815012f1 [CELEBORN-2218] Bump lz4-java version from 1.8.0 to 1.10.4
to resolve CVE‐2025‐12183 and CVE-2025-66566
e815012f1 is described below
commit e815012f1215c57533bcb67861a84b8c2bcfda03
Author: SteNicholas <[email protected]>
AuthorDate: Tue Mar 3 11:24:45 2026 +0800
[CELEBORN-2218] Bump lz4-java version from 1.8.0 to 1.10.4 to resolve
CVE‐2025‐12183 and CVE-2025-66566
- Bump lz4-java version from 1.8.0 to 1.10.4 to resolve CVE‐2025‐12183 and
CVE-2025-66566.
- `Lz4Decompressor` follows the
[suggestion](https://github.com/apache/spark/pull/53290#issuecomment-3607045004)
to move from `fastDecompressor` to `safeDecompressor` to mitigate the
performance.
Backport:
- https://github.com/apache/spark/pull/53327
- https://github.com/apache/spark/pull/53347
- https://github.com/apache/spark/pull/53971
- https://github.com/apache/spark/pull/53454
- https://github.com/apache/spark/pull/54585
-
[CVE‐2025‐12183](https://sites.google.com/sonatype.com/vulnerabilities/cve-2025-12183):
Various lz4-java compression and decompression implementations do not guard
against out-of-bounds memory access. Untrusted input may lead to denial of
service and information disclosure. Vulnerable Maven coordinates:
org.lz4:lz4-java up to and including 1.8.0.
- [CVE-2025-66566](https://github.com/advisories/GHSA-cmp6-m4wj-q63q):
Insufficient clearing of the output buffer in Java-based decompressor
implementations in lz4-java 1.10.0 and earlier allows remote attackers to read
previous buffer contents via crafted compressed input. In applications where
the output buffer is reused without being cleared, this may lead to disclosure
of sensitive data. JNI-based implementations are not affected.
Therefore, lz4-java version should upgrade to 1.10.4.
No.
No.
CI.
Closes #3555 from SteNicholas/CELEBORN-2218.
Lead-authored-by: SteNicholas <[email protected]>
Co-authored-by: Cheng Pan <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
(cherry picked from commit dca37496ce59bd67526548957d2f607af8eee6cc)
Signed-off-by: SteNicholas <[email protected]>
---
client-mr/mr-shaded/pom.xml | 6 +++---
client-mr/mr-shaded/src/main/resources/META-INF/LICENSE | 2 +-
client-tez/tez-shaded/pom.xml | 2 +-
.../tez-shaded/src/main/resources/META-INF/LICENSE | 2 +-
.../benchmarks/LZ4TPCDSDataBenchmark-jdk17-results.txt | 16 ++++++++--------
client/pom.xml | 2 +-
.../apache/celeborn/client/compress/Lz4Decompressor.java | 16 ++++++++--------
dev/deps/dependencies-client-flink-1.16 | 2 +-
dev/deps/dependencies-client-flink-1.17 | 2 +-
dev/deps/dependencies-client-flink-1.18 | 2 +-
dev/deps/dependencies-client-flink-1.19 | 2 +-
dev/deps/dependencies-client-flink-1.20 | 2 +-
dev/deps/dependencies-client-flink-2.0 | 2 +-
dev/deps/dependencies-client-mr | 2 +-
dev/deps/dependencies-client-tez | 2 +-
dev/deps/dependencies-server | 2 +-
pom.xml | 13 +++++++++++--
project/CelebornBuild.scala | 6 ++++--
18 files changed, 47 insertions(+), 36 deletions(-)
diff --git a/client-mr/mr-shaded/pom.xml b/client-mr/mr-shaded/pom.xml
index 089d83019..68fbde43e 100644
--- a/client-mr/mr-shaded/pom.xml
+++ b/client-mr/mr-shaded/pom.xml
@@ -64,8 +64,8 @@
<shadedPattern>${shading.prefix}.org.scala-lang</shadedPattern>
</relocation>
<relocation>
- <pattern>org.lz4</pattern>
- <shadedPattern>${shading.prefix}.org.lz4</shadedPattern>
+ <pattern>${lz4-java.group}</pattern>
+
<shadedPattern>${shading.prefix}.${lz4-java.group}</shadedPattern>
</relocation>
<relocation>
<pattern>org.roaringbitmap</pattern>
@@ -81,7 +81,7 @@
<include>io.netty:*</include>
<include>org.apache.commons:commons-lang3</include>
<include>org.scala-lang:scala-library</include>
- <include>org.lz4:lz4-java</include>
+ <include>${lz4-java.group}:lz4-java</include>
<include>com.github.luben:zstd-jni</include>
<include>org.roaringbitmap:RoaringBitmap</include>
</includes>
diff --git a/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE
b/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE
index ec665dcc8..7435dd2e5 100644
--- a/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE
+++ b/client-mr/mr-shaded/src/main/resources/META-INF/LICENSE
@@ -208,6 +208,7 @@ This project bundles the following dependencies under the
Apache License 2.0 (ht
Apache License 2.0
--------------------------------------
+at.yawk.lz4:lz4-java
com.google.guava:failureaccess
com.google.guava:guava
io.netty:netty
@@ -240,7 +241,6 @@ io.netty:netty-transport-rxtx
io.netty:netty-transport-sctp
io.netty:netty-transport-udt
org.apache.commons:commons-lang3
-org.lz4:lz4-java
org.roaringbitmap:RoaringBitmap
org.scala-lang:scala-library
diff --git a/client-tez/tez-shaded/pom.xml b/client-tez/tez-shaded/pom.xml
index e192d3bdb..9ff54c815 100644
--- a/client-tez/tez-shaded/pom.xml
+++ b/client-tez/tez-shaded/pom.xml
@@ -94,7 +94,7 @@
<include>org.roaringbitmap:RoaringBitmap</include>
<include>org.scala-lang:scala-library</include>
<include>org.scala-lang:scala-reflect</include>
- <include>org.lz4:lz4-java</include>
+ <include>${lz4-java.group}:lz4-java</include>
<include>io.dropwizard.metrics:metrics-core</include>
<include>com.codahale.metrics:metrics-core</include>
<include>com.github.luben:zstd-jni</include>
diff --git a/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE
b/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE
index ec665dcc8..7435dd2e5 100644
--- a/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE
+++ b/client-tez/tez-shaded/src/main/resources/META-INF/LICENSE
@@ -208,6 +208,7 @@ This project bundles the following dependencies under the
Apache License 2.0 (ht
Apache License 2.0
--------------------------------------
+at.yawk.lz4:lz4-java
com.google.guava:failureaccess
com.google.guava:guava
io.netty:netty
@@ -240,7 +241,6 @@ io.netty:netty-transport-rxtx
io.netty:netty-transport-sctp
io.netty:netty-transport-udt
org.apache.commons:commons-lang3
-org.lz4:lz4-java
org.roaringbitmap:RoaringBitmap
org.scala-lang:scala-library
diff --git a/client/benchmarks/LZ4TPCDSDataBenchmark-jdk17-results.txt
b/client/benchmarks/LZ4TPCDSDataBenchmark-jdk17-results.txt
index a6ce37aba..1ae36bb16 100644
--- a/client/benchmarks/LZ4TPCDSDataBenchmark-jdk17-results.txt
+++ b/client/benchmarks/LZ4TPCDSDataBenchmark-jdk17-results.txt
@@ -6,48 +6,48 @@ OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux
6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Compression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
-------------------------------------------------------------------------------------------------------------------------
-Compression with chunk size 65536 4 times 2131 2134
5 0.0 532707902.0 1.0X
+Compression with chunk size 65536 4 times 2193 2210
24 0.0 548315522.5 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Decompression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Decompression with chunk size 65536 4 times 536 541
9 0.0 133951799.0 1.0X
+Decompression with chunk size 65536 4 times 460 463
3 0.0 114986376.3 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Compression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Compression with chunk size 262144 4 times 1754 1756
2 0.0 438523185.2 1.0X
+Compression with chunk size 262144 4 times 1683 1683
0 0.0 420711475.3 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Decompression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Decompression with chunk size 262144 4 times 436 439
4 0.0 109013659.0 1.0X
+Decompression with chunk size 262144 4 times 367 369
1 0.0 91804273.0 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Compression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Compression with chunk size 1048576 4 times 1774 1780
9 0.0 443426664.3 1.0X
+Compression with chunk size 1048576 4 times 1725 1726
3 0.0 431152298.8 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Decompression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Decompression with chunk size 1048576 4 times 431 434
4 0.0 107823243.0 1.0X
+Decompression with chunk size 1048576 4 times 367 368
1 0.0 91743487.5 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Compression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Compression with chunk size 4194304 4 times 1785 1791
8 0.0 446360006.8 1.0X
+Compression with chunk size 4194304 4 times 1697 1702
7 0.0 424249326.5 1.0X
OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.17.9-76061709-generic
Intel(R) Core(TM) i5-9500 CPU @ 3.00GHz
Decompression: Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Decompression with chunk size 4194304 4 times 436 438
1 0.0 109117732.3 1.0X
+Decompression with chunk size 4194304 4 times 371 376
4 0.0 92720790.5 1.0X
diff --git a/client/pom.xml b/client/pom.xml
index 4b69dfedc..8f227ab85 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -51,7 +51,7 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
- <groupId>org.lz4</groupId>
+ <groupId>${lz4-java.group}</groupId>
<artifactId>lz4-java</artifactId>
</dependency>
<dependency>
diff --git
a/client/src/main/java/org/apache/celeborn/client/compress/Lz4Decompressor.java
b/client/src/main/java/org/apache/celeborn/client/compress/Lz4Decompressor.java
index 8a538ef34..a579c0122 100644
---
a/client/src/main/java/org/apache/celeborn/client/compress/Lz4Decompressor.java
+++
b/client/src/main/java/org/apache/celeborn/client/compress/Lz4Decompressor.java
@@ -26,7 +26,7 @@ import scala.Option;
import com.google.common.collect.ImmutableMap;
import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.lz4.LZ4SafeDecompressor;
import net.jpountz.xxhash.XXHashFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
public class Lz4Decompressor extends Lz4Trait implements Decompressor {
private static final Logger logger =
LoggerFactory.getLogger(Lz4Decompressor.class);
- private final LZ4FastDecompressor decompressor;
+ private final LZ4SafeDecompressor decompressor;
private final Checksum checksum;
private final Map<String, Supplier<XXHashFactory>> xxHashFactories =
@@ -47,7 +47,7 @@ public class Lz4Decompressor extends Lz4Trait implements
Decompressor {
XXHashFactory::unsafeInstance);
public Lz4Decompressor(Option<String> xxHashInstance) {
- decompressor = LZ4Factory.fastestInstance().fastDecompressor();
+ decompressor = LZ4Factory.fastestInstance().safeDecompressor();
checksum =
getXXHashFactory(xxHashInstance).newStreamingHash32(DEFAULT_SEED).asChecksum();
}
@@ -68,13 +68,13 @@ public class Lz4Decompressor extends Lz4Trait implements
Decompressor {
System.arraycopy(src, HEADER_LENGTH, dst, dstOff, originalLen);
break;
case COMPRESSION_METHOD_LZ4:
- int compressedLen2 = decompressor.decompress(src, HEADER_LENGTH, dst,
dstOff, originalLen);
- if (compressedLen != compressedLen2) {
+ int originalLen2 = decompressor.decompress(src, HEADER_LENGTH,
compressedLen, dst, dstOff);
+ if (originalLen != originalLen2) {
throw new IOException(
- "Compressed length corrupted! expected: "
- + compressedLen
+ "Original length corrupted! expected: "
+ + originalLen
+ ", actual: "
- + compressedLen2
+ + originalLen2
+ ".");
}
}
diff --git a/dev/deps/dependencies-client-flink-1.16
b/dev/deps/dependencies-client-flink-1.16
index ab6caa152..5fd86b231 100644
--- a/dev/deps/dependencies-client-flink-1.16
+++ b/dev/deps/dependencies-client-flink-1.16
@@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-client-flink-1.17
b/dev/deps/dependencies-client-flink-1.17
index ab6caa152..5fd86b231 100644
--- a/dev/deps/dependencies-client-flink-1.17
+++ b/dev/deps/dependencies-client-flink-1.17
@@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-client-flink-1.18
b/dev/deps/dependencies-client-flink-1.18
index ab6caa152..5fd86b231 100644
--- a/dev/deps/dependencies-client-flink-1.18
+++ b/dev/deps/dependencies-client-flink-1.18
@@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-client-flink-1.19
b/dev/deps/dependencies-client-flink-1.19
index ab6caa152..5fd86b231 100644
--- a/dev/deps/dependencies-client-flink-1.19
+++ b/dev/deps/dependencies-client-flink-1.19
@@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-client-flink-1.20
b/dev/deps/dependencies-client-flink-1.20
index ab6caa152..5fd86b231 100644
--- a/dev/deps/dependencies-client-flink-1.20
+++ b/dev/deps/dependencies-client-flink-1.20
@@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-client-flink-2.0
b/dev/deps/dependencies-client-flink-2.0
index 031b3821e..6355f4bf8 100644
--- a/dev/deps/dependencies-client-flink-2.0
+++ b/dev/deps/dependencies-client-flink-2.0
@@ -32,7 +32,7 @@ jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jsr305/1.3.9//jsr305-1.3.9.jar
jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar
diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
index 35d26cec7..3d353a290 100644
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@@ -134,7 +134,7 @@ kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
kotlin-stdlib-common/1.4.10//kotlin-stdlib-common-1.4.10.jar
kotlin-stdlib/1.4.10//kotlin-stdlib-1.4.10.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-client-tez b/dev/deps/dependencies-client-tez
index 21edcd5e2..b65b52798 100644
--- a/dev/deps/dependencies-client-tez
+++ b/dev/deps/dependencies-client-tez
@@ -107,7 +107,7 @@ kerby-util/1.0.1//kerby-util-1.0.1.jar
kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
log4j/1.2.17//log4j-1.2.17.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index 3095350c1..98e97dcea 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -79,7 +79,7 @@ log4j-1.2-api/2.24.3//log4j-1.2-api-2.24.3.jar
log4j-api/2.24.3//log4j-api-2.24.3.jar
log4j-core/2.24.3//log4j-core-2.24.3.jar
log4j-slf4j-impl/2.24.3//log4j-slf4j-impl-2.24.3.jar
-lz4-java/1.8.0//lz4-java-1.8.0.jar
+lz4-java/1.10.4//lz4-java-1.10.4.jar
maven-jdk-tools-wrapper/0.1//maven-jdk-tools-wrapper-0.1.jar
metrics-core/4.2.25//metrics-core-4.2.25.jar
metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar
diff --git a/pom.xml b/pom.xml
index 78aa6821d..1980be7cb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,7 +91,8 @@
<leveldb.version>1.8</leveldb.version>
<log4j2.version>2.24.3</log4j2.version>
<disruptor.version>3.4.4</disruptor.version>
- <lz4-java.version>1.8.0</lz4-java.version>
+ <lz4-java.group>at.yawk.lz4</lz4-java.group>
+ <lz4-java.version>1.10.4</lz4-java.version>
<mockito.version>4.11.0</mockito.version>
<mockito-scalatest.version>1.17.14</mockito-scalatest.version>
<netty.version>4.1.118.Final</netty.version>
@@ -441,7 +442,7 @@
<version>${leveldb.version}</version>
</dependency>
<dependency>
- <groupId>org.lz4</groupId>
+ <groupId>${lz4-java.group}</groupId>
<artifactId>lz4-java</artifactId>
<version>${lz4-java.version}</version>
</dependency>
@@ -1438,6 +1439,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.4.0</lz4-java.version>
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
@@ -1457,6 +1459,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.7.1</lz4-java.version>
<scala.version>2.12.10</scala.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1477,6 +1480,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.7.1</lz4-java.version>
<scala.version>2.12.10</scala.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1497,6 +1501,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.7.1</lz4-java.version>
<scala.version>2.12.15</scala.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1516,6 +1521,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.8.0</lz4-java.version>
<scala.version>2.12.15</scala.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1535,6 +1541,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.8.0</lz4-java.version>
<scala.version>2.12.17</scala.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1554,6 +1561,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.8.0</lz4-java.version>
<scala.version>2.12.18</scala.version>
<scala.binary.version>2.12</scala.binary.version>
@@ -1573,6 +1581,7 @@
<module>tests/spark-it</module>
</modules>
<properties>
+ <lz4-java.group>org.lz4</lz4-java.group>
<lz4-java.version>1.8.0</lz4-java.version>
<scala.version>2.13.16</scala.version>
<scala.binary.version>2.13</scala.binary.version>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index a798c4ab5..914d4f3fb 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -38,7 +38,8 @@ import CelebornCommonSettings._
object Dependencies {
val zstdJniVersion =
sparkClientProjects.map(_.zstdJniVersion).getOrElse("1.5.7-1")
- val lz4JavaVersion =
sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.8.0")
+ val lz4JavaGroup =
sparkClientProjects.map(_.lz4JavaGroup).getOrElse("at.yawk.lz4")
+ val lz4JavaVersion =
sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.10.4")
// Dependent library versions
val apLoaderVersion = "3.0-9"
@@ -152,7 +153,7 @@ object Dependencies {
val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version
val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" %
log4j2Version
val disruptor = "com.lmax" % "disruptor" % disruptorVersion
- val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion
+ val lz4Java = lz4JavaGroup % "lz4-java" % lz4JavaVersion
val protobufJava = "com.google.protobuf" % "protobuf-java" % protoVersion
val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion
val ratisCommon = "org.apache.ratis" % "ratis-common" % ratisVersion
@@ -945,6 +946,7 @@ trait SparkClientProjects {
val sparkClientShadedProjectPath: String
val sparkClientShadedProjectName: String
+ val lz4JavaGroup: String = "org.lz4"
val lz4JavaVersion: String
val sparkProjectScalaVersion: String
val sparkVersion: String
