This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new b44730771 [CELEBORN-1413][FOLLOWUP] Bump spark 4.0 version to 4.0.0
b44730771 is described below
commit b44730771d435238812f44a888d7b460452cb5db
Author: Fei Wang <[email protected]>
AuthorDate: Wed May 28 17:56:08 2025 +0800
[CELEBORN-1413][FOLLOWUP] Bump spark 4.0 version to 4.0.0
### What changes were proposed in this pull request?
Bump spark 4.0 version to 4.0.0.
### Why are the changes needed?
Spark 4.0.0 is ready.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
GA.
Closes #3282 from turboFei/spark_4.0.
Lead-authored-by: Fei Wang <[email protected]>
Co-authored-by: Wang, Fei <[email protected]>
Co-authored-by: Fei Wang <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
---
.github/workflows/maven.yml | 37 +++++++++
.github/workflows/sbt.yml | 36 +++++++++
LICENSE-binary | 4 +
README.md | 13 ++-
build/make-distribution.sh | 6 ++
build/release/release.sh | 6 ++
client-mr/mr/pom.xml | 4 +
client-spark/spark-3-columnar-shuffle/pom.xml | 10 +++
client-spark/spark-3/pom.xml | 10 +++
.../execution/columnar/CelebornColumnType.scala | 2 +-
client-tez/tez/pom.xml | 4 +
.../celeborn/client/ApplicationHeartbeater.scala | 2 +-
dev/deps/dependencies-client-mr | 2 +-
dev/deps/dependencies-client-tez | 2 +-
dev/deps/dependencies-server | 3 +-
docs/developers/sbt.md | 6 ++
pom.xml | 31 +++++--
project/CelebornBuild.scala | 65 +++++++++++----
service/pom.xml | 24 +++---
tests/spark-it/pom.xml | 94 +++++++++++++++++++++-
.../celeborn/tests/spark/SparkTestBase.scala | 9 +--
21 files changed, 324 insertions(+), 46 deletions(-)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index b39ee9c9a..26404f8fc 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -146,6 +146,43 @@ jobs:
**/target/test-reports/**
**/target/unit-tests.log
+ spark4:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ java:
+ - 17
+ spark:
+ - '4.0'
+ shuffle-plugin-class:
+ - 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ cache: maven
+ check-latest: false
+ - name: Test with Maven
+ run: |
+ SPARK_BINARY_VERSION=${{ matrix.spark }}
+ SPARK_MAJOR_VERSION=${SPARK_BINARY_VERSION%%.*}
+ PROFILES="-Pgoogle-mirror,spark-${{ matrix.spark }}"
+
TEST_MODULES="client-spark/common,client-spark/spark-3,client-spark/spark-3-columnar-common,client-spark/spark-${SPARK_MAJOR_VERSION}-shaded,tests/spark-it"
+ build/mvn $PROFILES -pl $TEST_MODULES -am clean install -DskipTests
+ build/mvn $PROFILES -pl $TEST_MODULES
-Dspark.shuffle.sort.io.plugin.class=${{ matrix.shuffle-plugin-class }} test
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-unit-test-log
+ path: |
+ **/target/test-reports/**
+ **/target/unit-tests.log
+
flink1:
runs-on: ubuntu-22.04
strategy:
diff --git a/.github/workflows/sbt.yml b/.github/workflows/sbt.yml
index 309d96cd3..94c97078d 100644
--- a/.github/workflows/sbt.yml
+++ b/.github/workflows/sbt.yml
@@ -196,6 +196,42 @@ jobs:
**/target/test-reports/**
**/target/unit-tests.log
+ spark4:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: false
+ matrix:
+ spark:
+ - '4.0'
+ java:
+ - 17
+ scala-binary:
+ - '2.13'
+ shuffle-plugin-class:
+ - 'org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO'
+ include:
+ - spark: '4.0'
+ scala: '2.13.16'
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: zulu
+ java-version: ${{ matrix.java }}
+ check-latest: false
+ - name: Test with SBT
+ run: |
+ build/sbt -Dspark.shuffle.plugin.class=${{
matrix.shuffle-plugin-class }} -Pspark-${{ matrix.spark }} ++${{ matrix.scala
}} "clean; celeborn-spark-group/test"
+ - name: Upload test log
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: spark-${{ matrix.spark }}-java-${{ matrix.java }}-scala-${{
matrix.scala }}-unit-test-log
+ path: |
+ **/target/test-reports/**
+ **/target/unit-tests.log
+
flink1:
runs-on: ubuntu-22.04
strategy:
diff --git a/LICENSE-binary b/LICENSE-binary
index c167f5cda..5c29c642b 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -362,3 +362,7 @@ EDL 1.0
--------------------------------
See licenses-binary/LICENSE-mimepull.txt for detail.
org.jvnet.mimepull:mimepull
+
+CDDL License 1.1
+----------------
+javax.servlet-api
diff --git a/README.md b/README.md
index fc5d834e1..da1418652 100644
--- a/README.md
+++ b/README.md
@@ -41,12 +41,12 @@ Celeborn Worker's slot count is decided by `total usable
disk size / average shu
Celeborn worker's slot count decreases when a partition is allocated and
increments when a partition is freed.
## Build
-1. Celeborn supports Spark 2.4/3.0/3.1/3.2/3.3/3.4/3.5, Flink
1.16/1.17/1.18/1.19/1.20 and Hadoop MapReduce 2/3.
+1. Celeborn supports Spark 2.4/3.0/3.1/3.2/3.3/3.4/3.5/4.0, Flink
1.16/1.17/1.18/1.19/1.20 and Hadoop MapReduce 2/3.
2. Celeborn tested under Scala 2.11/2.12/2.13 and Java 8/11/17 environment.
Build Celeborn via `make-distribution.sh`:
```shell
-./build/make-distribution.sh
-Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pmr
+./build/make-distribution.sh
-Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pmr
```
Package `apache-celeborn-${project.version}-bin.tgz` will be generated.
@@ -62,6 +62,7 @@ Package `apache-celeborn-${project.version}-bin.tgz` will be
generated.
| Spark 3.3 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 3.4 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 3.5 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
+| Spark 4.0 | ❌ | ❌ | ❌ |
❌ | ❌ | ❌ | ✔
|
| Flink 1.16 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.17 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.18 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
@@ -88,6 +89,14 @@ To compile for Spark 3.5 with Java21, please use the
following command
./build/make-distribution.sh --sbt-enabled -Pspark-3.5 -Pjdk-21
```
+To compile for Spark 4.0 with Java21, please use the following command
+```shell
+./build/make-distribution.sh -Pspark-4.0 -Pjdk-21
+```
+```shell
+./build/make-distribution.sh --sbt-enabled -Pspark-4.0 -Pjdk-21
+```
+
> **_NOTE:_** Celeborn supports automatic builds on linux aarch64 platform via
> `aarch64` profile. `aarch64` profile requires glibc version 3.4.21. There is
> potential problematic frame `C [libc.so.6+0x8412a]` for other glibc version
> like 2.x etc.
To build Celeborn with AWS S3 support MPU, please use the following command
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index 96b0b4aa6..37aeef118 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -346,6 +346,9 @@ if [ "$SBT_ENABLED" == "true" ]; then
sbt_build_client -Pspark-2.4
sbt_build_client -Pspark-3.4
sbt_build_client -Pspark-3.5
+ export JAVA_HOME=$JAVA17_HOME
+ sbt_build_client -Pspark-4.0
+ export JAVA_HOME=$JAVA8_HOME
sbt_build_client -Pflink-1.16
sbt_build_client -Pflink-1.17
sbt_build_client -Pflink-1.18
@@ -384,6 +387,9 @@ else
build_spark_client -Pspark-2.4
build_spark_client -Pspark-3.4
build_spark_client -Pspark-3.5
+ export JAVA_HOME=$JAVA17_HOME
+ build_spark_client -Pspark-4.0
+ export JAVA_HOME=$JAVA8_HOME
build_flink_client -Pflink-1.16
build_flink_client -Pflink-1.17
build_flink_client -Pflink-1.18
diff --git a/build/release/release.sh b/build/release/release.sh
index 6e98f46e0..37a6bb066 100755
--- a/build/release/release.sh
+++ b/build/release/release.sh
@@ -28,6 +28,7 @@ ASF_PASSWORD=${ASF_PASSWORD:?"ASF_PASSWORD is required"}
RELEASE_RC_NO=${RELEASE_RC_NO:?"RELEASE_RC_NO is required, e.g. 0"}
JAVA8_HOME=${JAVA8_HOME:?"JAVA8_HOME is required"}
JAVA11_HOME=${JAVA11_HOME:?"JAVA11_HOME is required"}
+JAVA17_HOME=${JAVA17_HOME:?"JAVA17_HOME is required"}
RELEASE_VERSION=$(awk -F'"' '/ThisBuild \/ version/ {print $2}' version.sbt)
@@ -110,6 +111,11 @@ upload_nexus_staging() {
echo "Deploying celeborn-client-spark-3-shaded_2.13"
${PROJECT_DIR}/build/sbt -Pspark-3.4 ++2.13.8
"clean;celeborn-client-spark-3-shaded/publishSigned"
+ export JAVA_HOME=$JAVA17_HOME
+ echo "Deploying celeborn-client-spark-4-shaded_2.13"
+ ${PROJECT_DIR}/build/sbt -Pspark-4.0
"clean;celeborn-client-spark-4-shaded/publishSigned"
+ export JAVA_HOME=$JAVA8_HOME
+
echo "Deploying celeborn-client-flink-1.16-shaded_2.12"
${PROJECT_DIR}/build/sbt -Pflink-1.16
"clean;celeborn-client-flink-1_16-shaded/publishSigned"
diff --git a/client-mr/mr/pom.xml b/client-mr/mr/pom.xml
index 9ebc5623f..7321c9dbb 100644
--- a/client-mr/mr/pom.xml
+++ b/client-mr/mr/pom.xml
@@ -55,6 +55,10 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
</dependencies>
<profiles>
diff --git a/client-spark/spark-3-columnar-shuffle/pom.xml
b/client-spark/spark-3-columnar-shuffle/pom.xml
index 5f0824437..2a5fdebfb 100644
--- a/client-spark/spark-3-columnar-shuffle/pom.xml
+++ b/client-spark/spark-3-columnar-shuffle/pom.xml
@@ -54,6 +54,16 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/client-spark/spark-3/pom.xml b/client-spark/spark-3/pom.xml
index 47fb034ee..0c3d798ea 100644
--- a/client-spark/spark-3/pom.xml
+++ b/client-spark/spark-3/pom.xml
@@ -86,6 +86,16 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git
a/client-spark/spark-4-columnar-shuffle/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala
b/client-spark/spark-4-columnar-shuffle/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala
index 706078684..70d174fe4 100644
---
a/client-spark/spark-4-columnar-shuffle/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala
+++
b/client-spark/spark-4-columnar-shuffle/src/main/scala/org/apache/spark/sql/execution/columnar/CelebornColumnType.scala
@@ -440,7 +440,7 @@ private[columnar] trait
DirectCopyCelebornColumnType[JvmType] extends CelebornCo
private[columnar] object CELEBORN_STRING
extends NativeCelebornColumnType(
- PhysicalStringType(SqlApiConf.get.defaultStringType.collationId),
+ PhysicalStringType(StringType.collationId),
8)
with DirectCopyCelebornColumnType[UTF8String] {
diff --git a/client-tez/tez/pom.xml b/client-tez/tez/pom.xml
index 1211303c7..3a85865d7 100644
--- a/client-tez/tez/pom.xml
+++ b/client-tez/tez/pom.xml
@@ -188,6 +188,10 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git
a/client/src/main/scala/org/apache/celeborn/client/ApplicationHeartbeater.scala
b/client/src/main/scala/org/apache/celeborn/client/ApplicationHeartbeater.scala
index 94c33d146..d8eef2ded 100644
---
a/client/src/main/scala/org/apache/celeborn/client/ApplicationHeartbeater.scala
+++
b/client/src/main/scala/org/apache/celeborn/client/ApplicationHeartbeater.scala
@@ -170,7 +170,7 @@ class ApplicationHeartbeater(
}
def stop(): Unit = {
- stopped.synchronized {
+ this.synchronized {
if (!stopped) {
// Stop appHeartbeat first
logInfo(s"Stop Application heartbeat $appId")
diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
index f953baa0c..35d26cec7 100644
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@@ -83,7 +83,7 @@
javax-websocket-client-impl/9.4.51.v20230217//javax-websocket-client-impl-9.4.51
javax-websocket-server-impl/9.4.51.v20230217//javax-websocket-server-impl-9.4.51.v20230217.jar
javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
javax.inject/1//javax.inject-1.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
+javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar
javax.websocket-api/1.0//javax.websocket-api-1.0.jar
javax.websocket-client-api/1.0//javax.websocket-client-api-1.0.jar
jaxb-api/2.2.11//jaxb-api-2.2.11.jar
diff --git a/dev/deps/dependencies-client-tez b/dev/deps/dependencies-client-tez
index e39a52587..21edcd5e2 100644
--- a/dev/deps/dependencies-client-tez
+++ b/dev/deps/dependencies-client-tez
@@ -64,7 +64,7 @@
jackson-module-scala_2.12/2.15.3//jackson-module-scala_2.12-2.15.3.jar
jackson-xc/1.9.2//jackson-xc-1.9.2.jar
jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
jakarta.activation/1.2.2//jakarta.activation-1.2.2.jar
-javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
+javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar
jaxb-api/2.2.2//jaxb-api-2.2.2.jar
jaxb-impl/2.2.3-1//jaxb-impl-2.2.3-1.jar
jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index eb08ed80f..25c923f23 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -48,11 +48,12 @@
jackson-module-jaxb-annotations/2.15.3//jackson-module-jaxb-annotations-2.15.3.j
jackson-module-scala_2.12/2.15.3//jackson-module-scala_2.12-2.15.3.jar
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
-jakarta.servlet-api/4.0.4//jakarta.servlet-api-4.0.4.jar
+jakarta.servlet-api/5.0.0//jakarta.servlet-api-5.0.0.jar
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
jakarta.xml.bind-api/2.3.3//jakarta.xml.bind-api-2.3.3.jar
javassist/3.29.0-GA//javassist-3.29.0-GA.jar
+javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar
jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar
jersey-client/2.39.1//jersey-client-2.39.1.jar
jersey-common/2.39.1//jersey-common-2.39.1.jar
diff --git a/docs/developers/sbt.md b/docs/developers/sbt.md
index d2751955e..e2f89a8c6 100644
--- a/docs/developers/sbt.md
+++ b/docs/developers/sbt.md
@@ -35,6 +35,7 @@ The following table indicates the compatibility of Celeborn
Spark and Flink clie
| Spark 3.3 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 3.4 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
| Spark 3.5 | ❌ | ✔ | ✔ |
✔ | ✔ | ✔ | ✔
|
+| Spark 4.0 | ❌ | ❌ | ❌ |
❌ | ❌ | ❌ | ✔
|
| Flink 1.16 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.17 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.18 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
@@ -311,6 +312,11 @@ Publish the shade client for Spark 3.5:
$ ./build/sbt -Pspark-3.5 celeborn-client-spark-3-shaded/publish
```
+Publish the shade client for Spark 4.0:
+```shell
+$ ./build/sbt -Pspark-4.0 celeborn-client-spark-4-shaded/publish
+```
+
Publish the shade client for Flink 1.18:
```shell
$ ./build/sbt -Pflink-1.18 celeborn-client-flink-1_18-shaded/publish
diff --git a/pom.xml b/pom.xml
index a75716f72..491705735 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,8 +121,11 @@
<swagger-ui.version>4.9.1</swagger-ui.version>
<jersey.version>2.39.1</jersey.version>
<jetty.version>9.4.56.v20240826</jetty.version>
- <jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
+ <javax.servlet-api.version>4.0.1</javax.servlet-api.version>
+ <!-- 6.0.0 requires JDK 11 -->
+ <jakarta.servlet-api.version>5.0.0</jakarta.servlet-api.version>
<jakarta.annotation-api.version>1.3.5</jakarta.annotation-api.version>
+ <jakarta.ws.rs-api.version>2.1.6</jakarta.ws.rs-api.version>
<!-- openapi-generator dependencies -->
<openapitools.jackson-databind-nullable.version>0.2.6</openapitools.jackson-databind-nullable.version>
<httpclient5.version>5.3.1</httpclient5.version>
@@ -149,7 +152,7 @@
<maven.plugin.shade.version>3.5.2</maven.plugin.shade.version>
<maven.plugin.spotless.version>2.24.1</maven.plugin.spotless.version>
<maven.plugin.surefire.version>3.0.0-M7</maven.plugin.surefire.version>
- <maven.plugin.silencer.version>1.7.13</maven.plugin.silencer.version>
+ <maven.plugin.silencer.version>1.7.19</maven.plugin.silencer.version>
<maven.plugin.resources.version>3.3.1</maven.plugin.resources.version>
<openapi.generator.version>7.8.0</openapi.generator.version>
@@ -498,6 +501,12 @@
</dependency>
<!-- RESTful service dependencies -->
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>${javax.servlet-api.version}</version>
+ </dependency>
+
<dependency>
<groupId>jakarta.servlet</groupId>
<artifactId>jakarta.servlet-api</artifactId>
@@ -510,6 +519,12 @@
<version>${jakarta.annotation-api.version}</version>
</dependency>
+ <dependency>
+ <groupId>jakarta.ws.rs</groupId>
+ <artifactId>jakarta.ws.rs-api</artifactId>
+ <version>${jakarta.ws.rs-api.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.openapitools</groupId>
<artifactId>jackson-databind-nullable</artifactId>
@@ -585,6 +600,10 @@
<groupId>jakarta.activation</groupId>
<artifactId>jakarta.activation-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -909,6 +928,7 @@
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
<spark.driver.memory>8g</spark.driver.memory>
<spark.shuffle.sort.io.plugin.class>${spark.shuffle.plugin.class}</spark.shuffle.sort.io.plugin.class>
+ <spark.ui.enabled>false</spark.ui.enabled>
</systemProperties>
<environmentVariables>
<CELEBORN_LOCAL_HOSTNAME>localhost</CELEBORN_LOCAL_HOSTNAME>
@@ -948,6 +968,7 @@
<java.io.tmpdir>${project.build.directory}/tmp</java.io.tmpdir>
<spark.driver.memory>8g</spark.driver.memory>
<spark.shuffle.sort.io.plugin.class>${spark.shuffle.plugin.class}</spark.shuffle.sort.io.plugin.class>
+ <spark.ui.enabled>false</spark.ui.enabled>
</systemProperties>
<environmentVariables>
<CELEBORN_LOCAL_HOSTNAME>localhost</CELEBORN_LOCAL_HOSTNAME>
@@ -1527,10 +1548,10 @@
</modules>
<properties>
<lz4-java.version>1.8.0</lz4-java.version>
- <scala.version>2.13.11</scala.version>
+ <scala.version>2.13.16</scala.version>
<scala.binary.version>2.13</scala.binary.version>
- <spark.version>4.0.0-preview2</spark.version>
- <zstd-jni.version>1.5.6-5</zstd-jni.version>
+ <spark.version>4.0.0</spark.version>
+ <zstd-jni.version>1.5.6-9</zstd-jni.version>
</properties>
</profile>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 15da584bc..32d6367c0 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -80,11 +80,13 @@ object Dependencies {
val swaggerUiVersion = "4.9.1"
val jerseyVersion = "2.39.1"
val jettyVersion = "9.4.56.v20240826"
- val jakartaServeletApiVersion = "4.0.4"
+ val javaxServletApiVersion = "4.0.1"
+ val jakartaServeletApiVersion = "5.0.0"
val openApiToolsJacksonBindNullableVersion = "0.2.6"
val httpClient5Version = "5.3.1"
val httpCore5Version = "5.2.4"
val jakartaAnnotationApiVersion = "1.3.5"
+ val jakartaWsRsApiVersion = "2.1.6"
val picocliVersion = "4.7.6"
val jmhVersion = "1.37"
@@ -183,11 +185,11 @@ object Dependencies {
val zstdJni = "com.github.luben" % "zstd-jni" % zstdJniVersion
val mybatis = "org.mybatis" % "mybatis" % mybatisVersion
val hikaricp = "com.zaxxer" % "HikariCP" % hikaricpVersion
- val jettyServer = "org.eclipse.jetty" % "jetty-server" % jettyVersion
excludeAll(
- ExclusionRule("javax.servlet", "javax.servlet-api"))
+ val jettyServer = "org.eclipse.jetty" % "jetty-server" % jettyVersion
val jettyServlet = "org.eclipse.jetty" % "jetty-servlet" % jettyVersion
excludeAll(
ExclusionRule("javax.servlet", "javax.servlet-api"))
val jettyProxy = "org.eclipse.jetty" % "jetty-proxy" % jettyVersion
+ val javaxServletApi = "javax.servlet" % "javax.servlet-api" %
javaxServletApiVersion
val jakartaServletApi = "jakarta.servlet" % "jakarta.servlet-api" %
jakartaServeletApiVersion
val jerseyServer = "org.glassfish.jersey.core" % "jersey-server" %
jerseyVersion excludeAll(
ExclusionRule("jakarta.xml.bind", "jakarta.xml.bind-api"))
@@ -206,6 +208,7 @@ object Dependencies {
val httpCore5 = "org.apache.httpcomponents.core5" % "httpcore5" %
httpCore5Version
val httpCore5H2 = "org.apache.httpcomponents.core5" % "httpcore5-h2" %
httpCore5Version
val jakartaAnnotationApi = "jakarta.annotation" % "jakarta.annotation-api" %
jakartaAnnotationApiVersion
+ val jakartaWsRsApi = "jakarta.ws.rs" % "jakarta.ws.rs-api" %
jakartaWsRsApiVersion
// Test dependencies
// https://www.scala-sbt.org/1.x/docs/Testing.html
@@ -280,10 +283,10 @@ object CelebornCommonSettings {
val SCALA_2_12_15 = "2.12.15"
val SCALA_2_12_17 = "2.12.17"
val SCALA_2_12_18 = "2.12.18"
- val scala213 = "2.13.5"
+ val SCALA_2_13_5 = "2.13.5"
val SCALA_2_13_8 = "2.13.8"
- val scala213_11 = "2.13.11"
- val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, SCALA_2_12_18, scala213, SCALA_2_13_8, scala213_11)
+ val SCALA_2_13_16 = "2.13.16"
+ val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, SCALA_2_12_18, SCALA_2_13_5, SCALA_2_13_8, SCALA_2_13_16)
val DEFAULT_SCALA_VERSION = SCALA_2_12_18
@@ -358,6 +361,7 @@ object CelebornCommonSettings {
Test / javaOptions ++= Seq(
"-Dspark.shuffle.sort.io.plugin.class="
+ sys.props.getOrElse("spark.shuffle.plugin.class",
"org.apache.spark.shuffle.sort.io.LocalDiskShuffleDataIO"),
+ "-Dspark.ui.enabled=false"
),
Test / envVars += ("IS_TESTING", "1")
@@ -706,7 +710,10 @@ object CelebornService {
Dependencies.jacksonDataFormatYam,
Dependencies.swaggerJaxrs2,
Dependencies.swaggerUi,
+ Dependencies.javaxServletApi,
Dependencies.jakartaServletApi,
+ Dependencies.jakartaAnnotationApi,
+ Dependencies.jakartaWsRsApi,
Dependencies.jerseyServer,
Dependencies.jerseyContainerServletCore,
Dependencies.jerseyHk2,
@@ -919,10 +926,10 @@ object Spark40 extends SparkClientProjects {
val sparkClientShadedProjectName = "celeborn-client-spark-4-shaded"
val lz4JavaVersion = "1.8.0"
- val sparkProjectScalaVersion = "2.13.11"
+ val sparkProjectScalaVersion = "2.13.16"
- val sparkVersion = "4.0.0-preview2"
- val zstdJniVersion = "1.5.6-5"
+ val sparkVersion = "4.0.0"
+ val zstdJniVersion = "1.5.6-9"
val scalaBinaryVersion = "2.13"
override val sparkColumnarShuffleVersion: String = "4"
@@ -983,6 +990,8 @@ trait SparkClientProjects {
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
+ Dependencies.javaxServletApi % "test",
+ Dependencies.jakartaServletApi % "test"
) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline %
"test")
)
}
@@ -1006,11 +1015,13 @@ trait SparkClientProjects {
// ref:
https://www.scala-sbt.org/1.x/docs/Multi-Project.html#Classpath+dependencies
.dependsOn(sparkColumnarCommon)
.dependsOn(sparkClient % "test->test;compile->compile")
- .dependsOn(CelebornClient.client % "test")
+ .dependsOn(CelebornClient.client % "test->test;compile->compile")
.settings(
commonSettings,
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
+ Dependencies.javaxServletApi % "test",
+ Dependencies.jakartaServletApi % "test"
) ++ commonUnitTestDependencies ++ Seq(Dependencies.mockitoInline %
"test")
)
}
@@ -1026,12 +1037,32 @@ trait SparkClientProjects {
.settings (
commonSettings,
libraryDependencies ++= Seq(
- "org.apache.spark" %% "spark-core" % sparkVersion % "test",
- "org.apache.spark" %% "spark-sql" % sparkVersion % "test",
- "org.apache.spark" %% "spark-core" % sparkVersion % "test"
classifier "tests" excludeAll(
+ "org.apache.spark" %% "spark-core" % sparkVersion % "test"
excludeAll(
+ ExclusionRule("jakarta.annotation", "jakarta.annotation-api"),
+ ExclusionRule("jakarta.servlet", "jakarta.servlet-api"),
+ ExclusionRule("jakarta.validation", "jakarta.validation-api"),
+ ExclusionRule("jakarta.ws.rs", "jakarta.ws.rs-api"),
+ ExclusionRule("jakarta.xml.bind", "jakarta.xml.bind-api"),
+ ExclusionRule("org.eclipse.jetty", "*"),
+ ExclusionRule("org.glassfish.hk2", "*"),
+ ExclusionRule("org.glassfish.jersey.core", "*"),
+ ExclusionRule("org.glassfish.jersey.containers", "*"),
+ ExclusionRule("org.glassfish.jersey.inject", "*"),
+ ExclusionRule("org.glassfish.jersey.media", "*")),
+ "org.apache.spark" %% "spark-sql" % sparkVersion % "test" excludeAll(
+ ExclusionRule("jakarta.annotation", "jakarta.annotation-api"),
+ ExclusionRule("jakarta.servlet", "jakarta.servlet-api"),
+ ExclusionRule("jakarta.validation", "jakarta.validation-api"),
+ ExclusionRule("jakarta.ws.rs", "jakarta.ws.rs-api"),
+ ExclusionRule("jakarta.xml.bind", "jakarta.xml.bind-api"),
+ ExclusionRule("org.eclipse.jetty", "*"),
+ ExclusionRule("org.glassfish.hk2", "*"),
+ ExclusionRule("org.glassfish.jersey.core", "*"),
+ ExclusionRule("org.glassfish.jersey.containers", "*"),
ExclusionRule("org.glassfish.jersey.inject", "*"),
- ExclusionRule("org.glassfish.jersey.core", "*")),
- "org.apache.spark" %% "spark-sql" % sparkVersion % "test" classifier
"tests"
+ ExclusionRule("org.glassfish.jersey.media", "*")),
+ Dependencies.javaxServletApi % "test",
+ Dependencies.jakartaServletApi % "test"
) ++ commonUnitTestDependencies
)
}
@@ -1346,7 +1377,8 @@ object MRClientProjects {
Dependencies.hadoopClientRuntime,
Dependencies.hadoopMapreduceClientApp,
Dependencies.jacksonJaxrsJsonProvider,
- Dependencies.jakartaActivationApi
+ Dependencies.jakartaActivationApi,
+ Dependencies.javaxServletApi
) ++ commonUnitTestDependencies,
dependencyOverrides += Dependencies.commonsCompress
)
@@ -1697,6 +1729,7 @@ object TezClientProjects {
Dependencies.tezApi,
Dependencies.hadoopCommon,
Dependencies.slf4jApi,
+ Dependencies.javaxServletApi
) ++ commonUnitTestDependencies,
dependencyOverrides += Dependencies.commonsCompress
)
diff --git a/service/pom.xml b/service/pom.xml
index 8d97d20a8..bc1c69547 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -36,15 +36,6 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
- <exclusions>
- <!--
- Use `jakarta.servlet-api` instead.
- -->
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
@@ -130,11 +121,26 @@
<artifactId>swagger-ui</artifactId>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
+
<dependency>
<groupId>jakarta.servlet</groupId>
<artifactId>jakarta.servlet-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>jakarta.ws.rs</groupId>
+ <artifactId>jakarta.ws.rs-api</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml
index 8b15b9c4f..c0a37ea6c 100644
--- a/tests/spark-it/pom.xml
+++ b/tests/spark-it/pom.xml
@@ -76,19 +76,111 @@
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
- <groupId>org.glassfish.jersey.inject</groupId>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.validation</groupId>
+ <artifactId>jakarta.validation-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.ws.rs</groupId>
+ <artifactId>jakarta.ws.rs-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.xml.bind</groupId>
+ <artifactId>jakarta.xml.bind-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta.annotation</groupId>
+ <artifactId>jakarta.annotation-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.validation</groupId>
+ <artifactId>jakarta.validation-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.ws.rs</groupId>
+ <artifactId>jakarta.ws.rs-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.xml.bind</groupId>
+ <artifactId>jakarta.xml.bind-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
<profiles>
diff --git
a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
index 41cbe072b..dd58934ac 100644
---
a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
+++
b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
@@ -17,21 +17,14 @@
package org.apache.celeborn.tests.spark
-import java.io.File
-import java.util.concurrent.atomic.AtomicBoolean
-
import scala.util.Random
-import org.apache.spark.{SPARK_VERSION, SparkConf, TaskContext}
-import org.apache.spark.shuffle.ShuffleHandle
-import org.apache.spark.shuffle.celeborn.{CelebornShuffleHandle,
ShuffleManagerHook, SparkUtils}
+import org.apache.spark.{SPARK_VERSION, SparkConf}
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.internal.SQLConf
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.scalatest.funsuite.AnyFunSuite
-import org.apache.celeborn.client.ShuffleClient
-import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.CelebornConf._
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.protocol.ShuffleMode
