This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 7af3126c7 Support Spark3.5 with JDK21
7af3126c7 is described below
commit 7af3126c7e09f01a43fd5f46df5956ae4ed2a524
Author: zky.zhoukeyong <[email protected]>
AuthorDate: Wed Mar 27 18:42:16 2024 +0800
Support Spark3.5 with JDK21
### What changes were proposed in this pull request?
Compile Spark-3.5 with
`./build/make-distribution.sh -Pspark-3.5 -Pjdk-21`
or
`./build/make-distribution.sh --sbt-enabled -Pspark-3.5 -Pjdk-21`
### Why are the changes needed?
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
manual tests
Closes #2385 from waitinfuture/1327.
Authored-by: zky.zhoukeyong <[email protected]>
Signed-off-by: zky.zhoukeyong <[email protected]>
---
README.md | 9 +++++
build/make-distribution.sh | 2 +
dev/dependencies.sh | 3 +-
dev/deps/dependencies-client-flink-1.14 | 4 +-
dev/deps/dependencies-client-flink-1.15 | 4 +-
dev/deps/dependencies-client-flink-1.17 | 4 +-
dev/deps/dependencies-client-flink-1.18 | 4 +-
dev/deps/dependencies-client-flink-1.19 | 4 +-
dev/deps/dependencies-client-mr | 4 +-
dev/deps/dependencies-server | 4 +-
pom.xml | 66 ++++++++++++++++++++++++++++++---
project/CelebornBuild.scala | 2 +-
12 files changed, 89 insertions(+), 21 deletions(-)
diff --git a/README.md b/README.md
index 9da2d5160..121640bd2 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,15 @@ To compile the client for Spark 2.4 with Scala 2.12, please
use the following co
./build/make-distribution.sh -DskipTests -Pspark-2.4
-Dscala.version=${scala.version} -Dscala.binary.version=2.12
```
+To compile for Spark 3.5 with Java21, please use the following command
+```shell
+./build/make-distribution.sh -Pspark-3.5 -Pjdk-21
+```
+```shell
+./build/make-distribution.sh --sbt-enabled -Pspark-3.5 -Pjdk-21
+```
+
+
### Package Details
Build procedure will create a compressed package.
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index ce45126fe..5a9333199 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -306,6 +306,7 @@ if [ "$SBT_ENABLED" == "true" ]; then
if [ "$RELEASE" == "true" ]; then
sbt_build_client -Pspark-2.4
sbt_build_client -Pspark-3.4
+ sbt_build_client -Pspark-3.5
sbt_build_client -Pflink-1.14
sbt_build_client -Pflink-1.15
sbt_build_client -Pflink-1.17
@@ -338,6 +339,7 @@ else
build_service
build_spark_client -Pspark-2.4
build_spark_client -Pspark-3.4
+ build_spark_client -Pspark-3.5
build_flink_client -Pflink-1.14
build_flink_client -Pflink-1.15
build_flink_client -Pflink-1.17
diff --git a/dev/dependencies.sh b/dev/dependencies.sh
index 5da33d71a..11fada9ef 100755
--- a/dev/dependencies.sh
+++ b/dev/dependencies.sh
@@ -250,6 +250,7 @@ if [ "$REPLACE" == "true" ]; then
exit 0
fi
-if [ "$CHECK" == "true" ]; then
+# skip check for sbt server now until sbt_build_server_classpath is refactored
+if [ "$CHECK" == "true" -a "$MODULE" != "server" ]; then
check_diff
fi
diff --git a/dev/deps/dependencies-client-flink-1.14
b/dev/deps/dependencies-client-flink-1.14
index 1c6256450..d57febdfb 100644
--- a/dev/deps/dependencies-client-flink-1.14
+++ b/dev/deps/dependencies-client-flink-1.14
@@ -77,8 +77,8 @@ ratis-client/2.5.1//ratis-client-2.5.1.jar
ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/2.2//snakeyaml-2.2.jar
diff --git a/dev/deps/dependencies-client-flink-1.15
b/dev/deps/dependencies-client-flink-1.15
index 1c6256450..d57febdfb 100644
--- a/dev/deps/dependencies-client-flink-1.15
+++ b/dev/deps/dependencies-client-flink-1.15
@@ -77,8 +77,8 @@ ratis-client/2.5.1//ratis-client-2.5.1.jar
ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/2.2//snakeyaml-2.2.jar
diff --git a/dev/deps/dependencies-client-flink-1.17
b/dev/deps/dependencies-client-flink-1.17
index 1c6256450..d57febdfb 100644
--- a/dev/deps/dependencies-client-flink-1.17
+++ b/dev/deps/dependencies-client-flink-1.17
@@ -77,8 +77,8 @@ ratis-client/2.5.1//ratis-client-2.5.1.jar
ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/2.2//snakeyaml-2.2.jar
diff --git a/dev/deps/dependencies-client-flink-1.18
b/dev/deps/dependencies-client-flink-1.18
index 1c6256450..d57febdfb 100644
--- a/dev/deps/dependencies-client-flink-1.18
+++ b/dev/deps/dependencies-client-flink-1.18
@@ -77,8 +77,8 @@ ratis-client/2.5.1//ratis-client-2.5.1.jar
ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/2.2//snakeyaml-2.2.jar
diff --git a/dev/deps/dependencies-client-flink-1.19
b/dev/deps/dependencies-client-flink-1.19
index 1c6256450..d57febdfb 100644
--- a/dev/deps/dependencies-client-flink-1.19
+++ b/dev/deps/dependencies-client-flink-1.19
@@ -77,8 +77,8 @@ ratis-client/2.5.1//ratis-client-2.5.1.jar
ratis-common/2.5.1//ratis-common-2.5.1.jar
ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/2.2//snakeyaml-2.2.jar
diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
index 010599e0b..4f7b59ff9 100644
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@@ -186,8 +186,8 @@ ratis-proto/2.5.1//ratis-proto-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
re2j/1.1//re2j-1.1.jar
reload4j/1.2.22//reload4j-1.2.22.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
slf4j-reload4j/1.7.36//slf4j-reload4j-1.7.36.jar
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index c0f7f0a26..eadf37b17 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -95,8 +95,8 @@ ratis-shell/2.5.1//ratis-shell-2.5.1.jar
ratis-thirdparty-misc/1.0.4//ratis-thirdparty-misc-1.0.4.jar
reflections/0.10.2//reflections-0.10.2.jar
rocksdbjni/8.11.3//rocksdbjni-8.11.3.jar
-scala-library/2.12.15//scala-library-2.12.15.jar
-scala-reflect/2.12.15//scala-reflect-2.12.15.jar
+scala-library/2.12.18//scala-library-2.12.18.jar
+scala-reflect/2.12.18//scala-reflect-2.12.18.jar
shims/0.9.32//shims-0.9.32.jar
slf4j-api/1.7.36//slf4j-api-1.7.36.jar
snakeyaml/2.2//snakeyaml-2.2.jar
diff --git a/pom.xml b/pom.xml
index 14ec1775e..fdd60e69f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,11 +58,11 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>8</java.version>
- <scala.version>2.12.15</scala.version>
+ <scala.version>2.12.18</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
- <maven.version>3.8.8</maven.version>
+ <maven.version>3.9.6</maven.version>
<flink.version>1.14.6</flink.version>
<spark.version>3.3.4</spark.version>
@@ -111,17 +111,17 @@
<maven.plugin.antrun.version>3.0.0</maven.plugin.antrun.version>
<maven.plugin.clean.version>3.2.0</maven.plugin.clean.version>
- <maven.plugin.compiler.version>3.11.0</maven.plugin.compiler.version>
+ <maven.plugin.compiler.version>3.12.1</maven.plugin.compiler.version>
<maven.plugin.dependency.version>3.1.1</maven.plugin.dependency.version>
<maven.plugin.flatten.version>1.3.0</maven.plugin.flatten.version>
- <maven.plugin.jacoco.version>0.8.7</maven.plugin.jacoco.version>
+ <maven.plugin.jacoco.version>0.8.11</maven.plugin.jacoco.version>
<maven.plugin.jar.version>3.0.2</maven.plugin.jar.version>
<maven.plugin.os.version>1.7.0</maven.plugin.os.version>
<maven.plugin.protobuf.version>0.6.1</maven.plugin.protobuf.version>
<maven.plugin.rat.version>0.13</maven.plugin.rat.version>
<maven.plugin.scala.version>4.7.2</maven.plugin.scala.version>
<maven.plugin.scalatest.version>2.1.0</maven.plugin.scalatest.version>
- <maven.plugin.shade.version>3.4.0</maven.plugin.shade.version>
+ <maven.plugin.shade.version>3.5.2</maven.plugin.shade.version>
<maven.plugin.spotless.version>2.24.1</maven.plugin.spotless.version>
<maven.plugin.surefire.version>3.0.0-M7</maven.plugin.surefire.version>
<maven.plugin.silencer.version>1.7.13</maven.plugin.silencer.version>
@@ -1326,6 +1326,62 @@
</build>
</profile>
+ <profile>
+ <id>jdk-21</id>
+ <activation>
+ <jdk>21</jdk>
+ </activation>
+ <properties>
+ <java.version>21</java.version>
+ <error-prone.version>2.25.0</error-prone.version>
+ </properties>
+ <build>
+ <plugins>
+ <!-- Based on
https://github.com/google/error-prone/blob/f8e33bc460be82ab22256a7ef8b979d7a2cacaba/docs/installation.md#jdk-16
-->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven.plugin.compiler.version}</version>
+ <configuration>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
+ <encoding>UTF-8</encoding>
+ <compilerArgs>
+ <arg>-XDcompilePolicy=simple</arg>
+ <arg>-Xplugin:ErrorProne \
+ -XepExcludedPaths:.*/target/generated-sources/protobuf/.* \
+ -Xep:FutureReturnValueIgnored:OFF \
+ -Xep:TypeParameterUnusedInFormals:OFF \
+ -Xep:UnusedVariable:OFF \
+ -Xep:StringSplitter:OFF \
+ -Xep:EmptyBlockTag:OFF \
+ -Xep:EqualsGetClass:OFF \
+ -Xep:MissingSummary:OFF \
+ -Xep:BadImport:OFF</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED</arg>
+
<arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED</arg>
+
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED</arg>
+
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED</arg>
+ </compilerArgs>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_core</artifactId>
+ <version>${error-prone.version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<profile>
<id>flink-1.14</id>
<modules>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 7b5474345..0cd12ac3e 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -160,7 +160,7 @@ object CelebornCommonSettings {
val scala213 = "2.13.5"
val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, SCALA_2_12_18, scala213)
- val DEFAULT_SCALA_VERSION = SCALA_2_12_15
+ val DEFAULT_SCALA_VERSION = SCALA_2_12_18
val projectScalaVersion = defaultScalaVersion()
