This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new d701d3ae2 [CELEBORN-912] Support build with Spark 3.5
d701d3ae2 is described below
commit d701d3ae2c40e42de7a937362a895333b1763463
Author: zhouyifan279 <[email protected]>
AuthorDate: Tue Aug 29 03:15:12 2023 +0000
[CELEBORN-912] Support build with Spark 3.5
### What changes were proposed in this pull request?
Support build with Spark 3.5
### Why are the changes needed?
Keep up with upstream.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Build with `mvn` and `sbt` locally.
Closes #1850 from zhouyifan279/build-spark-3.5.
Authored-by: zhouyifan279 <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/maven.yml | 1 +
.github/workflows/sbt.yml | 19 +++++++++++++++++++
build/sbt-config/repositories | 1 +
pom.xml | 24 ++++++++++++++++++++++++
project/CelebornBuild.scala | 18 +++++++++++++++++-
tests/spark-it/pom.xml | 11 +++++++++++
6 files changed, 73 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index e443ae005..06aa5cb04 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -94,6 +94,7 @@ jobs:
- '3.2'
- '3.3'
- '3.4'
+ - '3.5'
exclude:
# SPARK-33772: Spark supports JDK 17 since 3.3.0
- java: 17
diff --git a/.github/workflows/sbt.yml b/.github/workflows/sbt.yml
index c4ae29268..f6271e63e 100644
--- a/.github/workflows/sbt.yml
+++ b/.github/workflows/sbt.yml
@@ -161,6 +161,25 @@ jobs:
- spark: '3.4'
scala: '2.13.5'
java: 17
+ # Spark 3.5
+ - spark: '3.5'
+ scala: '2.12.18'
+ java: 8
+ - spark: '3.5'
+ scala: '2.12.18'
+ java: 11
+ - spark: '3.5'
+ scala: '2.12.18'
+ java: 17
+ - spark: '3.5'
+ scala: '2.13.5'
+ java: 8
+ - spark: '3.5'
+ scala: '2.13.5'
+ java: 11
+ - spark: '3.5'
+ scala: '2.13.5'
+ java: 17
steps:
- uses: actions/checkout@v2
- name: Setup JDK ${{ matrix.java }}
diff --git a/build/sbt-config/repositories b/build/sbt-config/repositories
index 713581a32..ffa3fc3f7 100644
--- a/build/sbt-config/repositories
+++ b/build/sbt-config/repositories
@@ -32,3 +32,4 @@
bintray-typesafe-sbt-plugin-releases:
https://dl.bintray.com/typesafe/sbt-plugins/,
[organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
bintray-spark-packages: https://dl.bintray.com/spark-packages/maven/
typesafe-releases: https://repo.typesafe.com/typesafe/releases/
+ apache-release-candidates:
https://repository.apache.org/content/repositories/staging/
diff --git a/pom.xml b/pom.xml
index 5ec9800d4..246219871 100644
--- a/pom.xml
+++ b/pom.xml
@@ -443,6 +443,11 @@
<name>${mavenCentralName}</name>
<url>${mavenCentralUrl}</url>
</repository>
+ <repository>
+ <id>apache-release-candidates</id>
+ <name>Apache Release Candidates Repository</name>
+ <url>https://repository.apache.org/content/repositories/staging</url>
+ </repository>
</repositories>
<pluginRepositories>
@@ -991,6 +996,25 @@
</properties>
</profile>
+ <profile>
+ <id>spark-3.5</id>
+ <modules>
+ <module>client-spark/common</module>
+ <module>client-spark/spark-3</module>
+ <!-- TODO solve compilation error when build spark-3-columnar-shuffle
with Spark 3.5 -->
+ <!-- <module>client-spark/spark-3-columnar-shuffle</module> -->
+ <module>client-spark/spark-3-shaded</module>
+ <module>tests/spark-it</module>
+ </modules>
+ <properties>
+ <lz4-java.version>1.8.0</lz4-java.version>
+ <scala.version>2.12.18</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
+ <spark.version>3.5.0</spark.version>
+ <zstd-jni.version>1.5.5-4</zstd-jni.version>
+ </properties>
+ </profile>
+
<profile>
<id>jdk-8</id>
<activation>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index 56e05e509..e00f4d754 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -112,8 +112,9 @@ object CelebornCommonSettings {
val SCALA_2_12_10 = "2.12.10"
val SCALA_2_12_15 = "2.12.15"
val SCALA_2_12_17 = "2.12.17"
+ val SCALA_2_12_18 = "2.12.18"
val scala213 = "2.13.5"
- val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, scala213)
+ val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15,
SCALA_2_12_17, SCALA_2_12_18, scala213)
val DEFAULT_SCALA_VERSION = SCALA_2_12_15
@@ -241,6 +242,7 @@ object Utils {
case Some("spark-3.2") => Some(Spark32)
case Some("spark-3.3") => Some(Spark33)
case Some("spark-3.4") => Some(Spark34)
+ case Some("spark-3.5") => Some(Spark35)
case _ => None
}
@@ -530,6 +532,20 @@ object Spark34 extends SparkClientProjects {
}
}
+object Spark35 extends SparkClientProjects {
+
+ val sparkClientProjectPath = "client-spark/spark-3"
+ val sparkClientProjectName = "celeborn-client-spark-3"
+ val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
+ val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
+
+ val lz4JavaVersion = "1.8.0"
+ val sparkProjectScalaVersion = "2.12.18"
+
+ val sparkVersion = "3.5.0"
+ val zstdJniVersion = "1.5.5-4"
+}
+
trait SparkClientProjects {
val sparkClientProjectPath: String
diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml
index b3b0b8e92..783cbe100 100644
--- a/tests/spark-it/pom.xml
+++ b/tests/spark-it/pom.xml
@@ -150,5 +150,16 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>spark-3.5</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+
<artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
