This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new caf710433 [#1083] feat(spark): Support Spark 3.5 (#1223)
caf710433 is described below
commit caf71043378a2aeedee43b9436f614a884d18935
Author: summaryzb <[email protected]>
AuthorDate: Wed Oct 4 05:15:08 2023 -0500
[#1083] feat(spark): Support Spark 3.5 (#1223)
### What changes were proposed in this pull request?
1. Add maven profile `spark3.5`
2. Enable tests for Spark 3.5 in CI
### Why are the changes needed?
https://github.com/apache/incubator-uniffle/issues/1083
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass Test
---
.github/workflows/parallel.yml | 1 +
README.md | 2 +-
pom.xml | 90 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 92 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/parallel.yml b/.github/workflows/parallel.yml
index 32d3f47ea..1e6e0b429 100644
--- a/.github/workflows/parallel.yml
+++ b/.github/workflows/parallel.yml
@@ -59,6 +59,7 @@ jobs:
- spark3.2.0
- spark3.3
- spark3.4
+ - spark3.5
- mr-hadoop2.8
- mr-hadoop3.2
- tez
diff --git a/README.md b/README.md
index 6686dde82..0007510e5 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ The shuffle data is stored with index file and data file.
Data file has all bloc
## Supported Spark Version
-Currently supports Spark 2.3.x, Spark 2.4.x, Spark 3.0.x, Spark 3.1.x, Spark
3.2.x, Spark 3.3.x, Spark 3.4.x
+Currently supports Spark 2.3.x, Spark 2.4.x, Spark 3.0.x, Spark 3.1.x, Spark
3.2.x, Spark 3.3.x, Spark 3.4.x, Spark 3.5.x
Note: To support dynamic allocation, the patch(which is included in
patch/spark folder) should be applied to Spark
diff --git a/pom.xml b/pom.xml
index 719cd1397..b4431109a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1628,6 +1628,96 @@
</dependencyManagement>
</profile>
+ <profile>
+ <id>spark3.5</id>
+ <properties>
+ <scala.binary.version>2.12</scala.binary.version>
+ <spark.version>3.5.0</spark.version>
+ <client.type>3</client.type>
+ <jackson.version>2.15.2</jackson.version>
+ <log4j.core.version>2.19.0</log4j.core.version>
+ </properties>
+ <modules>
+ <module>client-spark/common</module>
+ <module>client-spark/spark3</module>
+ <module>integration-test/spark-common</module>
+ <module>integration-test/spark3</module>
+ </modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark3</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-spark-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
+
<profile>
<id>spark3.2.0</id>
<properties>
