This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 791e3e09 [#1006] feat(spark): Support Spark 3.4 (#1082)
791e3e09 is described below
commit 791e3e09e386391a3e000a9a9b005bc3f4fe8ab3
Author: summaryzb <[email protected]>
AuthorDate: Mon Aug 7 19:26:32 2023 +0800
[#1006] feat(spark): Support Spark 3.4 (#1082)
### What changes were proposed in this pull request?
1. Add maven profile `spark3.4`
2. Enable tests for Spark 3.4 in CI
3. Fix broken tests
### Why are the changes needed?
https://github.com/apache/incubator-uniffle/issues/1006
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass Test
---
.github/workflows/parallel.yml | 1 +
README.md | 2 +-
.../org/apache/uniffle/test/GetReaderTest.java | 4 +
pom.xml | 90 ++++++++++++++++++++++
4 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/parallel.yml b/.github/workflows/parallel.yml
index 132b56db..32d3f47e 100644
--- a/.github/workflows/parallel.yml
+++ b/.github/workflows/parallel.yml
@@ -58,6 +58,7 @@ jobs:
- spark3.2
- spark3.2.0
- spark3.3
+ - spark3.4
- mr-hadoop2.8
- mr-hadoop3.2
- tez
diff --git a/README.md b/README.md
index 39ecce98..17eb872b 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ The shuffle data is stored with index file and data file.
Data file has all bloc
## Supported Spark Version
-Currently supports Spark 2.3.x, Spark 2.4.x, Spark 3.0.x, Spark 3.1.x, Spark
3.2.x, Spark 3.3.x
+Currently supports Spark 2.3.x, Spark 2.4.x, Spark 3.0.x, Spark 3.1.x, Spark
3.2.x, Spark 3.3.x, Spark 3.4.x
Note: To support dynamic allocation, the patch(which is included in
patch/spark folder) should be applied to Spark
diff --git
a/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
b/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
index f0c8d879..1b92530e 100644
---
a/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
+++
b/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
@@ -304,6 +304,10 @@ public class GetReaderTest extends IntegrationTestBase {
return 0;
}
+ public int numPartitions() {
+ return 0;
+ }
+
@Override
public Option<String> getKillReason() {
return null;
diff --git a/pom.xml b/pom.xml
index 4dfe1c61..2ac91097 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1535,6 +1535,96 @@
</dependencyManagement>
</profile>
+ <profile>
+ <id>spark3.4</id>
+ <properties>
+ <scala.binary.version>2.12</scala.binary.version>
+ <spark.version>3.4.1</spark.version>
+ <client.type>3</client.type>
+ <jackson.version>2.14.2</jackson.version>
+ <log4j.core.version>2.19.0</log4j.core.version>
+ </properties>
+ <modules>
+ <module>client-spark/common</module>
+ <module>client-spark/spark3</module>
+ <module>integration-test/spark-common</module>
+ <module>integration-test/spark3</module>
+ </modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark3</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-spark-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
+
<profile>
<id>spark3.2.0</id>
<properties>
