This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 94ecd9e25a [GLUTEN-7336][CORE] Bump Spark version to v3.5.3 (#7537)
94ecd9e25a is described below
commit 94ecd9e25af1e036c6431b1873b72efb0a907d6d
Author: Yang Zhang <[email protected]>
AuthorDate: Tue Oct 22 16:53:38 2024 +0800
[GLUTEN-7336][CORE] Bump Spark version to v3.5.3 (#7537)
---
.github/workflows/util/install_spark_resources.sh | 20 ++++++++++----------
.github/workflows/velox_backend.yml | 14 +++++++-------
pom.xml | 8 ++++----
.../gluten/sql/shims/spark35/SparkShimProvider.scala | 2 +-
tools/gluten-it/pom.xml | 2 +-
5 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/util/install_spark_resources.sh
b/.github/workflows/util/install_spark_resources.sh
index dd2afec821..d5f413ee21 100755
--- a/.github/workflows/util/install_spark_resources.sh
+++ b/.github/workflows/util/install_spark_resources.sh
@@ -63,26 +63,26 @@ case "$1" in
3.5)
# Spark-3.5
cd ${INSTALL_DIR} && \
- wget -nv
https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz
&& \
- tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz
spark-3.5.2-bin-hadoop3/jars/ && \
- rm -rf spark-3.5.2-bin-hadoop3.tgz && \
+ wget -nv
https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz
&& \
+ tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz
spark-3.5.3-bin-hadoop3/jars/ && \
+ rm -rf spark-3.5.3-bin-hadoop3.tgz && \
mkdir -p
${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12
&& \
- wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz
&& \
- tar --strip-components=1 -xf v3.5.2.tar.gz
spark-3.5.2/sql/core/src/test/resources/ && \
+ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz
&& \
+ tar --strip-components=1 -xf v3.5.3.tar.gz
spark-3.5.3/sql/core/src/test/resources/ && \
mkdir -p shims/spark35/spark_home/ && \
mv sql shims/spark35/spark_home/
;;
3.5-scala2.13)
# Spark-3.5, scala 2.13
cd ${INSTALL_DIR} && \
- wget -nv
https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz
&& \
- tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz
spark-3.5.2-bin-hadoop3/jars/ && \
- rm -rf spark-3.5.2-bin-hadoop3.tgz && \
+ wget -nv
https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz
&& \
+ tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz
spark-3.5.3-bin-hadoop3/jars/ && \
+ rm -rf spark-3.5.3-bin-hadoop3.tgz && \
mkdir -p
${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13
&& \
- wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz
&& \
- tar --strip-components=1 -xf v3.5.2.tar.gz
spark-3.5.2/sql/core/src/test/resources/ && \
+ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz
&& \
+ tar --strip-components=1 -xf v3.5.3.tar.gz
spark-3.5.3/sql/core/src/test/resources/ && \
mkdir -p shims/spark35/spark_home/ && \
mv sql shims/spark35/spark_home/
;;
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 8637c92ffe..dd31b8e77b 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -860,13 +860,13 @@ jobs:
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.2 cython && \
+ pip3 install pyspark==3.5.3 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.2 (other tests)
+ - name: Build and Run unit test for Spark 3.5.3 (other tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -906,9 +906,9 @@ jobs:
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.2 cython && \
+ pip3 install pyspark==3.5.3 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.2 with scala-2.13 (other
tests)
+ - name: Build and Run unit test for Spark 3.5.3 with scala-2.13 (other
tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -949,10 +949,10 @@ jobs:
- name: Prepare spark.test.home for Spark 3.5.2 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
- - name: Build and Run unit test for Spark 3.5.2 (slow tests)
+ - name: Build and Run unit test for Spark 3.5.3 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
diff --git a/pom.xml b/pom.xml
index a3ce1c6f70..ea8a9f9808 100644
--- a/pom.xml
+++ b/pom.xml
@@ -341,12 +341,12 @@
<sparkbundle.version>3.5</sparkbundle.version>
<sparkshim.module.name>spark35</sparkshim.module.name>
<sparkshim.artifactId>spark-sql-columnar-shims-spark35</sparkshim.artifactId>
- <spark.version>3.5.2</spark.version>
+ <spark.version>3.5.3</spark.version>
<iceberg.version>1.5.0</iceberg.version>
<delta.package.name>delta-spark</delta.package.name>
- <delta.version>3.2.0</delta.version>
- <delta.binary.version>32</delta.binary.version>
- <hudi.version>0.15.0</hudi.version>
+ <delta.version>3.2.1</delta.version>
+ <delta.binary.version>32</delta.binary.version>
+ <hudi.version>0.15.0</hudi.version>
<fasterxml.version>2.15.1</fasterxml.version>
<hadoop.version>3.3.4</hadoop.version>
<antlr4.version>4.9.3</antlr4.version>
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
index eab32ab9d0..ad99f7be25 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.sql.shims.{SparkShimDescriptor,
SparkShims}
import org.apache.gluten.sql.shims.spark35.SparkShimProvider.DESCRIPTOR
object SparkShimProvider {
- val DESCRIPTOR = SparkShimDescriptor(3, 5, 2)
+ val DESCRIPTOR = SparkShimDescriptor(3, 5, 3)
}
class SparkShimProvider extends org.apache.gluten.sql.shims.SparkShimProvider {
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index bad4d6087f..70b8d20c5b 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -164,7 +164,7 @@
<profile>
<id>spark-3.5</id>
<properties>
- <spark.version>3.5.2</spark.version>
+ <spark.version>3.5.3</spark.version>
<scala.library.version>2.12.18</scala.library.version>
</properties>
</profile>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
