This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d6c6bba877 Revert "[GLUTEN-7336][CORE] Bump Spark version to v3.5.3
(#7537) (#7930)
d6c6bba877 is described below
commit d6c6bba877b677e00ff37e086522a492d9e64a30
Author: Yang Zhang <[email protected]>
AuthorDate: Wed Nov 13 14:45:24 2024 +0800
Revert "[GLUTEN-7336][CORE] Bump Spark version to v3.5.3 (#7537) (#7930)
Revert Spark release version from v3.5.3 to v3.5.2 due to datalake issue.
---
.github/workflows/util/install_spark_resources.sh | 20 +++++++--------
.github/workflows/velox_backend.yml | 30 +++++++++++-----------
pom.xml | 6 ++---
.../sql/shims/spark35/SparkShimProvider.scala | 2 +-
tools/gluten-it/pom.xml | 2 +-
5 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/util/install_spark_resources.sh
b/.github/workflows/util/install_spark_resources.sh
index d5f413ee21..dd2afec821 100755
--- a/.github/workflows/util/install_spark_resources.sh
+++ b/.github/workflows/util/install_spark_resources.sh
@@ -63,26 +63,26 @@ case "$1" in
3.5)
# Spark-3.5
cd ${INSTALL_DIR} && \
- wget -nv
https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz
&& \
- tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz
spark-3.5.3-bin-hadoop3/jars/ && \
- rm -rf spark-3.5.3-bin-hadoop3.tgz && \
+ wget -nv
https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz
&& \
+ tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz
spark-3.5.2-bin-hadoop3/jars/ && \
+ rm -rf spark-3.5.2-bin-hadoop3.tgz && \
mkdir -p
${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12
&& \
- wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz
&& \
- tar --strip-components=1 -xf v3.5.3.tar.gz
spark-3.5.3/sql/core/src/test/resources/ && \
+ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz
&& \
+ tar --strip-components=1 -xf v3.5.2.tar.gz
spark-3.5.2/sql/core/src/test/resources/ && \
mkdir -p shims/spark35/spark_home/ && \
mv sql shims/spark35/spark_home/
;;
3.5-scala2.13)
# Spark-3.5, scala 2.13
cd ${INSTALL_DIR} && \
- wget -nv
https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz
&& \
- tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz
spark-3.5.3-bin-hadoop3/jars/ && \
- rm -rf spark-3.5.3-bin-hadoop3.tgz && \
+ wget -nv
https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz
&& \
+ tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz
spark-3.5.2-bin-hadoop3/jars/ && \
+ rm -rf spark-3.5.2-bin-hadoop3.tgz && \
mkdir -p
${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13
&& \
- wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz
&& \
- tar --strip-components=1 -xf v3.5.3.tar.gz
spark-3.5.3/sql/core/src/test/resources/ && \
+ wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz
&& \
+ tar --strip-components=1 -xf v3.5.2.tar.gz
spark-3.5.2/sql/core/src/test/resources/ && \
mkdir -p shims/spark35/spark_home/ && \
mv sql shims/spark35/spark_home/
;;
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index c316dba6f8..4b7f5836e7 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -754,19 +754,19 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Prepare spark.test.home for Spark 3.5.3 (other tests)
+ - name: Prepare spark.test.home for Spark 3.5.2 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.3 cython && \
+ pip3 install pyspark==3.5.2 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.3 (other tests)
+ - name: Build and Run unit test for Spark 3.5.2 (other tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
- name: Upload test report
@@ -792,15 +792,15 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Prepare spark.test.home for Spark 3.5.3 (other tests)
+ - name: Prepare spark.test.home for Spark 3.5.2 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5-scala2.13
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.3 cython && \
+ pip3 install pyspark==3.5.2 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.3 with scala-2.13 (other
tests)
+ - name: Build and Run unit test for Spark 3.5.2 with scala-2.13 (other
tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.13
@@ -830,13 +830,13 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Prepare spark.test.home for Spark 3.5.3 (slow tests)
+ - name: Prepare spark.test.home for Spark 3.5.2 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
- - name: Build and Run unit test for Spark 3.5.3 (slow tests)
+ - name: Build and Run unit test for Spark 3.5.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
- $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Phudi -Pspark-ut \
-DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
@@ -862,15 +862,15 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Prepare spark.test.home for Spark 3.5.3 (other tests)
+ - name: Prepare spark.test.home for Spark 3.5.2 (other tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
dnf module -y install python39 && \
alternatives --set python3 /usr/bin/python3.9 && \
pip3 install setuptools && \
- pip3 install pyspark==3.5.3 cython && \
+ pip3 install pyspark==3.5.2 cython && \
pip3 install pandas pyarrow
- - name: Build and Run unit test for Spark 3.5.3 (other tests)
+ - name: Build and Run unit test for Spark 3.5.2 (other tests)
run: |
cd $GITHUB_WORKSPACE/
export SPARK_SCALA_VERSION=2.12
@@ -899,10 +899,10 @@ jobs:
with:
name: arrow-jars-centos-7-${{github.sha}}
path: /root/.m2/repository/org/apache/arrow/
- - name: Prepare spark.test.home for Spark 3.5.3 (slow tests)
+ - name: Prepare spark.test.home for Spark 3.5.2 (slow tests)
run: |
bash .github/workflows/util/install_spark_resources.sh 3.5
- - name: Build and Run unit test for Spark 3.5.3 (slow tests)
+ - name: Build and Run unit test for Spark 3.5.2 (slow tests)
run: |
cd $GITHUB_WORKSPACE/
$MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn
-Piceberg -Pdelta -Pspark-ut \
diff --git a/pom.xml b/pom.xml
index d3c67f231e..030828a327 100644
--- a/pom.xml
+++ b/pom.xml
@@ -336,11 +336,11 @@
<properties>
<sparkbundle.version>3.5</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark35</sparkshim.artifactId>
- <spark.version>3.5.3</spark.version>
+ <spark.version>3.5.2</spark.version>
<iceberg.version>1.5.0</iceberg.version>
<delta.package.name>delta-spark</delta.package.name>
- <delta.version>3.2.1</delta.version>
- <delta.binary.version>32</delta.binary.version>
+ <delta.version>3.2.0</delta.version>
+ <delta.binary.version>32</delta.binary.version>
<hudi.version>0.15.0</hudi.version>
<fasterxml.version>2.15.1</fasterxml.version>
<hadoop.version>3.3.4</hadoop.version>
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
index ad99f7be25..eab32ab9d0 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala
@@ -20,7 +20,7 @@ import org.apache.gluten.sql.shims.{SparkShimDescriptor,
SparkShims}
import org.apache.gluten.sql.shims.spark35.SparkShimProvider.DESCRIPTOR
object SparkShimProvider {
- val DESCRIPTOR = SparkShimDescriptor(3, 5, 3)
+ val DESCRIPTOR = SparkShimDescriptor(3, 5, 2)
}
class SparkShimProvider extends org.apache.gluten.sql.shims.SparkShimProvider {
diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
index 70b8d20c5b..bad4d6087f 100644
--- a/tools/gluten-it/pom.xml
+++ b/tools/gluten-it/pom.xml
@@ -164,7 +164,7 @@
<profile>
<id>spark-3.5</id>
<properties>
- <spark.version>3.5.3</spark.version>
+ <spark.version>3.5.2</spark.version>
<scala.library.version>2.12.18</scala.library.version>
</properties>
</profile>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
