This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 991e72dc7 chore: Bump Spark from 3.5.6 to 3.5.7 (#2574)
991e72dc7 is described below
commit 991e72dc77a40bc0b0bfc9f585d494d37144ec5e
Author: Fu Chen <[email protected]>
AuthorDate: Thu Oct 16 22:28:42 2025 +0800
chore: Bump Spark from 3.5.6 to 3.5.7 (#2574)
---
.github/actions/setup-spark-builder/action.yaml | 2 +-
.github/workflows/iceberg_spark_test.yml | 8 ++++----
.github/workflows/pr_build_linux.yml | 12 ++++++------
.github/workflows/spark_sql_test.yml | 6 +++---
.github/workflows/spark_sql_test_native_datafusion.yml | 2 +-
.github/workflows/spark_sql_test_native_iceberg_compat.yml | 2 +-
dev/diffs/{3.5.6.diff => 3.5.7.diff} | 0
docs/source/user-guide/latest/installation.md | 4 ++--
kube/Dockerfile | 4 ++--
pom.xml | 4 ++--
10 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/.github/actions/setup-spark-builder/action.yaml
b/.github/actions/setup-spark-builder/action.yaml
index 677d61a56..459baeef7 100644
--- a/.github/actions/setup-spark-builder/action.yaml
+++ b/.github/actions/setup-spark-builder/action.yaml
@@ -22,7 +22,7 @@ inputs:
description: 'The Apache Spark short version (e.g., 3.5) to build'
required: true
spark-version:
- description: 'The Apache Spark version (e.g., 3.5.6) to build'
+ description: 'The Apache Spark version (e.g., 3.5.7) to build'
required: true
runs:
using: "composite"
diff --git a/.github/workflows/iceberg_spark_test.yml
b/.github/workflows/iceberg_spark_test.yml
index aa59d2ff7..36c94fa6e 100644
--- a/.github/workflows/iceberg_spark_test.yml
+++ b/.github/workflows/iceberg_spark_test.yml
@@ -47,7 +47,7 @@ jobs:
os: [ubuntu-24.04]
java-version: [11, 17]
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full:
'1.9.1'}]
- spark-version: [{short: '3.5', full: '3.5.6'}]
+ spark-version: [{short: '3.5', full: '3.5.7'}]
scala-version: ['2.13']
fail-fast: false
name: iceberg-spark/${{ matrix.os }}/iceberg-${{
matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{
matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -86,7 +86,7 @@ jobs:
os: [ubuntu-24.04]
java-version: [11, 17]
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full:
'1.9.1'}]
- spark-version: [{short: '3.5', full: '3.5.6'}]
+ spark-version: [{short: '3.5', full: '3.5.7'}]
scala-version: ['2.13']
fail-fast: false
name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{
matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{
matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -125,7 +125,7 @@ jobs:
os: [ubuntu-24.04]
java-version: [11, 17]
iceberg-version: [{short: '1.8', full: '1.8.1'}, {short: '1.9', full:
'1.9.1'}]
- spark-version: [{short: '3.5', full: '3.5.6'}]
+ spark-version: [{short: '3.5', full: '3.5.7'}]
scala-version: ['2.13']
fail-fast: false
name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{
matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{
matrix.scala-version }}/java-${{ matrix.java-version }}
@@ -155,4 +155,4 @@ jobs:
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet
cache requires cleanups
ENABLE_COMET=true ENABLE_COMET_ONHEAP=true ./gradlew
-DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{
matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
:iceberg-spark:iceberg-spark-runtime-${{
matrix.spark-version.short }}_${{ matrix.scala-version }}:integrationTest \
- -Pquick=true -x javadoc
\ No newline at end of file
+ -Pquick=true -x javadoc
diff --git a/.github/workflows/pr_build_linux.yml
b/.github/workflows/pr_build_linux.yml
index 208bf87f6..118b756ee 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -74,16 +74,16 @@ jobs:
maven_opts: "-Pspark-3.4 -Pscala-2.12"
scan_impl: "native_comet"
- - name: "Spark 3.5.4, JDK 11, Scala 2.12"
- java_version: "11"
- maven_opts: "-Pspark-3.5 -Dspark.version=3.5.4 -Pscala-2.12"
- scan_impl: "native_comet"
-
- name: "Spark 3.5.5, JDK 17, Scala 2.13"
java_version: "17"
maven_opts: "-Pspark-3.5 -Dspark.version=3.5.5 -Pscala-2.13"
scan_impl: "native_comet"
+ - name: "Spark 3.5.6, JDK 17, Scala 2.13"
+ java_version: "17"
+ maven_opts: "-Pspark-3.5 -Dspark.version=3.5.6 -Pscala-2.13"
+ scan_impl: "native_comet"
+
- name: "Spark 3.5, JDK 17, Scala 2.12 native_datafusion"
java_version: "17"
maven_opts: "-Pspark-3.5 -Pscala-2.12"
@@ -175,4 +175,4 @@ jobs:
suites: ${{ matrix.suite.name == 'sql' && matrix.profile.name ==
'Spark 3.4, JDK 11, Scala 2.12' && '' || matrix.suite.value }}
maven_opts: ${{ matrix.profile.maven_opts }}
scan_impl: ${{ matrix.profile.scan_impl }}
- upload-test-reports: true
\ No newline at end of file
+ upload-test-reports: true
diff --git a/.github/workflows/spark_sql_test.yml
b/.github/workflows/spark_sql_test.yml
index 113fb3072..58a49ffe3 100644
--- a/.github/workflows/spark_sql_test.yml
+++ b/.github/workflows/spark_sql_test.yml
@@ -51,7 +51,7 @@ jobs:
matrix:
os: [ubuntu-24.04]
java-version: [11]
- spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.6'}]
+ spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.7'}]
module:
- {name: "catalyst", args1: "catalyst/test", args2: ""}
- {name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l
org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
@@ -100,7 +100,7 @@ jobs:
matrix:
os: [ ubuntu-24.04 ]
java-version: [ 11 ]
- spark-version: [ { short: '3.4', full: '3.4.3' }, { short: '3.5',
full: '3.5.6' } ]
+ spark-version: [ { short: '3.4', full: '3.4.3' }, { short: '3.5',
full: '3.5.7' } ]
module:
- { name: "catalyst", args1: "catalyst/test", args2: "" }
- { name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l
org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest }
@@ -149,7 +149,7 @@ jobs:
matrix:
os: [ubuntu-24.04]
java-version: [11]
- spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.6'}]
+ spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.7'}]
module:
- {name: "catalyst", args1: "catalyst/test", args2: ""}
- {name: "sql_core-1", args1: "", args2: sql/testOnly * -- -l
org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
diff --git a/.github/workflows/spark_sql_test_native_datafusion.yml
b/.github/workflows/spark_sql_test_native_datafusion.yml
index 3c2054591..6d81e144e 100644
--- a/.github/workflows/spark_sql_test_native_datafusion.yml
+++ b/.github/workflows/spark_sql_test_native_datafusion.yml
@@ -35,7 +35,7 @@ jobs:
matrix:
os: [ubuntu-24.04]
java-version: [11]
- spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.6'}]
+ spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.7'}]
module:
- {name: "catalyst", args1: "catalyst/test", args2: ""}
- {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l
org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
diff --git a/.github/workflows/spark_sql_test_native_iceberg_compat.yml
b/.github/workflows/spark_sql_test_native_iceberg_compat.yml
index 7457e9ae3..0681283f9 100644
--- a/.github/workflows/spark_sql_test_native_iceberg_compat.yml
+++ b/.github/workflows/spark_sql_test_native_iceberg_compat.yml
@@ -35,7 +35,7 @@ jobs:
matrix:
os: [ubuntu-24.04]
java-version: [11]
- spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.6'}]
+ spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full:
'3.5.7'}]
module:
- {name: "catalyst", args1: "catalyst/test", args2: ""}
- {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l
org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
diff --git a/dev/diffs/3.5.6.diff b/dev/diffs/3.5.7.diff
similarity index 100%
rename from dev/diffs/3.5.6.diff
rename to dev/diffs/3.5.7.diff
diff --git a/docs/source/user-guide/latest/installation.md
b/docs/source/user-guide/latest/installation.md
index 780e7cfe6..40efdeb62 100644
--- a/docs/source/user-guide/latest/installation.md
+++ b/docs/source/user-guide/latest/installation.md
@@ -38,9 +38,9 @@ Other versions may work well enough for development and
evaluation purposes.
| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL
Tests in CI |
|---------------| ------------ | -------------
|-------------------|-----------------------|
| 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes
|
-| 3.5.4 | 11/17 | 2.12/2.13 | Yes | No
|
| 3.5.5 | 11/17 | 2.12/2.13 | Yes | No
|
-| 3.5.6 | 11/17 | 2.12/2.13 | Yes | Yes
|
+| 3.5.6 | 11/17 | 2.12/2.13 | Yes | No
|
+| 3.5.7 | 11/17 | 2.12/2.13 | Yes | Yes
|
Note that we do not test the full matrix of supported Java and Scala versions
in CI for every Spark version.
diff --git a/kube/Dockerfile b/kube/Dockerfile
index 6c6be5d75..1425aefe4 100644
--- a/kube/Dockerfile
+++ b/kube/Dockerfile
@@ -15,7 +15,7 @@
# limitations under the License.
#
-FROM apache/spark:3.5.6 AS builder
+FROM apache/spark:3.5.7 AS builder
USER root
@@ -70,7 +70,7 @@ RUN mkdir -p /root/.m2 && \
RUN cd /comet \
&& JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make
release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
-FROM apache/spark:3.5.6
+FROM apache/spark:3.5.7
ENV SPARK_VERSION=3.5
ENV SCALA_VERSION=2.12
USER root
diff --git a/pom.xml b/pom.xml
index 109c24fee..6a6254dfc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,7 +70,7 @@ under the License.
<scala.plugin.version>4.9.6</scala.plugin.version>
<scalatest.version>3.2.16</scalatest.version>
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
- <spark.version>3.5.6</spark.version>
+ <spark.version>3.5.7</spark.version>
<spark.version.short>3.5</spark.version.short>
<spark.maven.scope>provided</spark.maven.scope>
<protobuf.version>3.25.5</protobuf.version>
@@ -609,7 +609,7 @@ under the License.
<id>spark-3.5</id>
<properties>
<scala.version>2.12.18</scala.version>
- <spark.version>3.5.6</spark.version>
+ <spark.version>3.5.7</spark.version>
<spark.version.short>3.5</spark.version.short>
<parquet.version>1.13.1</parquet.version>
<slf4j.version>2.0.7</slf4j.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
