This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new f4bde1c40 chore: Change default Spark version to 3.5 (#1620)
f4bde1c40 is described below
commit f4bde1c4070f6ff619e88d62eded397c7926596f
Author: Andy Grove <[email protected]>
AuthorDate: Thu Apr 10 08:26:13 2025 -0600
chore: Change default Spark version to 3.5 (#1620)
---
.github/actions/setup-spark-builder/action.yaml | 6 ++----
.github/workflows/docker-publish.yml | 2 +-
docs/source/contributor-guide/debugging.md | 2 +-
docs/source/contributor-guide/development.md | 6 +++---
docs/source/user-guide/datasources.md | 4 ++--
docs/source/user-guide/installation.md | 4 ++--
docs/source/user-guide/source.md | 8 ++++----
kube/Dockerfile | 8 ++++----
pom.xml | 13 ++++++++-----
9 files changed, 27 insertions(+), 26 deletions(-)
diff --git a/.github/actions/setup-spark-builder/action.yaml
b/.github/actions/setup-spark-builder/action.yaml
index dc333e5d1..2b867bada 100644
--- a/.github/actions/setup-spark-builder/action.yaml
+++ b/.github/actions/setup-spark-builder/action.yaml
@@ -19,13 +19,11 @@ name: Setup Spark Builder
description: 'Setup Apache Spark to run SQL tests'
inputs:
spark-short-version:
- description: 'The Apache Spark short version (e.g., 3.4) to build'
+ description: 'The Apache Spark short version (e.g., 3.5) to build'
required: true
- default: '3.4'
spark-version:
- description: 'The Apache Spark version (e.g., 3.4.3) to build'
+ description: 'The Apache Spark version (e.g., 3.5.5) to build'
required: true
- default: '3.4.3'
runs:
using: "composite"
steps:
diff --git a/.github/workflows/docker-publish.yml
b/.github/workflows/docker-publish.yml
index 39741d688..e9af6ac52 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -73,6 +73,6 @@ jobs:
with:
platforms: linux/amd64,linux/arm64
push: true
- tags: ghcr.io/apache/datafusion-comet:spark-3.4-scala-2.12-${{
env.COMET_VERSION }}
+ tags: ghcr.io/apache/datafusion-comet:spark-3.5-scala-2.12-${{
env.COMET_VERSION }}
file: kube/Dockerfile
no-cache: true
diff --git a/docs/source/contributor-guide/debugging.md
b/docs/source/contributor-guide/debugging.md
index 2f958ce4b..94bdb072b 100644
--- a/docs/source/contributor-guide/debugging.md
+++ b/docs/source/contributor-guide/debugging.md
@@ -130,7 +130,7 @@ Then build the Comet as
[described](https://github.com/apache/arrow-datafusion-c
Start Comet with `RUST_BACKTRACE=1`
```console
-RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars
spark/target/comet-spark-spark3.4_2.12-0.8.0-SNAPSHOT.jar --conf
spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true
--conf spark.comet.exec.enabled=true
+RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars
spark/target/comet-spark-spark3.5_2.12-0.8.0-SNAPSHOT.jar --conf
spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true
--conf spark.comet.exec.enabled=true
```
Get the expanded exception details
diff --git a/docs/source/contributor-guide/development.md
b/docs/source/contributor-guide/development.md
index a9c43103b..93d2c709d 100644
--- a/docs/source/contributor-guide/development.md
+++ b/docs/source/contributor-guide/development.md
@@ -109,7 +109,7 @@ The tests can be run with:
```sh
export SPARK_HOME=`pwd`
-./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -nsu
test
+./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite"
-Pspark-3.4 -nsu test
./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite"
-Pspark-3.5 -nsu test
./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite"
-Pspark-4.0 -nsu test
```
@@ -117,7 +117,7 @@ export SPARK_HOME=`pwd`
and
```sh
export SPARK_HOME=`pwd`
-./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite" -nsu
test
+./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite"
-Pspark-3.4 -nsu test
./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite"
-Pspark-3.5 -nsu test
./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite"
-Pspark-4.0 -nsu test
```
@@ -127,7 +127,7 @@ To regenerate the golden files, you can run the following
commands.
```sh
export SPARK_HOME=`pwd`
-SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite" -nsu
test
+SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite"
-Pspark-3.4 -nsu test
SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite"
-Pspark-3.5 -nsu test
SPARK_GENERATE_GOLDEN_FILES=1 ./mvnw -pl spark
-Dsuites="org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite"
-Pspark-4.0 -nsu test
```
diff --git a/docs/source/user-guide/datasources.md
b/docs/source/user-guide/datasources.md
index 5cca4b505..5634df8e0 100644
--- a/docs/source/user-guide/datasources.md
+++ b/docs/source/user-guide/datasources.md
@@ -51,12 +51,12 @@ Unlike to native Comet reader the Datafusion reader fully
supports nested types
To build Comet with native DataFusion reader and remote HDFS support it is
required to have a JDK installed
Example:
-Build a Comet for `spark-3.4` provide a JDK path in `JAVA_HOME`
+Build a Comet for `spark-3.5` provide a JDK path in `JAVA_HOME`
Provide the JRE linker path in `RUSTFLAGS`, the path can vary depending on the
system. Typically JRE linker is a part of installed JDK
```shell
export JAVA_HOME="/opt/homebrew/opt/openjdk@11"
-make release PROFILES="-Pspark-3.4" COMET_FEATURES=hdfs RUSTFLAGS="-L
$JAVA_HOME/libexec/openjdk.jdk/Contents/Home/lib/server"
+make release PROFILES="-Pspark-3.5" COMET_FEATURES=hdfs RUSTFLAGS="-L
$JAVA_HOME/libexec/openjdk.jdk/Contents/Home/lib/server"
```
Start Comet with experimental reader and HDFS support as
[described](installation.md/#run-spark-shell-with-comet-enabled)
diff --git a/docs/source/user-guide/installation.md
b/docs/source/user-guide/installation.md
index d42b8de9a..74488bc60 100644
--- a/docs/source/user-guide/installation.md
+++ b/docs/source/user-guide/installation.md
@@ -85,7 +85,7 @@ See the [Comet Kubernetes Guide](kubernetes.md) guide.
Make sure `SPARK_HOME` points to the same Spark version as Comet was built for.
```shell
-export COMET_JAR=spark/target/comet-spark-spark3.4_2.12-0.8.0-SNAPSHOT.jar
+export COMET_JAR=spark/target/comet-spark-spark3.5_2.12-0.8.0-SNAPSHOT.jar
$SPARK_HOME/bin/spark-shell \
--jars $COMET_JAR \
@@ -141,7 +141,7 @@ explicitly contain Comet otherwise Spark may use a
different class-loader for th
components which will then fail at runtime. For example:
```
---driver-class-path spark/target/comet-spark-spark3.4_2.12-0.8.0-SNAPSHOT.jar
+--driver-class-path spark/target/comet-spark-spark3.5_2.12-0.8.0-SNAPSHOT.jar
```
Some cluster managers may require additional configuration, see
<https://spark.apache.org/docs/latest/cluster-overview.html>
diff --git a/docs/source/user-guide/source.md b/docs/source/user-guide/source.md
index ab7f89745..b7038d341 100644
--- a/docs/source/user-guide/source.md
+++ b/docs/source/user-guide/source.md
@@ -38,7 +38,7 @@ cd apache-datafusion-comet-$COMET_VERSION
Build
```console
-make release-nogit PROFILES="-Pspark-3.4"
+make release-nogit PROFILES="-Pspark-3.5"
```
## Building from the GitHub repository
@@ -53,17 +53,17 @@ Build Comet for a specific Spark version:
```console
cd datafusion-comet
-make release PROFILES="-Pspark-3.4"
+make release PROFILES="-Pspark-3.5"
```
Note that the project builds for Scala 2.12 by default but can be built for
Scala 2.13 using an additional profile:
```console
-make release PROFILES="-Pspark-3.4 -Pscala-2.13"
+make release PROFILES="-Pspark-3.5 -Pscala-2.13"
```
To build Comet from the source distribution on an isolated environment without
an access to `github.com` it is necessary to disable
`git-commit-id-maven-plugin`, otherwise you will face errors that there is no
access to the git during the build process. In that case you may use:
```console
-make release-nogit PROFILES="-Pspark-3.4"
+make release-nogit PROFILES="-Pspark-3.5"
```
diff --git a/kube/Dockerfile b/kube/Dockerfile
index 00a526c33..74e087e18 100644
--- a/kube/Dockerfile
+++ b/kube/Dockerfile
@@ -15,7 +15,7 @@
# limitations under the License.
#
-FROM apache/spark:3.4.3 AS builder
+FROM apache/spark:3.5.5 AS builder
USER root
@@ -28,7 +28,7 @@ RUN apt update \
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
ENV RUSTFLAGS="-C debuginfo=line-tables-only -C incremental=false"
-ENV SPARK_VERSION=3.4
+ENV SPARK_VERSION=3.5
ENV SCALA_VERSION=2.12
# copy source files to Docker image
@@ -61,8 +61,8 @@ RUN mkdir -p /root/.m2 && \
RUN cd /comet \
&& JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make
release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
-FROM apache/spark:3.4.3
-ENV SPARK_VERSION=3.4
+FROM apache/spark:3.5.5
+ENV SPARK_VERSION=3.5
ENV SCALA_VERSION=2.12
USER root
diff --git a/pom.xml b/pom.xml
index 9c0339f16..7bdfc782a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -47,13 +47,13 @@ under the License.
<java.version>11</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
- <scala.version>2.12.17</scala.version>
+ <scala.version>2.12.18</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.plugin.version>4.7.2</scala.plugin.version>
<scalatest.version>3.2.16</scalatest.version>
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
- <spark.version>3.4.3</spark.version>
- <spark.version.short>3.4</spark.version.short>
+ <spark.version>3.5.5</spark.version>
+ <spark.version.short>3.5</spark.version.short>
<spark.maven.scope>provided</spark.maven.scope>
<protobuf.version>3.25.5</protobuf.version>
<parquet.version>1.13.1</parquet.version>
@@ -64,7 +64,7 @@ under the License.
<spotless.version>2.43.0</spotless.version>
<jacoco.version>0.8.11</jacoco.version>
<semanticdb.version>4.8.8</semanticdb.version>
- <slf4j.version>2.0.6</slf4j.version>
+ <slf4j.version>2.0.7</slf4j.version>
<guava.version>33.2.1-jre</guava.version>
<jni.dir>${project.basedir}/../native/target/debug</jni.dir>
<platform>darwin</platform>
@@ -97,7 +97,7 @@ under the License.
</extraJavaTestArgs>
<argLine>-ea -Xmx4g -Xss4m ${extraJavaTestArgs}</argLine>
<shims.majorVerSrc>spark-3.x</shims.majorVerSrc>
- <shims.minorVerSrc>spark-3.4</shims.minorVerSrc>
+ <shims.minorVerSrc>spark-3.5</shims.minorVerSrc>
</properties>
<dependencyManagement>
@@ -555,8 +555,11 @@ under the License.
<id>spark-3.4</id>
<properties>
<scala.version>2.12.17</scala.version>
+ <spark.version>3.4.3</spark.version>
<spark.version.short>3.4</spark.version.short>
<parquet.version>1.13.1</parquet.version>
+ <slf4j.version>2.0.6</slf4j.version>
+ <shims.minorVerSrc>spark-3.4</shims.minorVerSrc>
</properties>
</profile>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
