This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 064dcf64b9 chore: bump Spark 4.1 to 4.1.3 (#5183)
064dcf64b9 is described below
commit 064dcf64b990feef56c1693ee7ec71ccf87f8aed
Author: Andy Grove <[email protected]>
AuthorDate: Sat Aug 1 05:22:14 2026 -0600
chore: bump Spark 4.1 to 4.1.3 (#5183)
---
.github/workflows/ci.yml | 4 +-
.github/workflows/pyarrow_udf_test.yml | 2 +-
.github/workflows/spark_sql_writer_tests.yml | 2 +-
dev/ci/compute-changes.py | 2 +-
dev/diffs/{4.1.2.diff => 4.1.3.diff} | 78 +++++++++++-----------
.../source/contributor-guide/benchmarking_macos.md | 10 +--
.../contributor-guide/iceberg-spark-tests.md | 2 +-
.../latest/compatibility/spark-versions.md | 2 +-
docs/source/user-guide/latest/installation.md | 2 +-
docs/source/user-guide/latest/kubernetes.md | 8 +--
kube/Dockerfile | 4 +-
pom.xml | 4 +-
spark/pom.xml | 2 +-
13 files changed, 61 insertions(+), 61 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4f38a2c515..476dddd3bb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -260,7 +260,7 @@ jobs:
uses: ./.github/workflows/spark_sql_test_reusable.yml
with:
spark-short: '4.1'
- spark-full: '4.1.2'
+ spark-full: '4.1.3'
java: 17
iceberg_1_8:
@@ -332,5 +332,5 @@ jobs:
iceberg-short: '1.11'
iceberg-full: '1.11.0'
spark-short: '4.1'
- spark-full: '4.1.2'
+ spark-full: '4.1.3'
java: 17
diff --git a/.github/workflows/pyarrow_udf_test.yml
b/.github/workflows/pyarrow_udf_test.yml
index 807056cf9e..8493ebb9f0 100644
--- a/.github/workflows/pyarrow_udf_test.yml
+++ b/.github/workflows/pyarrow_udf_test.yml
@@ -73,7 +73,7 @@ jobs:
pyspark: "4.0.2"
- name: Spark 4.1
maven_profiles: "-Pspark-4.1"
- pyspark: "4.1.2"
+ pyspark: "4.1.3"
container:
# Pinned to the Debian 12 (bookworm) base so the system `python3` is
3.11. The default
# `amd64/rust` image is Debian 13 (trixie) which ships Python 3.13 and
no python3.11 apt
diff --git a/.github/workflows/spark_sql_writer_tests.yml
b/.github/workflows/spark_sql_writer_tests.yml
index 0406aa7ff8..c813a8b39e 100644
--- a/.github/workflows/spark_sql_writer_tests.yml
+++ b/.github/workflows/spark_sql_writer_tests.yml
@@ -71,7 +71,7 @@ jobs:
# versions only; 3.4 and 4.0 are label-gated and not offered here).
case "${{ inputs.spark-version }}" in
3.5) spark_full=3.5.9; java=17 ;;
- 4.1) spark_full=4.1.2; java=17 ;;
+ 4.1) spark_full=4.1.3; java=17 ;;
*) echo "Unsupported spark-version: ${{ inputs.spark-version }}"
>&2; exit 1 ;;
esac
echo "spark-full=$spark_full" >> "$GITHUB_OUTPUT"
diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py
index 195d27e828..bc28e0e789 100644
--- a/dev/ci/compute-changes.py
+++ b/dev/ci/compute-changes.py
@@ -169,7 +169,7 @@ FILTERS = {
"!spark/src/main/spark-4.2/**",
"!spark/src/main/scala/org/apache/comet/GenerateDocs.scala",
"spark/pom.xml",
- "dev/diffs/4.1.2.diff",
+ "dev/diffs/4.1.3.diff",
"pom.xml",
"rust-toolchain.toml",
".github/workflows/ci.yml",
diff --git a/dev/diffs/4.1.2.diff b/dev/diffs/4.1.3.diff
similarity index 99%
rename from dev/diffs/4.1.2.diff
rename to dev/diffs/4.1.3.diff
index b0a32baaba..79d008c989 100644
--- a/dev/diffs/4.1.2.diff
+++ b/dev/diffs/4.1.3.diff
@@ -39,7 +39,7 @@ index 6df8bc85b51..dabb75e2b75 100644
withSpark(sc) { sc =>
TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
diff --git a/pom.xml b/pom.xml
-index dc201151999..20ee0e7482a 100644
+index 370bfa4397f..8447b5c375e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -152,6 +152,8 @@
@@ -78,7 +78,7 @@ index dc201151999..20ee0e7482a 100644
<dependency>
<groupId>org.apache.datasketches</groupId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
-index c25b83c355b..5e23b863dcf 100644
+index febc7d747bc..63c8e4e22c0 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -97,6 +97,10 @@
@@ -514,7 +514,7 @@ index 93ff7becaec..87537a25b3b 100644
assert(unionExec.size == 1)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-index 5b88eeefeca..d4f07bc182a 100644
+index d9ce3000a0c..f2d044ed6b8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -36,11 +36,12 @@ import
org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference,
@@ -574,13 +574,13 @@ index 5b88eeefeca..d4f07bc182a 100644
assert(exchanges.size == 2)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-index 6e9f3385571..5af115c9a7d 100644
+index 4a070becfa6..61d515d127e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
-@@ -24,8 +24,10 @@ import
org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression
- import org.apache.spark.sql.catalyst.optimizer.TransposeWindow
+@@ -25,8 +25,10 @@ import
org.apache.spark.sql.catalyst.optimizer.TransposeWindow
import org.apache.spark.sql.catalyst.plans.logical.{Window => LogicalWindow}
import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
+ import org.apache.spark.sql.catalyst.trees.UnaryLike
+import org.apache.spark.sql.comet.CometWindowExec
+import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
@@ -589,7 +589,7 @@ index 6e9f3385571..5af115c9a7d 100644
import org.apache.spark.sql.execution.window.WindowExec
import org.apache.spark.sql.expressions.{Aggregator,
MutableAggregationBuffer, UserDefinedAggregateFunction, Window}
import org.apache.spark.sql.functions._
-@@ -951,7 +953,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest
+@@ -974,7 +976,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest
}
}
@@ -599,7 +599,7 @@ index 6e9f3385571..5af115c9a7d 100644
val df = Seq((1, "1"), (2, "2"), (1, "3"), (2, "4")).toDF("key", "value")
val window = Window.partitionBy($"key").orderBy($"value")
-@@ -963,7 +966,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest
+@@ -986,7 +989,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest
}
}
@@ -608,7 +608,7 @@ index 6e9f3385571..5af115c9a7d 100644
// Make sure we trigger the spilling path.
withSQLConf(SQLConf.WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD.key -> "1",
SQLConf.WINDOW_EXEC_BUFFER_SPILL_THRESHOLD.key -> "17") {
-@@ -1142,10 +1145,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest
+@@ -1165,10 +1168,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest
}
def isShuffleExecByRequirement(
@@ -622,7 +622,7 @@ index 6e9f3385571..5af115c9a7d 100644
case _ => false
}
-@@ -1168,7 +1173,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest
+@@ -1191,7 +1196,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest
val shuffleByRequirement = windowed.queryExecution.executedPlan.exists {
case w: WindowExec =>
w.child.exists {
@@ -636,7 +636,7 @@ index 6e9f3385571..5af115c9a7d 100644
case _ => false
}
case _ => false
-@@ -1620,7 +1630,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest
+@@ -1643,7 +1653,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest
}
}
@@ -1660,7 +1660,7 @@ index a09b7e0827c..ffc29f764bc 100644
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-index 2a0ab21ddb0..6030e7c2b9b 100644
+index 370cd931809..d90bc1e8d54 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
@@ -21,6 +21,7 @@ import scala.collection.mutable.ArrayBuffer
@@ -1671,7 +1671,7 @@ index 2a0ab21ddb0..6030e7c2b9b 100644
import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite,
Table, TableCapability}
import org.apache.spark.sql.connector.read.ScanBuilder
import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
-@@ -188,7 +189,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest
with SharedSparkSession {
+@@ -191,7 +192,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest
with SharedSparkSession {
val df = spark.read.format(format).load(path.getCanonicalPath)
checkAnswer(df, inputData.toDF())
assert(
@@ -2298,7 +2298,7 @@ index a3cfdc5a240..3793b6191bf 100644
})
checkAnswer(distinctWithId, Seq(Row(1, 0), Row(1, 0)))
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
-index 3e7d26f74bd..79232dc3664 100644
+index 188a28ff1c0..3cacc4efbdf 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -27,12 +27,14 @@ import org.apache.spark.SparkException
@@ -2566,7 +2566,7 @@ index 3e7d26f74bd..79232dc3664 100644
Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { joinHint =>
def getJoinNode(plan: SparkPlan): Seq[ShuffledJoin] = if (joinHint ==
"SHUFFLE_MERGE") {
findTopLevelSortMergeJoin(plan)
-@@ -1257,7 +1286,8 @@ class AdaptiveQueryExecSuite
+@@ -1266,7 +1295,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2576,7 +2576,7 @@ index 3e7d26f74bd..79232dc3664 100644
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
"SELECT key FROM testData GROUP BY key")
-@@ -1891,7 +1921,7 @@ class AdaptiveQueryExecSuite
+@@ -1900,7 +1930,7 @@ class AdaptiveQueryExecSuite
val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
"SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id")
assert(collect(adaptivePlan) {
@@ -2585,7 +2585,7 @@ index 3e7d26f74bd..79232dc3664 100644
}.length == 1)
}
}
-@@ -1972,7 +2002,8 @@ class AdaptiveQueryExecSuite
+@@ -1981,7 +2011,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2595,7 +2595,7 @@ index 3e7d26f74bd..79232dc3664 100644
def hasRepartitionShuffle(plan: SparkPlan): Boolean = {
find(plan) {
case s: ShuffleExchangeLike =>
-@@ -2159,6 +2190,9 @@ class AdaptiveQueryExecSuite
+@@ -2168,6 +2199,9 @@ class AdaptiveQueryExecSuite
def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin):
Unit = {
assert(collect(ds.queryExecution.executedPlan) {
case s: ShuffleExchangeExec if s.shuffleOrigin == origin &&
s.numPartitions == 2 => s
@@ -2605,7 +2605,7 @@ index 3e7d26f74bd..79232dc3664 100644
}.size == 1)
ds.collect()
val plan = ds.queryExecution.executedPlan
-@@ -2167,6 +2201,9 @@ class AdaptiveQueryExecSuite
+@@ -2176,6 +2210,9 @@ class AdaptiveQueryExecSuite
}.isEmpty)
assert(collect(plan) {
case s: ShuffleExchangeExec if s.shuffleOrigin == origin &&
s.numPartitions == 2 => s
@@ -2615,7 +2615,7 @@ index 3e7d26f74bd..79232dc3664 100644
}.size == 1)
checkAnswer(ds, testData)
}
-@@ -2331,7 +2368,8 @@ class AdaptiveQueryExecSuite
+@@ -2340,7 +2377,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2625,7 +2625,7 @@ index 3e7d26f74bd..79232dc3664 100644
withTempView("t1", "t2") {
def checkJoinStrategy(shouldShuffleHashJoin: Boolean): Unit = {
Seq("100", "100000").foreach { size =>
-@@ -2417,7 +2455,8 @@ class AdaptiveQueryExecSuite
+@@ -2426,7 +2464,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2635,7 +2635,7 @@ index 3e7d26f74bd..79232dc3664 100644
withTempView("v") {
withSQLConf(
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-@@ -2516,7 +2555,7 @@ class AdaptiveQueryExecSuite
+@@ -2525,7 +2564,7 @@ class AdaptiveQueryExecSuite
runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM
skewData1 " +
s"JOIN skewData2 ON key1 = key2 GROUP BY key1")
val shuffles1 = collect(adaptive1) {
@@ -2644,7 +2644,7 @@ index 3e7d26f74bd..79232dc3664 100644
}
assert(shuffles1.size == 3)
// shuffles1.head is the top-level shuffle under the Aggregate
operator
-@@ -2529,7 +2568,7 @@ class AdaptiveQueryExecSuite
+@@ -2538,7 +2577,7 @@ class AdaptiveQueryExecSuite
runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM
skewData1 " +
s"JOIN skewData2 ON key1 = key2")
val shuffles2 = collect(adaptive2) {
@@ -2653,7 +2653,7 @@ index 3e7d26f74bd..79232dc3664 100644
}
if (hasRequiredDistribution) {
assert(shuffles2.size == 3)
-@@ -2563,7 +2602,8 @@ class AdaptiveQueryExecSuite
+@@ -2572,7 +2611,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2663,7 +2663,7 @@ index 3e7d26f74bd..79232dc3664 100644
CostEvaluator.instantiate(
classOf[SimpleShuffleSortCostEvaluator].getCanonicalName,
spark.sparkContext.getConf)
intercept[IllegalArgumentException] {
-@@ -2729,6 +2769,7 @@ class AdaptiveQueryExecSuite
+@@ -2738,6 +2778,7 @@ class AdaptiveQueryExecSuite
val (_, adaptive) = runAdaptiveAndVerifyResult(query)
assert(adaptive.collect {
case sort: SortExec => sort
@@ -2671,7 +2671,7 @@ index 3e7d26f74bd..79232dc3664 100644
}.size == 1)
val read = collect(adaptive) {
case read: AQEShuffleReadExec => read
-@@ -2746,7 +2787,8 @@ class AdaptiveQueryExecSuite
+@@ -2755,7 +2796,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2681,7 +2681,7 @@ index 3e7d26f74bd..79232dc3664 100644
withTempView("v") {
withSQLConf(
SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key
-> "true",
-@@ -2858,7 +2900,7 @@ class AdaptiveQueryExecSuite
+@@ -2867,7 +2909,7 @@ class AdaptiveQueryExecSuite
runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN
skewData2 ON key1 = key2 " +
"JOIN skewData3 ON value2 = value3")
val shuffles1 = collect(adaptive1) {
@@ -2690,7 +2690,7 @@ index 3e7d26f74bd..79232dc3664 100644
}
assert(shuffles1.size == 4)
val smj1 = findTopLevelSortMergeJoin(adaptive1)
-@@ -2869,7 +2911,7 @@ class AdaptiveQueryExecSuite
+@@ -2878,7 +2920,7 @@ class AdaptiveQueryExecSuite
runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN
skewData2 ON key1 = key2 " +
"JOIN skewData3 ON value1 = value3")
val shuffles2 = collect(adaptive2) {
@@ -2699,7 +2699,7 @@ index 3e7d26f74bd..79232dc3664 100644
}
assert(shuffles2.size == 4)
val smj2 = findTopLevelSortMergeJoin(adaptive2)
-@@ -3127,6 +3169,7 @@ class AdaptiveQueryExecSuite
+@@ -3136,6 +3178,7 @@ class AdaptiveQueryExecSuite
}.size == (if (firstAccess) 1 else 0))
assert(collect(initialExecutedPlan) {
case s: SortExec => s
@@ -2707,7 +2707,7 @@ index 3e7d26f74bd..79232dc3664 100644
}.size == (if (firstAccess) 2 else 0))
assert(collect(initialExecutedPlan) {
case i: InMemoryTableScanLike => i
-@@ -3139,6 +3182,7 @@ class AdaptiveQueryExecSuite
+@@ -3148,6 +3191,7 @@ class AdaptiveQueryExecSuite
}.isEmpty)
assert(collect(finalExecutedPlan) {
case s: SortExec => s
@@ -2715,7 +2715,7 @@ index 3e7d26f74bd..79232dc3664 100644
}.isEmpty)
assert(collect(initialExecutedPlan) {
case i: InMemoryTableScanLike => i
-@@ -3220,7 +3264,8 @@ class AdaptiveQueryExecSuite
+@@ -3229,7 +3273,8 @@ class AdaptiveQueryExecSuite
}
}
@@ -2725,7 +2725,7 @@ index 3e7d26f74bd..79232dc3664 100644
val emptyDf = spark.range(1).where("false")
val aggDf1 = emptyDf.agg(sum("id").as("id")).withColumn("name",
lit("df1"))
val aggDf2 = emptyDf.agg(sum("id").as("id")).withColumn("name",
lit("df2"))
-@@ -3310,7 +3355,8 @@ class AdaptiveQueryExecSuite
+@@ -3319,7 +3364,8 @@ class AdaptiveQueryExecSuite
val plan =
df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec]
assert(plan.inputPlan.isInstanceOf[TakeOrderedAndProjectExec])
@@ -3060,10 +3060,10 @@ index 6ba790deddf..34b2f424c8f 100644
checkAnswer(
// "fruit" column in this file is encoded using
DELTA_LENGTH_BYTE_ARRAY.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index c530dc0d3df..d23069689da 100644
+index 97aad5b30ea..77de969b80b 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-@@ -996,7 +996,11 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1032,7 +1032,11 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
Seq(Some("A"), Some("A"), None).toDF().repartition(1)
.write.parquet(path.getAbsolutePath)
val df = spark.read.parquet(path.getAbsolutePath)
@@ -3076,7 +3076,7 @@ index c530dc0d3df..d23069689da 100644
}
}
}
-@@ -1060,7 +1064,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1096,7 +1100,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
checkAnswer(readParquet(schema2, path), df)
}
@@ -3086,7 +3086,7 @@ index c530dc0d3df..d23069689da 100644
val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
checkAnswer(readParquet(schema1, path), df)
val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
-@@ -1084,7 +1089,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1120,7 +1125,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c,
CAST('1.2' AS BINARY) d")
df.write.parquet(path.toString)
@@ -3392,7 +3392,7 @@ index e31e0e70cf3..034ad5b953e 100644
import testImplicits._
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
-index 38e5b15465b..ca3e8fef27a 100644
+index a32084bf74b..39eca1173b7 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
@@ -24,6 +24,7 @@ import scala.jdk.CollectionConverters.SetHasAsScala
@@ -3681,7 +3681,7 @@ index 4c06a0109e3..0b0e4990c0a 100644
fail(s"No FileScan in query\n${df.queryExecution}")
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala
-index 4306e5a8601..ee80e65ec26 100644
+index a9b8d54a6c6..01b5a13d735 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala
@@ -20,11 +20,13 @@ package org.apache.spark.sql.streaming
@@ -3689,7 +3689,7 @@ index 4306e5a8601..ee80e65ec26 100644
import org.apache.spark.SparkIllegalArgumentException
+import org.apache.spark.sql.IgnoreCometSuite
- import org.apache.spark.sql.execution.streaming.LowLatencyMemoryStream
+ import org.apache.spark.sql.execution.streaming.sources.LowLatencyMemoryStream
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
diff --git a/docs/source/contributor-guide/benchmarking_macos.md
b/docs/source/contributor-guide/benchmarking_macos.md
index 07e4b27105..916fa25940 100644
--- a/docs/source/contributor-guide/benchmarking_macos.md
+++ b/docs/source/contributor-guide/benchmarking_macos.md
@@ -55,13 +55,13 @@ export DF_BENCH=`pwd`
## Install Spark
-Install Apache Spark. This example refers to 4.1.2 version.
+Install Apache Spark. This example refers to 4.1.3 version.
```shell
-wget
https://archive.apache.org/dist/spark/spark-4.1.2/spark-4.1.2-bin-hadoop3.tgz
-tar xzf spark-4.1.2-bin-hadoop3.tgz
-sudo mv spark-4.1.2-bin-hadoop3 /opt
-export SPARK_HOME=/opt/spark-4.1.2-bin-hadoop3/
+wget
https://archive.apache.org/dist/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz
+tar xzf spark-4.1.3-bin-hadoop3.tgz
+sudo mv spark-4.1.3-bin-hadoop3 /opt
+export SPARK_HOME=/opt/spark-4.1.3-bin-hadoop3/
```
Start Spark in standalone mode:
diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md
b/docs/source/contributor-guide/iceberg-spark-tests.md
index f7468e4240..fccbdeb54b 100644
--- a/docs/source/contributor-guide/iceberg-spark-tests.md
+++ b/docs/source/contributor-guide/iceberg-spark-tests.md
@@ -95,7 +95,7 @@ diff must be generated against its own tag.
The `iceberg_spark_test_<version>.yml` workflows apply these diffs and run the
three Gradle targets above
against each Iceberg version. Iceberg 1.8.1 runs against Spark 3.4.3 with Java
11; Iceberg 1.9.1 and 1.10.0
-run against Spark 3.5.9 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.2
with Java 17. Iceberg 1.11
+run against Spark 3.5.9 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.3
with Java 17. Iceberg 1.11
(the only version testing Spark 4.1) runs on every pull request and on pushes
to main; the older versions
(1.8, 1.9, 1.10) run only on pushes to main, or on a pull request labeled
`run-iceberg-tests`. All caller
workflows delegate to `iceberg_spark_test_reusable.yml`, which holds the build
and test job logic.
diff --git a/docs/source/user-guide/latest/compatibility/spark-versions.md
b/docs/source/user-guide/latest/compatibility/spark-versions.md
index d96294b5c9..9fe7a2b339 100644
--- a/docs/source/user-guide/latest/compatibility/spark-versions.md
+++ b/docs/source/user-guide/latest/compatibility/spark-versions.md
@@ -76,7 +76,7 @@ Spark 4.0.2 is supported with Java 17 and Scala 2.13.
## Spark 4.1
-Spark 4.1.2 is supported with Java 17/21 and Scala 2.13.
+Spark 4.1.3 is supported with Java 17/21 and Scala 2.13.
### Known Limitations
diff --git a/docs/source/user-guide/latest/installation.md
b/docs/source/user-guide/latest/installation.md
index c25ce8da75..93da456421 100644
--- a/docs/source/user-guide/latest/installation.md
+++ b/docs/source/user-guide/latest/installation.md
@@ -55,7 +55,7 @@ We recommend moving to JDK 17 or later and Spark 3.5 or later.
| 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes
|
| 3.5.9 | 11/17 | 2.12/2.13 | Yes | Yes
|
| 4.0.2 | 17/21 | 2.13 | Yes | Yes
|
-| 4.1.2 | 17/21 | 2.13 | Yes | Yes
|
+| 4.1.3 | 17/21 | 2.13 | Yes | Yes
|
Note that we do not test the full matrix of supported Java and Scala versions
in CI for every Spark version.
diff --git a/docs/source/user-guide/latest/kubernetes.md
b/docs/source/user-guide/latest/kubernetes.md
index 5dfb723e65..b91bb93e80 100644
--- a/docs/source/user-guide/latest/kubernetes.md
+++ b/docs/source/user-guide/latest/kubernetes.md
@@ -72,7 +72,7 @@ spec:
image: apache/datafusion-comet:$COMET_VERSION-spark3.5.5-scala2.12-java11
imagePullPolicy: IfNotPresent
mainClass: org.apache.spark.examples.SparkPi
- mainApplicationFile:
local:///opt/spark/examples/jars/spark-examples_2.13-4.1.2.jar
+ mainApplicationFile:
local:///opt/spark/examples/jars/spark-examples_2.13-4.1.3.jar
sparkConf:
"spark.executor.extraClassPath":
"/opt/spark/jars/comet-spark-spark3.5_2.12-$COMET_VERSION.jar"
"spark.driver.extraClassPath":
"/opt/spark/jars/comet-spark-spark3.5_2.12-$COMET_VERSION.jar"
@@ -82,17 +82,17 @@ spec:
"spark.comet.shuffle.enabled": "true"
"spark.comet.shuffle.mode": "auto"
"spark.shuffle.manager":
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
- sparkVersion: 4.1.2
+ sparkVersion: 4.1.3
driver:
labels:
- version: 4.1.2
+ version: 4.1.3
cores: 1
coreLimit: 1200m
memory: 512m
serviceAccount: spark-operator-spark
executor:
labels:
- version: 4.1.2
+ version: 4.1.3
instances: 1
cores: 1
coreLimit: 1200m
diff --git a/kube/Dockerfile b/kube/Dockerfile
index 5ee3eac35c..f28514681e 100644
--- a/kube/Dockerfile
+++ b/kube/Dockerfile
@@ -15,7 +15,7 @@
# limitations under the License.
#
-FROM apache/spark:4.1.2 AS builder
+FROM apache/spark:4.1.3 AS builder
USER root
@@ -69,7 +69,7 @@ RUN mkdir -p /root/.m2 && \
RUN cd /comet \
&& JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make
release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION"
-FROM apache/spark:4.1.2
+FROM apache/spark:4.1.3
ENV SPARK_VERSION=4.1
ENV SCALA_VERSION=2.13
USER root
diff --git a/pom.xml b/pom.xml
index 927c07ca9c..1ab9a6d69a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -69,7 +69,7 @@ under the License.
<scala.plugin.version>4.9.6</scala.plugin.version>
<scalatest.version>3.2.16</scalatest.version>
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
- <spark.version>4.1.2</spark.version>
+ <spark.version>4.1.3</spark.version>
<spark.version.short>4.1</spark.version.short>
<spark.maven.scope>provided</spark.maven.scope>
<protobuf.version>3.25.5</protobuf.version>
@@ -717,7 +717,7 @@ under the License.
yet published, so the -Psemanticdb / scalafix lint job is skipped
for spark-4.1. -->
<scala.version>2.13.17</scala.version>
<scala.binary.version>2.13</scala.binary.version>
- <spark.version>4.1.2</spark.version>
+ <spark.version>4.1.3</spark.version>
<spark.version.short>4.1</spark.version.short>
<parquet.version>1.16.0</parquet.version>
<semanticdb.version>4.13.6</semanticdb.version>
diff --git a/spark/pom.xml b/spark/pom.xml
index fbe059c739..33e6d102c1 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -297,7 +297,7 @@ under the License.
<version>1.11.0</version>
<scope>test</scope>
</dependency>
- <!-- Jetty 11.x for Spark 4.1 (jakarta.servlet); matches Spark 4.1.2's
jetty.version -->
+ <!-- Jetty 11.x for Spark 4.1 (jakarta.servlet); matches Spark 4.1.3's
jetty.version -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
