This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 0abe9667ba chore: bump Spark 3.5 to 3.5.9 (#5181)
0abe9667ba is described below
commit 0abe9667baece1a7fbbe9cd9dd4427a3fbcccafb
Author: Andy Grove <[email protected]>
AuthorDate: Fri Jul 31 17:27:55 2026 -0600
chore: bump Spark 3.5 to 3.5.9 (#5181)
Rename dev/diffs/3.5.8.diff to 3.5.9.diff and regenerate it against the
v3.5.9 tag, and point the build, CI workflows, and docs at 3.5.9.
---
.github/actions/setup-spark-builder/action.yaml | 2 +-
.github/workflows/ci.yml | 6 +-
.github/workflows/iceberg_spark_test_reusable.yml | 2 +-
.github/workflows/spark_sql_test_reusable.yml | 2 +-
.github/workflows/spark_sql_writer_tests.yml | 2 +-
dev/ci/compute-changes.py | 2 +-
dev/diffs/{3.5.8.diff => 3.5.9.diff} | 68 ++++++++++++----------
.../contributor-guide/iceberg-spark-tests.md | 2 +-
.../latest/compatibility/spark-versions.md | 2 +-
docs/source/user-guide/latest/installation.md | 2 +-
pom.xml | 2 +-
.../resources/pyspark/benchmark_pyarrow_udf.py | 2 +-
.../src/test/resources/pyspark/test_pyarrow_udf.py | 2 +-
13 files changed, 52 insertions(+), 44 deletions(-)
diff --git a/.github/actions/setup-spark-builder/action.yaml
b/.github/actions/setup-spark-builder/action.yaml
index 6d9ad6e1d2..84804c0a79 100644
--- a/.github/actions/setup-spark-builder/action.yaml
+++ b/.github/actions/setup-spark-builder/action.yaml
@@ -22,7 +22,7 @@ inputs:
description: 'The Apache Spark short version (e.g., 3.5) to build'
required: true
spark-version:
- description: 'The Apache Spark version (e.g., 3.5.8) to build'
+ description: 'The Apache Spark version (e.g., 3.5.9) to build'
required: true
skip-native-build:
description: 'Skip native build (when using pre-built artifact)'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index be277efa29..4f38a2c515 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -228,7 +228,7 @@ jobs:
uses: ./.github/workflows/spark_sql_test_reusable.yml
with:
spark-short: '3.5'
- spark-full: '3.5.8'
+ spark-full: '3.5.9'
java: 17
spark_4_0:
@@ -296,7 +296,7 @@ jobs:
iceberg-short: '1.9'
iceberg-full: '1.9.1'
spark-short: '3.5'
- spark-full: '3.5.8'
+ spark-full: '3.5.9'
java: 17
iceberg_1_10:
@@ -315,7 +315,7 @@ jobs:
iceberg-short: '1.10'
iceberg-full: '1.10.0'
spark-short: '3.5'
- spark-full: '3.5.8'
+ spark-full: '3.5.9'
java: 17
iceberg_1_11:
diff --git a/.github/workflows/iceberg_spark_test_reusable.yml
b/.github/workflows/iceberg_spark_test_reusable.yml
index 5f2dfeb5ab..4b72cf47c2 100644
--- a/.github/workflows/iceberg_spark_test_reusable.yml
+++ b/.github/workflows/iceberg_spark_test_reusable.yml
@@ -37,7 +37,7 @@ on:
required: true
type: string
spark-full:
- description: 'Spark full version, e.g. 3.5.8'
+ description: 'Spark full version, e.g. 3.5.9'
required: true
type: string
java:
diff --git a/.github/workflows/spark_sql_test_reusable.yml
b/.github/workflows/spark_sql_test_reusable.yml
index 7279a13078..5d4f8f0473 100644
--- a/.github/workflows/spark_sql_test_reusable.yml
+++ b/.github/workflows/spark_sql_test_reusable.yml
@@ -29,7 +29,7 @@ on:
required: true
type: string
spark-full:
- description: 'Spark full version, e.g. 3.5.8'
+ description: 'Spark full version, e.g. 3.5.9'
required: true
type: string
java:
diff --git a/.github/workflows/spark_sql_writer_tests.yml
b/.github/workflows/spark_sql_writer_tests.yml
index 07c028757d..0406aa7ff8 100644
--- a/.github/workflows/spark_sql_writer_tests.yml
+++ b/.github/workflows/spark_sql_writer_tests.yml
@@ -70,7 +70,7 @@ jobs:
# Mirrors ci.yml's per-version reusable invocations (default-on PR
# versions only; 3.4 and 4.0 are label-gated and not offered here).
case "${{ inputs.spark-version }}" in
- 3.5) spark_full=3.5.8; java=17 ;;
+ 3.5) spark_full=3.5.9; java=17 ;;
4.1) spark_full=4.1.2; java=17 ;;
*) echo "Unsupported spark-version: ${{ inputs.spark-version }}"
>&2; exit 1 ;;
esac
diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py
index 47af20d450..195d27e828 100644
--- a/dev/ci/compute-changes.py
+++ b/dev/ci/compute-changes.py
@@ -125,7 +125,7 @@ FILTERS = {
"!spark/src/main/spark-4.x/**",
"!spark/src/main/scala/org/apache/comet/GenerateDocs.scala",
"spark/pom.xml",
- "dev/diffs/3.5.8.diff",
+ "dev/diffs/3.5.9.diff",
"pom.xml",
"rust-toolchain.toml",
".github/workflows/ci.yml",
diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.9.diff
similarity index 98%
rename from dev/diffs/3.5.8.diff
rename to dev/diffs/3.5.9.diff
index 4214767ff8..f15ed14054 100644
--- a/dev/diffs/3.5.8.diff
+++ b/dev/diffs/3.5.9.diff
@@ -1,17 +1,17 @@
diff --git a/pom.xml b/pom.xml
-index edd2ad57880..45f8fd01538 100644
+index 49eca0f7555..ba234805889 100644
--- a/pom.xml
+++ b/pom.xml
-@@ -152,6 +152,8 @@
- -->
- <ivy.version>2.5.1</ivy.version>
+@@ -148,6 +148,8 @@
+ <chill.version>0.10.0</chill.version>
+ <ivy.version>2.5.3</ivy.version>
<oro.version>2.0.8</oro.version>
+ <spark.version.short>3.5</spark.version.short>
+ <comet.version>1.0.0-SNAPSHOT</comet.version>
<!--
If you changes codahale.metrics.version, you also need to change
the link to metrics.dropwizard.io in docs/monitoring.md.
-@@ -2840,6 +2842,25 @@
+@@ -2836,6 +2838,25 @@
<artifactId>okio</artifactId>
<version>${okio.version}</version>
</dependency>
@@ -38,7 +38,7 @@ index edd2ad57880..45f8fd01538 100644
</dependencyManagement>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
-index bc00c448b80..82068d7a2eb 100644
+index d7f34db599a..5f676071ec0 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -77,6 +77,10 @@
@@ -522,10 +522,10 @@ index f33432ddb6f..b375e285dde 100644
}
assert(scanOption.isDefined)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-index a206e97c353..8bd3ab5985a 100644
+index 739557bef30..718fbc5b786 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
-@@ -264,7 +264,8 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
+@@ -265,7 +265,8 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
}
}
@@ -535,7 +535,7 @@ index a206e97c353..8bd3ab5985a 100644
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
withTempView("df") {
val df1 = spark.range(1, 100)
-@@ -467,7 +468,8 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
+@@ -468,7 +469,8 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
}
}
@@ -545,7 +545,7 @@ index a206e97c353..8bd3ab5985a 100644
withTempDir { dir =>
Seq("parquet", "orc", "csv", "json").foreach { fmt =>
val basePath = dir.getCanonicalPath + "/" + fmt
-@@ -545,7 +547,9 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
+@@ -546,7 +548,9 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
}
}
@@ -1184,7 +1184,7 @@ index d269290e616..13726a31e07 100644
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
-index cfc8b2cc845..c4be7eb3731 100644
+index 881992fe7ea..eb5dab406ca 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
@@ -21,6 +21,7 @@ import scala.collection.mutable.ArrayBuffer
@@ -1195,7 +1195,7 @@ index cfc8b2cc845..c4be7eb3731 100644
import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite,
Table, TableCapability}
import org.apache.spark.sql.connector.read.ScanBuilder
import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
-@@ -184,7 +185,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest
with SharedSparkSession {
+@@ -187,7 +188,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest
with SharedSparkSession {
val df = spark.read.format(format).load(path.getCanonicalPath)
checkAnswer(df, inputData.toDF())
assert(
@@ -1209,10 +1209,10 @@ index cfc8b2cc845..c4be7eb3731 100644
} finally {
spark.listenerManager.unregister(listener)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-index 71e030f535e..d5ae6cbf3d5 100644
+index 4819c210da1..3f30ec927b7 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala
-@@ -22,6 +22,7 @@ import org.apache.spark.sql.{DataFrame, Row}
+@@ -23,6 +23,7 @@ import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Literal,
TransformExpression}
import org.apache.spark.sql.catalyst.plans.physical
@@ -1220,8 +1220,8 @@ index 71e030f535e..d5ae6cbf3d5 100644
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog
import org.apache.spark.sql.connector.catalog.functions._
-@@ -31,7 +32,7 @@ import
org.apache.spark.sql.connector.expressions.Expressions._
- import org.apache.spark.sql.execution.SparkPlan
+@@ -32,7 +33,7 @@ import
org.apache.spark.sql.connector.expressions.Expressions._
+ import org.apache.spark.sql.execution.{RDDScanExec, SparkPlan}
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
@@ -1229,7 +1229,7 @@ index 71e030f535e..d5ae6cbf3d5 100644
import org.apache.spark.sql.execution.joins.SortMergeJoinExec
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf._
-@@ -282,13 +283,14 @@ class KeyGroupedPartitioningSuite extends
DistributionAndOrderingSuiteBase {
+@@ -283,19 +284,20 @@ class KeyGroupedPartitioningSuite extends
DistributionAndOrderingSuiteBase {
Row("bbb", 20, 250.0), Row("bbb", 20, 350.0), Row("ccc", 30, 400.50)))
}
@@ -1246,8 +1246,16 @@ index 71e030f535e..d5ae6cbf3d5 100644
})
}
+- private def collectAllShuffles(plan: SparkPlan): Seq[ShuffleExchangeExec] =
{
++ private def collectAllShuffles(plan: SparkPlan): Seq[ShuffleExchangeLike] =
{
+ collect(plan) {
+- case s: ShuffleExchangeExec => s
++ case s: ShuffleExchangeLike => s
+ }
+ }
+
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
-index 12007cd94cd..07020f201fb 100644
+index 3aa896386b3..671c437cafc 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/WriteDistributionAndOrderingSuite.scala
@@ -21,7 +21,7 @@ package org.apache.spark.sql.connector
@@ -1256,10 +1264,10 @@ index 12007cd94cd..07020f201fb 100644
-import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame, Row}
+import org.apache.spark.sql.{catalyst, AnalysisException, DataFrame,
IgnoreCometSuite, Row}
- import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression,
Cast, Literal}
+ import org.apache.spark.sql.catalyst.expressions.{ApplyFunctionExpression,
Cast, Literal, TransformExpression}
import org.apache.spark.sql.catalyst.expressions.objects.Invoke
import org.apache.spark.sql.catalyst.plans.physical
-@@ -45,7 +45,8 @@ import org.apache.spark.sql.util.QueryExecutionListener
+@@ -46,7 +46,8 @@ import org.apache.spark.sql.util.QueryExecutionListener
import org.apache.spark.tags.SlowSQLTest
@SlowSQLTest
@@ -2199,7 +2207,7 @@ index 8e88049f51e..f8e2194a8ac 100644
case _ =>
throw new AnalysisException("Can not match ParquetTable in the
query.")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-index 8ed9ef1630e..eed2a6f5ad5 100644
+index 710ab21090f..604af376a02 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -1345,7 +1345,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest
with SharedSparkSession
@@ -2213,7 +2221,7 @@ index 8ed9ef1630e..eed2a6f5ad5 100644
checkAnswer(
// "fruit" column in this file is encoded using
DELTA_LENGTH_BYTE_ARRAY.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index f6472ba3d9d..e8ccb8dec54 100644
+index ecec7d7b217..f6a4c746226 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -185,7 +185,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
@@ -2226,7 +2234,7 @@ index f6472ba3d9d..e8ccb8dec54 100644
val data = (1 to 1000).map { i =>
val ts = new java.sql.Timestamp(i)
Row(ts)
-@@ -998,7 +999,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1034,7 +1035,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
}
}
@@ -2236,7 +2244,7 @@ index f6472ba3d9d..e8ccb8dec54 100644
withAllParquetReaders {
withTempPath { path =>
// Repeated values for dictionary encoding.
-@@ -1051,7 +1053,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1087,7 +1089,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96")
}
@@ -2246,7 +2254,7 @@ index f6472ba3d9d..e8ccb8dec54 100644
def readParquet(schema: String, path: File): DataFrame = {
spark.read.schema(schema).parquet(path.toString)
}
-@@ -1067,7 +1070,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1103,7 +1106,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
checkAnswer(readParquet(schema, path), df)
}
@@ -2256,7 +2264,7 @@ index f6472ba3d9d..e8ccb8dec54 100644
val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
checkAnswer(readParquet(schema1, path), df)
val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
-@@ -1089,7 +1093,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1125,7 +1129,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c,
CAST('1.2' AS BINARY) d")
df.write.parquet(path.toString)
@@ -2266,7 +2274,7 @@ index f6472ba3d9d..e8ccb8dec54 100644
checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT
123456.0"))
-@@ -1133,7 +1138,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1169,7 +1174,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
}
}
@@ -2276,7 +2284,7 @@ index f6472ba3d9d..e8ccb8dec54 100644
withTempPath { path =>
Seq(0).toDF("a").write.parquet(path.toString)
// The vectorized and non-vectorized readers will produce different
exceptions, we don't need
-@@ -1148,7 +1154,7 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1184,7 +1190,7 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
.where(s"a < ${Long.MaxValue}")
.collect()
}
@@ -2960,10 +2968,10 @@ index e937173a590..263934fbe7b 100644
spark.internalCreateDataFrame(withoutFilters.execute(), schema)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-index ed2e309fa07..2dc45ad821f 100644
+index c23bf4204f7..f91ba524e6e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-@@ -74,6 +74,20 @@ trait SharedSparkSessionBase
+@@ -97,6 +97,20 @@ trait SharedSparkSessionBase
// this rule may potentially block testing of other optimization rules
such as
// ConstantPropagation etc.
.set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key,
ConvertToLocalRelation.ruleName)
diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md
b/docs/source/contributor-guide/iceberg-spark-tests.md
index 8a76b652b0..f7468e4240 100644
--- a/docs/source/contributor-guide/iceberg-spark-tests.md
+++ b/docs/source/contributor-guide/iceberg-spark-tests.md
@@ -95,7 +95,7 @@ diff must be generated against its own tag.
The `iceberg_spark_test_<version>.yml` workflows apply these diffs and run the
three Gradle targets above
against each Iceberg version. Iceberg 1.8.1 runs against Spark 3.4.3 with Java
11; Iceberg 1.9.1 and 1.10.0
-run against Spark 3.5.8 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.2
with Java 17. Iceberg 1.11
+run against Spark 3.5.9 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.2
with Java 17. Iceberg 1.11
(the only version testing Spark 4.1) runs on every pull request and on pushes
to main; the older versions
(1.8, 1.9, 1.10) run only on pushes to main, or on a pull request labeled
`run-iceberg-tests`. All caller
workflows delegate to `iceberg_spark_test_reusable.yml`, which holds the build
and test job logic.
diff --git a/docs/source/user-guide/latest/compatibility/spark-versions.md
b/docs/source/user-guide/latest/compatibility/spark-versions.md
index a0b739994e..d96294b5c9 100644
--- a/docs/source/user-guide/latest/compatibility/spark-versions.md
+++ b/docs/source/user-guide/latest/compatibility/spark-versions.md
@@ -47,7 +47,7 @@ Spark 3.4 support is deprecated as of the 1.0.0 release and
will be removed in t
## Spark 3.5
-Spark 3.5.8 is supported with Java 11/17 and Scala 2.12/2.13.
+Spark 3.5.9 is supported with Java 11/17 and Scala 2.12/2.13.
```{warning}
JDK 11 support is deprecated as of the 1.0.0 release and will be removed in
the 1.1.0 release.
diff --git a/docs/source/user-guide/latest/installation.md
b/docs/source/user-guide/latest/installation.md
index 454ec527c2..c25ce8da75 100644
--- a/docs/source/user-guide/latest/installation.md
+++ b/docs/source/user-guide/latest/installation.md
@@ -53,7 +53,7 @@ We recommend moving to JDK 17 or later and Spark 3.5 or later.
| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL
Tests in CI |
| ------------- | ------------ | ------------- | ----------------- |
--------------------- |
| 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes
|
-| 3.5.8 | 11/17 | 2.12/2.13 | Yes | Yes
|
+| 3.5.9 | 11/17 | 2.12/2.13 | Yes | Yes
|
| 4.0.2 | 17/21 | 2.13 | Yes | Yes
|
| 4.1.2 | 17/21 | 2.13 | Yes | Yes
|
diff --git a/pom.xml b/pom.xml
index 2b262ae9d8..927c07ca9c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -675,7 +675,7 @@ under the License.
<properties>
<scala.version>2.12.18</scala.version>
<scala.binary.version>2.12</scala.binary.version>
- <spark.version>3.5.8</spark.version>
+ <spark.version>3.5.9</spark.version>
<spark.version.short>3.5</spark.version.short>
<parquet.version>1.13.1</parquet.version>
<semanticdb.version>4.8.8</semanticdb.version>
diff --git a/spark/src/test/resources/pyspark/benchmark_pyarrow_udf.py
b/spark/src/test/resources/pyspark/benchmark_pyarrow_udf.py
index 08f2c6540f..dd6d8b3717 100644
--- a/spark/src/test/resources/pyspark/benchmark_pyarrow_udf.py
+++ b/spark/src/test/resources/pyspark/benchmark_pyarrow_udf.py
@@ -49,7 +49,7 @@ Usage:
# Build Comet (release for representative numbers):
make release
- pip install pyspark==3.5.8 pyarrow pandas
+ pip install pyspark==3.5.9 pyarrow pandas
python3 spark/src/test/resources/pyspark/benchmark_pyarrow_udf.py
diff --git a/spark/src/test/resources/pyspark/test_pyarrow_udf.py
b/spark/src/test/resources/pyspark/test_pyarrow_udf.py
index 4571932266..a7bdd598b3 100644
--- a/spark/src/test/resources/pyspark/test_pyarrow_udf.py
+++ b/spark/src/test/resources/pyspark/test_pyarrow_udf.py
@@ -33,7 +33,7 @@ Usage:
# explicitly via COMET_JAR:
export
COMET_JAR=$PWD/spark/target/comet-spark-spark3.5_2.12-0.16.0-SNAPSHOT.jar
- pip install pyspark==3.5.8 pyarrow pandas pytest
+ pip install pyspark==3.5.9 pyarrow pandas pytest
pytest -v spark/src/test/resources/pyspark/test_pyarrow_udf.py
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
