This is an automated email from the ASF dual-hosted git repository.
jackylee-ch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c823f816a3 [MINOR] Align supported Spark version references (#12525)
c823f816a3 is described below
commit c823f816a32c51130a7e70793c2d459a4854cb41
Author: YangJie <[email protected]>
AuthorDate: Fri Jul 17 10:16:30 2026 +0800
[MINOR] Align supported Spark version references (#12525)
---
.github/ISSUE_TEMPLATE/bug.yml | 4 ++--
.../apache/gluten/execution/RowToVeloxColumnarExec.scala | 1 -
.../apache/gluten/extension/CudfNodeValidationRule.scala | 5 ++---
.../gluten/execution/VeloxAggregateFunctionsSuite.scala | 1 -
.../gluten/functions/ArithmeticAnsiValidateSuite.scala | 2 +-
docs/developers/HowToRelease.md | 1 -
docs/get-started/Velox.md | 16 +++++++++-------
docs/get-started/VeloxQAT.md | 2 +-
docs/velox-backend-limitations.md | 10 +++++-----
.../sql/execution/datasources/orc/OrcFileFormat.scala | 11 +++++------
tools/gluten-it/README.md | 2 +-
.../org/apache/gluten/integration/SparkJvmOptions.java | 16 ++++++++++------
12 files changed, 36 insertions(+), 35 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
index 21e12f2fff..09baf2a826 100644
--- a/.github/ISSUE_TEMPLATE/bug.yml
+++ b/.github/ISSUE_TEMPLATE/bug.yml
@@ -65,11 +65,11 @@ body:
label: Spark version
description: Please provide the spark version in your environment.
options:
- - Spark-3.2.x
- Spark-3.3.x
- Spark-3.4.x
- Spark-3.5.x
- - spark-4.0.x
+ - Spark-4.0.x
+ - Spark-4.1.x
validations:
required: false
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
index f11de5894a..70518d3f71 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
@@ -91,7 +91,6 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends
RowToColumnarExecBas
)
}
- // For spark 3.2.
protected def withNewChildInternal(newChild: SparkPlan):
RowToVeloxColumnarExec =
copy(child = newChild)
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/extension/CudfNodeValidationRule.scala
b/backends-velox/src/main/scala/org/apache/gluten/extension/CudfNodeValidationRule.scala
index 14029cf28f..b3857c68f5 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/extension/CudfNodeValidationRule.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/extension/CudfNodeValidationRule.scala
@@ -55,11 +55,10 @@ case class CudfNodeValidationRule(glutenConf: GlutenConfig)
extends Rule[SparkPl
object CudfNodeValidationRule {
def setTagForWholeStageTransformer(transformer: WholeStageTransformer): Unit
= {
if (!VeloxConfig.get.cudfEnableTableScan) {
- // Spark3.2 does not have exists
- val hasLeaf = transformer.find {
+ val hasLeaf = transformer.exists {
case _: LeafTransformSupport => true
case _ => false
- }.isDefined
+ }
if (!hasLeaf && VeloxConfig.get.cudfEnableValidation) {
if (
VeloxCudfPlanValidatorJniWrapper.validate(
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
index 98b46255ff..66a7b12a87 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
@@ -1156,7 +1156,6 @@ abstract class VeloxAggregateFunctionsSuite extends
VeloxWholeStageTransformerSu
}
test("drop redundant partial sort which has pre-project when offload
sortAgg") {
- // Spark 3.2 does not have this configuration, but it does not affect the
test results.
withSQLConf("spark.sql.test.forceApplySortAggregate" -> "true") {
withTempView("t1") {
Seq((-1, 2), (-1, 3), (2, 3), (3, 4), (-3, 5), (4, 5))
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
index a1633c4cb4..25eefb9ff0 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ArithmeticAnsiValidateSuite.scala
@@ -84,7 +84,7 @@ class ArithmeticAnsiValidateSuite extends
FunctionsValidateSuite {
sql("SELECT 1 / 0").collect()
}
} else {
- // Spark 3.2 and 3.3 don't throw exception for division by zero in ANSI
mode
+ // Spark 3.3 doesn't throw exception for division by zero in ANSI mode
sql("SELECT 1 / 0").collect()
}
}
diff --git a/docs/developers/HowToRelease.md b/docs/developers/HowToRelease.md
index 57dedb6a72..fdf042d57c 100644
--- a/docs/developers/HowToRelease.md
+++ b/docs/developers/HowToRelease.md
@@ -79,7 +79,6 @@ Confirm that all the needed sources and binaries are
successfully created at the
[root@8de83f716f0f workspace]# ls -l release/
total 481628
-rw-r--r--. 1 root root 74396439 Oct 14 14:19
apache-gluten-1.6.0-example-src.tar.gz
--rw-r--r--. 1 root root 104790092 Oct 14 14:19
apache-gluten-1.6.0-example-bin-spark-3.2.tar.gz
-rw-r--r--. 1 root root 104767582 Oct 14 14:19
apache-gluten-1.6.0-example-bin-spark-3.3.tar.gz
-rw-r--r--. 1 root root 104625356 Oct 14 14:19
apache-gluten-1.6.0-example-bin-spark-3.4.tar.gz
-rw-r--r--. 1 root root 104595103 Oct 14 14:19
apache-gluten-1.6.0-example-bin-spark-3.5.tar.gz
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
index 427a774958..5c504607fb 100644
--- a/docs/get-started/Velox.md
+++ b/docs/get-started/Velox.md
@@ -7,18 +7,20 @@ parent: Getting-Started
# Supported Version
-| Type | Version |
-|-------|------------------------------|
-| Spark | 3.2.2, 3.3.1, 3.4.4, 3.5.5 |
-| OS | Ubuntu20.04/22.04, Centos7/8 |
-| jdk | openjdk8/jdk17 |
-| scala | 2.12 |
+| Type | Version |
+|-------|-------------------------------------|
+| Spark | 3.3.1, 3.4.4, 3.5.5, 4.0.2, 4.1.1 |
+| OS | Ubuntu20.04/22.04, Centos7/8 |
+| jdk | openjdk8/jdk17 |
+| scala | 2.12 |
+
+Note: Spark 4.0 and 4.1 require JDK 17+ and Scala 2.13 (build with
`-Pspark-4.0` or `-Pspark-4.1` plus `-Pjava-17 -Pscala-2.13`). Spark 3.3 to 3.5
can be built with JDK 8/17 and Scala 2.12.
# Prerequisite
Currently, with static build Gluten+Velox backend supports all the Linux OSes,
but is only tested on **Ubuntu20.04/Ubuntu22.04/Centos7/Centos8**. With dynamic
build, Gluten+Velox backend support **Ubuntu20.04/Ubuntu22.04/Centos7/Centos8**
and their variants.
-Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.4 and
3.5.5.
+Currently, the officially supported Spark versions are 3.3.1, 3.4.4, 3.5.5,
4.0.2 and 4.1.1.
We need to set up the `JAVA_HOME` env. Currently, Gluten supports **java 8**
and **java 17**.
diff --git a/docs/get-started/VeloxQAT.md b/docs/get-started/VeloxQAT.md
index 6d33654e59..0280775993 100644
--- a/docs/get-started/VeloxQAT.md
+++ b/docs/get-started/VeloxQAT.md
@@ -83,7 +83,7 @@ exit
```bash
cd /path/to/gluten
-## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1.
+## The script builds jars for all supported Spark versions.
./dev/buildbundle-veloxbe.sh --enable_qat=ON
```
diff --git a/docs/velox-backend-limitations.md
b/docs/velox-backend-limitations.md
index 9dc830fb33..eee1e05432 100644
--- a/docs/velox-backend-limitations.md
+++ b/docs/velox-backend-limitations.md
@@ -6,12 +6,12 @@ nav_order: 5
This document describes the limitations of velox backend by listing some known
cases where exception will be thrown, gluten behaves incompatibly with spark,
or certain plan's execution
must fall back to vanilla spark, etc.
-### Override of Spark classes (For Spark3.2 and Spark3.3)
-Gluten avoids to modify Spark's existing code and use Spark APIs if possible.
However, some APIs aren't exposed in Vanilla spark and we have to copy the
Spark file and do the hardcode changes. The list of override classes can be
found as ignoreClasses in package/pom.xml . If you use customized Spark, you
may check if the files are modified in your spark, otherwise your changes will
be overrided.
+### Override of Spark classes
+Gluten avoids modifying Spark's existing code and prefers Spark APIs when
possible. However, some APIs are not exposed by vanilla Spark, so we have to
copy the Spark file and apply hardcoded changes. The list of overridden classes
can be found as `ignoreClasses` in `package/pom.xml`. If you use a customized
Spark, check whether these files are modified in your Spark distribution,
otherwise your changes will be overridden.
So you need to ensure preferentially load the Gluten jar to overwrite the jar
of vanilla spark. Refer to [How to prioritize loading Gluten jars in
Spark](https://github.com/apache/gluten/blob/main/docs/velox-backend-troubleshooting.md#incompatible-class-error-when-using-native-writer).
-If not officially supported spark3.2/3.3 version is used, NoSuchMethodError
can be thrown at runtime. More details see
[issue-4514](https://github.com/apache/gluten/issues/4514).
+If an unofficially supported Spark version is used, NoSuchMethodError can be
thrown at runtime. More details see
[issue-4514](https://github.com/apache/gluten/issues/4514).
### Fallbacks
Except the unsupported operators, functions, file formats, data sources listed
in , there are some known cases also fall back to Vanilla Spark.
@@ -59,7 +59,7 @@ Spark has `spark.sql.parquet.datetimeRebaseModeInWrite`
config to decide whether
or Proleptic Gregorian calendar should be used during parquet writing for
dates/timestamps. If the parquet to read is written
by Spark with this config as true, Velox's TableScan will output different
result when reading it back.
-#### Partition write (For Spark3.2 and Spark3.3)
+#### Partition write (For Spark3.3)
Gluten only supports static partition writes and does not support dynamic
partition writes.
@@ -96,7 +96,7 @@ spark.range(100).selectExpr("id as c1", "id % 7 as p")
.save(f.getCanonicalPath)
```
-#### CTAS write (For Spark3.2 and Spark3.3)
+#### CTAS write (For Spark3.3)
Gluten does not create table as select. It may raise exception. e.g.,
diff --git
a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
index d1b9d5099d..bb1eca0ec9 100644
---
a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
+++
b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
@@ -43,12 +43,11 @@ import org.apache.orc.mapreduce._
import java.io._
import java.net.URI
-/* -
- * This class is copied from Spark 3.2 and modified for Gluten. \n
- * Gluten should make sure this class is loaded before the original class.
- * If new Spark releases accepts changes and make this class incompatible,
- * we can move this class to shims-spark32,
- * shims-spark33, etc.
+/**
+ * This class is copied from Spark and modified for Gluten. Gluten should make
sure this class is
+ * loaded before the original class. If a future Spark release introduces
breaking changes to this
+ * class, we can move this file into a version-specific shim (e.g.
shims/spark34) so it is only used
+ * by the Spark versions that need it.
*/
class OrcFileFormat extends FileFormat with DataSourceRegister with
Serializable {
diff --git a/tools/gluten-it/README.md b/tools/gluten-it/README.md
index 745c8523a3..d2963fcc6d 100644
--- a/tools/gluten-it/README.md
+++ b/tools/gluten-it/README.md
@@ -22,7 +22,7 @@ mvn clean package -P{Spark-Version}
sbin/gluten-it.sh
```
-Note: **Spark-Version** can only be **spark-3.2**, **spark-3.3**,
**spark-3.4** or **spark-3.5**.
+Note: **Spark-Version** can only be **spark-3.3**, **spark-3.4** or
**spark-3.5**.
## Usage
diff --git
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
index aa9336d0f0..d2c2934eaa 100644
---
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
+++
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
@@ -25,14 +25,18 @@ public class SparkJvmOptions {
public static String read() {
try {
- final Class<?> clazz =
Class.forName("org.apache.spark.launcher.JavaModuleOptions");
+ final Class<?> clazz = Class.forName(MODULE_OPTIONS_CLASS_NAME);
final Method method = clazz.getMethod("defaultModuleOptions");
return (String) method.invoke(null);
- } catch (ClassNotFoundException e) {
- // Could happen in Spark 3.2 which doesn't have this class yet.
- return "";
- } catch (NoSuchMethodException | InvocationTargetException |
IllegalAccessException e) {
- throw new RuntimeException(e);
+ } catch (ClassNotFoundException
+ | NoSuchMethodException
+ | InvocationTargetException
+ | IllegalAccessException e) {
+ throw new RuntimeException(
+ "Failed to read Spark JVM module options via "
+ + MODULE_OPTIONS_CLASS_NAME
+ + "#defaultModuleOptions",
+ e);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
