This is an automated email from the ASF dual-hosted git repository.
jackylee-ch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6df103a6a7 [MINOR] Clean up Spark 3.2 / 3.1-era compat leftovers in
non-CH modules (#12543)
6df103a6a7 is described below
commit 6df103a6a70fe5ff9f75894c6d44958f7b878903
Author: YangJie <[email protected]>
AuthorDate: Mon Jul 27 11:40:56 2026 +0800
[MINOR] Clean up Spark 3.2 / 3.1-era compat leftovers in non-CH modules
(#12543)
---
.../datasources/velox/VeloxFormatWriterInjects.scala | 3 +--
.../gluten/vectorized/ArrowColumnarBatch.scala | 6 ++++--
.../gluten/celeborn/CelebornShuffleManager.java | 1 -
.../gluten/execution/WriteFilesExecTransformer.scala | 3 ++-
.../apache/gluten/integration/SparkJvmOptions.java | 20 ++------------------
5 files changed, 9 insertions(+), 24 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
index 06e9d91c0f..f67446d846 100644
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
@@ -92,8 +92,7 @@ trait VeloxFormatWriterInjects extends
GlutenFormatWriterInjectsBase {
datasourceJniWrapper.close(dsHandle)
}
- // Do NOT add override keyword for compatibility on spark 3.1.
- def path(): String = {
+ override def path(): String = {
filePath
}
}
diff --git
a/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala
b/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala
index decd87e78f..67e823452e 100644
---
a/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala
+++
b/gluten-arrow/src/main/scala/org/apache/gluten/vectorized/ArrowColumnarBatch.scala
@@ -20,8 +20,10 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.vectorized.ColumnVector
/**
- * Because Spark-3.2 declares ColumnarBatch as final, so `ArrowColumnarBatch`
can't extend
- * `ColumnarBatch`. The code is mainly copied from Spark-3.2
+ * A `ColumnarBatch`-like container of [[ArrowWritableColumnVector]]s.
Originally forked from
+ * Spark's `ColumnarBatch` before Gluten could depend on that class directly;
it is kept as a
+ * standalone class rather than extending `ColumnarBatch` to preserve the
writable-column contract
+ * and existing call sites.
*
* @param writableColumns
* the columns this class wraps
diff --git
a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
index 31d82ce0c7..0a82e32726 100644
---
a/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
+++
b/gluten-celeborn/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
@@ -379,7 +379,6 @@ public class CelebornShuffleManager
}
}
- // Added in SPARK-32055, for Spark 3.1 and above
public <K, C> ShuffleReader<K, C> getReader(
ShuffleHandle handle,
int startMapIndex,
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
index 13689f1b28..98593a3e12 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
@@ -287,7 +287,8 @@ object WriteFilesExecTransformer {
.toLowerCase(Locale.ROOT)
}
- // To be compatible with Spark3.2/3.3/3.4, we do cleanup spark internal
metadata manually.
+ // To be compatible with Spark 3.3/3.4, we clean up Spark internal metadata
manually.
+ // Spark 3.5 already strips these fields via SPARK-43123.
// See https://github.com/apache/spark/pull/40776
private val INTERNAL_METADATA_KEYS = Seq(
"__autoGeneratedAlias",
diff --git
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
index d2c2934eaa..e29a7df6d1 100644
---
a/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
+++
b/tools/gluten-it/common/src/main/java/org/apache/gluten/integration/SparkJvmOptions.java
@@ -16,28 +16,12 @@
*/
package org.apache.gluten.integration;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
+import org.apache.spark.launcher.JavaModuleOptions;
public class SparkJvmOptions {
- private static final String MODULE_OPTIONS_CLASS_NAME =
- "org.apache.spark.launcher.JavaModuleOptions";
public static String read() {
- try {
- final Class<?> clazz = Class.forName(MODULE_OPTIONS_CLASS_NAME);
- final Method method = clazz.getMethod("defaultModuleOptions");
- return (String) method.invoke(null);
- } catch (ClassNotFoundException
- | NoSuchMethodException
- | InvocationTargetException
- | IllegalAccessException e) {
- throw new RuntimeException(
- "Failed to read Spark JVM module options via "
- + MODULE_OPTIONS_CLASS_NAME
- + "#defaultModuleOptions",
- e);
- }
+ return JavaModuleOptions.defaultModuleOptions();
}
public static void main(String[] args) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
