This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 5d35d9ec26 [MINOR] Clean up residual Spark 3.3 references (#13089)
5d35d9ec26 is described below
commit 5d35d9ec26e53bef2c90f950dd2ecc299d87e641
Author: YangJie <[email protected]>
AuthorDate: Wed Sep 30 17:24:37 2026 -0400
[MINOR] Clean up residual Spark 3.3 references (#13089)
---
.../apache/gluten/execution/WriteFilesExecTransformer.scala | 2 +-
tools/workload/benchmark_velox/initialize.ipynb | 12 ++++++------
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
index 98593a3e12..a2040d34d8 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala
@@ -287,7 +287,7 @@ object WriteFilesExecTransformer {
.toLowerCase(Locale.ROOT)
}
- // To be compatible with Spark 3.3/3.4, we clean up Spark internal metadata
manually.
+ // To be compatible with Spark 3.4, we clean up Spark internal metadata
manually.
// Spark 3.5 already strips these fields via SPARK-43123.
// See https://github.com/apache/spark/pull/40776
private val INTERNAL_METADATA_KEYS = Seq(
diff --git a/tools/workload/benchmark_velox/initialize.ipynb
b/tools/workload/benchmark_velox/initialize.ipynb
index 1d3e6397b6..3c504137e4 100644
--- a/tools/workload/benchmark_velox/initialize.ipynb
+++ b/tools/workload/benchmark_velox/initialize.ipynb
@@ -671,7 +671,7 @@
},
"outputs": [],
"source": [
- "!wget
https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz >
/dev/null 2>&1"
+ "!wget
https://archive.apache.org/dist/spark/spark-3.5.5/spark-3.5.5-bin-hadoop3.tgz >
/dev/null 2>&1"
]
},
{
@@ -759,12 +759,12 @@
"source": [
"for l in hclients:\n",
" !scp hadoop-3.2.4.tar.gz {l}:~/\n",
- " !scp spark-3.3.1-bin-hadoop3.tgz {l}:~/\n",
+ " !scp spark-3.5.5-bin-hadoop3.tgz {l}:~/\n",
" !ssh {l} \"mv -f hadoop hadoop.bak; mv -f spark spark.bak\"\n",
" !ssh {l} \"tar zxvf hadoop-3.2.4.tar.gz > /dev/null 2>&1\"\n",
- " !ssh {l} \"tar -zxvf spark-3.3.1-bin-hadoop3.tgz > /dev/null
2>&1\"\n",
+ " !ssh {l} \"tar -zxvf spark-3.5.5-bin-hadoop3.tgz > /dev/null
2>&1\"\n",
" !ssh root@{l} \"apt install -y openjdk-8-jdk > /dev/null 2>&1\"\n",
- " !ssh {l} \"ln -s hadoop-3.2.4 hadoop; ln -s spark-3.3.1-bin-hadoop3
spark\"\n",
+ " !ssh {l} \"ln -s hadoop-3.2.4 hadoop; ln -s spark-3.5.5-bin-hadoop3
spark\"\n",
" if is_arm:\n",
" !ssh {l} \"tar zxvf hadoop-3.3.5-aarch64.tar.gz > /dev/null
2>&1\"\n",
" !ssh {l} \"cd hadoop && mv lib lib.bak && cp -rf
~/hadoop-3.3.5/lib ~/hadoop\""
@@ -795,7 +795,7 @@
"export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop\n",
"\n",
"export SPARK_HOME=~/spark\n",
- "export
PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.5-src.zip:$PYTHONPATH\n",
+ "export
PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH\n",
"export PATH=$SPARK_HOME/bin:$PATH\n",
"\n",
"'''"
@@ -1598,7 +1598,7 @@
"outputs": [],
"source": [
"for l in hclients:\n",
- " !scp spark/yarn/spark-3.3.1-yarn-shuffle.jar
{l}:~/hadoop/share/hadoop/common/lib/"
+ " !scp spark/yarn/spark-3.5.5-yarn-shuffle.jar
{l}:~/hadoop/share/hadoop/common/lib/"
]
},
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
