This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 42b78780c [GLUTEN-4424][CORE] Follow up upgrading spark version to
3.5.1 (#4845)
42b78780c is described below
commit 42b78780c71fe81f47be9be01cdedddb84388f9a
Author: JiaKe <[email protected]>
AuthorDate: Fri Mar 8 12:17:33 2024 +0800
[GLUTEN-4424][CORE] Follow up upgrading spark version to 3.5.1 (#4845)
---
dev/buildbundle-veloxbe.sh | 1 +
dev/package.sh | 1 +
.../main/scala/io/glutenproject/execution/WholeStageTransformer.scala | 2 --
.../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++--
.../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++--
.../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++--
.../scala/io/glutenproject/execution/GenerateTreeStringShim.scala | 4 ++--
tools/gluten-te/ubuntu/dockerfile-buildenv | 4 ++++
8 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh
index 3bfd6994a..1c77b52a6 100755
--- a/dev/buildbundle-veloxbe.sh
+++ b/dev/buildbundle-veloxbe.sh
@@ -7,3 +7,4 @@ cd $GLUTEN_DIR
mvn clean package -Pbackends-velox -Prss -Pspark-3.2 -DskipTests
mvn clean package -Pbackends-velox -Prss -Pspark-3.3 -DskipTests
mvn clean package -Pbackends-velox -Prss -Pspark-3.4 -DskipTests
+mvn clean package -Pbackends-velox -Prss -Pspark-3.5 -DskipTests
diff --git a/dev/package.sh b/dev/package.sh
index 0e1f362bc..2f6602d00 100755
--- a/dev/package.sh
+++ b/dev/package.sh
@@ -14,6 +14,7 @@ $GLUTEN_DIR/dev/builddeps-veloxbe.sh --build_tests=ON
--build_benchmarks=ON --en
mvn clean package -Pbackends-velox -Prss -Pspark-3.2 -DskipTests
mvn clean package -Pbackends-velox -Prss -Pspark-3.3 -DskipTests
mvn clean package -Pbackends-velox -Prss -Pspark-3.4 -DskipTests
+mvn clean package -Pbackends-velox -Prss -Pspark-3.5 -DskipTests
mkdir -p $THIRDPARTY_LIB
function process_setup_ubuntu_2004 {
diff --git
a/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala
b/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala
index 2bbc80afc..24957d1c2 100644
---
a/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala
+++
b/gluten-core/src/main/scala/io/glutenproject/execution/WholeStageTransformer.scala
@@ -102,8 +102,6 @@ case class WholeStageTransformer(child: SparkPlan,
materializeInput: Boolean = f
def stageId: Int = transformStageId
- def substraitPlanJsonValue: String = substraitPlanJson
-
def wholeStageTransformerContextDefined: Boolean =
wholeStageTransformerContext.isDefined
// For WholeStageCodegen-like operator, only pipeline time will be handled
in graph plotting.
diff --git
a/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
b/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
index d99fb18d7..d06728bc7 100644
---
a/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
+++
b/shims/spark32/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
@@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
def stageId: Int
- def substraitPlanJsonValue: String
+ def substraitPlanJson: String
def wholeStageTransformerContextDefined: Boolean
@@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
if (verbose && wholeStageTransformerContextDefined) {
append(prefix + "Substrait plan:\n")
- append(substraitPlanJsonValue)
+ append(substraitPlanJson)
append("\n")
}
}
diff --git
a/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
b/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
index d99fb18d7..d06728bc7 100644
---
a/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
+++
b/shims/spark33/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
@@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
def stageId: Int
- def substraitPlanJsonValue: String
+ def substraitPlanJson: String
def wholeStageTransformerContextDefined: Boolean
@@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
if (verbose && wholeStageTransformerContextDefined) {
append(prefix + "Substrait plan:\n")
- append(substraitPlanJsonValue)
+ append(substraitPlanJson)
append("\n")
}
}
diff --git
a/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
b/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
index d99fb18d7..d06728bc7 100644
---
a/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
+++
b/shims/spark34/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
@@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
def stageId: Int
- def substraitPlanJsonValue: String
+ def substraitPlanJson: String
def wholeStageTransformerContextDefined: Boolean
@@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
if (verbose && wholeStageTransformerContextDefined) {
append(prefix + "Substrait plan:\n")
- append(substraitPlanJsonValue)
+ append(substraitPlanJson)
append("\n")
}
}
diff --git
a/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
b/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
index 3753d1df4..9d0419967 100644
---
a/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
+++
b/shims/spark35/src/main/scala/io/glutenproject/execution/GenerateTreeStringShim.scala
@@ -29,7 +29,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
def stageId: Int
- def substraitPlanJsonValue: String
+ def substraitPlanJson: String
def wholeStageTransformerContextDefined: Boolean
@@ -57,7 +57,7 @@ trait GenerateTreeStringShim extends UnaryExecNode {
if (verbose && wholeStageTransformerContextDefined) {
append(prefix + "Substrait plan:\n")
- append(substraitPlanJsonValue)
+ append(substraitPlanJson)
append("\n")
}
}
diff --git a/tools/gluten-te/ubuntu/dockerfile-buildenv
b/tools/gluten-te/ubuntu/dockerfile-buildenv
index 280837bb3..dbb9fc65b 100644
--- a/tools/gluten-te/ubuntu/dockerfile-buildenv
+++ b/tools/gluten-te/ubuntu/dockerfile-buildenv
@@ -115,6 +115,10 @@ RUN cd /opt/spark331 && ./build/mvn -Pyarn -DskipTests
clean install
RUN cd /opt && git clone --depth 1 --branch v3.4.2
https://github.com/apache/spark.git spark342
RUN cd /opt/spark342 && ./build/mvn -Pyarn -DskipTests clean install
+# Build & install Spark 3.5.1
+# RUN cd /opt && git clone --depth 1 --branch v3.5.1
https://github.com/apache/spark.git spark351
+# RUN cd /opt/spark351 && ./build/mvn -Pyarn -DskipTests clean install
+
# Prepare entry command
COPY scripts/cmd.sh /root/.cmd.sh
CMD ["/root/.cmd.sh"]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
