This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 36a3b8fc6a [GLUTEN-11088][VL] Fix NativeBenchmarkPlanGenerator (#11144)
36a3b8fc6a is described below
commit 36a3b8fc6a59c4b9cc24fd320b53a5310018dbe3
Author: Rong Ma <[email protected]>
AuthorDate: Mon Nov 24 12:55:14 2025 +0000
[GLUTEN-11088][VL] Fix NativeBenchmarkPlanGenerator (#11144)
---
.github/workflows/velox_backend_x86.yml | 17 +++++++++++++++--
.../benchmarks/NativeBenchmarkPlanGenerator.scala | 5 ++---
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/velox_backend_x86.yml
b/.github/workflows/velox_backend_x86.yml
index 98c33f7ee2..c99d936732 100644
--- a/.github/workflows/velox_backend_x86.yml
+++ b/.github/workflows/velox_backend_x86.yml
@@ -1257,7 +1257,7 @@ jobs:
cd ./cpp/build && ctest -V
- name: Run CPP benchmark test
run: |
- $MVN_CMD test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \
+ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -pl backends-velox
-am \
-DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none
-DfailIfNoTests=false -Dexec.skip
# This test depends on files generated by the above mvn test.
./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle
--partitioning hash --threads 1 --iterations 1 \
@@ -1266,14 +1266,27 @@ jobs:
--data $(realpath
backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath
backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
- name: Run UDF test
run: |
- # Depends on --build_example=ON.
yum install -y java-17-openjdk-devel
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
+ # Depends on --build_example=ON.
$MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg
-Pdelta -Ppaimon -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \
-DtagsToInclude=org.apache.gluten.tags.UDFTest \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/"
+ - name: Run CPP benchmark test for Spark 4.0
+ run: |
+ export SPARK_SCALA_VERSION=2.13
+ yum install -y java-17-openjdk-devel
+ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ export PATH=$JAVA_HOME/bin:$PATH
+ $MVN_CMD clean test -Pspark-4.0 -Pjava-17 -Pscala-2.13
-Pbackends-velox -pl backends-velox -am \
+ -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none
-DfailIfNoTests=false -Dexec.skip
+ # This test depends on files generated by the above mvn test.
+ ./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle
--partitioning hash --threads 1 --iterations 1 \
+ --conf $(realpath
backends-velox/generated-native-benchmark/conf_12_0_*.ini) \
+ --plan $(realpath
backends-velox/generated-native-benchmark/plan_12_0_*.json) \
+ --data $(realpath
backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath
backends-velox/generated-native-benchmark/data_12_0_*_1.parquet)
- name: Upload test report
uses: actions/upload-artifact@v4
with:
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala
b/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala
index 7ef1a1c06d..6886adbd26 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala
@@ -71,8 +71,7 @@ class NativeBenchmarkPlanGenerator extends
VeloxWholeStageTransformerSuite {
}
}
- // TODO: fix on spark-4.0
- testWithMaxSparkVersion("Test plan json non-empty - AQE on", "3.5") {
+ test("Test plan json non-empty - AQE on") {
withSQLConf(
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
GlutenConfig.CACHE_WHOLE_STAGE_TRANSFORMER_CONTEXT.key -> "true") {
@@ -85,7 +84,7 @@ class NativeBenchmarkPlanGenerator extends
VeloxWholeStageTransformerSuite {
executedPlan.execute()
val finalPlan =
executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
- val lastStageTransformer =
finalPlan.find(_.isInstanceOf[WholeStageTransformer])
+ val lastStageTransformer =
stripAQEPlan(finalPlan).find(_.isInstanceOf[WholeStageTransformer])
assert(lastStageTransformer.nonEmpty)
val planJson =
lastStageTransformer.get.asInstanceOf[WholeStageTransformer].substraitPlanJson
assert(planJson.nonEmpty)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
