This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3f463a39b7 [GLUTEN][UT] Fix PlanStability test suites for Velox
backend (#11799)
3f463a39b7 is described below
commit 3f463a39b72a155f5bea5b885e601279f4f9d281
Author: Chang Chen <[email protected]>
AuthorDate: Fri Mar 20 20:16:32 2026 +0800
[GLUTEN][UT] Fix PlanStability test suites for Velox backend (#11799)
Fix BroadcastHashJoinExecTransformerBase.requiredChildDistribution to use
bound+rewritten keys (matching Spark's BroadcastHashJoinExec), so that
ValidateRequirements.validate(plan) passes when AQE is disabled.
Add GlutenPlanStabilityTestTrait to override testQuery() - validates plans
via ValidateRequirements instead of golden file comparison, since Gluten
produces different physical plans with native Transformer operators.
Apply fixes to all PlanStability test suites for both Spark 4.0 and 4.1:
- GlutenTPCDSV1_4_PlanStabilitySuite
- GlutenTPCDSV1_4_PlanStabilityWithStatsSuite
- GlutenTPCDSV2_7_PlanStabilitySuite
- GlutenTPCDSV2_7_PlanStabilityWithStatsSuite
- GlutenTPCDSModifiedPlanStabilitySuite
- GlutenTPCDSModifiedPlanStabilityWithStatsSuite
- GlutenTPCHPlanStabilitySuite
Co-authored-by: Copilot <[email protected]>
---
.../gluten/execution/JoinExecTransformer.scala | 10 +++-
.../spark/sql/GlutenPlanStabilitySuite.scala | 53 +++++++++++++++++++---
.../spark/sql/GlutenPlanStabilitySuite.scala | 53 +++++++++++++++++++---
3 files changed, 101 insertions(+), 15 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala
index ed98bdc1a5..149a0ca729 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/JoinExecTransformer.scala
@@ -371,7 +371,15 @@ abstract class BroadcastHashJoinExecTransformerBase(
JoinUtils.getDirectJoinOutputSeq(joinType, left.output, right.output,
getClass.getSimpleName)
override def requiredChildDistribution: Seq[Distribution] = {
- val mode = HashedRelationBroadcastMode(buildKeyExprs, isNullAwareAntiJoin)
+ // Use the same bound+rewritten keys as Spark's BroadcastHashJoinExec to
ensure
+ // the mode matches the BroadcastExchange created by EnsureRequirements.
+ val (bKeys, bOutput) = buildSide match {
+ case BuildLeft => (leftKeys, left.output)
+ case BuildRight => (rightKeys, right.output)
+ }
+ val boundBuildKeys =
+ BindReferences.bindReferences(HashJoin.rewriteKeyExpr(bKeys), bOutput)
+ val mode = HashedRelationBroadcastMode(boundBuildKeys, isNullAwareAntiJoin)
buildSide match {
case BuildLeft =>
BroadcastDistribution(mode) :: UnspecifiedDistribution :: Nil
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
index 8eaa795090..82ff3afc11 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
@@ -16,28 +16,67 @@
*/
package org.apache.spark.sql
+import org.apache.spark.sql.catalyst.util.resourceToString
+import org.apache.spark.sql.execution.exchange.ValidateRequirements
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Gluten plan stability suites verify that Gluten-transformed plans satisfy
Spark's distribution
+ * and ordering requirements (via ValidateRequirements). Golden file
comparison against vanilla
+ * Spark plans is skipped because Gluten intentionally produces different
physical plans using
+ * native Transformer operators.
+ */
+trait GlutenPlanStabilityTestTrait {
+ self: PlanStabilitySuite =>
+
+ override protected def testQuery(tpcdsGroup: String, query: String, suffix:
String = ""): Unit = {
+ val queryString = resourceToString(
+ s"$tpcdsGroup/$query.sql",
+ classLoader = Thread.currentThread().getContextClassLoader)
+ withSQLConf(
+ SQLConf.READ_SIDE_CHAR_PADDING.key -> "false",
+ SQLConf.LEGACY_NO_CHAR_PADDING_IN_PREDICATE.key -> "true",
+ SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10MB") {
+ val qe = sql(queryString).queryExecution
+ val plan = qe.executedPlan
+ assert(
+ ValidateRequirements.validate(plan),
+ s"ValidateRequirements failed for $tpcdsGroup/$query$suffix")
+ }
+ }
+}
+
class GlutenTPCDSV1_4_PlanStabilitySuite
extends TPCDSV1_4_PlanStabilitySuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSV1_4_PlanStabilityWithStatsSuite
extends TPCDSV1_4_PlanStabilityWithStatsSuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSV2_7_PlanStabilitySuite
extends TPCDSV2_7_PlanStabilitySuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSV2_7_PlanStabilityWithStatsSuite
extends TPCDSV2_7_PlanStabilityWithStatsSuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSModifiedPlanStabilitySuite
extends TPCDSModifiedPlanStabilitySuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSModifiedPlanStabilityWithStatsSuite
extends TPCDSModifiedPlanStabilityWithStatsSuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
-class GlutenTPCHPlanStabilitySuite extends TPCHPlanStabilitySuite with
GlutenTestsCommonTrait {}
+class GlutenTPCHPlanStabilitySuite
+ extends TPCHPlanStabilitySuite
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
index 8eaa795090..82ff3afc11 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala
@@ -16,28 +16,67 @@
*/
package org.apache.spark.sql
+import org.apache.spark.sql.catalyst.util.resourceToString
+import org.apache.spark.sql.execution.exchange.ValidateRequirements
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Gluten plan stability suites verify that Gluten-transformed plans satisfy
Spark's distribution
+ * and ordering requirements (via ValidateRequirements). Golden file
comparison against vanilla
+ * Spark plans is skipped because Gluten intentionally produces different
physical plans using
+ * native Transformer operators.
+ */
+trait GlutenPlanStabilityTestTrait {
+ self: PlanStabilitySuite =>
+
+ override protected def testQuery(tpcdsGroup: String, query: String, suffix:
String = ""): Unit = {
+ val queryString = resourceToString(
+ s"$tpcdsGroup/$query.sql",
+ classLoader = Thread.currentThread().getContextClassLoader)
+ withSQLConf(
+ SQLConf.READ_SIDE_CHAR_PADDING.key -> "false",
+ SQLConf.LEGACY_NO_CHAR_PADDING_IN_PREDICATE.key -> "true",
+ SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10MB") {
+ val qe = sql(queryString).queryExecution
+ val plan = qe.executedPlan
+ assert(
+ ValidateRequirements.validate(plan),
+ s"ValidateRequirements failed for $tpcdsGroup/$query$suffix")
+ }
+ }
+}
+
class GlutenTPCDSV1_4_PlanStabilitySuite
extends TPCDSV1_4_PlanStabilitySuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSV1_4_PlanStabilityWithStatsSuite
extends TPCDSV1_4_PlanStabilityWithStatsSuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSV2_7_PlanStabilitySuite
extends TPCDSV2_7_PlanStabilitySuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSV2_7_PlanStabilityWithStatsSuite
extends TPCDSV2_7_PlanStabilityWithStatsSuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSModifiedPlanStabilitySuite
extends TPCDSModifiedPlanStabilitySuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
class GlutenTPCDSModifiedPlanStabilityWithStatsSuite
extends TPCDSModifiedPlanStabilityWithStatsSuite
- with GlutenTestsCommonTrait {}
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
-class GlutenTPCHPlanStabilitySuite extends TPCHPlanStabilitySuite with
GlutenTestsCommonTrait {}
+class GlutenTPCHPlanStabilitySuite
+ extends TPCHPlanStabilitySuite
+ with GlutenSQLTestsBaseTrait
+ with GlutenPlanStabilityTestTrait {}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
