This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3c749e50ff [MINOR] Refactor operator-enable configs and remove dead
code (#12212)
3c749e50ff is described below
commit 3c749e50ffdd66535a160e7ee8a416051b9ab30f
Author: Philo He <[email protected]>
AuthorDate: Tue Jun 9 09:50:57 2026 -0700
[MINOR] Refactor operator-enable configs and remove dead code (#12212)
---
.../gluten/backendsapi/clickhouse/CHBackend.scala | 2 --
.../extension/FallbackBroadcastHashJoinRules.scala | 2 +-
.../gluten/backendsapi/velox/VeloxBackend.scala | 2 --
docs/Configuration.md | 4 ++--
.../gluten/backendsapi/BackendSettingsApi.scala | 2 --
.../org/apache/gluten/config/GlutenConfig.scala | 20 ++++++++++----------
.../BroadcastNestedLoopJoinExecTransformer.scala | 6 ------
.../gluten/execution/ExpandExecTransformer.scala | 3 ---
.../extension/columnar/validator/Validators.scala | 4 ++--
9 files changed, 15 insertions(+), 30 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
index ba540914fb..395ed6fd17 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
@@ -363,8 +363,6 @@ object CHBackendSettings extends BackendSettingsApi with
Logging {
override def structFieldToLowerCase(): Boolean = false
- override def supportExpandExec(): Boolean = true
-
override def excludeScanExecFromCollapsedStage(): Boolean =
SQLConf.get
.getConfString(GLUTEN_CLICKHOUSE_SEP_SCAN_RDD,
GLUTEN_CLICKHOUSE_SEP_SCAN_RDD_DEFAULT)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala
index 2b728a9464..4a9513b429 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala
@@ -151,7 +151,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession)
extends Rule[SparkPl
GlutenConfig.get.enableColumnarBroadcastExchange
private val enableColumnarBroadcastNestedLoopJoin: Boolean =
- GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled &&
+ GlutenConfig.get.enableColumnarBroadcastNestedLoopJoin &&
GlutenConfig.get.enableColumnarBroadcastExchange
override def apply(plan: SparkPlan): SparkPlan = {
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index 599b91851e..6ecbb745b2 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -397,8 +397,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}
- override def supportExpandExec(): Boolean = true
-
override def supportSortExec(): Boolean = true
override def supportSortMergeJoinExec(): Boolean = {
diff --git a/docs/Configuration.md b/docs/Configuration.md
index 60d555e7c9..6b46692650 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -41,15 +41,15 @@ nav_order: 15
| spark.gluten.soft-affinity.replications.num | 🔄
Dynamic | 2 | Calculate the number of the replications for
scheduling to the target executors per file
[...]
| spark.gluten.sql.adaptive.costEvaluator.enabled | âš“
Static | true | If true, use
org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost
evaluator class, else follow the configuration
spark.sql.adaptive.customCostEvaluatorClass.
[...]
| spark.gluten.sql.ansiFallback.enabled | 🔄
Dynamic | true | When true (default), Gluten will fall back to
Spark when ANSI mode is enabled. When false, Gluten will attempt to execute in
ANSI mode.
[...]
-| spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled | 🔄
Dynamic | true | Config to enable
BroadcastNestedLoopJoinExecTransformer.
[...]
| spark.gluten.sql.cacheWholeStageTransformerContext | 🔄
Dynamic | false | When true, `WholeStageTransformer` will cache
the `WholeStageTransformerContext` when executing. It is used to get substrait
plan node and native plan string.
[...]
-| spark.gluten.sql.cartesianProductTransformerEnabled | 🔄
Dynamic | true | Config to enable
CartesianProductExecTransformer.
[...]
| spark.gluten.sql.collapseGetJsonObject.enabled | 🔄
Dynamic | false | Collapse nested get_json_object functions as
one for optimization.
[...]
| spark.gluten.sql.columnar.appendData | 🔄
Dynamic | true | Enable or disable columnar v2 command append
data.
[...]
| spark.gluten.sql.columnar.arrowUdf | 🔄
Dynamic | true | Enable or disable columnar arrow udf.
[...]
| spark.gluten.sql.columnar.batchscan | 🔄
Dynamic | true | Enable or disable columnar batchscan.
[...]
| spark.gluten.sql.columnar.broadcastExchange | 🔄
Dynamic | true | Enable or disable columnar broadcastExchange.
[...]
| spark.gluten.sql.columnar.broadcastJoin | 🔄
Dynamic | true | Enable or disable columnar broadcastJoin.
[...]
+| spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled | 🔄
Dynamic | true | Enable or disable columnar
broadcastNestedLoopJoin.
[...]
+| spark.gluten.sql.columnar.cartesianProduct.enabled | 🔄
Dynamic | true | Enable or disable columnar cartesianProduct.
[...]
| spark.gluten.sql.columnar.cast.avg | 🔄
Dynamic | true |
| spark.gluten.sql.columnar.coalesce | 🔄
Dynamic | true | Enable or disable columnar coalesce.
[...]
| spark.gluten.sql.columnar.collectLimit | 🔄
Dynamic | true | Enable or disable columnar collectLimit.
[...]
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
index 7f5d5a9c21..980dec57db 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
@@ -63,8 +63,6 @@ trait BackendSettingsApi {
def supportNativeRowIndexColumn(): Boolean = true
- def supportExpandExec(): Boolean = false
-
def supportSortExec(): Boolean = false
def supportSortMergeJoinExec(): Boolean = true
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index d198e6ffc4..f32f225be6 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -285,11 +285,11 @@ class GlutenConfig(conf: SQLConf) extends
GlutenCoreConfig(conf) {
def fallbackPreferColumnar: Boolean =
getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR)
- def cartesianProductTransformerEnabled: Boolean =
- getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED)
+ def enableColumnarCartesianProduct: Boolean =
+ getConf(COLUMNAR_CARTESIAN_PRODUCT_ENABLED)
- def broadcastNestedLoopJoinTransformerTransformerEnabled: Boolean =
- getConf(BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED)
+ def enableColumnarBroadcastNestedLoopJoin: Boolean =
+ getConf(COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED)
def transformPlanLogLevel: String = getConf(TRANSFORM_PLAN_LOG_LEVEL)
@@ -1503,15 +1503,15 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(true)
- val CARTESIAN_PRODUCT_TRANSFORMER_ENABLED =
- buildConf("spark.gluten.sql.cartesianProductTransformerEnabled")
- .doc("Config to enable CartesianProductExecTransformer.")
+ val COLUMNAR_CARTESIAN_PRODUCT_ENABLED =
+ buildConf("spark.gluten.sql.columnar.cartesianProduct.enabled")
+ .doc("Enable or disable columnar cartesianProduct.")
.booleanConf
.createWithDefault(true)
- val BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED =
- buildConf("spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled")
- .doc("Config to enable BroadcastNestedLoopJoinExecTransformer.")
+ val COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED =
+ buildConf("spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled")
+ .doc("Enable or disable columnar broadcastNestedLoopJoin.")
.booleanConf
.createWithDefault(true)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala
index 282cf8581e..32a5793678 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala
@@ -17,7 +17,6 @@
package org.apache.gluten.execution
import org.apache.gluten.backendsapi.BackendsApiManager
-import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.metrics.MetricsUpdater
import org.apache.gluten.substrait.{JoinParams, SubstraitContext}
import org.apache.gluten.utils.SubstraitUtil
@@ -167,11 +166,6 @@ abstract class BroadcastNestedLoopJoinExecTransformer(
}
override protected def doValidateInternal(): ValidationResult = {
- if
(!GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled) {
- return ValidationResult.failed(
- s"Config
${GlutenConfig.BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED.key} not enabled")
- }
-
if (substraitJoinType == CrossRel.JoinType.UNRECOGNIZED) {
return ValidationResult.failed(
s"$joinType join is not supported with BroadcastNestedLoopJoin")
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala
index 47aa66f16b..7cf49d97af 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala
@@ -97,9 +97,6 @@ case class ExpandExecTransformer(
}
override protected def doValidateInternal(): ValidationResult = {
- if (!BackendsApiManager.getSettings.supportExpandExec()) {
- return ValidationResult.failed("Current backend does not support expand")
- }
if (projections.isEmpty) {
return ValidationResult.failed("Current backend does not support empty
projections in expand")
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
index faa0d9e2e5..4f056969c4 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
@@ -189,7 +189,7 @@ object Validators {
fail(p)
case p: GenerateExec if !glutenConf.enableColumnarGenerate => fail(p)
case p: CoalesceExec if !glutenConf.enableColumnarCoalesce => fail(p)
- case p: CartesianProductExec if
!glutenConf.cartesianProductTransformerEnabled => fail(p)
+ case p: CartesianProductExec if
!glutenConf.enableColumnarCartesianProduct => fail(p)
case p: TakeOrderedAndProjectExec
if !(glutenConf.enableTakeOrderedAndProject &&
glutenConf.enableColumnarSort &&
glutenConf.enableColumnarShuffle &&
glutenConf.enableColumnarProject) =>
@@ -198,7 +198,7 @@ object Validators {
fail(p)
case p: BroadcastNestedLoopJoinExec
if !(glutenConf.enableColumnarBroadcastJoin &&
- glutenConf.broadcastNestedLoopJoinTransformerTransformerEnabled) =>
+ glutenConf.enableColumnarBroadcastNestedLoopJoin) =>
fail(p)
case p @ (_: HashAggregateExec | _: SortAggregateExec | _:
ObjectHashAggregateExec)
if !glutenConf.enableColumnarHashAgg =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
