This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f97ca6654d [VL] Support user fallback option for CollectTail (#9531)
f97ca6654d is described below
commit f97ca6654dcf3af666046699459c22145a93fbd6
Author: Arnav Balyan <[email protected]>
AuthorDate: Wed May 7 13:26:58 2025 +0530
[VL] Support user fallback option for CollectTail (#9531)
---
.../gluten/extension/columnar/CollectTailTransformerRule.scala | 2 +-
.../org/apache/gluten/extension/columnar/validator/Validators.scala | 1 +
.../src/main/scala/org/apache/gluten/config/GlutenConfig.scala | 6 ++++++
3 files changed, 8 insertions(+), 1 deletion(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollectTailTransformerRule.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollectTailTransformerRule.scala
index 7b2cf7d6f5..bbb33fede6 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollectTailTransformerRule.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollectTailTransformerRule.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.execution.{CollectTailExec,
SparkPlan}
case class CollectTailTransformerRule() extends Rule[SparkPlan] {
override def apply(plan: SparkPlan): SparkPlan = {
- if (!GlutenConfig.get.enableColumnarCollectLimit) {
+ if (!GlutenConfig.get.enableColumnarCollectTail) {
return plan
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
index cc9258d85c..73f2da891e 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
@@ -190,6 +190,7 @@ object Validators {
fail(p)
case p: RangeExec if !glutenConf.enableColumnarRange => fail(p)
case p: CollectLimitExec if !glutenConf.enableColumnarCollectLimit =>
fail(p)
+ case p: CollectTailExec if !glutenConf.enableColumnarCollectTail =>
fail(p)
case _ => pass()
}
}
diff --git
a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 835245bfb6..a1f3674509 100644
--- a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -366,6 +366,7 @@ class GlutenConfig(conf: SQLConf) extends Logging {
def parquetEncryptionValidationFileLimit: Int =
getConf(ENCRYPTED_PARQUET_FALLBACK_FILE_LIMIT)
def enableColumnarRange: Boolean = getConf(COLUMNAR_RANGE_ENABLED)
def enableColumnarCollectLimit: Boolean =
getConf(COLUMNAR_COLLECT_LIMIT_ENABLED)
+ def enableColumnarCollectTail: Boolean =
getConf(COLUMNAR_COLLECT_TAIL_ENABLED)
def getSupportedFlattenedExpressions: String =
getConf(GLUTEN_SUPPORTED_FLATTENED_FUNCTIONS)
def maxBroadcastTableSize: Long =
@@ -1759,4 +1760,9 @@ object GlutenConfig {
.booleanConf
.createWithDefault(false)
+ val COLUMNAR_COLLECT_TAIL_ENABLED =
+ buildConf("spark.gluten.sql.columnar.collectTail")
+ .doc("Enable or disable columnar collectTail.")
+ .booleanConf
+ .createWithDefault(true)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
