This is an automated email from the ASF dual-hosted git repository.
mingliang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c5280f8b73 [CORE] Move ClickHouse-only extended rule configs from
GlutenConfig to CHConfig (#11897)
c5280f8b73 is described below
commit c5280f8b73fadebb1f9393cc1d14676a740e2c7a
Author: Mingliang Zhu <[email protected]>
AuthorDate: Mon Apr 13 09:08:49 2026 +0800
[CORE] Move ClickHouse-only extended rule configs from GlutenConfig to
CHConfig (#11897)
---
.../gluten/backendsapi/clickhouse/CHConfig.scala | 30 ++++++++++++++++++++
.../backendsapi/clickhouse/CHListenerApi.scala | 2 +-
.../gluten/backendsapi/clickhouse/CHRuleApi.scala | 6 ++--
.../extension/GlutenCustomAggExpressionSuite.scala | 4 +--
...ckhouseCustomerExpressionTransformerSuite.scala | 4 +--
.../org/apache/gluten/config/GlutenConfig.scala | 33 ----------------------
.../extension/GlutenCustomerExtensionSuite.scala | 9 ++++--
.../extension/GlutenCustomerExtensionSuite.scala | 9 ++++--
.../extension/GlutenCustomerExtensionSuite.scala | 9 ++++--
.../extension/GlutenCustomerExtensionSuite.scala | 9 ++++--
.../extension/GlutenCustomerExtensionSuite.scala | 9 ++++--
11 files changed, 73 insertions(+), 51 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala
index e3ad3aedc6..4d82a5dfc0 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConfig.scala
@@ -102,6 +102,28 @@ object CHConfig extends ConfigRegistry {
.doc("Enable local cache for CH backend.")
.booleanConf
.createWithDefault(false)
+
+ val EXTENDED_COLUMNAR_TRANSFORM_RULES =
+ buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
+ .internal()
+ .withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
+ .doc("A comma-separated list of classes for the extended columnar
transform rules.")
+ .stringConf
+ .createWithDefaultString("")
+
+ val EXTENDED_COLUMNAR_POST_RULES =
+ buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
+ .internal()
+ .doc("A comma-separated list of classes for the extended columnar post
rules.")
+ .stringConf
+ .createWithDefaultString("")
+
+ val EXTENDED_EXPRESSION_TRAN_CONF =
+ buildConf("spark.gluten.sql.columnar.extended.expressions.transformer")
+ .internal()
+ .doc("A class for the extended expressions transformer.")
+ .stringConf
+ .createWithDefaultString("")
}
class CHConfig(conf: SQLConf) extends GlutenConfig(conf) {
@@ -128,6 +150,14 @@ class CHConfig(conf: SQLConf) extends GlutenConfig(conf) {
getConf(ENABLE_CH_REWRITE_DATE_CONVERSION)
def enableGlutenLocalFileCache: Boolean =
getConf(ENABLE_GLUTEN_LOCAL_FILE_CACHE)
+
+ // A comma-separated list of classes for the extended columnar pre rules
+ def extendedColumnarTransformRules: String =
getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES)
+
+ // A comma-separated list of classes for the extended columnar post rules
+ def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES)
+
+ def extendedExpressionTransformer: String =
getConf(EXTENDED_EXPRESSION_TRAN_CONF)
}
object GlutenObjectStorageConfig {
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
index ba74884d2c..2d21a08ca0 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
@@ -49,7 +49,7 @@ class CHListenerApi extends ListenerApi with Logging {
initialize(pc.conf, isDriver = true)
val expressionExtensionTransformer =
ExpressionUtil.extendedExpressionTransformer(
- pc.conf.get(GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF)
+ pc.conf.get(CHConfig.EXTENDED_EXPRESSION_TRAN_CONF)
)
if (expressionExtensionTransformer != null) {
ExpressionExtensionTrait.registerExpressionExtension(expressionExtensionTransformer)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala
index f6b9e2e317..c960d26c7d 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala
@@ -130,7 +130,7 @@ object CHRuleApi {
c =>
intercept(
SparkPlanRules.extendedColumnarRule(
- new
GlutenConfig(c.sqlConf).extendedColumnarTransformRules)(c.session)))
+ new
CHConfig(c.sqlConf).extendedColumnarTransformRules)(c.session)))
injector.injectPostTransform(_ => CollectLimitTransformerRule())
injector.injectPostTransform(_ => CollectTailTransformerRule())
injector.injectPostTransform(c =>
InsertTransitions.create(c.outputsColumnar, CHBatchType))
@@ -153,8 +153,8 @@ object CHRuleApi {
injector.injectPost(
c =>
intercept(
- SparkPlanRules.extendedColumnarRule(
- new GlutenConfig(c.sqlConf).extendedColumnarPostRules)(c.session)))
+ SparkPlanRules
+ .extendedColumnarRule(new
CHConfig(c.sqlConf).extendedColumnarPostRules)(c.session)))
injector.injectPost(c => GlutenNoopWriterRule.apply(c.session))
// Gluten columnar: Final rules.
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala
index f0d371f1aa..07f8952ec1 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/extension/GlutenCustomAggExpressionSuite.scala
@@ -16,7 +16,7 @@
*/
package org.apache.gluten.execution.extension
-import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.backendsapi.clickhouse.CHConfig
import org.apache.gluten.execution._
import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.utils.SubstraitPlanPrinterUtil
@@ -33,7 +33,7 @@ class GlutenCustomAggExpressionSuite extends ParquetSuite {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
+ CHConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
"org.apache.gluten.execution.extension.CustomAggExpressionTransformer")
}
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/extension/GlutenClickhouseCustomerExpressionTransformerSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/extension/GlutenClickhouseCustomerExpressionTransformerSuite.scala
index e7e2f3ac52..52d83ce5e2 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/extension/GlutenClickhouseCustomerExpressionTransformerSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/extension/GlutenClickhouseCustomerExpressionTransformerSuite.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.extension
-import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.backendsapi.clickhouse.CHConfig
import
org.apache.gluten.execution.{GlutenClickHouseWholeStageTransformerSuite,
ProjectExecTransformer}
import org.apache.gluten.expression.ExpressionConverter
@@ -88,7 +88,7 @@ class GlutenClickhouseCustomerExpressionTransformerSuite
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
+ CHConfig.EXTENDED_EXPRESSION_TRAN_CONF.key,
"org.apache.spark.sql.extension.CustomerExpressionTransformer")
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 371c2b9609..198fc3f025 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -286,14 +286,6 @@ class GlutenConfig(conf: SQLConf) extends
GlutenCoreConfig(conf) {
def softAffinityLogLevel: String = getConf(SOFT_AFFINITY_LOG_LEVEL)
- // A comma-separated list of classes for the extended columnar pre rules
- def extendedColumnarTransformRules: String =
getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES)
-
- // A comma-separated list of classes for the extended columnar post rules
- def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES)
-
- def extendedExpressionTransformer: String =
getConf(EXTENDED_EXPRESSION_TRAN_CONF)
-
def smallFileThreshold: Double = getConf(SMALL_FILE_THRESHOLD)
def expressionBlacklist: Set[String] = {
@@ -1314,31 +1306,6 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(true)
- // FIXME: This only works with CH backend.
- val EXTENDED_COLUMNAR_TRANSFORM_RULES =
- buildConf("spark.gluten.sql.columnar.extended.columnar.transform.rules")
- .internal()
- .withAlternative("spark.gluten.sql.columnar.extended.columnar.pre.rules")
- .doc("A comma-separated list of classes for the extended columnar
transform rules.")
- .stringConf
- .createWithDefaultString("")
-
- // FIXME: This only works with CH backend.
- val EXTENDED_COLUMNAR_POST_RULES =
- buildConf("spark.gluten.sql.columnar.extended.columnar.post.rules")
- .internal()
- .doc("A comma-separated list of classes for the extended columnar post
rules.")
- .stringConf
- .createWithDefaultString("")
-
- // FIXME: This only works with CH backend.
- val EXTENDED_EXPRESSION_TRAN_CONF =
- buildConf("spark.gluten.sql.columnar.extended.expressions.transformer")
- .internal()
- .doc("A class for the extended expressions transformer.")
- .stringConf
- .createWithDefaultString("")
-
val EXPRESSION_BLACK_LIST =
buildConf("spark.gluten.expression.blacklist")
.doc("A black list of expression to skip transform, multiple values
separated by commas.")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 1d25f2a40e..4bd104f04c 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
+ // These configs only take effect on ClickHouse backend.
+ private val ExtendedColumnarTransformRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.transform.rules"
+ private val ExtendedColumnarPostRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.post.rules"
override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
+ ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
- .set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
+ .set(ExtendedColumnarPostRulesKey, "")
}
testGluten("test customer column rules") {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 1d25f2a40e..4bd104f04c 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
+ // These configs only take effect on ClickHouse backend.
+ private val ExtendedColumnarTransformRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.transform.rules"
+ private val ExtendedColumnarPostRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.post.rules"
override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
+ ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
- .set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
+ .set(ExtendedColumnarPostRulesKey, "")
}
testGluten("test customer column rules") {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 6dc265bc4f..cb37cec895 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
+ // These configs only take effect on ClickHouse backend.
+ private val ExtendedColumnarTransformRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.transform.rules"
+ private val ExtendedColumnarPostRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.post.rules"
override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
+ ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
- .set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
+ .set(ExtendedColumnarPostRulesKey, "")
}
testGluten("test customer column rules") {
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 6dc265bc4f..cb37cec895 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
+ // These configs only take effect on ClickHouse backend.
+ private val ExtendedColumnarTransformRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.transform.rules"
+ private val ExtendedColumnarPostRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.post.rules"
override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
+ ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
- .set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
+ .set(ExtendedColumnarPostRulesKey, "")
}
testGluten("test customer column rules") {
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 6dc265bc4f..cb37cec895 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -22,15 +22,20 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
class GlutenCustomerExtensionSuite extends GlutenSQLTestsTrait {
+ // These configs only take effect on ClickHouse backend.
+ private val ExtendedColumnarTransformRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.transform.rules"
+ private val ExtendedColumnarPostRulesKey =
+ "spark.gluten.sql.columnar.extended.columnar.post.rules"
override def sparkConf: SparkConf = {
super.sparkConf
.set("spark.sql.adaptive.enabled", "false")
.set(
- GlutenConfig.EXTENDED_COLUMNAR_TRANSFORM_RULES.key,
+ ExtendedColumnarTransformRulesKey,
"org.apache.spark.sql" +
".extension.CustomerColumnarPreRules")
- .set(GlutenConfig.EXTENDED_COLUMNAR_POST_RULES.key, "")
+ .set(ExtendedColumnarPostRulesKey, "")
}
testGluten("test customer column rules") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
