This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 73c06100e4 [GLUTEN-7856][CORE] Ensure correct enabling of
GlutenCostEvaluator (#7857)
73c06100e4 is described below
commit 73c06100e454723546b396b02110504b6c76c61e
Author: Xiuli Wei <[email protected]>
AuthorDate: Thu Nov 14 12:27:55 2024 +0800
[GLUTEN-7856][CORE] Ensure correct enabling of GlutenCostEvaluator (#7857)
Closes #7856
---
gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala | 9 ++++++---
shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 9 ++++-----
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index d5f5fd2e21..e52c53d1d5 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -65,7 +65,7 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin
with Logging {
postBuildInfoEvent(sc)
- setPredefinedConfigs(sc, conf)
+ setPredefinedConfigs(conf)
// Initialize Backend.
Backend.get().onDriverStart(sc, pluginContext)
@@ -123,7 +123,7 @@ private[gluten] class GlutenDriverPlugin extends
DriverPlugin with Logging {
GlutenEventUtils.post(sc, event)
}
- private def setPredefinedConfigs(sc: SparkContext, conf: SparkConf): Unit = {
+ private def setPredefinedConfigs(conf: SparkConf): Unit = {
// Spark SQL extensions
val extensions = if (conf.contains(SPARK_SESSION_EXTENSIONS.key)) {
s"${conf.get(SPARK_SESSION_EXTENSIONS.key)}," +
@@ -134,7 +134,10 @@ private[gluten] class GlutenDriverPlugin extends
DriverPlugin with Logging {
conf.set(SPARK_SESSION_EXTENSIONS.key, extensions)
// adaptive custom cost evaluator class
- if (GlutenConfig.getConf.enableGluten &&
GlutenConfig.getConf.enableGlutenCostEvaluator) {
+ val enableGlutenCostEvaluator = conf.getBoolean(
+ GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED,
+ GLUTEN_COST_EVALUATOR_ENABLED_DEFAULT_VALUE)
+ if (enableGlutenCostEvaluator) {
val costEvaluator =
"org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator"
conf.set(SQLConf.ADAPTIVE_CUSTOM_COST_EVALUATOR_CLASS.key, costEvaluator)
}
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 0bfe16270e..0e41fc6537 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -466,8 +466,6 @@ class GlutenConfig(conf: SQLConf) extends Logging {
def enableCastAvgAggregateFunction: Boolean =
conf.getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED)
- def enableGlutenCostEvaluator: Boolean = conf.getConf(COST_EVALUATOR_ENABLED)
-
def dynamicOffHeapSizingEnabled: Boolean =
conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED)
@@ -648,6 +646,7 @@ object GlutenConfig {
"spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction"
val GLUTEN_COST_EVALUATOR_ENABLED =
"spark.gluten.sql.adaptive.costEvaluator.enabled"
+ val GLUTEN_COST_EVALUATOR_ENABLED_DEFAULT_VALUE = true
var ins: GlutenConfig = _
@@ -2116,15 +2115,15 @@ object GlutenConfig {
.createWithDefault(true)
val COST_EVALUATOR_ENABLED =
- buildConf(GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED)
+ buildStaticConf(GlutenConfig.GLUTEN_COST_EVALUATOR_ENABLED)
.internal()
.doc(
- "If true and gluten enabled, use " +
+ "If true, use " +
"org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as
custom cost " +
"evaluator class, else follow the configuration " +
"spark.sql.adaptive.customCostEvaluatorClass.")
.booleanConf
- .createWithDefault(true)
+ .createWithDefault(GLUTEN_COST_EVALUATOR_ENABLED_DEFAULT_VALUE)
val DYNAMIC_OFFHEAP_SIZING_ENABLED =
buildConf(GlutenConfig.GLUTEN_DYNAMIC_OFFHEAP_SIZING_ENABLED)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
