This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b798ef5f1 [GLUTEN-5844][CORE] Refactor the usage of
spark.gluten.enabled (#5845)
b798ef5f1 is described below
commit b798ef5f1bab3a6674d3c69473b5ec781ffe2ca3
Author: sharmaplkt <[email protected]>
AuthorDate: Thu May 23 04:51:02 2024 +0530
[GLUTEN-5844][CORE] Refactor the usage of spark.gluten.enabled (#5845)
[GLUTEN-5844][CORE] Refactor the usage of spark.gluten.enabled.
Co-authored-by: Pulkit Sharma <[email protected]>
---
.../GlutenClickHouseMergeTreeOptimizeSuite.scala | 50 +++++++++++-----------
.../GlutenClickHouseNativeWriteTableSuite.scala | 20 ++++-----
...ckHouseTPCHColumnarShuffleParquetAQESuite.scala | 3 +-
.../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 3 +-
.../GlutenClickHouseTableAfterRestart.scala | 6 ++-
.../execution/GlutenClickhouseFunctionSuite.scala | 2 +-
.../scala/org/apache/gluten/s3/S3AuthSuite.scala | 2 +-
.../benchmarks/CHAggAndShuffleBenchmark.scala | 3 +-
.../benchmarks/CHParquetReadBenchmark.scala | 3 +-
.../execution/WholeStageTransformerSuite.scala | 3 +-
.../benchmarks/ParquetReadBenchmark.scala | 2 +-
.../extension/GlutenCustomerExtensionSuite.scala | 4 +-
.../benchmarks/ParquetReadBenchmark.scala | 2 +-
.../extension/GlutenCustomerExtensionSuite.scala | 4 +-
.../benchmarks/ParquetReadBenchmark.scala | 2 +-
.../extension/GlutenCustomerExtensionSuite.scala | 4 +-
.../benchmarks/ParquetReadBenchmark.scala | 2 +-
.../extension/GlutenCustomerExtensionSuite.scala | 4 +-
18 files changed, 68 insertions(+), 51 deletions(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
index ae0cd170d..d4302193f 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseMergeTreeOptimizeSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.gluten.execution
+import org.apache.gluten.GlutenConfig
+
import org.apache.spark.SparkConf
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
@@ -122,11 +124,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p").collect()
assert(ret.apply(0).get(0) == 600572)
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) ==
22728)
spark.sql("VACUUM lineitem_mergetree_optimize_p RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p")) ==
22728)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret2 = spark.sql("select count(*) from
lineitem_mergetree_optimize_p").collect()
assert(ret2.apply(0).get(0) == 600572)
@@ -154,14 +156,14 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p2").collect()
assert(ret.apply(0).get(0) == 600572)
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))
== 812)
spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))
== 232)
spark.sql("VACUUM lineitem_mergetree_optimize_p2 RETAIN 0 HOURS")
// the second VACUUM will remove some empty folders
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p2"))
== 220)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret2 = spark.sql("select count(*) from
lineitem_mergetree_optimize_p2").collect()
assert(ret2.apply(0).get(0) == 600572)
@@ -185,13 +187,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p3").collect()
assert(ret.apply(0).get(0) == 600572)
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))
== 398)
spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))
== 286)
spark.sql("VACUUM lineitem_mergetree_optimize_p3 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p3"))
== 270)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret2 = spark.sql("select count(*) from
lineitem_mergetree_optimize_p3").collect()
assert(ret2.apply(0).get(0) == 600572)
@@ -216,13 +218,13 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p4").collect()
assert(ret.apply(0).get(0) == 600572)
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))
== 398)
spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))
== 286)
spark.sql("VACUUM lineitem_mergetree_optimize_p4 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p4"))
== 270)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret2 = spark.sql("select count(*) from
lineitem_mergetree_optimize_p4").collect()
assert(ret2.apply(0).get(0) == 600572)
@@ -246,11 +248,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite
spark.sql("optimize lineitem_mergetree_optimize_p5")
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS")
spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))
== 99)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p5").collect()
assert(ret.apply(0).get(0) == 600572)
@@ -266,11 +268,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite
spark.sql("optimize lineitem_mergetree_optimize_p5")
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS")
spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))
== 93)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p5").collect()
assert(ret.apply(0).get(0) == 600572)
@@ -279,11 +281,11 @@ class GlutenClickHouseMergeTreeOptimizeSuite
// now merge all parts (testing merging from merged parts)
spark.sql("optimize lineitem_mergetree_optimize_p5")
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS")
spark.sql("VACUUM lineitem_mergetree_optimize_p5 RETAIN 0 HOURS")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p5"))
== 77)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p5").collect()
assert(ret.apply(0).get(0) == 600572)
@@ -309,7 +311,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val ret = spark.sql("select count(*) from
lineitem_mergetree_optimize_p6").collect()
assert(ret.apply(0).get(0) == 600572)
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6"))
== {
if (sparkVersion.equals("3.2")) 931 else 1014
})
@@ -318,7 +320,7 @@ class GlutenClickHouseMergeTreeOptimizeSuite
assert(countFiles(new File(s"$basePath/lineitem_mergetree_optimize_p6"))
== {
if (sparkVersion.equals("3.2")) 439 else 445
})
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val ret2 = spark.sql("select count(*) from
lineitem_mergetree_optimize_p6").collect()
assert(ret2.apply(0).get(0) == 600572)
@@ -341,9 +343,9 @@ class GlutenClickHouseMergeTreeOptimizeSuite
|""".stripMargin)
spark.sql("optimize lineitem_mergetree_index")
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
spark.sql("vacuum lineitem_mergetree_index")
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
val df = spark
.sql(s"""
@@ -387,10 +389,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
clickhouseTable.optimize().executeCompaction()
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
clickhouseTable.vacuum(0.0)
clickhouseTable.vacuum(0.0)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
assert(countFiles(new File(dataPath)) == 99)
val ret = spark.sql(s"select count(*) from
clickhouse.`$dataPath`").collect()
@@ -408,10 +410,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
clickhouseTable.optimize().executeCompaction()
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
clickhouseTable.vacuum(0.0)
clickhouseTable.vacuum(0.0)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
assert(countFiles(new File(dataPath)) == 93)
val ret = spark.sql(s"select count(*) from
clickhouse.`$dataPath`").collect()
@@ -422,10 +424,10 @@ class GlutenClickHouseMergeTreeOptimizeSuite
val clickhouseTable = ClickhouseTable.forPath(spark, dataPath)
clickhouseTable.optimize().executeCompaction()
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
clickhouseTable.vacuum(0.0)
clickhouseTable.vacuum(0.0)
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
assert(countFiles(new File(dataPath)) == 77)
val ret = spark.sql(s"select count(*) from
clickhouse.`$dataPath`").collect()
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala
index 6b5924297..9269303d9 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeWriteTableSuite.scala
@@ -173,7 +173,7 @@ class GlutenClickHouseNativeWriteTableSuite
test("test insert into dir") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val originDF = spark.createDataFrame(genTestData())
originDF.createOrReplaceTempView("origin_table")
@@ -209,7 +209,7 @@ class GlutenClickHouseNativeWriteTableSuite
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
("spark.sql.orc.compression.codec", "lz4"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val originDF = spark.createDataFrame(genTestData())
originDF.createOrReplaceTempView("origin_table")
@@ -260,7 +260,7 @@ class GlutenClickHouseNativeWriteTableSuite
test("test CTAS") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val originDF = spark.createDataFrame(genTestData())
originDF.createOrReplaceTempView("origin_table")
@@ -311,7 +311,7 @@ class GlutenClickHouseNativeWriteTableSuite
("spark.gluten.sql.native.writer.enabled", "true"),
("spark.sql.hive.convertMetastoreParquet", "false"),
("spark.sql.hive.convertMetastoreOrc", "false"),
- ("spark.gluten.enabled", "true")
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")
) {
val originDF = spark.createDataFrame(genTestData())
@@ -429,7 +429,7 @@ class GlutenClickHouseNativeWriteTableSuite
test("test 2-col partitioned table") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val fields: ListMap[String, String] = ListMap(
("string_field", "string"),
@@ -467,7 +467,7 @@ class GlutenClickHouseNativeWriteTableSuite
" ignore because takes too long") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val fields: ListMap[String, String] = ListMap(
("date_field", "date"),
@@ -508,7 +508,7 @@ class GlutenClickHouseNativeWriteTableSuite
ignore("test hive parquet/orc table, all columns being partitioned. ") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val fields: ListMap[String, String] = ListMap(
("date_field", "date"),
@@ -547,7 +547,7 @@ class GlutenClickHouseNativeWriteTableSuite
test(("test hive parquet/orc table with aggregated results")) {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val fields: ListMap[String, String] = ListMap(
("sum(int_field)", "bigint")
@@ -573,7 +573,7 @@ class GlutenClickHouseNativeWriteTableSuite
test("test 1-col partitioned + 1-col bucketed table") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
val fields: ListMap[String, String] = ListMap(
("string_field", "string"),
@@ -911,7 +911,7 @@ class GlutenClickHouseNativeWriteTableSuite
test("GLUTEN-4316: fix crash on dynamic partition inserting") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
formats.foreach(
format => {
val tbl = "t_" + format
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala
index 1b3929dcc..6caac9918 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala
@@ -16,6 +16,7 @@
*/
package org.apache.gluten.execution
+import org.apache.gluten.GlutenConfig
import org.apache.gluten.extension.GlutenPlan
import org.apache.spark.SparkConf
@@ -298,7 +299,7 @@ class GlutenClickHouseTPCHColumnarShuffleParquetAQESuite
}
test("Test 'spark.gluten.enabled' false") {
- withSQLConf(("spark.gluten.enabled", "false")) {
+ withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) {
runTPCHQuery(2, noFallBack = false) {
df =>
val glutenPlans = collect(df.queryExecution.executedPlan) {
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
index a1bba300e..eec0ad874 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHSaltNullParquetSuite.scala
@@ -16,6 +16,7 @@
*/
package org.apache.gluten.execution
+import org.apache.gluten.GlutenConfig
import org.apache.gluten.extension.GlutenPlan
import org.apache.spark.{SparkConf, SparkException}
@@ -1231,7 +1232,7 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends
GlutenClickHouseTPCHAbstr
}
test("Test 'spark.gluten.enabled' false") {
- withSQLConf(("spark.gluten.enabled", "false")) {
+ withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) {
runTPCHQuery(2, noFallBack = false) {
df =>
val glutenPlans = df.queryExecution.executedPlan.collect {
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala
index a673d4ba3..9e55df0fa 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTableAfterRestart.scala
@@ -16,6 +16,8 @@
*/
package org.apache.gluten.execution
+import org.apache.gluten.GlutenConfig
+
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SparkSession.{getActiveSession, getDefaultSession}
@@ -248,9 +250,9 @@ class GlutenClickHouseTableAfterRestart
restartSpark()
- spark.sql("set spark.gluten.enabled=false")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=false")
spark.sql("vacuum table_restart_vacuum")
- spark.sql("set spark.gluten.enabled=true")
+ spark.sql(s"set ${GlutenConfig.GLUTEN_ENABLED.key}=true")
assert(spark.sql("select count(*) from
table_restart_vacuum").collect().apply(0).get(0) == 4)
}
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
index c90e9131a..63e105e12 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
@@ -121,7 +121,7 @@ class GlutenClickhouseFunctionSuite extends
GlutenClickHouseTPCHAbstractSuite {
test("test uuid - write and read") {
withSQLConf(
("spark.gluten.sql.native.writer.enabled", "true"),
- ("spark.gluten.enabled", "true")) {
+ (GlutenConfig.GLUTEN_ENABLED.key, "true")) {
spark.sql("drop table if exists uuid_test")
spark.sql("create table if not exists uuid_test (id string) stored as
parquet")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala
index b2ac3672a..32c64c78a 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala
@@ -102,7 +102,7 @@ class S3AuthSuite extends AnyFunSuite {
}
def withGluten(enable: Boolean): Builder = {
- builder.config("spark.gluten.enabled", enable.toString)
+ builder.config(GlutenConfig.GLUTEN_ENABLED.key, enable.toString)
}
}
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala
index 19ced9517..358b78518 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala
@@ -16,6 +16,7 @@
*/
package org.apache.spark.sql.execution.benchmarks
+import org.apache.gluten.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
ProjectExecTransformer, WholeStageTransformer}
import org.apache.gluten.sql.shims.SparkShimLoader
@@ -303,7 +304,7 @@ object CHAggAndShuffleBenchmark extends SqlBasedBenchmark
with CHSqlBasedBenchma
// Get the file partitions for generating the `FileScanRDD`
val filePartitions = fileScan.getPartitions
.map(_.asInstanceOf[FilePartition])
- spark.conf.set("spark.gluten.enabled", "false")
+ spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false")
val sparkExecutedPlan = allStages.queryExecution.executedPlan
// Get the `FileSourceScanExec`
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala
index 1cc8c8383..5e802eeed 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHParquetReadBenchmark.scala
@@ -16,6 +16,7 @@
*/
package org.apache.spark.sql.execution.benchmarks
+import org.apache.gluten.GlutenConfig
import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformContext}
import org.apache.gluten.expression.ConverterUtils
@@ -189,7 +190,7 @@ object CHParquetReadBenchmark extends SqlBasedBenchmark
with CHSqlBasedBenchmark
}
if (executedVanilla) {
- spark.conf.set("spark.gluten.enabled", "false")
+ spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false")
val vanillaParquet = spark.sql(s"""
|select $scanSchema from
parquet.`$parquetDir`
diff --git
a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala
b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala
index 8e8743857..5f60de27b 100644
---
a/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala
+++
b/gluten-core/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala
@@ -16,6 +16,7 @@
*/
package org.apache.gluten.execution
+import org.apache.gluten.GlutenConfig
import org.apache.gluten.extension.GlutenPlan
import org.apache.gluten.test.FallbackUtil
import org.apache.gluten.utils.Arm
@@ -320,7 +321,7 @@ abstract class WholeStageTransformerSuite
noFallBack)
protected def vanillaSparkConfs(): Seq[(String, String)] = {
- List(("spark.gluten.enabled", "false"))
+ List((GlutenConfig.GLUTEN_ENABLED.key, "false"))
}
protected def checkDataFrame(
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index ad08318bb..ef9699458 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -165,7 +165,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
if (executedVanilla) {
- spark.conf.set("spark.gluten.enabled", "false")
+ spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false")
val vanillaParquet = spark.sql(s"""
|select $scanSchema from
parquet.`$parquetDir`
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 18a9f5c9f..5e6c66265 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.extension
+import org.apache.gluten.GlutenConfig
+
import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
@@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends
GlutenSQLTestsTrait {
}
testGluten("test customer column rules") {
- withSQLConf(("spark.gluten.enabled", "false")) {
+ withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) {
sql("create table my_parquet(id int) using parquet")
sql("insert into my_parquet values (1)")
sql("insert into my_parquet values (2)")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index 7d8a29204..f61f7ebba 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -165,7 +165,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
if (executedVanilla) {
- spark.conf.set("spark.gluten.enabled", "false")
+ spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false")
val vanillaParquet = spark.sql(s"""
|select $scanSchema from
parquet.`$parquetDir`
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 18a9f5c9f..5e6c66265 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.extension
+import org.apache.gluten.GlutenConfig
+
import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
@@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends
GlutenSQLTestsTrait {
}
testGluten("test customer column rules") {
- withSQLConf(("spark.gluten.enabled", "false")) {
+ withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) {
sql("create table my_parquet(id int) using parquet")
sql("insert into my_parquet values (1)")
sql("insert into my_parquet values (2)")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index b5481f4d8..471bdf179 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -166,7 +166,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
if (executedVanilla) {
- spark.conf.set("spark.gluten.enabled", "false")
+ spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false")
val vanillaParquet = spark.sql(s"""
|select $scanSchema from
parquet.`$parquetDir`
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 18a9f5c9f..5e6c66265 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.extension
+import org.apache.gluten.GlutenConfig
+
import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
@@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends
GlutenSQLTestsTrait {
}
testGluten("test customer column rules") {
- withSQLConf(("spark.gluten.enabled", "false")) {
+ withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) {
sql("create table my_parquet(id int) using parquet")
sql("insert into my_parquet values (1)")
sql("insert into my_parquet values (2)")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index 7d8a29204..f61f7ebba 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -165,7 +165,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
if (executedVanilla) {
- spark.conf.set("spark.gluten.enabled", "false")
+ spark.conf.set(GlutenConfig.GLUTEN_ENABLED.key, "false")
val vanillaParquet = spark.sql(s"""
|select $scanSchema from
parquet.`$parquetDir`
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
index 18a9f5c9f..5e6c66265 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.extension
+import org.apache.gluten.GlutenConfig
+
import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsTrait
@@ -32,7 +34,7 @@ class GlutenCustomerExtensionSuite extends
GlutenSQLTestsTrait {
}
testGluten("test customer column rules") {
- withSQLConf(("spark.gluten.enabled", "false")) {
+ withSQLConf((GlutenConfig.GLUTEN_ENABLED.key, "false")) {
sql("create table my_parquet(id int) using parquet")
sql("insert into my_parquet values (1)")
sql("insert into my_parquet values (2)")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
