This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ec6808e8bc [GLUTEN-10351] Extract immutable collection as reusable
field (#10353)
ec6808e8bc is described below
commit ec6808e8bce61fbab56a306881d5fb7e479c4878
Author: Jiaan Geng <[email protected]>
AuthorDate: Mon Aug 11 15:05:59 2025 +0800
[GLUTEN-10351] Extract immutable collection as reusable field (#10353)
extract immutable collection as reusable field
---
.../org/apache/gluten/config/GlutenConfig.scala | 99 +++++++++++-----------
1 file changed, 50 insertions(+), 49 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index cc3b55a0df..913de09981 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -456,6 +456,55 @@ object GlutenConfig {
def prefixOf(backendName: String): String =
s"spark.gluten.sql.columnar.backend.$backendName"
+ private lazy val nativeKeys = Set(
+ DEBUG_ENABLED.key,
+ BENCHMARK_SAVE_DIR.key,
+ GlutenCoreConfig.COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES.key,
+ COLUMNAR_MAX_BATCH_SIZE.key,
+ SHUFFLE_WRITER_BUFFER_SIZE.key,
+ SQLConf.LEGACY_SIZE_OF_NULL.key,
+ SQLConf.LEGACY_STATISTICAL_AGGREGATE.key,
+ "spark.io.compression.codec",
+ "spark.sql.decimalOperations.allowPrecisionLoss",
+ "spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems",
+ "spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits",
+ "spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits",
+ // s3 config
+ SPARK_S3_ACCESS_KEY,
+ SPARK_S3_SECRET_KEY,
+ SPARK_S3_ENDPOINT,
+ SPARK_S3_CONNECTION_SSL_ENABLED,
+ SPARK_S3_PATH_STYLE_ACCESS,
+ SPARK_S3_USE_INSTANCE_CREDENTIALS,
+ SPARK_S3_IAM,
+ SPARK_S3_IAM_SESSION_NAME,
+ SPARK_S3_RETRY_MAX_ATTEMPTS,
+ SPARK_S3_CONNECTION_MAXIMUM,
+ SPARK_S3_ENDPOINT_REGION,
+ "spark.gluten.velox.fs.s3a.connect.timeout",
+ "spark.gluten.velox.fs.s3a.retry.mode",
+ "spark.gluten.velox.awsSdkLogLevel",
+ "spark.gluten.velox.s3UseProxyFromEnv",
+ "spark.gluten.velox.s3PayloadSigningPolicy",
+ "spark.gluten.velox.s3LogLocation",
+ // gcs config
+ SPARK_GCS_STORAGE_ROOT_URL,
+ SPARK_GCS_AUTH_TYPE,
+ SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE,
+ SPARK_REDACTION_REGEX,
+ "spark.gluten.sql.columnar.backend.velox.queryTraceEnabled",
+ "spark.gluten.sql.columnar.backend.velox.queryTraceDir",
+ "spark.gluten.sql.columnar.backend.velox.queryTraceNodeIds",
+ "spark.gluten.sql.columnar.backend.velox.queryTraceMaxBytes",
+ "spark.gluten.sql.columnar.backend.velox.queryTraceTaskRegExp",
+ "spark.gluten.sql.columnar.backend.velox.opTraceDirectoryCreateConfig",
+ "spark.gluten.sql.columnar.backend.velox.enableUserExceptionStacktrace",
+ "spark.gluten.sql.columnar.backend.velox.enableSystemExceptionStacktrace",
+ "spark.gluten.sql.columnar.backend.velox.memoryUseHugePages",
+ "spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct",
+
"spark.gluten.sql.columnar.backend.velox.memoryPoolCapacityTransferAcrossTasks"
+ )
+
/**
* Get dynamic configs.
*
@@ -465,55 +514,7 @@ object GlutenConfig {
backendName: String,
conf: Map[String, String]): util.Map[String, String] = {
val nativeConfMap = new util.HashMap[String, String]()
- val keys = Set(
- DEBUG_ENABLED.key,
- BENCHMARK_SAVE_DIR.key,
- GlutenCoreConfig.COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES.key,
- COLUMNAR_MAX_BATCH_SIZE.key,
- SHUFFLE_WRITER_BUFFER_SIZE.key,
- SQLConf.LEGACY_SIZE_OF_NULL.key,
- SQLConf.LEGACY_STATISTICAL_AGGREGATE.key,
- "spark.io.compression.codec",
- "spark.sql.decimalOperations.allowPrecisionLoss",
- "spark.gluten.sql.columnar.backend.velox.bloomFilter.expectedNumItems",
- "spark.gluten.sql.columnar.backend.velox.bloomFilter.numBits",
- "spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits",
- // s3 config
- SPARK_S3_ACCESS_KEY,
- SPARK_S3_SECRET_KEY,
- SPARK_S3_ENDPOINT,
- SPARK_S3_CONNECTION_SSL_ENABLED,
- SPARK_S3_PATH_STYLE_ACCESS,
- SPARK_S3_USE_INSTANCE_CREDENTIALS,
- SPARK_S3_IAM,
- SPARK_S3_IAM_SESSION_NAME,
- SPARK_S3_RETRY_MAX_ATTEMPTS,
- SPARK_S3_CONNECTION_MAXIMUM,
- SPARK_S3_ENDPOINT_REGION,
- "spark.gluten.velox.fs.s3a.connect.timeout",
- "spark.gluten.velox.fs.s3a.retry.mode",
- "spark.gluten.velox.awsSdkLogLevel",
- "spark.gluten.velox.s3UseProxyFromEnv",
- "spark.gluten.velox.s3PayloadSigningPolicy",
- "spark.gluten.velox.s3LogLocation",
- // gcs config
- SPARK_GCS_STORAGE_ROOT_URL,
- SPARK_GCS_AUTH_TYPE,
- SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE,
- SPARK_REDACTION_REGEX,
- "spark.gluten.sql.columnar.backend.velox.queryTraceEnabled",
- "spark.gluten.sql.columnar.backend.velox.queryTraceDir",
- "spark.gluten.sql.columnar.backend.velox.queryTraceNodeIds",
- "spark.gluten.sql.columnar.backend.velox.queryTraceMaxBytes",
- "spark.gluten.sql.columnar.backend.velox.queryTraceTaskRegExp",
- "spark.gluten.sql.columnar.backend.velox.opTraceDirectoryCreateConfig",
- "spark.gluten.sql.columnar.backend.velox.enableUserExceptionStacktrace",
-
"spark.gluten.sql.columnar.backend.velox.enableSystemExceptionStacktrace",
- "spark.gluten.sql.columnar.backend.velox.memoryUseHugePages",
- "spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct",
-
"spark.gluten.sql.columnar.backend.velox.memoryPoolCapacityTransferAcrossTasks"
- )
- nativeConfMap.putAll(conf.filter(e => keys.contains(e._1)).asJava)
+ nativeConfMap.putAll(conf.filter(e => nativeKeys.contains(e._1)).asJava)
val keyWithDefault = ImmutableList.of(
(SQLConf.CASE_SENSITIVE.key, SQLConf.CASE_SENSITIVE.defaultValueString),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
