This is an automated email from the ASF dual-hosted git repository.
taiyang-li pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ef309d210e [GLUTEN][CORE] Add backend hooks for native config
forwarding (#12437)
ef309d210e is described below
commit ef309d210edd8e2ee47a1a8c20785e65d5b6a276
Author: 李扬 <[email protected]>
AuthorDate: Tue Jul 7 18:13:21 2026 +0800
[GLUTEN][CORE] Add backend hooks for native config forwarding (#12437)
Allow backends to contribute extra non-prefixed session and backend config
keys so native config forwarding can be extended without hard-coding
backend-specific entries in GlutenConfig.
---
.../apache/gluten/backendsapi/BackendSettingsApi.scala | 12 ++++++++++++
.../scala/org/apache/gluten/config/GlutenConfig.scala | 16 ++++++++++++++--
2 files changed, 26 insertions(+), 2 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
index 75c94be4d4..85b146e195 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
@@ -168,4 +168,16 @@ trait BackendSettingsApi {
/** Whether the backend supports columnar shuffle with empty schema. */
def supportEmptySchemaColumnarShuffle(): Boolean = true
+
+ /**
+ * Backend-specific non-prefixed session configs that should be forwarded to
native runtime /
+ * memory manager creation.
+ */
+ def extraNativeSessionConfKeys(): Set[String] = Set.empty
+
+ /**
+ * Backend-specific non-prefixed backend initialization configs that should
be forwarded during
+ * native backend initialization.
+ */
+ def extraNativeBackendConfKeys(): Set[String] = Set.empty
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 37d8b2b3ab..bd5394095d 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -16,6 +16,7 @@
*/
package org.apache.gluten.config
+import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.shuffle.SupportsColumnarShuffle
import org.apache.spark.network.util.{ByteUnit, JavaUtils}
@@ -532,10 +533,18 @@ object GlutenConfig extends ConfigRegistry {
"spark.gluten.sql.columnar.backend.velox.columnarBatchSerializerCompression"
)
+ private def backendSettings(backendName: String) = {
+ // Only one backend is loaded in a running Gluten session. Use backend
settings hooks to avoid
+ // hard-coding backend-specific configs in common code.
+ BackendsApiManager.getSettings
+ }
+
/** Get dynamic configs. */
def getNativeSessionConf(backendName: String, conf: Map[String, String]):
Map[String, String] = {
+ val settings = backendSettings(backendName)
val nativeConfMap = mutable.Map[String, String](conf.filter {
- case (key, _) => nativeKeys.contains(key)
+ case (key, _) =>
+ nativeKeys.contains(key) ||
settings.extraNativeSessionConfKeys().contains(key)
}.toSeq: _*)
Seq(
@@ -649,6 +658,7 @@ object GlutenConfig extends ConfigRegistry {
(SQLConf.SESSION_LOCAL_TIMEZONE.key,
SQLConf.SESSION_LOCAL_TIMEZONE.defaultValueString)
).foreach { case (k, defaultValue) => nativeConfMap.put(k,
conf.getOrElse(k, defaultValue)) }
+ val settings = backendSettings(backendName)
val keys = Set(
DEBUG_ENABLED.key,
// datasource config
@@ -666,7 +676,9 @@ object GlutenConfig extends ConfigRegistry {
COLUMNAR_CUDF_ENABLED.key
)
- nativeConfMap ++= conf.filter { case (k, _) => keys.contains(k) }
+ nativeConfMap ++= conf.filter {
+ case (k, _) => keys.contains(k) ||
settings.extraNativeBackendConfKeys().contains(k)
+ }
val confPrefix = prefixOf(backendName)
val s3Prefix = HADOOP_PREFIX + S3A_PREFIX
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
