This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fd716bf142 [GLUTEN-7837][VL] Spark driver should not initialize cache
if not in local mode
fd716bf142 is described below
commit fd716bf1420566608f093e8354bcf577a2511c99
Author: Xuedong Luan <[email protected]>
AuthorDate: Fri Nov 15 13:01:12 2024 +0800
[GLUTEN-7837][VL] Spark driver should not initialize cache if not in local
mode
Closes #7837
---
.../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 850509db3e..e5c9afc096 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -89,7 +89,7 @@ class VeloxListenerApi extends ListenerApi with Logging {
SparkDirectoryUtil.init(conf)
UDFResolver.resolveUdfConf(conf, isDriver = true)
- initialize(conf)
+ initialize(conf, isDriver = true)
UdfJniWrapper.registerFunctionSignatures()
}
@@ -116,12 +116,12 @@ class VeloxListenerApi extends ListenerApi with Logging {
SparkDirectoryUtil.init(conf)
UDFResolver.resolveUdfConf(conf, isDriver = false)
- initialize(conf)
+ initialize(conf, isDriver = false)
}
override def onExecutorShutdown(): Unit = shutdown()
- private def initialize(conf: SparkConf): Unit = {
+ private def initialize(conf: SparkConf, isDriver: Boolean): Unit = {
// Force batch type initializations.
VeloxBatch.getClass
ArrowJavaBatch.getClass
@@ -157,7 +157,11 @@ class VeloxListenerApi extends ListenerApi with Logging {
}
// Initial native backend with configurations.
- val parsed = GlutenConfigUtil.parseConfig(conf.getAll.toMap)
+ var parsed = GlutenConfigUtil.parseConfig(conf.getAll.toMap)
+ // Workaround for https://github.com/apache/incubator-gluten/issues/7837
+ if (isDriver && !inLocalMode(conf)) {
+ parsed += (GlutenConfig.COLUMNAR_VELOX_CACHE_ENABLED.key -> "false")
+ }
NativeBackendInitializer.initializeBackend(parsed)
// Inject backend-specific implementations to override spark classes.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
