This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 2a2d2bdeb [VL] Following #6988, move a warning from core to Velox
backend
2a2d2bdeb is described below
commit 2a2d2bdeb8eb5d083d3b0260fa84dc93e2f71151
Author: Hongze Zhang <[email protected]>
AuthorDate: Mon Aug 26 16:04:59 2024 +0800
[VL] Following #6988, move a warning from core to Velox backend
---
.../org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 7 +++++++
gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala | 7 -------
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 2cfc4e9a9..56b439a37 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -42,6 +42,13 @@ class VeloxListenerApi extends ListenerApi with Logging {
override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = {
val conf = pc.conf()
+ // FIXME: The following is a workaround. Remove once the causes are fixed.
+ conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY,
Long.MaxValue.toString)
+ logWarning(
+ "Setting overhead memory that Gluten can use to UNLIMITED. This is
currently a" +
+ " temporary solution to avoid OOM by Velox's global memory pools." +
+ " See GLUTEN-6960 for more information.")
+
// Sql table cache serializer.
if (conf.getBoolean(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key,
defaultValue = false)) {
conf.set(
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index 6d0cdd0f8..47551764e 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -174,13 +174,6 @@ private[gluten] class GlutenDriverPlugin extends
DriverPlugin with Logging {
val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf)
conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY,
overheadSize.toString)
- // FIXME: The following is a workaround. Remove once the causes are fixed.
- conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY,
Long.MaxValue.toString)
- logWarning(
- "Setting overhead memory that Gluten can use to UNLIMITED. This is
currently a" +
- " temporary solution to avoid OOM by Velox's global memory pools." +
- " See GLUTEN-6960 for more information.")
-
// If dynamic off-heap sizing is enabled, the off-heap size is calculated
based on the on-heap
// size. Otherwise, the off-heap size is set to the value specified by the
user (if any).
// Note that this means that we will IGNORE the off-heap size specified by
the user if the
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
