This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 67a9c7666d [VL] Fix bug when setting Spark memory overhead
automatically
67a9c7666d is described below
commit 67a9c7666d7896cbe6faa4a155e5631224d6a97d
Author: Xuedong Luan <[email protected]>
AuthorDate: Thu Sep 19 16:44:07 2024 +0800
[VL] Fix bug when setting Spark memory overhead automatically
---
.../org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 2cec8d3923..926991ada7 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -49,14 +49,16 @@ class VeloxListenerApi extends ListenerApi with Logging {
// Overhead memory limits.
val offHeapSize = conf.getSizeAsBytes(GlutenConfig.SPARK_OFFHEAP_SIZE_KEY)
- val desiredOverheadSize = (0.1 *
offHeapSize).toLong.max(ByteUnit.MiB.toBytes(384))
+ val desiredOverheadSize = (0.3 *
offHeapSize).toLong.max(ByteUnit.MiB.toBytes(384))
if (!SparkResourceUtil.isMemoryOverheadSet(conf)) {
// If memory overhead is not set by user, automatically set it according
to off-heap settings.
logInfo(
s"Memory overhead is not set. Setting it to $desiredOverheadSize
automatically." +
" Gluten doesn't follow Spark's calculation on default value of this
option because the" +
" actual required memory overhead will depend on off-heap usage than
on on-heap usage.")
- conf.set(GlutenConfig.SPARK_OVERHEAD_SIZE_KEY,
desiredOverheadSize.toString)
+ conf.set(
+ GlutenConfig.SPARK_OVERHEAD_SIZE_KEY,
+ ByteUnit.BYTE.toMiB(desiredOverheadSize).toString)
}
val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf)
if (overheadSize < desiredOverheadSize) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
