This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cb3979e8a0 [GLUTEN-10932][VL] Pass memory changes to taskMemoryManager
for dynamic off heap (#10936)
cb3979e8a0 is described below
commit cb3979e8a0ea87c42ab227162a971ed3f7095d88
Author: Zhen Wang <[email protected]>
AuthorDate: Tue Oct 28 00:39:21 2025 +0800
[GLUTEN-10932][VL] Pass memory changes to taskMemoryManager for dynamic off
heap (#10936)
Closes #10932
---
.../memory/memtarget/DynamicOffHeapSizingMemoryTarget.java | 12 +++++++++++-
.../org/apache/gluten/memory/memtarget/MemoryTargets.java | 2 +-
.../main/scala/org/apache/spark/memory/SparkMemoryUtil.scala | 1 +
3 files changed, 13 insertions(+), 2 deletions(-)
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
index b1664223aa..3124f78fb7 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
@@ -130,7 +130,11 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
private final String name =
MemoryTargetUtil.toUniqueName("DynamicOffHeapSizing");
private final SimpleMemoryUsageRecorder recorder = new
SimpleMemoryUsageRecorder();
- public DynamicOffHeapSizingMemoryTarget() {}
+ private final MemoryTarget target;
+
+ public DynamicOffHeapSizingMemoryTarget(MemoryTarget target) {
+ this.target = target;
+ }
@Override
public long borrow(long size) {
@@ -197,6 +201,7 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
USED_OFF_HEAP_BYTES.addAndGet(size);
recorder.inc(size);
+ target.borrow(size);
return size;
}
@@ -204,6 +209,7 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
public long repay(long size) {
USED_OFF_HEAP_BYTES.addAndGet(-size);
recorder.inc(-size);
+ target.repay(size);
return size;
}
@@ -227,6 +233,10 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
return recorder.toStats();
}
+ public MemoryTarget target() {
+ return target;
+ }
+
public static boolean isJava9OrLater() {
String spec = System.getProperty("java.specification.version", "1.8");
// "1.8" → 8, "9" → 9, "11" → 11, etc.
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
index b1e41bc9eb..e6cab4b6a8 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
@@ -52,7 +52,7 @@ public final class MemoryTargets {
@Experimental
public static MemoryTarget dynamicOffHeapSizingIfEnabled(MemoryTarget
memoryTarget) {
if (GlutenCoreConfig.get().dynamicOffHeapSizingEnabled()) {
- return new DynamicOffHeapSizingMemoryTarget();
+ return new DynamicOffHeapSizingMemoryTarget(memoryTarget);
}
return memoryTarget;
diff --git
a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
index ae287cef3f..a902c448fe 100644
--- a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
@@ -132,6 +132,7 @@ object SparkMemoryUtil {
prettyPrintStats(
"Dynamic off-heap sizing memory target stats: ",
dynamicOffHeapSizingMemoryTarget)
+ dynamicOffHeapSizingMemoryTarget.target().accept(this)
}
override def visit(retryOnOomMemoryTarget: RetryOnOomMemoryTarget):
String = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
