This is an automated email from the ASF dual-hosted git repository.
liujiayi771 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 59665e93ab [CORE] Add more log for spill (#10117)
59665e93ab is described below
commit 59665e93ab39cbf9a768932e49ff2833c70a0dac
Author: Joey <[email protected]>
AuthorDate: Fri Jul 4 21:46:21 2025 +0800
[CORE] Add more log for spill (#10117)
---
.../org/apache/gluten/vectorized/NativePlanEvaluator.java | 11 ++++++++++-
.../scala/org/apache/gluten/memory/NativeMemoryManager.scala | 4 +++-
.../org/apache/gluten/memory/memtarget/MemoryTargets.java | 2 +-
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git
a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
index 96643dd711..44d4107c56 100644
---
a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
+++
b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
@@ -25,12 +25,15 @@ import org.apache.gluten.utils.DebugUtil;
import org.apache.gluten.validate.NativePlanValidationInfo;
import org.apache.spark.TaskContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
public class NativePlanEvaluator {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(NativePlanEvaluator.class);
private static final AtomicInteger id = new AtomicInteger(0);
private final Runtime runtime;
@@ -86,7 +89,13 @@ public class NativePlanEvaluator {
if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
return 0L;
}
- return out.spill(size);
+ long spilled = out.spill(size);
+ LOGGER.info(
+ "NativePlanEvaluator-{}: Spilled {} / {} bytes of data.",
+ id.get(),
+ spilled,
+ size);
+ return spilled;
}
});
return out;
diff --git
a/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
b/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
index f2db33d8e1..5aae7f6c57 100644
---
a/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
+++
b/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
@@ -61,7 +61,9 @@ object NativeMemoryManager {
// Only respond for shrinking.
return 0L
}
- NativeMemoryManagerJniWrapper.shrink(handle, size)
+ val shrunk = NativeMemoryManagerJniWrapper.shrink(handle, size)
+ LOGGER.info(s"NativeMemoryManager: Shrunk $shrunk / $size bytes of
data.")
+ shrunk
}
})
mutableStats += "single" -> new MemoryUsageStatsBuilder {
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
index 135a82fa58..5f8013c62e 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
@@ -90,7 +90,7 @@ public final class MemoryTargets {
LOGGER.info("Request for spilling on consumer {}...",
consumer.name());
// Note: Spill from root node so other consumers also get spilled.
long spilled = TreeMemoryTargets.spillTree(root, Long.MAX_VALUE);
- LOGGER.info("Consumer {} spilled {} bytes.", consumer.name(),
spilled);
+ LOGGER.info("Consumer {} gets {} bytes from spilling.",
consumer.name(), spilled);
});
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
