(iotdb) branch master updated: Load: Fix memory size counting on cases when query engine's memory is not sufficient (#15049)

[rong]

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 59c1a8ef114 Load: Fix memory size counting on cases when query 
engine's memory is not sufficient (#15049)
59c1a8ef114 is described below

commit 59c1a8ef114916b62c15b242bf210f13b08f9ee2
Author: Zikun Ma <55695098+danielwang2...@users.noreply.github.com>
AuthorDate: Mon Mar 10 15:55:13 2025 +0800

    Load: Fix memory size counting on cases when query engine's memory is not 
sufficient (#15049)
---
 .../load/memory/LoadTsFileMemoryManager.java       | 27 +++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
index 095c24a8cc7..221c0bbea3c 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
@@ -88,8 +88,9 @@ public class LoadTsFileMemoryManager {
           sizeInBytes,
           usedMemorySizeInBytes.get());
     }
-    usedMemorySizeInBytes.addAndGet(-sizeInBytes);
-    QUERY_ENGINE_MEMORY_MANAGER.releaseToFreeMemoryForOperators(sizeInBytes);
+    final long sizeToRelease = Math.min(sizeInBytes, 
usedMemorySizeInBytes.get());
+    usedMemorySizeInBytes.addAndGet(-sizeToRelease);
+    QUERY_ENGINE_MEMORY_MANAGER.releaseToFreeMemoryForOperators(sizeToRelease);
     this.notifyAll();
   }
 
@@ -97,8 +98,16 @@ public class LoadTsFileMemoryManager {
       long sizeInBytes) throws LoadRuntimeOutOfMemoryException {
     try {
       forceAllocateFromQuery(sizeInBytes);
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug(
+            "Load: Allocated AnalyzeSchemaMemoryBlock from query engine, size: 
{}", sizeInBytes);
+      }
     } catch (LoadRuntimeOutOfMemoryException e) {
       if (dataCacheMemoryBlock != null && 
dataCacheMemoryBlock.doShrink(sizeInBytes)) {
+        LOGGER.info(
+            "Load: Query engine's memory is not sufficient, allocated 
AnalyzeSchemaMemoryBlock from DataCacheMemoryBlock, size: {}",
+            sizeInBytes);
+        usedMemorySizeInBytes.addAndGet(sizeInBytes);
         return new LoadTsFileAnalyzeSchemaMemoryBlock(sizeInBytes);
       }
       throw e;
@@ -131,8 +140,20 @@ public class LoadTsFileMemoryManager {
     long bytesNeeded = newSizeInBytes - 
memoryBlock.getTotalMemorySizeInBytes();
     try {
       forceAllocateFromQuery(bytesNeeded);
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.info(
+            "Load: Force resized LoadTsFileAnalyzeSchemaMemoryBlock with 
memory from query engine, size added: {}, new size: {}",
+            bytesNeeded,
+            newSizeInBytes);
+      }
     } catch (LoadRuntimeOutOfMemoryException e) {
-      if (dataCacheMemoryBlock == null || 
!dataCacheMemoryBlock.doShrink(bytesNeeded)) {
+      if (dataCacheMemoryBlock != null && 
dataCacheMemoryBlock.doShrink(bytesNeeded)) {
+        LOGGER.info(
+            "Load: Query engine's memory is not sufficient, force resized 
LoadTsFileAnalyzeSchemaMemoryBlock with memory from DataCacheMemoryBlock, size 
added: {}, new size: {}",
+            bytesNeeded,
+            newSizeInBytes);
+        usedMemorySizeInBytes.addAndGet(bytesNeeded);
+      } else {
         throw e;
       }
     }