This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 42b8ef50cbf Load: Fixed the memory allocation bug that may cause more
free query memory & Optimized logger for memory not enough problem (#16206)
42b8ef50cbf is described below
commit 42b8ef50cbf2b013b334bcca414cf70d3b85a53e
Author: Caideyipi <[email protected]>
AuthorDate: Wed Aug 20 11:46:16 2025 +0800
Load: Fixed the memory allocation bug that may cause more free query memory
& Optimized logger for memory not enough problem (#16206)
* load-fix
* push-n
* push
* fix
---
.../queryengine/plan/planner/LocalExecutionPlanner.java | 7 +++++--
.../load/memory/LoadTsFileDataCacheMemoryBlock.java | 8 +++-----
.../load/memory/LoadTsFileMemoryManager.java | 15 +++++++--------
3 files changed, 15 insertions(+), 15 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index afe24040909..4ac3b6d7554 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@ -273,11 +273,14 @@ public class LocalExecutionPlanner {
}
}
- public synchronized long tryAllocateFreeMemoryForOperators(long
memoryInBytes) {
+ public synchronized long tryAllocateFreeMemory4Load(final long
memoryInBytes) {
if (OPERATORS_MEMORY_BLOCK.getFreeMemoryInBytes() - memoryInBytes
<= MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD) {
- long result =
+ final long result =
OPERATORS_MEMORY_BLOCK.getFreeMemoryInBytes() -
MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD;
+ if (result <= 0) {
+ return 0;
+ }
OPERATORS_MEMORY_BLOCK.forceAllocateWithoutLimitation(result);
return result;
} else {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
index 4611df37aa8..47f8c53edc3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileDataCacheMemoryBlock.java
@@ -66,9 +66,7 @@ public class LoadTsFileDataCacheMemoryBlock extends
LoadTsFileAbstractMemoryBloc
@Override
public synchronized void reduceMemoryUsage(long memoryInBytes) {
- if (memoryUsageInBytes.addAndGet(-memoryInBytes) < 0) {
- LOGGER.warn("{} has reduce memory usage to negative", this);
- }
+ memoryUsageInBytes.addAndGet(-memoryInBytes);
}
@Override
@@ -97,11 +95,11 @@ public class LoadTsFileDataCacheMemoryBlock extends
LoadTsFileAbstractMemoryBloc
return true;
}
- if (limitedMemorySizeInBytes.get() - shrinkMemoryInBytes <=
MINIMUM_MEMORY_SIZE_IN_BYTES) {
+ if (limitedMemorySizeInBytes.get() - shrinkMemoryInBytes
+ <= Math.max(MINIMUM_MEMORY_SIZE_IN_BYTES, memoryUsageInBytes.get())) {
return false;
}
- MEMORY_MANAGER.releaseToQuery(shrinkMemoryInBytes);
limitedMemorySizeInBytes.addAndGet(-shrinkMemoryInBytes);
return true;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
index f55607ece75..5094cbf2868 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java
@@ -69,19 +69,19 @@ public class LoadTsFileMemoryManager {
+ "current load used memory size %s bytes, load requested
memory size %s bytes",
MEMORY_ALLOCATE_MAX_RETRIES,
QUERY_ENGINE_MEMORY_MANAGER.getAllocateMemoryForOperators(),
- QUERY_ENGINE_MEMORY_MANAGER.getFreeMemoryForLoadTsFile(),
+ Math.max(0L,
QUERY_ENGINE_MEMORY_MANAGER.getFreeMemoryForLoadTsFile()),
usedMemorySizeInBytes.get(),
sizeInBytes));
}
- public synchronized long tryAllocateFromQuery(long sizeInBytes) {
- long actuallyAllocateMemoryInBytes =
- Math.max(0L,
QUERY_ENGINE_MEMORY_MANAGER.tryAllocateFreeMemoryForOperators(sizeInBytes));
+ public synchronized long tryAllocateFromQuery(final long sizeInBytes) {
+ final long actuallyAllocateMemoryInBytes =
+ QUERY_ENGINE_MEMORY_MANAGER.tryAllocateFreeMemory4Load(sizeInBytes);
usedMemorySizeInBytes.addAndGet(actuallyAllocateMemoryInBytes);
return actuallyAllocateMemoryInBytes;
}
- public synchronized void releaseToQuery(long sizeInBytes) {
+ public synchronized void releaseToQuery(final long sizeInBytes) {
if (usedMemorySizeInBytes.get() < sizeInBytes) {
LOGGER.error(
"Load: Attempting to release more memory ({}) than allocated ({})",
@@ -106,7 +106,6 @@ public class LoadTsFileMemoryManager {
LOGGER.info(
"Load: Query engine's memory is not sufficient, allocated
MemoryBlock from DataCacheMemoryBlock, size: {}",
sizeInBytes);
- usedMemorySizeInBytes.addAndGet(sizeInBytes);
return new LoadTsFileMemoryBlock(sizeInBytes);
}
throw e;
@@ -151,7 +150,6 @@ public class LoadTsFileMemoryManager {
"Load: Query engine's memory is not sufficient, force resized
LoadTsFileMemoryBlock with memory from DataCacheMemoryBlock, size added: {},
new size: {}",
bytesNeeded,
newSizeInBytes);
- usedMemorySizeInBytes.addAndGet(bytesNeeded);
} else {
throw e;
}
@@ -162,9 +160,10 @@ public class LoadTsFileMemoryManager {
public synchronized LoadTsFileDataCacheMemoryBlock
allocateDataCacheMemoryBlock()
throws LoadRuntimeOutOfMemoryException {
if (dataCacheMemoryBlock == null) {
- long actuallyAllocateMemoryInBytes =
+ final long actuallyAllocateMemoryInBytes =
tryAllocateFromQuery(MEMORY_TOTAL_SIZE_FROM_QUERY_IN_BYTES >> 2);
dataCacheMemoryBlock = new
LoadTsFileDataCacheMemoryBlock(actuallyAllocateMemoryInBytes);
+ usedMemorySizeInBytes.addAndGet(actuallyAllocateMemoryInBytes);
LOGGER.info(
"Create Data Cache Memory Block {}, allocate memory {}",
dataCacheMemoryBlock,
