This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch patch-2094 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7df501a26b16d3186783efcee52bdce192e4cbfa Author: shizy <[email protected]> AuthorDate: Thu Apr 30 08:52:34 2026 +0800 mon: add diagnose information for MemoryNotEnoughException (#17578) (cherry picked from commit 06621d0f045a15a66e2f16434fd0724caef226db) (cherry picked from commit 027f081ea2dbbbe4952056cbcd706d5228c7b1f5) --- .../fragment/FragmentInstanceContext.java | 2 +- .../schemaregion/utils/ResourceByPathUtils.java | 15 ++++++-- .../memtable/AbstractWritableMemChunk.java | 2 +- .../memtable/AlignedReadOnlyMemChunk.java | 4 +-- .../dataregion/memtable/ReadOnlyMemChunk.java | 4 +-- .../db/utils/datastructure/AlignedTVList.java | 5 +-- .../iotdb/db/utils/datastructure/TVList.java | 42 ++++++++++++++++++++-- .../fragment/FragmentInstanceExecutionTest.java | 4 +-- .../dataregion/memtable/PrimitiveMemTableTest.java | 2 +- 9 files changed, 64 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 346d06bf910..f78665d718b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -931,7 +931,7 @@ public class FragmentInstanceContext extends QueryContext { */ private void releaseTVListOwnedByQuery() { for (TVList tvList : tvListSet) { - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); Set<QueryContext> queryContextSet = tvList.getQueryContextSet(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index adb76d9bbdd..566f43b01d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; @@ -155,7 +156,7 @@ public abstract class ResourceByPathUtils { // mutable tvlist TVList list = memChunk.getWorkingTVList(); TVList cloneList = null; - long tvListRamSize = list.calculateRamSize(); + TVList.RamInfo listRamInfo = list.calculateRamSize(); list.lockQueryList(); try { if (copyTimeFilter != null @@ -196,8 +197,8 @@ public abstract class ResourceByPathUtils { if (firstQuery instanceof FragmentInstanceContext) { MemoryReservationManager memoryReservationManager = ((FragmentInstanceContext) firstQuery).getMemoryReservationContext(); - memoryReservationManager.reserveMemoryCumulatively(tvListRamSize); - list.setReservedMemoryBytes(tvListRamSize); + memoryReservationManager.reserveMemoryCumulatively(listRamInfo.getRamSize()); + list.setReservedMemoryBytes(listRamInfo.getRamSize()); } list.setOwnerQuery(firstQuery); @@ -207,6 +208,14 @@ public abstract class ResourceByPathUtils { tvListQueryMap.put(cloneList, cloneList.rowCount()); } } + } catch (MemoryNotEnoughException ex) { + LOGGER.warn( + "Failed to reserve memory for TVList: ramSize {}, timestampsSize {}, arrayMemCost {}, rowCount {}, dataTypes {}", + listRamInfo.getRamSize(), + listRamInfo.getTimestampsSize(), + listRamInfo.getArrayMemCost(), + listRamInfo.getRowCount(), + listRamInfo.getDataTypes()); } finally { list.unlockQueryList(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java index 7a3233d58b6..91bad093222 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java @@ -103,7 +103,7 @@ public abstract class AbstractWritableMemChunk implements IWritableMemChunk { } private void tryReleaseTvList(TVList tvList) { - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); try { if (tvList.getQueryContextSet().isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 517566ed78c..05c1602bf2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -127,7 +127,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { // We must update queryRowCount here, otherwise, it may be used later to build // BitMaps, causing bitmap array size mismatch and possible out of bound. entry.setValue(alignedTvList.sort()); - long alignedTvListRamSize = alignedTvList.calculateRamSize(); + long alignedTvListRamSize = alignedTvList.calculateRamSize().getRamSize(); alignedTvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = @@ -384,7 +384,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { int queryLength = entry.getValue(); if (!alignedTvList.isSorted() && queryLength > alignedTvList.seqRowCount()) { entry.setValue(alignedTvList.sort()); - long alignedTvListRamSize = alignedTvList.calculateRamSize(); + long alignedTvListRamSize = alignedTvList.calculateRamSize().getRamSize(); alignedTvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index f6fa9403c7b..798633e5429 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -137,7 +137,7 @@ public class ReadOnlyMemChunk { int queryRowCount = entry.getValue(); if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) { entry.setValue(tvList.sort()); - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = (FragmentInstanceContext) tvList.getOwnerQuery(); @@ -294,7 +294,7 @@ public class ReadOnlyMemChunk { int queryLength = entry.getValue(); if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) { entry.setValue(tvList.sort()); - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = (FragmentInstanceContext) tvList.getOwnerQuery(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 6293a0fc4b9..cb1379dc19d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -1064,8 +1064,9 @@ public abstract class AlignedTVList extends TVList { } @Override - public synchronized long calculateRamSize() { - return timestamps.size() * alignedTvListArrayMemCost(); + public synchronized RamInfo calculateRamSize() { + return new RamInfo( + timestamps.size(), alignedTvListArrayMemCost(), rowCount, new ArrayList<>(dataTypes)); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 8b571fed01f..fb0cc005813 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.utils.MathUtils; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -61,6 +62,42 @@ import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; public abstract class TVList implements WALEntryValue { protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not consistent"; + + public static class RamInfo { + private final int timestampsSize; + private final long arrayMemCost; + private final int rowCount; + private final List<TSDataType> dataTypes; + + public RamInfo( + int timestampCount, long arrayMemCost, int rowCount, List<TSDataType> dataTypes) { + this.timestampsSize = timestampCount; + this.rowCount = rowCount; + this.arrayMemCost = arrayMemCost; + this.dataTypes = dataTypes; + } + + public long getRamSize() { + return timestampsSize * arrayMemCost; + } + + public int getTimestampsSize() { + return timestampsSize; + } + + public int getRowCount() { + return rowCount; + } + + public long getArrayMemCost() { + return arrayMemCost; + } + + public List<TSDataType> getDataTypes() { + return dataTypes; + } + } + // list of timestamp array, add 1 when expanded -> data point timestamp array // index relation: arrayIndex -> elementIndex protected List<long[]> timestamps; @@ -165,8 +202,9 @@ public abstract class TVList implements WALEntryValue { return size; } - public synchronized long calculateRamSize() { - return timestamps.size() * tvListArrayMemCost(); + public synchronized RamInfo calculateRamSize() { + return new RamInfo( + timestamps.size(), tvListArrayMemCost(), rowCount, ImmutableList.of(getDataType())); } public synchronized boolean isSorted() { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java index 61b24cd4ed9..de78365c21c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java @@ -146,7 +146,7 @@ public class FragmentInstanceExecutionTest { // mock flush's behavior fragmentInstanceContext1 .getMemoryReservationContext() - .reserveMemoryCumulatively(tvList.calculateRamSize()); + .reserveMemoryCumulatively(tvList.calculateRamSize().getRamSize()); tvList.setOwnerQuery(fragmentInstanceContext1); fragmentInstanceContext1.decrementNumOfUnClosedDriver(); @@ -226,7 +226,7 @@ public class FragmentInstanceExecutionTest { pointReader.nextTimeValuePair(); } assertTrue(tvList.isSorted()); - assertEquals(tvList.calculateRamSize(), tvList.getReservedMemoryBytes()); + assertEquals(tvList.calculateRamSize().getRamSize(), tvList.getReservedMemoryBytes()); } catch (QueryProcessException | IOException | MetadataException diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index 7fb46b14cf2..bfd1cfde5fa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -748,7 +748,7 @@ public class PrimitiveMemTableTest { Mockito.mock(MemoryReservationManager.class); Mockito.doThrow(new MemoryNotEnoughException("")) .when(memoryReservationManager) - .reserveMemoryCumulatively(list.calculateRamSize()); + .reserveMemoryCumulatively(list.calculateRamSize().getRamSize()); // create FragmentInstanceId QueryId queryId = new QueryId("stub_query");
