This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch force_ci/object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b7e1c02338f5cae13ac8f49e0fc33a5453120cba Author: Caideyipi <[email protected]> AuthorDate: Fri Nov 21 09:18:06 2025 +0800 Pipe: improved the memory calculation for tablets (#16739) * fix * fix * opti * fix * fix * fix function name * fix * fix * fix ut --------- Co-authored-by: Tian Jiang <[email protected]> (cherry picked from commit 0ebdb878e57bc1754bbbcf8c59790919e6d256b4) --- .../it/session/IoTDBSessionRelationalIT.java | 2 + .../resource/memory/InsertNodeMemoryEstimator.java | 55 +++---------------- .../pipe/resource/memory/PipeMemoryWeightUtil.java | 62 ++-------------------- .../plan/statement/crud/InsertBaseStatement.java | 2 +- .../plan/statement/crud/InsertTabletStatement.java | 2 +- .../iotdb/db/utils/datastructure/TVList.java | 2 +- pom.xml | 2 +- 7 files changed, 16 insertions(+), 111 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index bba5681b58b..5a26ca510a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -1812,6 +1812,7 @@ public class IoTDBSessionRelationalIT { dataTypes.remove(TSDataType.OBJECT); dataTypes.remove(TSDataType.VECTOR); dataTypes.remove(TSDataType.UNKNOWN); + dataTypes.remove(TSDataType.OBJECT); try { for (TSDataType from : dataTypes) { @@ -1903,6 +1904,7 @@ public class IoTDBSessionRelationalIT { dataTypes.remove(TSDataType.OBJECT); dataTypes.remove(TSDataType.VECTOR); dataTypes.remove(TSDataType.UNKNOWN); + dataTypes.remove(TSDataType.OBJECT); for (TSDataType from : dataTypes) { for (TSDataType to : dataTypes) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 60076c6139e..7f1d7357b02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -189,7 +189,7 @@ public class InsertNodeMemoryEstimator { // MeasurementSchemas size += sizeOfMeasurementSchemas(node.getMeasurementSchemas()); // Measurement - size += sizeOfStringArray(node.getMeasurements()); + size += RamUsageEstimator.sizeOf(node.getMeasurements()); // dataTypes size += RamUsageEstimator.shallowSizeOf(node.getDataTypes()); // columnCategories @@ -231,7 +231,7 @@ public class InsertNodeMemoryEstimator { long size = INSERT_TABLET_NODE_SIZE; size += calculateFullInsertNodeSize(node); size += RamUsageEstimator.sizeOf(node.getTimes()); - size += sizeOfBitMapArray(node.getBitMaps()); + size += RamUsageEstimator.sizeOf(node.getBitMaps()); size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); final List<Integer> range = node.getRange(); if (range != null) { @@ -247,7 +247,7 @@ public class InsertNodeMemoryEstimator { size += RamUsageEstimator.sizeOf(node.getTimes()); - size += sizeOfBitMapArray(node.getBitMaps()); + size += RamUsageEstimator.sizeOf(node.getBitMaps()); size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); @@ -390,7 +390,7 @@ public class InsertNodeMemoryEstimator { size += RamUsageEstimator.sizeOf(node.getTimes()); - size += sizeOfBitMapArray(node.getBitMaps()); + size += RamUsageEstimator.sizeOf(node.getBitMaps()); size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); @@ -413,7 +413,7 @@ public class InsertNodeMemoryEstimator { if (nodes != null) { // Since fullPath may be lazy loaded, lazy loading will not be triggered here, so it is // assumed that the memory size of fullPath is the same as that of nodes. - size += sizeOfStringArray(nodes) * 2; + size += RamUsageEstimator.sizeOf(nodes) * 2; size += TsFileConstant.PATH_SEPARATOR.length() * (nodes.length - 1) + NUM_BYTES_OBJECT_HEADER; } return size; @@ -555,34 +555,6 @@ public class InsertNodeMemoryEstimator { return Objects.nonNull(binary) ? binary.ramBytesUsed() : 0L; } - public static long sizeOfStringArray(final String[] values) { - return Objects.nonNull(values) ? RamUsageEstimator.sizeOf(values) : 0L; - } - - public static long sizeOfBitMapArray(BitMap[] bitMaps) { - if (bitMaps == null) { - return 0L; - } - long size = - RamUsageEstimator.alignObjectSize( - NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * bitMaps.length); - for (BitMap bitMap : bitMaps) { - size += sizeOfBitMap(bitMap); - } - return size; - } - - private static long sizeOfBitMap(final BitMap bitMaps) { - if (bitMaps == null) { - return 0L; - } - long size = BIT_MAP_SIZE; - - size += - RamUsageEstimator.alignObjectSize(NUM_BYTES_ARRAY_HEADER + bitMaps.getByteArray().length); - return size; - } - public static long sizeOfColumns( final Object[] columns, final MeasurementSchema[] measurementSchemas) { // Directly calculate if measurementSchemas are absent @@ -632,7 +604,7 @@ public class InsertNodeMemoryEstimator { case BLOB: case OBJECT: { - size += getBinarySize((Binary[]) columns[i]); + size += RamUsageEstimator.sizeOf((Binary[]) columns[i]); break; } } @@ -642,23 +614,10 @@ public class InsertNodeMemoryEstimator { private static long getNumBytesUnknownObject(final Object obj) { return obj instanceof Binary[] - ? getBinarySize((Binary[]) obj) + ? RamUsageEstimator.sizeOf((Binary[]) obj) : RamUsageEstimator.sizeOfObject(obj); } - private static long getBinarySize(final Binary[] binaries) { - if (binaries == null) { - return 0L; - } - - long size = 0L; - for (Binary binary : binaries) { - size += InsertNodeMemoryEstimator.sizeOfBinary(binary); - } - - return size + RamUsageEstimator.shallowSizeOf(binaries); - } - public static long sizeOfValues( final Object[] values, final MeasurementSchema[] measurementSchemas) { // Directly calculate if measurementSchemas are absent diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java index 30b68e41741..0ba204fcbd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java @@ -35,7 +35,6 @@ import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.Tablet; @@ -44,6 +43,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; import static org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; @@ -203,64 +203,8 @@ public class PipeMemoryWeightUtil { } } - public static long calculateTabletSizeInBytes(Tablet tablet) { - long totalSizeInBytes = 0; - - if (tablet == null) { - return totalSizeInBytes; - } - - long[] timestamps = tablet.getTimestamps(); - Object[] tabletValues = tablet.getValues(); - - // timestamps - if (timestamps != null) { - totalSizeInBytes += timestamps.length * 8L; - } - - // values - final List<IMeasurementSchema> timeseries = tablet.getSchemas(); - if (timeseries != null) { - for (int column = 0; column < timeseries.size(); column++) { - final IMeasurementSchema measurementSchema = timeseries.get(column); - if (measurementSchema == null) { - continue; - } - - final TSDataType tsDataType = measurementSchema.getType(); - if (tsDataType == null) { - continue; - } - - if (tsDataType.isBinary()) { - if (tabletValues == null || tabletValues.length <= column) { - continue; - } - final Binary[] values = ((Binary[]) tabletValues[column]); - if (values == null) { - continue; - } - for (Binary value : values) { - totalSizeInBytes += value == null ? 8 : value.ramBytesUsed(); - } - } else { - totalSizeInBytes += (long) tablet.getMaxRowNumber() * tsDataType.getDataTypeSize(); - } - } - } - - // bitMaps - BitMap[] bitMaps = tablet.getBitMaps(); - if (bitMaps != null) { - for (int i = 0; i < bitMaps.length; i++) { - totalSizeInBytes += bitMaps[i] == null ? 0 : bitMaps[i].getSize(); - } - } - - // estimate other dataStructures size - totalSizeInBytes += 100; - - return totalSizeInBytes; + public static long calculateTabletSizeInBytes(final Tablet tablet) { + return Objects.nonNull(tablet) ? tablet.ramBytesUsed() : 0L; } public static long calculateTableSchemaBytesUsed(TableSchema tableSchema) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index 177c05bde19..f115bde9d2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -686,7 +686,7 @@ public abstract class InsertBaseStatement extends Statement implements Accountab ramBytesUsed = InsertNodeMemoryEstimator.sizeOfPartialPath(devicePath) + InsertNodeMemoryEstimator.sizeOfMeasurementSchemas(measurementSchemas) - + InsertNodeMemoryEstimator.sizeOfStringArray(measurements) + + RamUsageEstimator.sizeOf(measurements) + RamUsageEstimator.shallowSizeOf(dataTypes) + RamUsageEstimator.shallowSizeOf(columnCategories) // We assume that the integers are all cached by JVM diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index efb4746b9b3..d3257d98d41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -615,7 +615,7 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem protected long calculateBytesUsed() { return INSTANCE_SIZE + RamUsageEstimator.sizeOf(times) - + InsertNodeMemoryEstimator.sizeOfBitMapArray(nullBitMaps) + + RamUsageEstimator.sizeOf(nullBitMaps) + InsertNodeMemoryEstimator.sizeOfColumns(columns, measurementSchemas) + (Objects.nonNull(deviceIDs) ? Arrays.stream(deviceIDs) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index e20575be362..a1f2842d34a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -973,7 +973,7 @@ public abstract class TVList implements WALEntryValue { TSDataType dataType = getDataType(); int maxRowCountOfCurrentBatch = Math.min( - paginationController.hasLimit() + paginationController.hasSetLimit() ? (int) paginationController.getCurLimit() : Integer.MAX_VALUE, Math.min(maxNumberOfPointsInPage, rows - index)); diff --git a/pom.xml b/pom.xml index b997a87580c..71e0f017c5d 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ <thrift.version>0.14.1</thrift.version> <xz.version>1.9</xz.version> <zstd-jni.version>1.5.6-3</zstd-jni.version> - <tsfile.version>2.2.0-251030-SNAPSHOT</tsfile.version> + <tsfile.version>2.2.0-251113-SNAPSHOT</tsfile.version> </properties> <!-- if we claim dependencies in dependencyManagement, then we do not claim
