This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 4690d2e3c2b [Performance] Reduce aligned MemTable bitmap memory usage
(#18249) (#18256)
4690d2e3c2b is described below
commit 4690d2e3c2b8d4fc75b47c057c20c2518b91c804
Author: Caideyipi <[email protected]>
AuthorDate: Mon Jul 20 19:02:10 2026 +0800
[Performance] Reduce aligned MemTable bitmap memory usage (#18249) (#18256)
---
.../dataregion/memtable/TsFileProcessor.java | 14 ++++---
.../db/utils/datastructure/AlignedTVList.java | 46 ++++++++++++++++++----
.../dataregion/memtable/TsFileProcessorTest.java | 28 ++++++-------
.../db/utils/datastructure/AlignedTVListTest.java | 42 ++++++++++++++++++++
4 files changed, 103 insertions(+), 27 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index da62e80252b..8c966231fb1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -725,8 +725,10 @@ public class TsFileProcessor {
}
// Here currentChunkPointNum >= 1
if ((alignedMemChunk.alignedListSize() %
PrimitiveArrayManager.ARRAY_SIZE) == 0) {
-
dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes());
memTableIncrement +=
alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost();
+ for (TSDataType dataType : dataTypesInTVList) {
+ memTableIncrement += AlignedTVList.valueListArrayMemCost(dataType);
+ }
}
}
updateMemoryInfo(memTableIncrement, chunkMetadataIncrement,
textDataIncrement);
@@ -810,13 +812,13 @@ public class TsFileProcessor {
int addingPointNum = addingPointNumInfo.right;
// Here currentChunkPointNum + addingPointNum >= 1
if (((currentChunkPointNum + addingPointNum) %
PrimitiveArrayManager.ARRAY_SIZE) == 0) {
- if (alignedMemChunk != null) {
-
dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes());
- }
dataTypesInTVList.addAll(addingPointNumInfo.left.values());
memTableIncrement +=
alignedMemChunk != null
?
alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost()
+ + dataTypesInTVList.stream()
+ .mapToLong(AlignedTVList::valueListArrayMemCost)
+ .sum()
: AlignedTVList.alignedTvListArrayMemCost(
dataTypesInTVList.toArray(new TSDataType[0]));
}
@@ -983,9 +985,11 @@ public class TsFileProcessor {
}
if (acquireArray != 0) {
// memory of extending the TVList
-
dataTypesInTVList.addAll(alignedMemChunk.getWorkingTVList().getTsDataTypes());
memIncrements[0] +=
acquireArray *
alignedMemChunk.getWorkingTVList().alignedTvListArrayMemCost();
+ for (TSDataType dataType : dataTypesInTVList) {
+ memIncrements[0] += acquireArray *
AlignedTVList.valueListArrayMemCost(dataType);
+ }
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index b99e784faad..9e3a556710c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -39,6 +39,7 @@ import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
@@ -64,6 +65,11 @@ import static
org.apache.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public abstract class AlignedTVList extends TVList {
+ private static final long BITMAP_RAM_COST_PER_BLOCK =
+ RamUsageEstimator.shallowSizeOfInstance(BitMap.class)
+ + RamUsageEstimator.sizeOfByteArray(ARRAY_SIZE / Byte.SIZE + 1)
+ + NUM_BYTES_OBJECT_REF;
+
// Data types of this aligned tvList
protected List<TSDataType> dataTypes;
@@ -755,12 +761,37 @@ public abstract class AlignedTVList extends TVList {
}
/* 2.mask the column bitmap */
- if (bitMaps != null && bitMaps[j] != null) {
+ if (bitMaps != null && bitMaps[j] != null &&
containsMarkedBit(bitMaps[j], idx, len)) {
getBitMap(j, arrayIndex).merge(bitMaps[j], idx, elementIdx, len);
}
}
}
+ private static boolean containsMarkedBit(BitMap bitMap, int start, int
length) {
+ if (length <= 0) {
+ return false;
+ }
+
+ byte[] bytes = bitMap.getByteArray();
+ int end = start + length - 1;
+ int firstByteIndex = start >>> 3;
+ int lastByteIndex = end >>> 3;
+ if (firstByteIndex == lastByteIndex) {
+ int mask = (0xFF << (start & 7)) & (0xFF >>> (7 - (end & 7)));
+ return (bytes[firstByteIndex] & mask) != 0;
+ }
+
+ if ((bytes[firstByteIndex] & (0xFF << (start & 7))) != 0) {
+ return true;
+ }
+ for (int i = firstByteIndex + 1; i < lastByteIndex; i++) {
+ if (bytes[i] != 0) {
+ return true;
+ }
+ }
+ return (bytes[lastByteIndex] & (0xFF >>> (7 - (end & 7)))) != 0;
+ }
+
private void arrayCopy(Object[] value, int idx, int arrayIndex, int
elementIndex, int remaining) {
for (int i = 0; i < values.size(); i++) {
if (value[i] == null) {
@@ -822,14 +853,14 @@ public abstract class AlignedTVList extends TVList {
if (bitMaps.get(columnIndex) == null) {
List<BitMap> columnBitMaps = new ArrayList<>();
for (int i = 0; i < values.get(columnIndex).size(); i++) {
- columnBitMaps.add(new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE]));
+ columnBitMaps.add(null);
}
bitMaps.set(columnIndex, columnBitMaps);
}
// if the bitmap in arrayIndex is null, init the bitmap
if (bitMaps.get(columnIndex).get(arrayIndex) == null) {
- bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE, new
byte[ARRAY_SIZE]));
+ bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE));
}
return bitMaps.get(columnIndex).get(arrayIndex);
@@ -864,6 +895,7 @@ public abstract class AlignedTVList extends TVList {
for (TSDataType type : types) {
if (type != null) {
size += (long) ARRAY_SIZE * (long) type.getDataTypeSize();
+ size += BITMAP_RAM_COST_PER_BLOCK;
measurementColumnNum++;
}
}
@@ -892,9 +924,7 @@ public abstract class AlignedTVList extends TVList {
TSDataType type = dataTypes.get(column);
if (type != null) {
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long)
type.getDataTypeSize();
- if (bitMaps != null && bitMaps.get(column) != null) {
- size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1;
- }
+ size += BITMAP_RAM_COST_PER_BLOCK;
}
}
// size is 0 when all types are null
@@ -922,8 +952,8 @@ public abstract class AlignedTVList extends TVList {
long size = 0;
// value array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long)
type.getDataTypeSize();
- // bitmap array mem size
- size += (long) PrimitiveArrayManager.ARRAY_SIZE / 8 + 1;
+ // bitmap object, byte array, and reference in the bitmap list
+ size += BITMAP_RAM_COST_PER_BLOCK;
// array headers mem size
size += NUM_BYTES_ARRAY_HEADER;
// Object references size in ArrayList
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
index 9a40ee0b36c..658faa971a3 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessorTest.java
@@ -686,11 +686,11 @@ public class TsFileProcessorTest {
// Test Tablet
processor.insertTablet(genInsertTableNode(0, true), 0, 10, new
TSStatus[10]);
IMemTable memTable = processor.getWorkMemTable();
- Assert.assertEquals(1596552, memTable.getTVListsRamCost());
+ Assert.assertEquals(1776552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNode(100, true), 0, 10, new
TSStatus[10]);
- Assert.assertEquals(1596552, memTable.getTVListsRamCost());
+ Assert.assertEquals(1776552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNode(200, true), 0, 10, new
TSStatus[10]);
- Assert.assertEquals(1596552, memTable.getTVListsRamCost());
+ Assert.assertEquals(1776552, memTable.getTVListsRamCost());
Assert.assertEquals(90000, memTable.getTotalPointsNum());
Assert.assertEquals(720360, memTable.memSize());
// Test records
@@ -699,7 +699,7 @@ public class TsFileProcessorTest {
record.addTuple(DataPoint.getDataPoint(dataType, measurementId,
String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[4]);
}
- Assert.assertEquals(1598168, memTable.getTVListsRamCost());
+ Assert.assertEquals(1778168, memTable.getTVListsRamCost());
Assert.assertEquals(90100, memTable.getTotalPointsNum());
Assert.assertEquals(721560, memTable.memSize());
}
@@ -722,21 +722,21 @@ public class TsFileProcessorTest {
// Test Tablet
processor.insertTablet(genInsertTableNode(0, true), 0, 10, new
TSStatus[10]);
IMemTable memTable = processor.getWorkMemTable();
- Assert.assertEquals(1596552, memTable.getTVListsRamCost());
+ Assert.assertEquals(1776552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNodeFors3000ToS6000(0, true), 0, 10,
new TSStatus[10]);
- Assert.assertEquals(3219552, memTable.getTVListsRamCost());
+ Assert.assertEquals(3552552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNode(100, true), 0, 10, new
TSStatus[10]);
- Assert.assertEquals(3219552, memTable.getTVListsRamCost());
+ Assert.assertEquals(3552552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNodeFors3000ToS6000(100, true), 0,
10, new TSStatus[10]);
- Assert.assertEquals(3219552, memTable.getTVListsRamCost());
+ Assert.assertEquals(3552552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNode(200, true), 0, 10, new
TSStatus[10]);
- Assert.assertEquals(3219552, memTable.getTVListsRamCost());
+ Assert.assertEquals(3552552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNodeFors3000ToS6000(200, true), 0,
10, new TSStatus[10]);
- Assert.assertEquals(3219552, memTable.getTVListsRamCost());
+ Assert.assertEquals(3552552, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNode(300, true), 0, 10, new
TSStatus[10]);
- Assert.assertEquals(6466104, memTable.getTVListsRamCost());
+ Assert.assertEquals(7105104, memTable.getTVListsRamCost());
processor.insertTablet(genInsertTableNodeFors3000ToS6000(300, true), 0,
10, new TSStatus[10]);
- Assert.assertEquals(6466104, memTable.getTVListsRamCost());
+ Assert.assertEquals(7105104, memTable.getTVListsRamCost());
Assert.assertEquals(240000, memTable.getTotalPointsNum());
Assert.assertEquals(1920960, memTable.memSize());
@@ -746,14 +746,14 @@ public class TsFileProcessorTest {
record.addTuple(DataPoint.getDataPoint(dataType, measurementId,
String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[4]);
}
- Assert.assertEquals(6467720, memTable.getTVListsRamCost());
+ Assert.assertEquals(7106720, memTable.getTVListsRamCost());
// Test records
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(i, deviceId);
record.addTuple(DataPoint.getDataPoint(dataType, "s1",
String.valueOf(i)));
processor.insert(buildInsertRowNodeByTSRecord(record), new long[4]);
}
- Assert.assertEquals(6469336, memTable.getTVListsRamCost());
+ Assert.assertEquals(7108336, memTable.getTVListsRamCost());
Assert.assertEquals(240200, memTable.getTotalPointsNum());
Assert.assertEquals(1923360, memTable.memSize());
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java
index 392209e6581..6a7206d39c5 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/AlignedTVListTest.java
@@ -27,8 +27,11 @@ import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import static
org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
+
public class AlignedTVListTest {
@Test
@@ -142,6 +145,45 @@ public class AlignedTVListTest {
}
}
+ @Test
+ public void testBitmapIsAllocatedLazilyWithCompactBackingArray() {
+ AlignedTVList tvList =
+ AlignedTVList.newAlignedList(Arrays.asList(TSDataType.INT64,
TSDataType.INT64));
+ Object[] values = new Object[] {1L, 1L};
+ for (int i = 0; i < ARRAY_SIZE * 2 + 1; i++) {
+ tvList.putAlignedValue(i, values);
+ }
+
+ Assert.assertNull(tvList.getBitMaps());
+ tvList.putAlignedValue(ARRAY_SIZE * 2 + 1L, new Object[] {null, 1L});
+
+ List<BitMap> firstColumnBitMaps = tvList.getBitMaps().get(0);
+ Assert.assertEquals(3, firstColumnBitMaps.size());
+ Assert.assertNull(firstColumnBitMaps.get(0));
+ Assert.assertNull(firstColumnBitMaps.get(1));
+ Assert.assertNotNull(firstColumnBitMaps.get(2));
+ Assert.assertEquals(
+ ARRAY_SIZE / Byte.SIZE + 1,
firstColumnBitMaps.get(2).getByteArray().length);
+ Assert.assertTrue(tvList.isNullValue(ARRAY_SIZE * 2 + 1, 0));
+ Assert.assertFalse(tvList.isNullValue(ARRAY_SIZE * 2, 0));
+ }
+
+ @Test
+ public void testEmptyInputBitmapsDoNotMaterializeMemTableBitmaps() {
+ AlignedTVList tvList =
AlignedTVList.newAlignedList(Arrays.asList(TSDataType.INT64));
+ long[] times = new long[ARRAY_SIZE];
+ long[][] values = new long[1][ARRAY_SIZE];
+ BitMap[] bitMaps = new BitMap[] {new BitMap(ARRAY_SIZE)};
+ for (int i = 0; i < ARRAY_SIZE; i++) {
+ times[i] = i;
+ values[0][i] = i;
+ }
+
+ tvList.putAlignedValues(times, values, bitMaps, 0, ARRAY_SIZE);
+
+ Assert.assertNull(tvList.getBitMaps());
+ }
+
@Test
public void testClone() {
List<TSDataType> dataTypes = new ArrayList<>();
