This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 51fe585dfcd perf: Optimize the mark bitmap operation of AlignedTVList
InsertTablet (#16199)
51fe585dfcd is described below
commit 51fe585dfcd2e1b6a42b4f164c1fa8ebb999c0fe
Author: Zhenyu Luo <[email protected]>
AuthorDate: Wed Aug 27 10:10:51 2025 +0800
perf: Optimize the mark bitmap operation of AlignedTVList InsertTablet
(#16199)
* perf: Optimize the mark bitmap operation of AlignedTVList InsertTablet
* add UT
* update
* fix
* fix
* update tsfile version
* fix
---
.../resource/memory/InsertNodeMemoryEstimator.java | 10 +-
.../db/utils/datastructure/AlignedTVList.java | 156 +++++++++++++++------
pom.xml | 2 +-
3 files changed, 118 insertions(+), 50 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
index ed34f0de310..476ec83b787 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
@@ -650,10 +650,12 @@ public class InsertNodeMemoryEstimator {
return 0L;
}
- return RamUsageEstimator.shallowSizeOf(binaries)
- + Arrays.stream(binaries)
- .mapToLong(InsertNodeMemoryEstimator::sizeOfBinary)
- .reduce(0L, Long::sum);
+ long size = 0L;
+ for (Binary binary : binaries) {
+ size += InsertNodeMemoryEstimator.sizeOfBinary(binary);
+ }
+
+ return size + RamUsageEstimator.shallowSizeOf(binaries);
}
public static long sizeOfValues(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 80649eb182d..62b20e3e6ba 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -786,17 +786,9 @@ public abstract class AlignedTVList extends TVList {
if (indices != null) {
indices.get(arrayIdx)[elementIdx + i] = rowCount;
}
- for (int j = 0; j < values.size(); j++) {
- if (value[j] == null
- || bitMaps != null && bitMaps[j] != null &&
bitMaps[j].isMarked(idx + i)
- || results != null
- && results[idx + i] != null
- && results[idx + i].code !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- markNullValue(j, arrayIdx, elementIdx + i);
- }
- }
rowCount++;
}
+ markNullBitmapRange(value, bitMaps, results, idx, elementIdx,
inputRemaining, arrayIdx);
break;
} else {
// the remaining inputs cannot fit the last array, fill the last array
and create a new
@@ -807,23 +799,67 @@ public abstract class AlignedTVList extends TVList {
if (indices != null) {
indices.get(arrayIdx)[elementIdx + i] = rowCount;
}
- for (int j = 0; j < values.size(); j++) {
- if (value[j] == null
- || bitMaps != null && bitMaps[j] != null &&
bitMaps[j].isMarked(idx + i)
- || results != null
- && results[idx + i] != null
- && results[idx + i].code !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- markNullValue(j, arrayIdx, elementIdx + i);
- }
- }
rowCount++;
}
+ markNullBitmapRange(value, bitMaps, results, idx, elementIdx,
internalRemaining, arrayIdx);
idx += internalRemaining;
checkExpansion();
}
}
}
+ private void markNullBitmapRange(
+ Object[] values,
+ BitMap[] bitMaps,
+ TSStatus[] results,
+ int idx,
+ int elementIdx,
+ int len,
+ int arrayIndex) {
+
+ /* 1. Build result-level bitmap (1 = failure row) */
+ byte[] resultBitMap =
+ (results != null) ? buildResultBitMapBytes(results, idx, elementIdx,
len) : null;
+
+ for (int j = 0; j < values.length; j++) {
+ /* Fast-path: column is entirely null */
+ if (values[j] == null) {
+ getBitMap(j, arrayIndex).markRange(elementIdx, len);
+ continue;
+ }
+
+ /* 2.mask the column bitmap */
+ if (bitMaps != null && bitMaps[j] != null) {
+ getBitMap(j, arrayIndex).merge(bitMaps[j], idx, elementIdx, len);
+ }
+
+ /* 3. Overlay result bitmap (failure rows) */
+ if (resultBitMap != null) {
+ markNullValue(j, arrayIndex, elementIdx, resultBitMap);
+ }
+ }
+ }
+
+ public static byte[] buildResultBitMapBytes(
+ TSStatus[] results, int idx, int elementIdx, int length) {
+ int start = elementIdx & 7;
+ int totalBits = start + length;
+ int size = (totalBits + 7) >> 3;
+ BitMap bitmap = new BitMap(size, new byte[size]);
+
+ if (results == null) {
+ return bitmap.getByteArray();
+ }
+
+ for (int i = 0; i < length; i++) {
+ if (results[idx + i] != null
+ && results[idx + i].code !=
TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ bitmap.mark(start + i);
+ }
+ }
+ return bitmap.getByteArray();
+ }
+
private void arrayCopy(Object[] value, int idx, int arrayIndex, int
elementIndex, int remaining) {
for (int i = 0; i < values.size(); i++) {
if (value[i] == null) {
@@ -871,7 +907,7 @@ public abstract class AlignedTVList extends TVList {
}
}
- private void markNullValue(int columnIndex, int arrayIndex, int
elementIndex) {
+ private BitMap getBitMap(int columnIndex, int arrayIndex) {
// init BitMaps if doesn't have
if (bitMaps == null) {
List<List<BitMap>> localBitMaps = new ArrayList<>(dataTypes.size());
@@ -885,18 +921,31 @@ public abstract class AlignedTVList extends TVList {
if (bitMaps.get(columnIndex) == null) {
List<BitMap> columnBitMaps = new
ArrayList<>(values.get(columnIndex).size());
for (int i = 0; i < values.get(columnIndex).size(); i++) {
- columnBitMaps.add(new BitMap(ARRAY_SIZE));
+ columnBitMaps.add(new BitMap(ARRAY_SIZE, new byte[ARRAY_SIZE]));
}
bitMaps.set(columnIndex, columnBitMaps);
}
// if the bitmap in arrayIndex is null, init the bitmap
if (bitMaps.get(columnIndex).get(arrayIndex) == null) {
- bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE));
+ bitMaps.get(columnIndex).set(arrayIndex, new BitMap(ARRAY_SIZE, new
byte[ARRAY_SIZE]));
}
+ return bitMaps.get(columnIndex).get(arrayIndex);
+ }
+
+ private void markNullValue(
+ int columnIndex, int arrayIndex, int elementIndex, byte[] resultBitMap) {
+ byte[] bitMap = getBitMap(columnIndex, arrayIndex).getByteArray();
+ int start = elementIndex >>> 3;
+ for (byte b : resultBitMap) {
+ bitMap[start++] |= b;
+ }
+ }
+
+ private void markNullValue(int columnIndex, int arrayIndex, int
elementIndex) {
// mark the null value in the current bitmap
- bitMaps.get(columnIndex).get(arrayIndex).mark(elementIndex);
+ getBitMap(columnIndex, arrayIndex).mark(elementIndex);
}
@Override
@@ -1451,33 +1500,50 @@ public abstract class AlignedTVList extends TVList {
}
byte[] rowBitsArr = new byte[rowCount / Byte.SIZE + 1];
- for (int row = 0; row < rowCount; row += Byte.SIZE) {
- boolean isFirstColumn = true;
- byte rowBits = 0x00;
- for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
- List<BitMap> columnBitMaps = bitMaps.get(columnIndex);
- byte columnBits;
- if (values.get(columnIndex) == null) {
- columnBits = (byte) 0xFF;
- } else if (columnBitMaps == null || columnBitMaps.get(row /
ARRAY_SIZE) == null) {
- // row exists when any column value exists
- rowBits = 0x00;
- break;
- } else {
- columnBits =
- columnBitMaps.get(row / ARRAY_SIZE).getByteArray()[(row %
ARRAY_SIZE) / Byte.SIZE];
+ int bitsMapSize =
+ rowCount % ARRAY_SIZE == 0 ? rowCount / ARRAY_SIZE : rowCount /
ARRAY_SIZE + 1;
+ boolean[] allNotNullArray = new boolean[bitsMapSize];
+ Arrays.fill(rowBitsArr, (byte) 0xFF);
+ for (int columnIndex = 0; columnIndex < values.size(); columnIndex++) {
+ List<BitMap> columnBitMaps = bitMaps.get(columnIndex);
+ if (columnBitMaps == null) {
+ Arrays.fill(rowBitsArr, (byte) 0x00);
+ break;
+ } else if (values.get(columnIndex) != null) {
+ int row = 0;
+ boolean isEnd = true;
+ for (int i = 0; i < bitsMapSize; i++) {
+ if (allNotNullArray[i]) {
+ row += ARRAY_SIZE;
+ continue;
+ }
+
+ BitMap bitMap = columnBitMaps.get(i);
+ int index = row / Byte.SIZE;
+ int size = ((Math.min((rowCount - row), ARRAY_SIZE)) + 7) >>> 3;
+ row += ARRAY_SIZE;
+
+ if (bitMap == null) {
+ Arrays.fill(rowBitsArr, index, index + size, (byte) 0x00);
+ allNotNullArray[i] = true;
+ continue;
+ }
+
+ byte bits = (byte) 0X00;
+ for (int j = 0; j < size; j++) {
+ rowBitsArr[index] &= bitMap.getByteArray()[j];
+ bits |= rowBitsArr[index++];
+ isEnd = false;
+ }
+
+ allNotNullArray[i] = bits == (byte) 0;
}
- // set row to null when all column values are null
- if (isFirstColumn) {
- rowBits = columnBits;
- isFirstColumn = false;
- } else {
- rowBits &= columnBits;
+
+ if (isEnd) {
+ break;
}
}
- rowBitsArr[row / Byte.SIZE] = rowBits;
}
-
return new BitMap(rowCount, rowBitsArr);
}
diff --git a/pom.xml b/pom.xml
index cbe309852db..cf0f8017745 100644
--- a/pom.xml
+++ b/pom.xml
@@ -176,7 +176,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
- <tsfile.version>2.2.0-250813-SNAPSHOT</tsfile.version>
+ <tsfile.version>2.2.0-250825-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
