This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 013b4093ea3 [to dev/1.3] Further optimize the efficiency of memtable
region scan #16979
013b4093ea3 is described below
commit 013b4093ea38b2e4a6d7d5a4192ff32136218a96
Author: shuwenwei <[email protected]>
AuthorDate: Mon Jan 5 18:06:02 2026 +0800
[to dev/1.3] Further optimize the efficiency of memtable region scan #16979
---
.../memtable/AlignedWritableMemChunk.java | 96 +++++++++++++---------
.../dataregion/memtable/WritableMemChunk.java | 44 +++++++---
.../iotdb/db/utils/datastructure/TVList.java | 4 +
.../memtable/WritableMemChunkRegionScanTest.java | 76 +++++++++++++++++
4 files changed, 167 insertions(+), 53 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 88d55e5bdf8..50ae45b432c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -55,6 +55,7 @@ import java.util.TreeMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
+import static
org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
@@ -780,54 +781,67 @@ public class AlignedWritableMemChunk extends
AbstractWritableMemChunk {
// timestamp: 1 bitmap: 011
// timestamp: 2 bitmap: 101
// timestamp: 4 bitmap: 110
- for (int row = 0; row < rowCount; row++) {
- // the row is deleted
- if (allValueColDeletedMap != null &&
allValueColDeletedMap.isMarked(row)) {
- continue;
- }
- long timestamp = alignedTVList.getTime(row);
- if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp,
null)) {
- continue;
- }
-
- // Note that this method will only perform bitmap unmarking on the first
occurrence of a
- // non-null value in multiple timestamps for the same column.
- BitMap currentRowNullValueBitmap = null;
-
- for (int column = 0; column < schemaList.size(); column++) {
- if (alignedTVList.isNullValue(alignedTVList.getValueIndex(row),
column)) {
+ List<long[]> timestampsList = alignedTVList.getTimestamps();
+ List<int[]> indicesList = alignedTVList.getIndices();
+ int row = -1;
+
+ for (int i = 0; i < timestampsList.size(); i++) {
+ long[] timestamps = timestampsList.get(i);
+ int[] indices = indicesList == null ? null : indicesList.get(i);
+ int limit = (i == timestampsList.size() - 1) ? rowCount - i * ARRAY_SIZE
: ARRAY_SIZE;
+ for (int j = 0; j < limit; j++) {
+ row++;
+ // the row is deleted
+ if (allValueColDeletedMap != null &&
allValueColDeletedMap.isMarked(row)) {
continue;
}
-
- // skip deleted row
- if (valueColumnsDeletionList != null
- && !valueColumnsDeletionList.isEmpty()
- && isPointDeleted(
- timestamp,
- valueColumnsDeletionList.get(column),
- valueColumnDeleteCursor.get(column))) {
+ long timestamp = timestamps[j];
+ if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp,
null)) {
continue;
}
- if (!columnHasNonNullValue.isMarked(column)) {
- hasNonNullValueColumnCount.incrementAndGet();
- columnHasNonNullValue.mark(column);
- currentRowNullValueBitmap =
- currentRowNullValueBitmap != null
- ? currentRowNullValueBitmap
- : timestampWithBitmap.computeIfAbsent(
- timestamp, k -> getAllMarkedBitmap(schemaList.size()));
- currentRowNullValueBitmap.unmark(column);
+
+ // Note that this method will only perform bitmap unmarking on the
first occurrence of a
+ // non-null value in multiple timestamps for the same column.
+ BitMap currentRowNullValueBitmap = null;
+ for (int column = 0; column < schemaList.size(); column++) {
+ if (alignedTVList.isNullValue(indices == null ? row : indices[j],
column)) {
+ continue;
+ }
+
+ // skip deleted row
+ if (valueColumnsDeletionList != null &&
!valueColumnsDeletionList.isEmpty()) {
+ List<TimeRange> columnDeletionList =
valueColumnsDeletionList.get(column);
+ int[] deleteCursor = valueColumnDeleteCursor.get(column);
+ if (columnDeletionList != null && !columnDeletionList.isEmpty()) {
+ if (!alignedTVList.isSorted()) {
+ deleteCursor[0] = 0;
+ }
+ if (isPointDeleted(timestamp, columnDeletionList, deleteCursor))
{
+ continue;
+ }
+ }
+ }
+ if (!columnHasNonNullValue.isMarked(column)) {
+ hasNonNullValueColumnCount.incrementAndGet();
+ columnHasNonNullValue.mark(column);
+ currentRowNullValueBitmap =
+ currentRowNullValueBitmap != null
+ ? currentRowNullValueBitmap
+ : timestampWithBitmap.computeIfAbsent(
+ timestamp, k -> getAllMarkedBitmap(schemaList.size()));
+ currentRowNullValueBitmap.unmark(column);
+ }
}
- }
- if (currentRowNullValueBitmap == null) {
- continue;
- }
- // found new column with non-null value
- timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
+ if (currentRowNullValueBitmap == null) {
+ continue;
+ }
+ // found new column with non-null value
+ timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
- if (hasNonNullValueColumnCount.get() == schemaList.size()) {
- return;
+ if (hasNonNullValueColumnCount.get() == schemaList.size()) {
+ return;
+ }
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index c19ad4f5a22..fd616ac94fa 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
+import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.ModificationUtils;
import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
@@ -578,19 +579,38 @@ public class WritableMemChunk extends
AbstractWritableMemChunk {
&& !globalTimeFilter.satisfyStartEndTime(tvlist.getMinTime(),
tvlist.getMaxTime())) {
return Optional.empty();
}
- for (int i = 0; i < rowCount; i++) {
- if (tvlist.getBitMap() != null &&
tvlist.isNullValue(tvlist.getValueIndex(i))) {
- continue;
- }
- long curTime = tvlist.getTime(i);
- if (deletionList != null
- && ModificationUtils.isPointDeleted(curTime, deletionList,
deletionCursor)) {
- continue;
- }
- if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime,
null)) {
- continue;
+
+ List<long[]> timestampsList = tvlist.getTimestamps();
+ List<BitMap> bitMaps = tvlist.getBitMap();
+ List<int[]> indicesList = tvlist.getIndices();
+ for (int i = 0; i < timestampsList.size(); i++) {
+ long[] timestamps = timestampsList.get(i);
+ BitMap bitMap = bitMaps == null ? null : bitMaps.get(i);
+ int[] indices = indicesList == null ? null : indicesList.get(i);
+ int limit =
+ (i == timestampsList.size() - 1)
+ ? rowCount - i * PrimitiveArrayManager.ARRAY_SIZE
+ : PrimitiveArrayManager.ARRAY_SIZE;
+ for (int j = 0; j < limit; j++) {
+ if (bitMap != null
+ && (indices == null ? bitMap.isMarked(j) :
tvlist.isNullValue(indices[j]))) {
+ continue;
+ }
+ long curTime = timestamps[j];
+ if (deletionList != null && !deletionList.isEmpty()) {
+ if (!tvlist.isSorted()) {
+ deletionCursor[0] = 0;
+ }
+ if (ModificationUtils.isPointDeleted(curTime, deletionList,
deletionCursor)) {
+ continue;
+ }
+ }
+ if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime,
null)) {
+ continue;
+ }
+
+ return Optional.of(curTime);
}
- return Optional.of(curTime);
}
return Optional.empty();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 073b03f0a96..8695182712f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -736,6 +736,10 @@ public abstract class TVList implements WALEntryValue {
return timestamps;
}
+ public List<int[]> getIndices() {
+ return indices;
+ }
+
public void setOwnerQuery(QueryContext queryCtx) {
this.ownerQuery = queryCtx;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
index 63bbac18e38..b62d0f9159f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
@@ -232,6 +232,48 @@ public class WritableMemChunkRegionScanTest {
}
}
+ @Test
+ public void testAlignedWritableMemChunkRegionScan2() throws
IllegalPathException {
+ PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
+ try {
+ List<IMeasurementSchema> measurementSchemas =
+ Arrays.asList(
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32),
+ new MeasurementSchema("s3", TSDataType.INT32));
+ AlignedWritableMemChunk writableMemChunk = null;
+ for (int i = 1000; i < 2000; i++) {
+ memTable.writeAlignedRow(
+ new PlainDeviceID("root.test.d1"), measurementSchemas, i, new
Object[] {i, i, i});
+ }
+ for (int i = 1; i < 100; i++) {
+ memTable.writeAlignedRow(
+ new PlainDeviceID("root.test.d1"), measurementSchemas, i, new
Object[] {i, null, i});
+ }
+
+ MeasurementPath originalPath = new MeasurementPath("root.test.d1.s1");
+ memTable.delete(originalPath, originalPath.getDevicePath(), 1, 1500);
+
+ writableMemChunk =
+ (AlignedWritableMemChunk)
+ memTable.getWritableMemChunk(new PlainDeviceID("root.test.d1"),
"");
+ writableMemChunk.sortTvListForFlush();
+ List<BitMap> bitMaps = new ArrayList<>();
+ long[] timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(), Collections.emptyList(),
Collections.emptyList()),
+ bitMaps,
+ null);
+ Assert.assertEquals(3, timestamps.length);
+ Assert.assertEquals(1, timestamps[0]);
+ Assert.assertEquals(1000, timestamps[1]);
+ Assert.assertEquals(1501, timestamps[2]);
+ } finally {
+ memTable.release();
+ }
+ }
+
@Test
public void testNonAlignedWritableMemChunkRegionScan() throws
IllegalPathException {
PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
@@ -296,4 +338,38 @@ public class WritableMemChunkRegionScanTest {
memTable.release();
}
}
+
+ @Test
+ public void testNonAlignedWritableMemChunkRegionScan2() throws
IllegalPathException {
+ PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
+ try {
+ MeasurementSchema measurementSchema = new MeasurementSchema("s1",
TSDataType.INT32);
+ for (int i = 1000; i < 2000; i++) {
+ memTable.write(
+ new PlainDeviceID("root.test.d1"),
+ Collections.singletonList(measurementSchema),
+ i,
+ new Object[] {i});
+ }
+ for (int i = 1; i < 100; i++) {
+ memTable.write(
+ new PlainDeviceID("root.test.d1"),
+ Collections.singletonList(measurementSchema),
+ i,
+ new Object[] {i});
+ }
+
+ MeasurementPath originalPath = new MeasurementPath("root.test.d1.s1");
+ memTable.delete(originalPath, originalPath.getDevicePath(), 1, 1500);
+ WritableMemChunk writableMemChunk =
+ (WritableMemChunk) memTable.getWritableMemChunk(new
PlainDeviceID("root.test.d1"), "s1");
+ writableMemChunk.sortTvListForFlush();
+ Optional<Long> timestamp =
writableMemChunk.getAnySatisfiedTimestamp(null, null);
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(1501, timestamp.get().longValue());
+
+ } finally {
+ memTable.release();
+ }
+ }
}
