This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch optimizeMemTableRegionScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/optimizeMemTableRegionScan by
this push:
new 92f49413bea fix codecov
92f49413bea is described below
commit 92f49413beaef946c611ef4c6cc3f5c64e6e6042
Author: shuwenwei <[email protected]>
AuthorDate: Tue Dec 9 17:03:24 2025 +0800
fix codecov
---
.../dataregion/memtable/AbstractMemTable.java | 3 +
.../memtable/WritableMemChunkRegionScanTest.java | 333 ++++++++++++++-------
2 files changed, 226 insertions(+), 110 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index 78c918f4463..22606f232a8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -590,6 +590,9 @@ public abstract class AbstractMemTable implements IMemTable
{
WritableMemChunk memChunk =
(WritableMemChunk)
memTableMap.get(deviceID).getMemChunkMap().get(measurementId);
+ if (memChunk == null) {
+ return;
+ }
Optional<Long> anySatisfiedTimestamp =
memChunk.getAnySatisfiedTimestamp(deletionList, globalTimeFilter);
if (!anySatisfiedTimestamp.isPresent()) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
index 06582e8f9f4..3967409db52 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java
@@ -19,9 +19,13 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.commons.path.AlignedFullPath;
+import org.apache.iotdb.commons.path.NonAlignedFullPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.storageengine.dataregion.read.filescan.IChunkHandle;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.read.filter.operator.TimeFilterOperators;
import org.apache.tsfile.utils.BitMap;
@@ -38,7 +42,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
@RunWith(Parameterized.class)
@@ -69,94 +75,167 @@ public class WritableMemChunkRegionScanTest {
@Test
public void testAlignedWritableMemChunkRegionScan() {
- List<IMeasurementSchema> measurementSchemas =
- Arrays.asList(
- new MeasurementSchema("s1", TSDataType.INT32),
- new MeasurementSchema("s2", TSDataType.INT32),
- new MeasurementSchema("s3", TSDataType.INT32));
- AlignedWritableMemChunk writableMemChunk =
- new AlignedWritableMemChunk(measurementSchemas, false);
- int size = 100000;
- for (int i = 0; i < size; i++) {
- if (i <= 10000) {
- writableMemChunk.writeAlignedPoints(i, new Object[] {1, null, 1},
measurementSchemas);
- } else if (i <= 20000) {
- writableMemChunk.writeAlignedPoints(i, new Object[] {null, null, 2},
measurementSchemas);
- } else if (i <= 30000) {
- writableMemChunk.writeAlignedPoints(i, new Object[] {3, null, null},
measurementSchemas);
- } else {
- writableMemChunk.writeAlignedPoints(i, new Object[] {4, 4, 4},
measurementSchemas);
+ PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
+ try {
+ List<IMeasurementSchema> measurementSchemas =
+ Arrays.asList(
+ new MeasurementSchema("s1", TSDataType.INT32),
+ new MeasurementSchema("s2", TSDataType.INT32),
+ new MeasurementSchema("s3", TSDataType.INT32));
+ AlignedWritableMemChunk writableMemChunk = null;
+ int size = 100000;
+ for (int i = 0; i < size; i++) {
+ if (i <= 10000) {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {1, null, 1});
+ } else if (i <= 20000) {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {null, null, 2});
+ } else if (i <= 30000) {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {3, null, null});
+ } else {
+ memTable.writeAlignedRow(
+ new StringArrayDeviceID("root.test.d1"),
+ measurementSchemas,
+ i,
+ new Object[] {4, 4, 4});
+ }
}
- }
- List<BitMap> bitMaps = new ArrayList<>();
- long[] timestamps =
- writableMemChunk.getAnySatisfiedTimestamp(
- Arrays.asList(
- Collections.emptyList(), Collections.emptyList(),
Collections.emptyList()),
- bitMaps,
- true,
- null);
- Assert.assertEquals(2, timestamps.length);
- Assert.assertEquals(0, timestamps[0]);
- Assert.assertFalse(bitMaps.get(0).isMarked(0));
- Assert.assertTrue(bitMaps.get(0).isMarked(1));
- Assert.assertFalse(bitMaps.get(0).isMarked(2));
- Assert.assertTrue(bitMaps.get(1).isMarked(0));
- Assert.assertFalse(bitMaps.get(1).isMarked(1));
- Assert.assertTrue(bitMaps.get(1).isMarked(2));
- Assert.assertEquals(30001, timestamps[1]);
+ writableMemChunk =
+ (AlignedWritableMemChunk)
+ memTable.getWritableMemChunk(new
StringArrayDeviceID("root.test.d1"), "");
+ List<BitMap> bitMaps = new ArrayList<>();
+ long[] timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(), Collections.emptyList(),
Collections.emptyList()),
+ bitMaps,
+ true,
+ null);
+ Assert.assertEquals(2, timestamps.length);
+ Assert.assertEquals(0, timestamps[0]);
+ Assert.assertFalse(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertFalse(bitMaps.get(0).isMarked(2));
+ Assert.assertTrue(bitMaps.get(1).isMarked(0));
+ Assert.assertFalse(bitMaps.get(1).isMarked(1));
+ Assert.assertTrue(bitMaps.get(1).isMarked(2));
+ Assert.assertEquals(30001, timestamps[1]);
- bitMaps = new ArrayList<>();
- timestamps =
- writableMemChunk.getAnySatisfiedTimestamp(
- Arrays.asList(
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.singletonList(new TimeRange(0, 12000))),
- bitMaps,
- true,
- new TimeFilterOperators.TimeGt(11000));
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(new TimeRange(0, 12000))),
+ bitMaps,
+ true,
+ new TimeFilterOperators.TimeGt(10000000));
+ Assert.assertEquals(0, timestamps.length);
- Assert.assertEquals(3, timestamps.length);
- Assert.assertEquals(12001, timestamps[0]);
- Assert.assertTrue(bitMaps.get(0).isMarked(0));
- Assert.assertTrue(bitMaps.get(0).isMarked(1));
- Assert.assertFalse(bitMaps.get(0).isMarked(2));
- Assert.assertEquals(20001, timestamps[1]);
- Assert.assertFalse(bitMaps.get(1).isMarked(0));
- Assert.assertTrue(bitMaps.get(1).isMarked(1));
- Assert.assertTrue(bitMaps.get(1).isMarked(2));
- Assert.assertEquals(30001, timestamps[2]);
- Assert.assertTrue(bitMaps.get(2).isMarked(0));
- Assert.assertFalse(bitMaps.get(2).isMarked(1));
- Assert.assertTrue(bitMaps.get(2).isMarked(2));
-
- writableMemChunk.writeAlignedPoints(1000001, new Object[] {1, null, null},
measurementSchemas);
- writableMemChunk.writeAlignedPoints(1000002, new Object[] {null, 1, null},
measurementSchemas);
- writableMemChunk.writeAlignedPoints(1000002, new Object[] {1, 1, null},
measurementSchemas);
- writableMemChunk.writeAlignedPoints(1000003, new Object[] {1, null, null},
measurementSchemas);
- writableMemChunk.writeAlignedPoints(1000004, new Object[] {1, null, 1},
measurementSchemas);
- bitMaps = new ArrayList<>();
- timestamps =
- writableMemChunk.getAnySatisfiedTimestamp(
- Arrays.asList(
- Collections.emptyList(), Collections.emptyList(),
Collections.emptyList()),
- bitMaps,
- true,
- new TimeFilterOperators.TimeGt(1000000));
- Assert.assertEquals(3, timestamps.length);
- Assert.assertEquals(1000001, timestamps[0]);
- Assert.assertFalse(bitMaps.get(0).isMarked(0));
- Assert.assertTrue(bitMaps.get(0).isMarked(1));
- Assert.assertTrue(bitMaps.get(0).isMarked(2));
- Assert.assertEquals(1000002, timestamps[1]);
- Assert.assertTrue(bitMaps.get(1).isMarked(0));
- Assert.assertFalse(bitMaps.get(1).isMarked(1));
- Assert.assertTrue(bitMaps.get(1).isMarked(2));
- Assert.assertEquals(1000004, timestamps[2]);
- Assert.assertTrue(bitMaps.get(2).isMarked(0));
- Assert.assertTrue(bitMaps.get(2).isMarked(1));
- Assert.assertFalse(bitMaps.get(2).isMarked(2));
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(new TimeRange(0, 12000))),
+ bitMaps,
+ true,
+ new TimeFilterOperators.TimeGt(11000));
+
+ Assert.assertEquals(3, timestamps.length);
+ Assert.assertEquals(12001, timestamps[0]);
+ Assert.assertTrue(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertFalse(bitMaps.get(0).isMarked(2));
+ Assert.assertEquals(20001, timestamps[1]);
+ Assert.assertFalse(bitMaps.get(1).isMarked(0));
+ Assert.assertTrue(bitMaps.get(1).isMarked(1));
+ Assert.assertTrue(bitMaps.get(1).isMarked(2));
+ Assert.assertEquals(30001, timestamps[2]);
+ Assert.assertTrue(bitMaps.get(2).isMarked(0));
+ Assert.assertFalse(bitMaps.get(2).isMarked(1));
+ Assert.assertTrue(bitMaps.get(2).isMarked(2));
+
+ writableMemChunk.writeAlignedPoints(
+ 1000001, new Object[] {1, null, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(
+ 1000002, new Object[] {null, 1, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(1000002, new Object[] {1, 1, null},
measurementSchemas);
+ writableMemChunk.writeAlignedPoints(
+ 1000003, new Object[] {1, null, null}, measurementSchemas);
+ writableMemChunk.writeAlignedPoints(1000004, new Object[] {1, null, 1},
measurementSchemas);
+ bitMaps = new ArrayList<>();
+ timestamps =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Arrays.asList(
+ Collections.emptyList(), Collections.emptyList(),
Collections.emptyList()),
+ bitMaps,
+ true,
+ new TimeFilterOperators.TimeGt(1000000));
+ Assert.assertEquals(3, timestamps.length);
+ Assert.assertEquals(1000001, timestamps[0]);
+ Assert.assertFalse(bitMaps.get(0).isMarked(0));
+ Assert.assertTrue(bitMaps.get(0).isMarked(1));
+ Assert.assertTrue(bitMaps.get(0).isMarked(2));
+ Assert.assertEquals(1000002, timestamps[1]);
+ Assert.assertTrue(bitMaps.get(1).isMarked(0));
+ Assert.assertFalse(bitMaps.get(1).isMarked(1));
+ Assert.assertTrue(bitMaps.get(1).isMarked(2));
+ Assert.assertEquals(1000004, timestamps[2]);
+ Assert.assertTrue(bitMaps.get(2).isMarked(0));
+ Assert.assertTrue(bitMaps.get(2).isMarked(1));
+ Assert.assertFalse(bitMaps.get(2).isMarked(2));
+
+ Map<String, List<IChunkHandle>> chunkHandleMap = new HashMap<>();
+ memTable.queryForDeviceRegionScan(
+ new StringArrayDeviceID("root.test.d1"),
+ true,
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1000000));
+ Assert.assertEquals(3, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {1000001, 1000001},
chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000002, 1000002},
chunkHandleMap.get("s2").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000004, 1000004},
chunkHandleMap.get("s3").get(0).getPageStatisticsTime());
+
+ memTable.queryForSeriesRegionScan(
+ new AlignedFullPath(
+ new StringArrayDeviceID("root.test.d1"),
+ IMeasurementSchema.getMeasurementNameList(measurementSchemas),
+ measurementSchemas),
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1000000));
+ Assert.assertEquals(3, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {1000001, 1000001},
chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000002, 1000002},
chunkHandleMap.get("s2").get(0).getPageStatisticsTime());
+ Assert.assertArrayEquals(
+ new long[] {1000004, 1000004},
chunkHandleMap.get("s3").get(0).getPageStatisticsTime());
+ } finally {
+ memTable.release();
+ }
}
@Test
@@ -214,33 +293,67 @@ public class WritableMemChunkRegionScanTest {
@Test
public void testNonAlignedWritableMemChunkRegionScan() {
- MeasurementSchema measurementSchema = new MeasurementSchema("s1",
TSDataType.INT32);
- WritableMemChunk writableMemChunk = new
WritableMemChunk(measurementSchema);
- int size = 100000;
- for (int i = 0; i < size; i++) {
- writableMemChunk.writeNonAlignedPoint(i, i);
- }
- Optional<Long> timestamp = writableMemChunk.getAnySatisfiedTimestamp(null,
null);
- Assert.assertTrue(timestamp.isPresent());
- Assert.assertEquals(0, timestamp.get().longValue());
+ PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
+ try {
+ MeasurementSchema measurementSchema = new MeasurementSchema("s1",
TSDataType.INT32);
+ int size = 100000;
+ for (int i = 0; i < size; i++) {
+ memTable.write(
+ new StringArrayDeviceID("root.test.d1"),
+ Collections.singletonList(measurementSchema),
+ i,
+ new Object[] {i});
+ }
+ WritableMemChunk writableMemChunk =
+ (WritableMemChunk)
+ memTable.getWritableMemChunk(new
StringArrayDeviceID("root.test.d1"), "s1");
+ Optional<Long> timestamp =
writableMemChunk.getAnySatisfiedTimestamp(null, null);
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(0, timestamp.get().longValue());
- timestamp =
- writableMemChunk.getAnySatisfiedTimestamp(
- null, new TimeFilterOperators.TimeBetweenAnd(1000L, 2000L));
- Assert.assertTrue(timestamp.isPresent());
- Assert.assertEquals(1000, timestamp.get().longValue());
+ timestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ null, new TimeFilterOperators.TimeBetweenAnd(1000L, 2000L));
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(1000, timestamp.get().longValue());
- timestamp =
- writableMemChunk.getAnySatisfiedTimestamp(
- Collections.singletonList(new TimeRange(1, 1500)),
- new TimeFilterOperators.TimeBetweenAnd(1000L, 2000L));
- Assert.assertTrue(timestamp.isPresent());
- Assert.assertEquals(1501, timestamp.get().longValue());
+ timestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Collections.singletonList(new TimeRange(1, 1500)),
+ new TimeFilterOperators.TimeBetweenAnd(1000L, 2000L));
+ Assert.assertTrue(timestamp.isPresent());
+ Assert.assertEquals(1501, timestamp.get().longValue());
- timestamp =
- writableMemChunk.getAnySatisfiedTimestamp(
- Collections.singletonList(new TimeRange(1, 1500)),
- new TimeFilterOperators.TimeBetweenAnd(100000L, 200000L));
- Assert.assertFalse(timestamp.isPresent());
+ timestamp =
+ writableMemChunk.getAnySatisfiedTimestamp(
+ Collections.singletonList(new TimeRange(1, 1500)),
+ new TimeFilterOperators.TimeBetweenAnd(100000L, 200000L));
+ Assert.assertFalse(timestamp.isPresent());
+
+ Map<String, List<IChunkHandle>> chunkHandleMap = new HashMap<>();
+ memTable.queryForDeviceRegionScan(
+ new StringArrayDeviceID("root.test.d1"),
+ false,
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1));
+ Assert.assertEquals(1, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {2, 2},
chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ memTable.queryForSeriesRegionScan(
+ new NonAlignedFullPath(new StringArrayDeviceID("root.test.d1"),
measurementSchema),
+ Long.MIN_VALUE,
+ new HashMap<>(),
+ chunkHandleMap,
+ Collections.emptyList(),
+ new TimeFilterOperators.TimeGt(1));
+ Assert.assertEquals(1, chunkHandleMap.size());
+ Assert.assertArrayEquals(
+ new long[] {2, 2},
chunkHandleMap.get("s1").get(0).getPageStatisticsTime());
+ } finally {
+ memTable.release();
+ }
}
}
