This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 01d88b7d120 Fix some bugs in AlignedTVListIterator
01d88b7d120 is described below
commit 01d88b7d120ff12bb043f3a33162990ff527560b
Author: shuwenwei <[email protected]>
AuthorDate: Tue Sep 2 09:32:00 2025 +0800
Fix some bugs in AlignedTVListIterator
---
.../db/it/aligned/IoTDBAlignedMemQueryIT.java | 97 ++++++++++++++++++++++
.../db/utils/datastructure/AlignedTVList.java | 58 +++++++------
.../MergeSortMultiAlignedTVListIterator.java | 23 +++--
.../memtable/AlignedTVListIteratorTest.java | 21 +++--
4 files changed, 153 insertions(+), 46 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedMemQueryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedMemQueryIT.java
new file mode 100644
index 00000000000..036698c28c0
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedMemQueryIT.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.aligned;
+
+import org.apache.iotdb.it.env.EnvFactory;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.assertEquals;
+
+public class IoTDBAlignedMemQueryIT {
+
+ @Before
+ public void setUp() throws Exception {
+ Locale.setDefault(Locale.ENGLISH);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setPartitionInterval(1000)
+ .setMemtableSizeThreshold(10000);
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void test() throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("insert into root.vehicle.d0(time,s0) aligned values
(10,310)");
+ statement.execute("insert into root.vehicle.d0(time,s3) aligned values
(10,'text')");
+ statement.execute("insert into root.vehicle.d0(time,s4) aligned values
(10,true)");
+
+ try (ResultSet set = statement.executeQuery("SELECT s0 FROM
root.vehicle.d0")) {
+ int cnt = 0;
+ while (set.next()) {
+ cnt++;
+ }
+ assertEquals(1, cnt);
+ }
+
+ statement.execute("insert into root.vehicle.d1(time,s0,s1) aligned
values (1,1,1)");
+ statement.execute("insert into root.vehicle.d1(time,s0) aligned values
(2,2)");
+ statement.execute("insert into root.vehicle.d1(time,s1) aligned values
(3,3)");
+
+ try (ResultSet set = statement.executeQuery("SELECT s0 FROM
root.vehicle.d1")) {
+ int cnt = 0;
+ while (set.next()) {
+ cnt++;
+ }
+ assertEquals(2, cnt);
+ }
+ statement.execute("flush");
+
+ statement.execute("insert into root.vehicle.d1(time,s0,s1) aligned
values (1,1,1)");
+ statement.execute("insert into root.vehicle.d1(time,s0) aligned values
(2,2)");
+ statement.execute("insert into root.vehicle.d1(time,s1) aligned values
(3,3)");
+
+ try (ResultSet set = statement.executeQuery("SELECT s0 FROM
root.vehicle.d1")) {
+ int cnt = 0;
+ while (set.next()) {
+ cnt++;
+ }
+ assertEquals(2, cnt);
+ }
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 5f45569d4b6..b7474d1583b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -1705,17 +1705,14 @@ public abstract class AlignedTVList extends TVList {
probeNext = true;
return;
}
- if (scanOrder.isAscending()) {
- // When traversing in ASC order, we only need to overwrite the
previous non-null value
- // with the non-null value encountered later
- Arrays.fill(selectedIndices, index);
- } else {
- // When traversing in DESC order, we need to keep the non-null value
encountered first,
- // and only overwrite it if the previous value is null and current
value is non-null. In
- // order to identify the previous null, we use index -1 here to
represent
- for (int i = 0; i < selectedIndices.length; i++) {
- selectedIndices[i] = isNullValue(index, i) ? -1 : index;
- }
+ // When traversing in ASC order, we only need to overwrite the
previous non-null value
+ // with the non-null value encountered later.
+ // When traversing in DESC order, we need to keep the non-null value
encountered first,
+ // and only overwrite it if the previous value is null and current
value is non-null.
+ for (int i = 0; i < selectedIndices.length; i++) {
+ // In order to identify the previous null, we use index -1 here to
represent.
+ // The -1 here is also used for checking all null rows
+ selectedIndices[i] = isNullValue(index, i) ? -1 : index;
}
findValidRow = true;
@@ -1740,35 +1737,36 @@ public abstract class AlignedTVList extends TVList {
}
}
}
- // For DESC traversal, we previously set some -1. If these values are
still -1 in the end,
- // it means that each index is invalid. At this time, we can use any
one at random.
- if (!scanOrder.isAscending()) {
- for (int i = 0; i < selectedIndices.length; i++) {
- if (selectedIndices[i] == -1) {
- selectedIndices[i] = index;
- }
- }
- }
// valueColumnsDeletionList is set when AlignedTVList iterator is
created by
- // MemPointIterator.single method. Otherwise, all-null rows is checked
by
+ // MemPointIterator.single method. Otherwise, it is checked by
// MergeSortMultiAlignedTVListIterator or
OrderedMultiAlignedTVListIterator.
- if (valueColumnsDeletionList != null) {
- BitMap bitMap = new BitMap(dataTypeList.size());
+ if (ignoreAllNullRows) {
+ BitMap bitMap = null;
time = getTime(getScanOrderIndex(index));
for (int columnIndex = 0; columnIndex < dataTypeList.size();
columnIndex++) {
- if (isNullValue(index, columnIndex)
- || isPointDeleted(
- time,
- valueColumnsDeletionList.get(columnIndex),
- valueColumnDeleteCursor.get(columnIndex),
- scanOrder)) {
+ if (selectedIndices[columnIndex] == -1
+ || (valueColumnsDeletionList != null
+ && isPointDeleted(
+ time,
+ valueColumnsDeletionList.get(columnIndex),
+ valueColumnDeleteCursor.get(columnIndex),
+ scanOrder))) {
+ bitMap = bitMap == null ? new BitMap(dataTypeList.size()) :
bitMap;
bitMap.mark(columnIndex);
}
}
- if (ignoreAllNullRows && bitMap.isAllMarked()) {
+ if (bitMap != null && bitMap.isAllMarked()) {
findValidRow = false;
index++;
+ continue;
+ }
+ }
+ // We previously set some -1. If these values are still -1 in the end,
+ // it means that each index is invalid. At this time, we can use any
one at random.
+ for (int i = 0; i < selectedIndices.length; i++) {
+ if (selectedIndices[i] == -1) {
+ selectedIndices[i] = index;
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
index b4b12d94386..153cb6856c1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
@@ -115,11 +115,20 @@ public class MergeSortMultiAlignedTVListIterator extends
MultiAlignedTVListItera
.isNullValue(rowIndices[columnIndex], columnIndex)) {
bitMap.mark(columnIndex);
}
+ // check valueColumnsDeletionList
+ if (valueColumnsDeletionList != null
+ && isPointDeleted(
+ currentTime,
+ valueColumnsDeletionList.get(columnIndex),
+ valueColumnDeleteCursor.get(columnIndex),
+ scanOrder)) {
+ iteratorIndices[columnIndex] = -1;
+ bitMap.mark(columnIndex);
+ }
}
hasNext = true;
// duplicated timestamps
- boolean[] valueDeleted = new boolean[tsDataTypeList.size()];
while (!heap.isEmpty() && heap.peek().left == currentTime) {
Pair<Long, Integer> element = heap.poll();
probeIterators.add(element.right);
@@ -127,10 +136,14 @@ public class MergeSortMultiAlignedTVListIterator extends
MultiAlignedTVListItera
for (int columnIndex = 0; columnIndex < tsDataTypeList.size();
columnIndex++) {
// if current column null, it needs update
int iteratorIndex = currentIteratorIndex(columnIndex);
- if (iteratorIndex == -1
- || alignedTvListIterators
- .get(iteratorIndex)
- .isNullValue(rowIndices[columnIndex], columnIndex)) {
+ if (iteratorIndex == -1) {
+ // -1 means all point of this timestamp was deleted by Deletion
and no further
+ // processing is required.
+ continue;
+ }
+ if (alignedTvListIterators
+ .get(iteratorIndex)
+ .isNullValue(rowIndices[columnIndex], columnIndex)) {
iteratorIndices[columnIndex] = element.right;
rowIndices[columnIndex] =
alignedTvListIterators.get(element.right).getSelectedIndex(columnIndex);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
index 047e7abe088..f0ae227c05f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedTVListIteratorTest.java
@@ -109,17 +109,16 @@ public class AlignedTVListIteratorTest {
public void otherTest() throws IOException {
Map<TVList, Integer> tvListMap =
buildAlignedSingleTvListMap(Collections.singletonList(new TimeRange(1,
1)));
- // testAligned(
- // tvListMap,
- // Ordering.ASC,
- // null,
- // null,
- // PaginationController.UNLIMITED_PAGINATION_CONTROLLER,
- // Collections.emptyList(),
- // Arrays.asList(Collections.emptyList(), Collections.emptyList(),
- // Collections.emptyList()),
- // false,
- // 1);
+ testAligned(
+ tvListMap,
+ Ordering.ASC,
+ null,
+ null,
+ PaginationController.UNLIMITED_PAGINATION_CONTROLLER,
+ Collections.emptyList(),
+ Arrays.asList(Collections.emptyList(), Collections.emptyList(),
Collections.emptyList()),
+ false,
+ 1);
testAligned(
tvListMap,
Ordering.DESC,
