This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fixMemTableConcurrentQueryWriteBug-1.3 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit eb26915a1517543830a8dada93b82b3c4e1ce74d Author: shuwenwei <[email protected]> AuthorDate: Wed Jan 28 17:38:53 2026 +0800 Fix allValueColDeletedMap index out of bound caused by stale rowCount after TVList sort (#17098) --- .../memtable/AlignedReadOnlyMemChunk.java | 12 ++++- .../dataregion/memtable/ReadOnlyMemChunk.java | 4 +- .../db/utils/datastructure/AlignedTVList.java | 6 ++- .../db/utils/datastructure/BackAlignedTVList.java | 3 +- .../db/utils/datastructure/BackBinaryTVList.java | 3 +- .../db/utils/datastructure/BackBooleanTVList.java | 3 +- .../db/utils/datastructure/BackDoubleTVList.java | 3 +- .../db/utils/datastructure/BackFloatTVList.java | 3 +- .../db/utils/datastructure/BackIntTVList.java | 3 +- .../db/utils/datastructure/BackLongTVList.java | 3 +- .../db/utils/datastructure/QuickAlignedTVList.java | 3 +- .../db/utils/datastructure/QuickBinaryTVList.java | 3 +- .../db/utils/datastructure/QuickBooleanTVList.java | 3 +- .../db/utils/datastructure/QuickDoubleTVList.java | 3 +- .../db/utils/datastructure/QuickFloatTVList.java | 3 +- .../db/utils/datastructure/QuickIntTVList.java | 3 +- .../db/utils/datastructure/QuickLongTVList.java | 3 +- .../iotdb/db/utils/datastructure/TVList.java | 2 +- .../db/utils/datastructure/TimAlignedTVList.java | 3 +- .../db/utils/datastructure/TimBinaryTVList.java | 3 +- .../db/utils/datastructure/TimBooleanTVList.java | 3 +- .../db/utils/datastructure/TimDoubleTVList.java | 3 +- .../db/utils/datastructure/TimFloatTVList.java | 3 +- .../iotdb/db/utils/datastructure/TimIntTVList.java | 3 +- .../db/utils/datastructure/TimLongTVList.java | 3 +- .../dataregion/memtable/PrimitiveMemTableTest.java | 54 ++++++++++++++++++++++ 26 files changed, 114 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index d00424856ba..bbab08d4442 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -113,7 +113,15 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryRowCount = entry.getValue(); if (!alignedTvList.isSorted() && queryRowCount > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + // sort() returns the current row count + // TVList may grow between prepareTvListMapForQuery and actual query execution(now). + // The queryRowCount recorded here is only a snapshot taken during prepareTvListMapForQuery + // phase. + // Additional written rows that are not covered by the original queryRowCount can be + // involved in current sort operation. + // We must update queryRowCount here, otherwise, it may be used later to build + // BitMaps, causing bitmap array size mismatch and possible out of bound. + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { @@ -358,7 +366,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryLength = entry.getValue(); if (!alignedTvList.isSorted() && queryLength > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 3438759cfa9..38c8e9e2d13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -135,7 +135,7 @@ public class ReadOnlyMemChunk { TVList tvList = entry.getKey(); int queryRowCount = entry.getValue(); if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { @@ -287,7 +287,7 @@ public class ReadOnlyMemChunk { TVList tvList = entry.getKey(); int queryLength = entry.getValue(); if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index e4230787a76..090b0574987 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -1259,6 +1259,10 @@ public abstract class AlignedTVList extends TVList { } public BitMap getAllValueColDeletedMap() { + return getAllValueColDeletedMap(rowCount); + } + + public BitMap getAllValueColDeletedMap(int rowCount) { // row exists when any column value exists if (bitMaps == null) { return null; @@ -1415,7 +1419,7 @@ public abstract class AlignedTVList extends TVList { (columnIndexList == null) ? IntStream.range(0, dataTypes.size()).boxed().collect(Collectors.toList()) : columnIndexList; - this.allValueColDeletedMap = getAllValueColDeletedMap(); + this.allValueColDeletedMap = getAllValueColDeletedMap(this.rows); this.valueColumnsDeletionList = valueColumnsDeletionList; this.floatPrecision = floatPrecision != null ? floatPrecision : 0; this.encodingList = encodingList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java index 2ad728a1a26..2603790bd66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java @@ -32,12 +32,13 @@ public class BackAlignedTVList extends QuickAlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java index cd85976a58e..9faedb162a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java @@ -26,12 +26,13 @@ public class BackBinaryTVList extends QuickBinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java index 969d3981285..2186c1fa66e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java @@ -27,12 +27,13 @@ public class BackBooleanTVList extends QuickBooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java index 6d79b9d0ab2..0cb7c438db4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java @@ -27,12 +27,13 @@ public class BackDoubleTVList extends QuickDoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java index 75b09507cc4..a3208b96f6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java @@ -27,12 +27,13 @@ public class BackFloatTVList extends QuickFloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java index 3ccc5cbc946..3a2af2841c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java @@ -27,12 +27,13 @@ public class BackIntTVList extends QuickIntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java index 58d4a4e2fc8..580af939c7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java @@ -26,12 +26,13 @@ public class BackLongTVList extends QuickLongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java index 69f24527d67..c5bd5550056 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java @@ -31,11 +31,12 @@ public class QuickAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java index 645d0dbce5a..b8abddaee5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java @@ -26,11 +26,12 @@ public class QuickBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java index 779159ca719..50df94c772a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java @@ -26,11 +26,12 @@ public class QuickBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java index 58f3cc1ce2f..44120672fce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java @@ -26,11 +26,12 @@ public class QuickDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java index 2fc22ab2990..6ad9b1d7257 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java @@ -26,11 +26,12 @@ public class QuickFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java index e83646e3202..af7e35eb655 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java @@ -26,11 +26,12 @@ public class QuickIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java index 5f629720c5d..98134984c0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java @@ -26,11 +26,12 @@ public class QuickLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 8695182712f..0f4ab88441a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -176,7 +176,7 @@ public abstract class TVList implements WALEntryValue { return reservedMemoryBytes; } - public abstract void sort(); + public abstract int sort(); public void increaseReferenceCount() { referenceCount.incrementAndGet(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java index 479e0a969a3..61d5c75096e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java @@ -31,7 +31,7 @@ public class TimAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -40,6 +40,7 @@ public class TimAlignedTVList extends AlignedTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java index c78d67034da..df0b1795371 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java @@ -26,7 +26,7 @@ public class TimBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimBinaryTVList extends BinaryTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java index 0e7dc48faaf..e5051ec0ede 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java @@ -26,7 +26,7 @@ public class TimBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimBooleanTVList extends BooleanTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java index 29fc62d91c6..8b4b40c2da5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java @@ -26,7 +26,7 @@ public class TimDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimDoubleTVList extends DoubleTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java index 83ad6aca1f6..522b8bf7d61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java @@ -26,7 +26,7 @@ public class TimFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimFloatTVList extends FloatTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java index 8216f2eb296..594980d4546 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java @@ -26,7 +26,7 @@ public class TimIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimIntTVList extends IntTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java index bcd5c3022f9..e741ca51504 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java @@ -26,7 +26,7 @@ public class TimLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimLongTVList extends LongTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index 572e6cd7ef8..f9186b774e1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -44,16 +44,20 @@ import org.apache.iotdb.db.queryengine.execution.schedule.IDriverScheduler; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.datastructure.MemPointIterator; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -123,6 +127,56 @@ public class PrimitiveMemTableTest { Assert.assertEquals(count, i); } + /** + * Regression test for concurrent writes between prepare and query execution. The test ensures + * that when new rows are written after the prepare phase but before TVList sorting, the query + * refreshes the rowCount after sort and avoids using a stale queryRowCount for bitmap + * construction, which would otherwise cause ArrayIndexOutOfBoundsException. + */ + @Test + public void testWriteDuringPrepareTVListAndActualQueryExecution() + throws QueryProcessException, IOException, IllegalPathException { + + PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0"); + List<IMeasurementSchema> measurementSchemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + for (int i = 1000; i < 2000; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + for (int i = 100; i < 200; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + MeasurementPath path = new MeasurementPath("root.test.d1.s1", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + path = new MeasurementPath("root.test.d1.s2", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + path = new MeasurementPath("root.test.d1.s3", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + ResourceByPathUtils resourcesByPathUtils = + ResourceByPathUtils.getResourceInstance( + new AlignedPath("root.test.d1", Arrays.asList("s1", "s2", "s3"), measurementSchemas)); + ReadOnlyMemChunk readOnlyMemChunk = + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(1), memTable, null, Long.MAX_VALUE, null); + + for (int i = 1; i <= 50; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + + readOnlyMemChunk.sortTvLists(); + + MemPointIterator memPointIterator = readOnlyMemChunk.createMemPointIterator(Ordering.ASC, null); + while (memPointIterator.hasNextBatch()) { + memPointIterator.nextBatch(); + } + } + @Test public void memSeriesToStringTest() throws IOException { TSDataType dataType = TSDataType.INT32;
