This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fixMemTableConcurrentQueryWriteBug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 478e9ec83878c3c7f5ad6fe787eed0b0a93785d8 Author: shuwenwei <[email protected]> AuthorDate: Wed Jan 28 14:30:51 2026 +0800 Fix allValueColDeletedMap index out of bound caused by stale rowCount after TVList sort --- .../memtable/AlignedReadOnlyMemChunk.java | 12 ++++- .../dataregion/memtable/ReadOnlyMemChunk.java | 4 +- .../db/utils/datastructure/AlignedTVList.java | 6 ++- .../db/utils/datastructure/BackAlignedTVList.java | 3 +- .../db/utils/datastructure/BackBinaryTVList.java | 3 +- .../db/utils/datastructure/BackBooleanTVList.java | 3 +- .../db/utils/datastructure/BackDoubleTVList.java | 3 +- .../db/utils/datastructure/BackFloatTVList.java | 3 +- .../db/utils/datastructure/BackIntTVList.java | 3 +- .../db/utils/datastructure/BackLongTVList.java | 3 +- .../db/utils/datastructure/QuickAlignedTVList.java | 3 +- .../db/utils/datastructure/QuickBinaryTVList.java | 3 +- .../db/utils/datastructure/QuickBooleanTVList.java | 3 +- .../db/utils/datastructure/QuickDoubleTVList.java | 3 +- .../db/utils/datastructure/QuickFloatTVList.java | 3 +- .../db/utils/datastructure/QuickIntTVList.java | 3 +- .../db/utils/datastructure/QuickLongTVList.java | 3 +- .../iotdb/db/utils/datastructure/TVList.java | 2 +- .../db/utils/datastructure/TimAlignedTVList.java | 3 +- .../db/utils/datastructure/TimBinaryTVList.java | 3 +- .../db/utils/datastructure/TimBooleanTVList.java | 3 +- .../db/utils/datastructure/TimDoubleTVList.java | 3 +- .../db/utils/datastructure/TimFloatTVList.java | 3 +- .../iotdb/db/utils/datastructure/TimIntTVList.java | 3 +- .../db/utils/datastructure/TimLongTVList.java | 3 +- .../dataregion/memtable/PrimitiveMemTableTest.java | 57 ++++++++++++++++++++++ 26 files changed, 117 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 2157623e1c6..89fe9facc71 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -118,7 +118,15 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryRowCount = entry.getValue(); if (!alignedTvList.isSorted() && queryRowCount > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + // sort() returns the current row count + // TVList may grow between prepareTvListMapForQuery and actual query execution(now). + // The queryRowCount recorded here is only a snapshot taken during prepareTvListMapForQuery + // phase. + // Additional written rows that are not covered by the original queryRowCount can be + // involved in current sort operation. + // We must update queryRowCount here, otherwise, it may be used later to build + // BitMaps, causing bitmap array size mismatch and possible out of bound. + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { @@ -375,7 +383,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { AlignedTVList alignedTvList = (AlignedTVList) entry.getKey(); int queryLength = entry.getValue(); if (!alignedTvList.isSorted() && queryLength > alignedTvList.seqRowCount()) { - alignedTvList.sort(); + entry.setValue(alignedTvList.sort()); long alignedTvListRamSize = alignedTvList.calculateRamSize(); alignedTvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 7304eab6fb8..ec8a7448063 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -136,7 +136,7 @@ public class ReadOnlyMemChunk { TVList tvList = entry.getKey(); int queryRowCount = entry.getValue(); if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { @@ -289,7 +289,7 @@ public class ReadOnlyMemChunk { TVList tvList = entry.getKey(); int queryLength = entry.getValue(); if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) { - tvList.sort(); + entry.setValue(tvList.sort()); long tvListRamSize = tvList.calculateRamSize(); tvList.lockQueryList(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 237d6d28a0f..8c732d4f08e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -1599,6 +1599,10 @@ public abstract class AlignedTVList extends TVList { } public BitMap getAllValueColDeletedMap() { + return getAllValueColDeletedMap(rowCount); + } + + public BitMap getAllValueColDeletedMap(int rowCount) { // row exists when any column value exists if (bitMaps == null) { return null; @@ -1768,7 +1772,7 @@ public abstract class AlignedTVList extends TVList { (columnIndexList == null) ? IntStream.range(0, dataTypes.size()).boxed().collect(Collectors.toList()) : columnIndexList; - this.allValueColDeletedMap = ignoreAllNullRows ? getAllValueColDeletedMap() : null; + this.allValueColDeletedMap = ignoreAllNullRows ? getAllValueColDeletedMap(this.rows) : null; this.floatPrecision = floatPrecision != null ? floatPrecision : 0; this.encodingList = encodingList; this.timeColumnDeletion = timeColumnDeletion; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java index 2ad728a1a26..2603790bd66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackAlignedTVList.java @@ -32,12 +32,13 @@ public class BackAlignedTVList extends QuickAlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java index cd85976a58e..9faedb162a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBinaryTVList.java @@ -26,12 +26,13 @@ public class BackBinaryTVList extends QuickBinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java index 969d3981285..2186c1fa66e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackBooleanTVList.java @@ -27,12 +27,13 @@ public class BackBooleanTVList extends QuickBooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java index 6d79b9d0ab2..0cb7c438db4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackDoubleTVList.java @@ -27,12 +27,13 @@ public class BackDoubleTVList extends QuickDoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java index 75b09507cc4..a3208b96f6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackFloatTVList.java @@ -27,12 +27,13 @@ public class BackFloatTVList extends QuickFloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java index 403314f5817..6994b8e76eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackIntTVList.java @@ -34,12 +34,13 @@ public class BackIntTVList extends QuickIntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java index 58d4a4e2fc8..580af939c7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BackLongTVList.java @@ -26,12 +26,13 @@ public class BackLongTVList extends QuickLongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.backwardSort(timestamps, rowCount); policy.clearTmp(); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java index 69f24527d67..c5bd5550056 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java @@ -31,11 +31,12 @@ public class QuickAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java index 645d0dbce5a..b8abddaee5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBinaryTVList.java @@ -26,11 +26,12 @@ public class QuickBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java index 779159ca719..50df94c772a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickBooleanTVList.java @@ -26,11 +26,12 @@ public class QuickBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java index 58f3cc1ce2f..44120672fce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickDoubleTVList.java @@ -26,11 +26,12 @@ public class QuickDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java index 2fc22ab2990..6ad9b1d7257 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickFloatTVList.java @@ -26,11 +26,12 @@ public class QuickFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java index 035dbc4f393..984b322872a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickIntTVList.java @@ -33,11 +33,12 @@ public class QuickIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java index 5f629720c5d..98134984c0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickLongTVList.java @@ -26,11 +26,12 @@ public class QuickLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { if (!sorted) { policy.qsort(0, rowCount - 1); } sorted = true; seqRowCount = rowCount; + return rowCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index cfeeac11968..d61611c4cc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -185,7 +185,7 @@ public abstract class TVList implements WALEntryValue { return reservedMemoryBytes; } - public abstract void sort(); + public abstract int sort(); public void increaseReferenceCount() { referenceCount.incrementAndGet(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java index 479e0a969a3..61d5c75096e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimAlignedTVList.java @@ -31,7 +31,7 @@ public class TimAlignedTVList extends AlignedTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -40,6 +40,7 @@ public class TimAlignedTVList extends AlignedTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java index c78d67034da..df0b1795371 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBinaryTVList.java @@ -26,7 +26,7 @@ public class TimBinaryTVList extends BinaryTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimBinaryTVList extends BinaryTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java index 0e7dc48faaf..e5051ec0ede 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimBooleanTVList.java @@ -26,7 +26,7 @@ public class TimBooleanTVList extends BooleanTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimBooleanTVList extends BooleanTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java index 29fc62d91c6..8b4b40c2da5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimDoubleTVList.java @@ -26,7 +26,7 @@ public class TimDoubleTVList extends DoubleTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimDoubleTVList extends DoubleTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java index 83ad6aca1f6..522b8bf7d61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimFloatTVList.java @@ -26,7 +26,7 @@ public class TimFloatTVList extends FloatTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimFloatTVList extends FloatTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java index 33ccb1b0e5d..e1165faa12f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimIntTVList.java @@ -33,7 +33,7 @@ public class TimIntTVList extends IntTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -42,6 +42,7 @@ public class TimIntTVList extends IntTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java index bcd5c3022f9..e741ca51504 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TimLongTVList.java @@ -26,7 +26,7 @@ public class TimLongTVList extends LongTVList { } @Override - public synchronized void sort() { + public synchronized int sort() { policy.checkSortedTimestampsAndIndices(); if (!sorted) { policy.sort(0, rowCount); @@ -35,6 +35,7 @@ public class TimLongTVList extends LongTVList { policy.clearSortedTime(); sorted = true; seqRowCount = rowCount; + return rowCount; } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index d64e7c7e5cd..b16e20d4f85 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -45,17 +45,21 @@ import org.apache.iotdb.db.queryengine.execution.schedule.IDriverScheduler; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.apache.iotdb.db.utils.MathUtils; +import org.apache.iotdb.db.utils.datastructure.MemPointIterator; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -148,6 +152,59 @@ public class PrimitiveMemTableTest { Assert.assertEquals(count, i); } + /** + * Regression test for concurrent writes between prepare and query execution. The test ensures + * that when new rows are written after the prepare phase but before TVList sorting, the query + * refreshes the rowCount after sort and avoids using a stale queryRowCount for bitmap + * construction, which would otherwise cause ArrayIndexOutOfBoundsException. + */ + @Test + public void testWriteDuringPrepareTVListAndActualQueryExecution() + throws QueryProcessException, IOException, IllegalPathException { + + PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0"); + List<IMeasurementSchema> measurementSchemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + for (int i = 1000; i < 2000; i++) { + memTable.writeAlignedRow( + new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + for (int i = 100; i < 200; i++) { + memTable.writeAlignedRow( + new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + memTable.delete( + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160)); + memTable.delete( + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160)); + memTable.delete( + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160)); + ResourceByPathUtils resourcesByPathUtils = + ResourceByPathUtils.getResourceInstance( + new AlignedFullPath( + new StringArrayDeviceID("root.test.d1"), + Arrays.asList("s1", "s2", "s3"), + measurementSchemas)); + ReadOnlyMemChunk readOnlyMemChunk = + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(1), memTable, null, Long.MAX_VALUE, null); + + for (int i = 1; i <= 50; i++) { + memTable.writeAlignedRow( + new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + + readOnlyMemChunk.sortTvLists(); + + MemPointIterator memPointIterator = readOnlyMemChunk.createMemPointIterator(Ordering.ASC, null); + while (memPointIterator.hasNextBatch()) { + memPointIterator.nextBatch(); + } + } + @Test public void memSeriesToStringTest() throws IOException { TSDataType dataType = TSDataType.INT32;
