This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 49c46279f63 [Bug] Flush the pending aligned page before sealing a 
chunk at the chunk cap (#19313)
49c46279f63 is described below

commit 49c46279f63209d547662cbed24d1fda3b95316b
Author: Zhenyu Luo <[email protected]>
AuthorDate: Wed Nov 18 19:05:10 2026 +0800

    [Bug] Flush the pending aligned page before sealing a chunk at the chunk 
cap (#19313)
    
    When tvlist_sort_threshold > 0, an aligned memchunk is flushed through the
    single-TVLIST iterator. AlignedTVList.TVListIterator.encodeBatch stops as 
soon
    as the chunk point cap is reached, but it only writes the values of the 
current
    page into the value chunk writers. The matching time page is written by the
    caller only when the page cap is reached as well. 
AlignedWritableMemChunk.encode
    then sealed the chunk without flushing that pending page, so the time chunk
    ended up with one page less than the value chunks. The aligned reader walks 
the
    time chunk, so the rows of the extra value page were silently lost and the
    time/value chunk statistics diverged.
    
    The effective chunk cap is min(target_chunk_point_num, target_chunk_size / 
avg
    point size), so it is usually not a multiple of the page cap (1000 by 
default)
    once a wide TEXT column narrows it, e.g. 262144 / 300 = 873. The MergeSort
    multi-list iterator and the default encodeWorkingAlignedTVList path write 
each
    page before sealing, so only this path was affected.
    
    Reproduce: 2400 points, page cap 1000, chunk cap 1500, tvlist_sort_threshold
    above the row count. Before: the first encoded aligned chunk had 1000 
points in
    its time chunk but 1500 in its value chunk. After: both have 1500 and all 
2400
    points are written.
    
    Share the flush-pending-page-then-seal logic between the chunk-cap branch 
and
    the last-batch branch so the two cannot diverge again.
---
 .../memtable/AlignedWritableMemChunk.java          | 48 +++++++++------
 .../memtable/AlignedWritableMemChunkTest.java      | 49 +++++++++++++++
 .../dataregion/memtable/MemTableFlushTaskTest.java | 69 ++++++++++++++++++++++
 3 files changed, 147 insertions(+), 19 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index e79f3c304ff..106311f80de 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -1132,32 +1132,42 @@ public class AlignedWritableMemChunk extends 
AbstractWritableMemChunk {
       }
 
       if (encodeInfo.pointNumInChunk >= encodeInfo.maxNumberOfPointsInChunk) {
-        alignedChunkWriter.sealCurrentPage();
-        alignedChunkWriter.clearPageWriter();
-        try {
-          ioTaskQueue.put(alignedChunkWriter);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-        }
+        flushPendingPageAndEnqueueChunk(ioTaskQueue, alignedChunkWriter, 
encodeInfo, times);
         alignedChunkWriter = new AlignedChunkWriterImpl(activeSchemaList, 
encryptParameter);
-        encodeInfo.reset();
       }
     }
 
     // last batch of points
     if (encodeInfo.pointNumInChunk > 0) {
-      if (encodeInfo.pointNumInPage > 0) {
-        alignedChunkWriter.write(times, encodeInfo.pointNumInPage, 0);
-      }
-      alignedChunkWriter.sealCurrentPage();
-      alignedChunkWriter.clearPageWriter();
-      try {
-        ioTaskQueue.put(alignedChunkWriter);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
-      encodeInfo.reset();
+      flushPendingPageAndEnqueueChunk(ioTaskQueue, alignedChunkWriter, 
encodeInfo, times);
+    }
+  }
+
+  /**
+   * Flush the pending page, if any, and seal the chunk before enqueueing it 
for IO.
+   *
+   * <p>{@code encodeBatch} may return with a partially filled page: when the 
chunk cap is reached
+   * in the middle of a page, the values of that page have already been 
written into the value chunk
+   * writers, while the corresponding time page has not been written yet. 
Sealing the chunk without
+   * flushing the partial page first would make the time chunk contain one 
page less than the value
+   * chunks, so the rows of that page could never be read and the chunk 
statistics would diverge.
+   */
+  private void flushPendingPageAndEnqueueChunk(
+      BlockingQueue<Object> ioTaskQueue,
+      AlignedChunkWriterImpl alignedChunkWriter,
+      BatchEncodeInfo encodeInfo,
+      long[] times) {
+    if (encodeInfo.pointNumInPage > 0) {
+      alignedChunkWriter.write(times, encodeInfo.pointNumInPage, 0);
+    }
+    alignedChunkWriter.sealCurrentPage();
+    alignedChunkWriter.clearPageWriter();
+    try {
+      ioTaskQueue.put(alignedChunkWriter);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
     }
+    encodeInfo.reset();
   }
 
   @Override
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
index 117a979bf4b..74fe091b32e 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.storageengine.dataregion.memtable;
 
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import 
org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest;
 import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
 import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
@@ -27,6 +28,7 @@ import org.apache.tsfile.common.conf.TSFileConfig;
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.utils.Binary;
 import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
 import org.apache.tsfile.write.schema.IMeasurementSchema;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 import org.junit.Assert;
@@ -40,6 +42,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
 public class AlignedWritableMemChunkTest {
@@ -239,4 +242,50 @@ public class AlignedWritableMemChunkTest {
     Assert.assertTrue(ioTaskQueue.size() > 1);
     chunk.release();
   }
+
+  @Test
+  public void testEncodeFlushesPendingPageBeforeSealingAtChunkCap() {
+    int originalThreshold = 
IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold();
+    // A positive threshold selects the single-TVLIST iterator path. Keep it 
above the row count so
+    // the points stay in the working TVList instead of being handed over to 
sortedList, which
+    // would switch to the multi-list iterator that flushes pages on its own.
+    IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(100_000);
+    AlignedWritableMemChunk chunk = null;
+    try {
+      IMeasurementSchema schema = new MeasurementSchema("s0", 
TSDataType.INT64);
+      List<IMeasurementSchema> schemas = Collections.singletonList(schema);
+      chunk = new AlignedWritableMemChunk(new ArrayList<>(schemas));
+      int rowCount = 2400;
+      for (int i = 0; i < rowCount; i++) {
+        chunk.writeAlignedPoints(i, new Object[] {(long) i}, schemas);
+      }
+      chunk.sortTvListForFlush();
+
+      BlockingQueue<Object> ioTaskQueue = new LinkedBlockingQueue<>();
+      // The 1500-point chunk cap is not a multiple of the 1000-point page 
cap, so the second chunk
+      // is sealed in the middle of a page. This is what target_chunk_size 
narrowing produces for
+      // wide TEXT columns: min(target_chunk_point_num, target_chunk_size / 
avg point size).
+      BatchEncodeInfo encodeInfo = new BatchEncodeInfo(0, 0, 0, 1000, 1500, 
10_000_000L);
+      chunk.encode(ioTaskQueue, encodeInfo, new long[1000], false);
+
+      long totalTimePoints = 0;
+      for (Object encodedChunk : ioTaskQueue) {
+        AlignedChunkWriterImpl chunkWriter = (AlignedChunkWriterImpl) 
encodedChunk;
+        long timePoints = 
chunkWriter.getTimeChunkWriter().getStatistics().getCount();
+        totalTimePoints += timePoints;
+        for (int column = 0; column < 
chunkWriter.getValueChunkWriterList().size(); column++) {
+          Assert.assertEquals(
+              "the time and value chunks of one aligned chunk must contain the 
same points",
+              timePoints,
+              
chunkWriter.getValueChunkWriterByIndex(column).getStatistics().getCount());
+        }
+      }
+      Assert.assertEquals(rowCount, totalTimePoints);
+    } finally {
+      
IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(originalThreshold);
+      if (chunk != null) {
+        chunk.release();
+      }
+    }
+  }
 }
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
index 7ca3554afce..fd77297b86e 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
@@ -22,13 +22,16 @@ package 
org.apache.iotdb.db.storageengine.dataregion.memtable;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.exception.WriteProcessException;
 import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
 import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask;
 import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
 import org.apache.iotdb.db.utils.EnvironmentUtils;
 import org.apache.iotdb.db.utils.constant.TestConstant;
+import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
 
+import org.apache.tsfile.common.conf.TSFileConfig;
 import org.apache.tsfile.common.conf.TSFileDescriptor;
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.file.metadata.ChunkMetadata;
@@ -466,6 +469,72 @@ public class MemTableFlushTaskTest {
     }
   }
 
+  @Test
+  public void testAlignedFlushKeepsAllRowsWhenChunkCapIsNotAPageMultiple() 
throws Exception {
+    // With tvlist_sort_threshold > 0 the aligned memchunk is encoded through 
the single-TVLIST
+    // iterator. A wide TEXT column narrows the chunk cap to target_chunk_size 
/ avg point size,
+    // which is not a multiple of the page cap, so a chunk is sealed in the 
middle of a page.
+    int originalThreshold = 
IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold();
+    IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(100_000);
+    int pageCap = 
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+    List<IMeasurementSchema> schemas =
+        Arrays.asList(
+            new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN),
+            new MeasurementSchema("s1", TSDataType.TEXT, TSEncoding.PLAIN));
+    AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas);
+    String alignedFilePath =
+        
TestConstant.OUTPUT_DATA_DIR.concat("testAlignedWideTextChunkCap.tsfile");
+    try {
+      int rowCount = 3000;
+      char[] chars = new char[300];
+      Arrays.fill(chars, 'x');
+      Binary binary = new Binary(new String(chars), 
TSFileConfig.STRING_CHARSET);
+      for (int index = 0; index < rowCount; index++) {
+        memChunk.putAlignedRow(index, new Object[] {(long) index, binary});
+      }
+      memChunk.sortTvListForFlush();
+
+      BlockingQueue<Object> ioTaskQueue = new LinkedBlockingQueue<>();
+      // target_chunk_size / avg point size is below the page cap, so a chunk 
is sealed with a
+      // pending partial page whose values are already in the value chunk 
writers.
+      BatchEncodeInfo encodeInfo = new BatchEncodeInfo(0, 0, 0, pageCap, 
10_000, 262_144);
+      memChunk.encode(ioTaskQueue, encodeInfo, new long[pageCap], false);
+      try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new 
File(alignedFilePath))) {
+        
alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d"));
+        Object task;
+        while ((task = ioTaskQueue.poll()) != null) {
+          if (task instanceof IChunkWriter chunkWriter) {
+            chunkWriter.writeToFileWriter(alignedWriter);
+          }
+        }
+        alignedWriter.endChunkGroup();
+        alignedWriter.endFile();
+      }
+
+      try (TsFileSequenceReader sequenceReader = new 
TsFileSequenceReader(alignedFilePath);
+          TsFileReader fileReader = new TsFileReader(sequenceReader)) {
+        QueryDataSet dataSet =
+            fileReader.query(
+                QueryExpression.create(
+                    Arrays.asList(new Path("root.d", "s0", false), new 
Path("root.d", "s1", false)),
+                    null));
+        int index = 0;
+        while (dataSet.hasNext()) {
+          RowRecord row = dataSet.next();
+          assertEquals(index, row.getTimestamp());
+          assertEquals((long) index, row.getFields().get(0).getLongV());
+          assertEquals(binary, row.getFields().get(1).getBinaryV());
+          index++;
+        }
+        // The rows of the pending pages must be readable, not silently 
dropped.
+        assertEquals(rowCount, index);
+      }
+    } finally {
+      
IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(originalThreshold);
+      memChunk.release();
+    }
+  }
+
   @Test
   public void testAlignedFlushKeepsValuesAlignedAfterTimeDeletion() throws 
IOException {
     // Deleted rows must be omitted from both time and value pages, including 
after sorting.

Reply via email to