This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 49c46279f63 [Bug] Flush the pending aligned page before sealing a
chunk at the chunk cap (#19313)
49c46279f63 is described below
commit 49c46279f63209d547662cbed24d1fda3b95316b
Author: Zhenyu Luo <[email protected]>
AuthorDate: Wed Nov 18 19:05:10 2026 +0800
[Bug] Flush the pending aligned page before sealing a chunk at the chunk
cap (#19313)
When tvlist_sort_threshold > 0, an aligned memchunk is flushed through the
single-TVLIST iterator. AlignedTVList.TVListIterator.encodeBatch stops as
soon
as the chunk point cap is reached, but it only writes the values of the
current
page into the value chunk writers. The matching time page is written by the
caller only when the page cap is reached as well.
AlignedWritableMemChunk.encode
then sealed the chunk without flushing that pending page, so the time chunk
ended up with one page less than the value chunks. The aligned reader walks
the
time chunk, so the rows of the extra value page were silently lost and the
time/value chunk statistics diverged.
The effective chunk cap is min(target_chunk_point_num, target_chunk_size /
avg
point size), so it is usually not a multiple of the page cap (1000 by
default)
once a wide TEXT column narrows it, e.g. 262144 / 300 = 873. The MergeSort
multi-list iterator and the default encodeWorkingAlignedTVList path write
each
page before sealing, so only this path was affected.
Reproduce: 2400 points, page cap 1000, chunk cap 1500, tvlist_sort_threshold
above the row count. Before: the first encoded aligned chunk had 1000
points in
its time chunk but 1500 in its value chunk. After: both have 1500 and all
2400
points are written.
Share the flush-pending-page-then-seal logic between the chunk-cap branch
and
the last-batch branch so the two cannot diverge again.
---
.../memtable/AlignedWritableMemChunk.java | 48 +++++++++------
.../memtable/AlignedWritableMemChunkTest.java | 49 +++++++++++++++
.../dataregion/memtable/MemTableFlushTaskTest.java | 69 ++++++++++++++++++++++
3 files changed, 147 insertions(+), 19 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index e79f3c304ff..106311f80de 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -1132,32 +1132,42 @@ public class AlignedWritableMemChunk extends
AbstractWritableMemChunk {
}
if (encodeInfo.pointNumInChunk >= encodeInfo.maxNumberOfPointsInChunk) {
- alignedChunkWriter.sealCurrentPage();
- alignedChunkWriter.clearPageWriter();
- try {
- ioTaskQueue.put(alignedChunkWriter);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
+ flushPendingPageAndEnqueueChunk(ioTaskQueue, alignedChunkWriter,
encodeInfo, times);
alignedChunkWriter = new AlignedChunkWriterImpl(activeSchemaList,
encryptParameter);
- encodeInfo.reset();
}
}
// last batch of points
if (encodeInfo.pointNumInChunk > 0) {
- if (encodeInfo.pointNumInPage > 0) {
- alignedChunkWriter.write(times, encodeInfo.pointNumInPage, 0);
- }
- alignedChunkWriter.sealCurrentPage();
- alignedChunkWriter.clearPageWriter();
- try {
- ioTaskQueue.put(alignedChunkWriter);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- encodeInfo.reset();
+ flushPendingPageAndEnqueueChunk(ioTaskQueue, alignedChunkWriter,
encodeInfo, times);
+ }
+ }
+
+ /**
+ * Flush the pending page, if any, and seal the chunk before enqueueing it
for IO.
+ *
+ * <p>{@code encodeBatch} may return with a partially filled page: when the
chunk cap is reached
+ * in the middle of a page, the values of that page have already been
written into the value chunk
+ * writers, while the corresponding time page has not been written yet.
Sealing the chunk without
+ * flushing the partial page first would make the time chunk contain one
page less than the value
+ * chunks, so the rows of that page could never be read and the chunk
statistics would diverge.
+ */
+ private void flushPendingPageAndEnqueueChunk(
+ BlockingQueue<Object> ioTaskQueue,
+ AlignedChunkWriterImpl alignedChunkWriter,
+ BatchEncodeInfo encodeInfo,
+ long[] times) {
+ if (encodeInfo.pointNumInPage > 0) {
+ alignedChunkWriter.write(times, encodeInfo.pointNumInPage, 0);
+ }
+ alignedChunkWriter.sealCurrentPage();
+ alignedChunkWriter.clearPageWriter();
+ try {
+ ioTaskQueue.put(alignedChunkWriter);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
}
+ encodeInfo.reset();
}
@Override
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
index 117a979bf4b..74fe091b32e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.dataregion.memtable;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
@@ -27,6 +28,7 @@ import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
@@ -40,6 +42,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
public class AlignedWritableMemChunkTest {
@@ -239,4 +242,50 @@ public class AlignedWritableMemChunkTest {
Assert.assertTrue(ioTaskQueue.size() > 1);
chunk.release();
}
+
+ @Test
+ public void testEncodeFlushesPendingPageBeforeSealingAtChunkCap() {
+ int originalThreshold =
IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold();
+ // A positive threshold selects the single-TVLIST iterator path. Keep it
above the row count so
+ // the points stay in the working TVList instead of being handed over to
sortedList, which
+ // would switch to the multi-list iterator that flushes pages on its own.
+ IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(100_000);
+ AlignedWritableMemChunk chunk = null;
+ try {
+ IMeasurementSchema schema = new MeasurementSchema("s0",
TSDataType.INT64);
+ List<IMeasurementSchema> schemas = Collections.singletonList(schema);
+ chunk = new AlignedWritableMemChunk(new ArrayList<>(schemas));
+ int rowCount = 2400;
+ for (int i = 0; i < rowCount; i++) {
+ chunk.writeAlignedPoints(i, new Object[] {(long) i}, schemas);
+ }
+ chunk.sortTvListForFlush();
+
+ BlockingQueue<Object> ioTaskQueue = new LinkedBlockingQueue<>();
+ // The 1500-point chunk cap is not a multiple of the 1000-point page
cap, so the second chunk
+ // is sealed in the middle of a page. This is what target_chunk_size
narrowing produces for
+ // wide TEXT columns: min(target_chunk_point_num, target_chunk_size /
avg point size).
+ BatchEncodeInfo encodeInfo = new BatchEncodeInfo(0, 0, 0, 1000, 1500,
10_000_000L);
+ chunk.encode(ioTaskQueue, encodeInfo, new long[1000], false);
+
+ long totalTimePoints = 0;
+ for (Object encodedChunk : ioTaskQueue) {
+ AlignedChunkWriterImpl chunkWriter = (AlignedChunkWriterImpl)
encodedChunk;
+ long timePoints =
chunkWriter.getTimeChunkWriter().getStatistics().getCount();
+ totalTimePoints += timePoints;
+ for (int column = 0; column <
chunkWriter.getValueChunkWriterList().size(); column++) {
+ Assert.assertEquals(
+ "the time and value chunks of one aligned chunk must contain the
same points",
+ timePoints,
+
chunkWriter.getValueChunkWriterByIndex(column).getStatistics().getCount());
+ }
+ }
+ Assert.assertEquals(rowCount, totalTimePoints);
+ } finally {
+
IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(originalThreshold);
+ if (chunk != null) {
+ chunk.release();
+ }
+ }
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
index 7ca3554afce..fd77297b86e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java
@@ -22,13 +22,16 @@ package
org.apache.iotdb.db.storageengine.dataregion.memtable;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.WriteProcessException;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.constant.TestConstant;
+import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
+import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.ChunkMetadata;
@@ -466,6 +469,72 @@ public class MemTableFlushTaskTest {
}
}
+ @Test
+ public void testAlignedFlushKeepsAllRowsWhenChunkCapIsNotAPageMultiple()
throws Exception {
+ // With tvlist_sort_threshold > 0 the aligned memchunk is encoded through
the single-TVLIST
+ // iterator. A wide TEXT column narrows the chunk cap to target_chunk_size
/ avg point size,
+ // which is not a multiple of the page cap, so a chunk is sealed in the
middle of a page.
+ int originalThreshold =
IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold();
+ IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(100_000);
+ int pageCap =
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ List<IMeasurementSchema> schemas =
+ Arrays.asList(
+ new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN),
+ new MeasurementSchema("s1", TSDataType.TEXT, TSEncoding.PLAIN));
+ AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas);
+ String alignedFilePath =
+
TestConstant.OUTPUT_DATA_DIR.concat("testAlignedWideTextChunkCap.tsfile");
+ try {
+ int rowCount = 3000;
+ char[] chars = new char[300];
+ Arrays.fill(chars, 'x');
+ Binary binary = new Binary(new String(chars),
TSFileConfig.STRING_CHARSET);
+ for (int index = 0; index < rowCount; index++) {
+ memChunk.putAlignedRow(index, new Object[] {(long) index, binary});
+ }
+ memChunk.sortTvListForFlush();
+
+ BlockingQueue<Object> ioTaskQueue = new LinkedBlockingQueue<>();
+ // target_chunk_size / avg point size is below the page cap, so a chunk
is sealed with a
+ // pending partial page whose values are already in the value chunk
writers.
+ BatchEncodeInfo encodeInfo = new BatchEncodeInfo(0, 0, 0, pageCap,
10_000, 262_144);
+ memChunk.encode(ioTaskQueue, encodeInfo, new long[pageCap], false);
+ try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new
File(alignedFilePath))) {
+
alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d"));
+ Object task;
+ while ((task = ioTaskQueue.poll()) != null) {
+ if (task instanceof IChunkWriter chunkWriter) {
+ chunkWriter.writeToFileWriter(alignedWriter);
+ }
+ }
+ alignedWriter.endChunkGroup();
+ alignedWriter.endFile();
+ }
+
+ try (TsFileSequenceReader sequenceReader = new
TsFileSequenceReader(alignedFilePath);
+ TsFileReader fileReader = new TsFileReader(sequenceReader)) {
+ QueryDataSet dataSet =
+ fileReader.query(
+ QueryExpression.create(
+ Arrays.asList(new Path("root.d", "s0", false), new
Path("root.d", "s1", false)),
+ null));
+ int index = 0;
+ while (dataSet.hasNext()) {
+ RowRecord row = dataSet.next();
+ assertEquals(index, row.getTimestamp());
+ assertEquals((long) index, row.getFields().get(0).getLongV());
+ assertEquals(binary, row.getFields().get(1).getBinaryV());
+ index++;
+ }
+ // The rows of the pending pages must be readable, not silently
dropped.
+ assertEquals(rowCount, index);
+ }
+ } finally {
+
IoTDBDescriptor.getInstance().getConfig().setTVListSortThreshold(originalThreshold);
+ memChunk.release();
+ }
+ }
+
@Test
public void testAlignedFlushKeepsValuesAlignedAfterTimeDeletion() throws
IOException {
// Deleted rows must be omitted from both time and value pages, including
after sorting.