This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b5de54a0ddd [Bug] Keep the aligned per-device chunk cap local to
encode (#19311)
b5de54a0ddd is described below
commit b5de54a0ddd4098a76036b4c9100a3dae2ab000f
Author: Zhenyu Luo <[email protected]>
AuthorDate: Wed Nov 18 14:22:35 2026 +0800
[Bug] Keep the aligned per-device chunk cap local to encode (#19311)
AlignedWritableMemChunk.encode narrowed the shared
BatchEncodeInfo.maxNumberOfPointsInChunk based on the current device's largest
column and never restored it. A wide TEXT column in the first aligned device
therefore shrank every following device's chunk size in the same flush task.
Compute and use the narrowed cap only for this encode call, and restore the
shared value in a finally block.
---
.../memtable/AlignedWritableMemChunk.java | 24 ++++++++++++++++----
.../memtable/AlignedWritableMemChunkTest.java | 26 ++++++++++++++++++++++
2 files changed, 46 insertions(+), 4 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 173eb4a6d76..e79f3c304ff 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -1073,11 +1073,27 @@ public class AlignedWritableMemChunk extends
AbstractWritableMemChunk {
BatchEncodeInfo encodeInfo,
long[] times,
boolean ignoreAllNullRows) {
- encodeInfo.maxNumberOfPointsInChunk =
- Math.min(
- encodeInfo.maxNumberOfPointsInChunk,
- (encodeInfo.targetChunkSize / getAvgPointSizeOfLargestColumn()));
+ // The cap is derived from this memchunk's largest column. BatchEncodeInfo
is shared by all
+ // devices of a flush task, so keep the narrowed cap local to this encode
call and restore the
+ // original value afterwards; otherwise one wide TEXT column would shrink
the chunk size of
+ // every following device in the same flush.
+ long originalMaxNumberOfPointsInChunk =
encodeInfo.maxNumberOfPointsInChunk;
+ try {
+ encodeInfo.maxNumberOfPointsInChunk =
+ Math.min(
+ originalMaxNumberOfPointsInChunk,
+ (encodeInfo.targetChunkSize / getAvgPointSizeOfLargestColumn()));
+ encodeWithComputedChunkCap(ioTaskQueue, encodeInfo, times,
ignoreAllNullRows);
+ } finally {
+ encodeInfo.maxNumberOfPointsInChunk = originalMaxNumberOfPointsInChunk;
+ }
+ }
+ private void encodeWithComputedChunkCap(
+ BlockingQueue<Object> ioTaskQueue,
+ BatchEncodeInfo encodeInfo,
+ long[] times,
+ boolean ignoreAllNullRows) {
if (TVLIST_SORT_THRESHOLD == 0) {
encodeWorkingAlignedTVList(
ioTaskQueue,
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
index 7e7e66dae7c..117a979bf4b 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkTest.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable;
import
org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
+import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
@@ -39,6 +40,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
public class AlignedWritableMemChunkTest {
@@ -213,4 +215,28 @@ public class AlignedWritableMemChunkTest {
Assert.assertEquals(4L, list.getLongByValueIndex(2, columnCount));
chunk.release();
}
+
+ @Test
+ public void testEncodeRestoresSharedMaxNumberOfPointsInChunk() {
+ IMeasurementSchema text = new MeasurementSchema("text", TSDataType.TEXT);
+ AlignedWritableMemChunk chunk =
+ new AlignedWritableMemChunk(new
ArrayList<>(Collections.singletonList(text)));
+ Binary value = new Binary(new byte[1024]);
+ for (int i = 0; i < 10; i++) {
+ chunk.writeAlignedPoints(i, new Object[] {value},
Collections.singletonList(text));
+ }
+ chunk.initWorkingListForFlushIfNecessary(chunk.getWorkingTVList(), false);
+
+ long originalMaxNumberOfPointsInChunk = 100_000L;
+ BatchEncodeInfo encodeInfo =
+ new BatchEncodeInfo(0, 0, 0, 2048, originalMaxNumberOfPointsInChunk,
4096);
+ LinkedBlockingQueue<Object> ioTaskQueue = new LinkedBlockingQueue<>();
+ chunk.encode(ioTaskQueue, encodeInfo, new long[2048], false);
+
+ // The per-chunk point cap computed from the largest column must not leak
into the shared
+ // BatchEncodeInfo used by the following devices of the same flush task.
+ Assert.assertEquals(originalMaxNumberOfPointsInChunk,
encodeInfo.maxNumberOfPointsInChunk);
+ Assert.assertTrue(ioTaskQueue.size() > 1);
+ chunk.release();
+ }
}