This is an automated email from the ASF dual-hosted git repository. chrisdutz pushed a commit to branch fix/wal-flush-oversized-memtable in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ff7b801ba1d493f975cce8ef4dfecfc416797210 Author: Christofer Dutz <[email protected]> AuthorDate: Sat Nov 7 15:26:54 2026 +0100 [WAL] Flush a memtable instead of snapshotting it when the snapshot exceeds the WAL entry limit With a small WAL buffer, as in the edge profile (wal_buffer_size_in_byte = 1 MB, which is also the default WAL entry size limit), a memtable below the 8 MB snapshot threshold can still be larger than one WAL entry. Every snapshot attempt then rolled the WAL file under the region's write lock, was refused by WALBuffer.write and logged an ERROR with a stack trace; only the next cleanup cycle flushed the memtable. The cleanup task now flushes at once when the memtable is larger than the entry limit. --- .../apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 10526aaec51..9eb97a62e89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -560,7 +560,11 @@ public class WALNode implements IWALNode { if (TsFileUtils.getTimePartition(new File(oldestMemTableInfo.getTsFilePath())) < dataRegion.getLatestTimePartition() || snapshotCount >= config.getMaxWalMemTableSnapshotNum() - || oldestMemTableTVListsRamCost > config.getWalMemTableSnapshotThreshold()) { + || oldestMemTableTVListsRamCost > config.getWalMemTableSnapshotThreshold() + // a snapshot larger than one WAL entry is refused by WALBuffer.write. With a small + // wal_buffer_size_in_byte (the entry limit's default), the limit is below the snapshot + // threshold, so flush right away instead of failing a snapshot first. + || oldestMemTableTVListsRamCost > config.getWalEntrySizeLimitInByte()) { flushMemTable(dataRegion, oldestTsFile, oldestMemTable); WRITING_METRICS.recordWalFlushMemTableCount(1); WRITING_METRICS.recordMemTableRamWhenCauseFlush(identifier, oldestMemTableTVListsRamCost);
