This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 66209334eb7 [WAL] Flush a memtable instead of snapshotting it when the 
snapshot exceeds the WAL entry limit (#19174)
66209334eb7 is described below

commit 66209334eb7627eaa2505728ec346cfa1d40ea6b
Author: Christofer Dutz <[email protected]>
AuthorDate: Wed Nov 11 02:38:50 2026 +0100

    [WAL] Flush a memtable instead of snapshotting it when the snapshot exceeds 
the WAL entry limit (#19174)
    
    * [WAL] Flush a memtable instead of snapshotting it when the snapshot 
exceeds the WAL entry limit
    
    With a small WAL buffer, as in the edge profile (wal_buffer_size_in_byte = 
1 MB, which is also
    the default WAL entry size limit), a memtable below the 8 MB snapshot 
threshold can still be
    larger than one WAL entry. Every snapshot attempt then rolled the WAL file 
under the region's
    write lock, was refused by WALBuffer.write and logged an ERROR with a stack 
trace; only the next
    cleanup cycle flushed the memtable. The cleanup task now flushes at once 
when the memtable is
    larger than the entry limit.
    
    * [WAL] Test that a memtable larger than one WAL entry is flushed instead 
of snapshotted
    
    With a WAL entry limit of 1 MB, the cleanup task flushes a 2 MB memtable in 
the latest time
    partition (below the 8 MB snapshot threshold) without taking the region's 
write lock for a
    snapshot, and still snapshots a 512 KB memtable. A second, flushed memtable 
fills the WAL so
    that the effective information ratio is low and the cleanup has to act on 
the oldest one.
    
    Co-Authored-By: Claude Opus 5.5 <[email protected]>
    
    ---------
    
    Co-authored-by: Claude Opus 5.5 <[email protected]>
---
 .../storageengine/dataregion/wal/node/WALNode.java |   6 +-
 .../dataregion/wal/node/WALNodeTest.java           | 107 +++++++++++++++++++++
 2 files changed, 112 insertions(+), 1 deletion(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
index 10526aaec51..9eb97a62e89 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
@@ -560,7 +560,11 @@ public class WALNode implements IWALNode {
       if (TsFileUtils.getTimePartition(new 
File(oldestMemTableInfo.getTsFilePath()))
               < dataRegion.getLatestTimePartition()
           || snapshotCount >= config.getMaxWalMemTableSnapshotNum()
-          || oldestMemTableTVListsRamCost > 
config.getWalMemTableSnapshotThreshold()) {
+          || oldestMemTableTVListsRamCost > 
config.getWalMemTableSnapshotThreshold()
+          // a snapshot larger than one WAL entry is refused by 
WALBuffer.write. With a small
+          // wal_buffer_size_in_byte (the entry limit's default), the limit is 
below the snapshot
+          // threshold, so flush right away instead of failing a snapshot 
first.
+          || oldestMemTableTVListsRamCost > 
config.getWalEntrySizeLimitInByte()) {
         flushMemTable(dataRegion, oldestTsFile, oldestMemTable);
         WRITING_METRICS.recordWalFlushMemTableCount(1);
         WRITING_METRICS.recordMemTableRamWhenCauseFlush(identifier, 
oldestMemTableTVListsRamCost);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java
index ee2e0253186..1aedbbe742b 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java
@@ -25,6 +25,7 @@ import 
org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.consensus.ConsensusFactory;
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.DataRegionException;
 import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
 import org.apache.iotdb.db.storageengine.StorageEngine;
 import org.apache.iotdb.db.storageengine.dataregion.DataRegionTest;
@@ -392,4 +393,110 @@ public class WALNodeTest {
                     + WALFileUtils.getLogFileName(1, 0, 
WALFileStatus.CONTAINS_SEARCH_INDEX))
             .exists());
   }
+
+  @Test
+  public void testMemTableLargerThanAWalEntryIsFlushedInsteadOfSnapshotted() 
throws Exception {
+    // below the snapshot threshold, but larger than one WAL entry
+    RecordingDataRegion dataRegion = cleanUpOldestMemTableOfRamCost(2 * 1024 * 
1024);
+    assertTrue(dataRegion.flushSubmitted);
+    assertFalse(dataRegion.writeLocked);
+  }
+
+  @Test
+  public void testMemTableWithinAWalEntryIsSnapshotted() throws Exception {
+    RecordingDataRegion dataRegion = cleanUpOldestMemTableOfRamCost(512 * 
1024);
+    assertFalse(dataRegion.flushSubmitted);
+    assertTrue(dataRegion.writeLocked);
+  }
+
+  /**
+   * Lets the WAL cleanup pick an active memTable with the given TVList RAM 
cost in the latest time
+   * partition, with a WAL entry limit of 1 MB (as with the edge profile's 
wal_buffer_size_in_byte).
+   * The WAL is filled by a second memTable, which is flushed, so the 
effective information ratio is
+   * low and the cleanup has to snapshot or flush the oldest one.
+   */
+  private RecordingDataRegion cleanUpOldestMemTableOfRamCost(long 
tvListsRamCost) throws Exception {
+    int prevEntrySizeLimit = config.getWalEntrySizeLimitInByte();
+    config.setWalEntrySizeLimitInByte(1024 * 1024);
+    try {
+      assertTrue(tvListsRamCost < config.getWalMemTableSnapshotThreshold());
+      RecordingDataRegion dataRegion = new RecordingDataRegion(logDirectory, 
databasePath);
+      StorageEngine.getInstance().setDataRegion(new DataRegionId(1), 
dataRegion);
+
+      IMemTable oldest =
+          new PrimitiveMemTable(databasePath, dataRegionId) {
+            @Override
+            public long getTVListsRamCost() {
+              return tvListsRamCost;
+            }
+          };
+      walNode.onMemTableCreated(oldest, tsFilePathInLatestPartition(oldest));
+      List<WALFlushListener> listeners = new ArrayList<>();
+      for (long time = 1; time <= 10; time++) {
+        InsertTabletNode node = getInsertTabletNode(devicePath, new long[] 
{time});
+        listeners.add(
+            walNode.log(
+                oldest.getMemTableId(),
+                node,
+                Collections.singletonList(new int[] {0, node.getRowCount()})));
+      }
+
+      IMemTable filler = new PrimitiveMemTable(databasePath, dataRegionId);
+      walNode.onMemTableCreated(filler, tsFilePathInLatestPartition(filler));
+      for (long time = 1; time <= 20000; time++) {
+        InsertTabletNode node = getInsertTabletNode(devicePath, new long[] 
{time});
+        listeners.add(
+            walNode.log(
+                filler.getMemTableId(),
+                node,
+                Collections.singletonList(new int[] {0, node.getRowCount()})));
+      }
+      walNode.onMemTableFlushed(filler);
+      Awaitility.await().until(() -> walNode.isAllWALEntriesConsumed());
+      for (WALFlushListener listener : listeners) {
+        assertEquals(WALFlushListener.Status.SUCCESS, 
listener.waitForResult());
+      }
+
+      walNode.deleteOutdatedFiles();
+      return dataRegion;
+    } finally {
+      config.setWalEntrySizeLimitInByte(prevEntrySizeLimit);
+    }
+  }
+
+  private static String tsFilePathInLatestPartition(IMemTable memTable) {
+    return logDirectory
+        + File.separator
+        + databasePath
+        + File.separator
+        + dataRegionId
+        + File.separator
+        + "0"
+        + File.separator
+        + memTable.getMemTableId()
+        + ".tsfile";
+  }
+
+  /** Records whether the WAL cleanup flushed a memTable or took the write 
lock to snapshot it. */
+  private static class RecordingDataRegion extends 
DataRegionTest.DummyDataRegion {
+    private volatile boolean flushSubmitted;
+    private volatile boolean writeLocked;
+
+    private RecordingDataRegion(String systemInfoDir, String databaseName)
+        throws DataRegionException {
+      super(systemInfoDir, databaseName);
+    }
+
+    @Override
+    public boolean submitAFlushTask(long timeRangeId, boolean sequence, 
IMemTable memTable) {
+      flushSubmitted = true;
+      return false;
+    }
+
+    @Override
+    public void writeLock(String holder) {
+      writeLocked = true;
+      super.writeLock(holder);
+    }
+  }
 }

Reply via email to