jt2594838 commented on code in PR #18653:
URL: https://github.com/apache/iotdb/pull/18653#discussion_r4025173168


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java:
##########
@@ -99,39 +106,55 @@ public long getCurrentWALOriginalFileSize() {
    * @throws IOException If failing to close or open the log writer
    */
   protected File rollLogWriter(long searchIndex, WALFileStatus fileStatus) 
throws IOException {
-    // close file
-    currentWALFileWriter.close();
-    addDiskUsage(currentWALFileWriter.size());
-    addFileNum(1);
-    File lastFile = currentWALFileWriter.getLogFile();
+    if (!hasPendingRoll()) {
+      // Record the boundary only after sealing and forcing the old WAL have 
both succeeded.

Review Comment:
   Preserves the sealed WAL and current rotation stage across retries, so a 
later sync task can continue after disk space is restored without closing or 
counting the old file twice.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java:
##########
@@ -585,6 +591,45 @@ public SyncBufferTask(
     public void run() {
       final long startTime = System.nanoTime();
 
+      if (syncFailure != null) {
+        failListeners(syncFailure);

Review Comment:
   Propagates a failed sync batch to all listeners and keeps the buffer 
recoverable. SET SYSTEM TO RUNNING alone cannot repair an incomplete WAL 
record, so retry state is cleared only after the pending roll completes.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java:
##########
@@ -124,12 +132,12 @@ public double write(ByteBuffer buffer, boolean 
allowCompress) throws IOException
       
WritingMetrics.getInstance().recordCompressWALBufferCost(System.nanoTime() - 
startTime);
     }
     startTime = System.nanoTime();
-    try {
-      headerBuffer.flip();
+    headerBuffer.flip();
+    while (headerBuffer.hasRemaining()) {
       logChannel.write(headerBuffer);
+    }
+    while (buffer.hasRemaining()) {
       logChannel.write(buffer);

Review Comment:
   Consumes the full buffer and lets ClosedChannelException reach the caller. A 
partial or failed write must trigger WAL recovery instead of being acknowledged 
as a successful flush.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java:
##########
@@ -90,7 +90,10 @@ private synchronized void endFile() throws IOException {
 
   private void writeMetadata(ByteBuffer buffer) throws IOException {
     buffer.flip();
-    logChannel.write(buffer);
+    // A successful seal is the recovery boundary for switching to the next 
WAL file.
+    while (buffer.hasRemaining()) {
+      logChannel.write(buffer);

Review Comment:
   Persists the complete metadata trailer before treating the file as sealed; 
this trailer is the recovery boundary for switching to the next WAL file.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferRollRecoveryTest.java:
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.storageengine.dataregion.wal.buffer;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
+import 
org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager;
+import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALMetaData;
+import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALReader;
+import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter;
+import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus;
+import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
+import 
org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.AbstractResultListener.Status;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class WALBufferRollRecoveryTest {
+  @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  private final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
+  private final List<File> sealedFiles = new CopyOnWriteArrayList<>();
+  private NodeStatus previousStatus;
+  private String previousReason;
+  private File directory;
+  private WALBuffer buffer;
+  private WALWriter firstWriter;
+
+  @Before
+  public void setUp() throws Exception {
+    previousStatus = commonConfig.getNodeStatus();
+    previousReason = commonConfig.getStatusReason();
+    commonConfig.setNodeStatus(NodeStatus.Running);
+    directory = temporaryFolder.newFolder("wal");
+    buffer =
+        new WALBuffer(
+            "roll-recovery",
+            directory.getPath(),
+            new CheckpointManager("roll-recovery", directory.getPath()),
+            0,
+            0,
+            (sealedFile, currentFile) -> sealedFiles.add(sealedFile));
+    firstWriter = spy(buffer.currentWALFileWriter);
+    buffer.currentWALFileWriter = firstWriter;
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    try {
+      // Tests may inject a failure before close(); restore cleanup without 
reopening any file.
+      doCallRealMethod().when(firstWriter).close();
+      buffer.close();
+    } finally {
+      commonConfig.setNodeStatus(previousStatus);
+      commonConfig.setStatusReason(previousReason);
+    }
+  }
+
+  /**
+   * Failed successor creation must not reseal the old WAL, inflate counters, 
or lose later writes.
+   */
+  @Test
+  public void testResumeBeforeWritingAfterRepeatedOpenFailures() throws 
Exception {
+    writeAndAwait(entry(1, 1, "before"), Status.SUCCESS);

Review Comment:
   Covers repeated successor creation failures and verifies recovery resumes at 
the saved roll stage without duplicate sealing, counter inflation, or data loss.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java:
##########
@@ -86,6 +88,19 @@ public void tearDown() throws Exception {
     }
   }
 
+  /** Unexpected channel closure must propagate to the buffer instead of 
acknowledging a write. */
+  @Test
+  public void testClosedChannelWriteAndForceFail() throws IOException {
+    WALWriter writer = new WALWriter(walFile);

Review Comment:
   Confirms closed-channel write and force operations fail visibly and leave 
the WAL unchanged.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to