This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.3.1 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit af1b661f18d7af1306b6856ca9c66e6421979bec Author: Haonan <[email protected]> AuthorDate: Wed Mar 6 21:38:02 2024 +0800 Fix fail to read memTable ids from the wal file (#12132) --- .../org/apache/iotdb/db/service/IoTDBShutdownHook.java | 2 +- .../apache/iotdb/db/storageengine/StorageEngine.java | 2 +- .../db/storageengine/dataregion/wal/WALManager.java | 17 ++++++++++++++++- .../storageengine/dataregion/wal/buffer/WALBuffer.java | 8 ++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java index bdea1bab52a..214142a9e13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java @@ -78,7 +78,7 @@ public class IoTDBShutdownHook extends Thread { .equals(ConsensusFactory.RATIS_CONSENSUS)) { StorageEngine.getInstance().syncCloseAllProcessor(); } - WALManager.getInstance().deleteOutdatedFilesInWALNodes(); + WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes(); // We did this work because the RatisConsensus recovery mechanism is different from other // consensus algorithms, which will replace the underlying storage engine based on its diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 025b891fc5a..fce7f95b6c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -631,7 +631,7 @@ public class StorageEngine implements IService { public void operateFlush(TFlushReq req) { if (req.storageGroups == null) { StorageEngine.getInstance().syncCloseAllProcessor(); - WALManager.getInstance().deleteOutdatedFilesInWALNodes(); + WALManager.getInstance().syncDeleteOutdatedFilesInWALNodes(); } else { for (String storageGroup : req.storageGroups) { if (req.isSeq == null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java index a36e16037ea..2268b745121 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.wal.allocation.ElasticStrategy; import org.apache.iotdb.db.storageengine.dataregion.wal.allocation.FirstCreateStrategy; @@ -44,7 +45,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -165,7 +168,7 @@ public class WALManager implements IService { } } - public void deleteOutdatedFilesInWALNodes() { + protected void deleteOutdatedFilesInWALNodes() { if (config.getWalMode() == WALMode.DISABLE) { return; } @@ -262,6 +265,18 @@ public class WALManager implements IService { walDeleteThread, this::deleteOutdatedFiles, initDelayMs, periodMs, TimeUnit.MILLISECONDS); } + public void syncDeleteOutdatedFilesInWALNodes() { + Future<?> future = walDeleteThread.submit(this::deleteOutdatedFilesInWALNodes); + try { + future.get(); + } catch (ExecutionException e) { + throw new StorageEngineFailureException("Failed to delete outdated wal file", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new StorageEngineFailureException("Failed to delete outdated wal file", e); + } + } + @TestOnly public void clear() { totalDiskUsage.set(0); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index d6deb613e6f..8e6f18a713a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -723,8 +723,12 @@ public class WALBuffer extends AbstractWALBuffer { file, FileChannel.open(file.toPath(), StandardOpenOption.READ)) .getMemTablesId(); } catch (IOException e) { - logger.error("Fail to read memTable ids from the wal file {}.", id); - return new HashSet<>(); + logger.warn( + "Fail to read memTable ids from the wal file {} of wal node {}.", + id, + identifier, + e); + return Collections.emptySet(); } }); }
