This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new d20ef864a80 [Pipe] Guard DAL lifecycle and drain producers before 
region cleanup (#19321)
d20ef864a80 is described below

commit d20ef864a809e870f5213bf323876e47921035e9
Author: Peng Junzhi <[email protected]>
AuthorDate: Fri Nov 20 11:06:27 2026 +0800

    [Pipe] Guard DAL lifecycle and drain producers before region cleanup 
(#19321)
    
    * [Pipe] Avoid creating DAL managers during region cleanup and suppress 
expected close warnings
    
    * [Pipe] Serialize DAL lifecycle and drain region producers before cleanup
---
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |   4 +
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |   4 +
 .../pipe/consensus/deletion/DeletionResource.java  |   5 +-
 .../deletion/DeletionResourceManager.java          | 236 ++++++++++----
 .../deletion/persist/PageCacheDeletionBuffer.java  |  54 +++-
 ...istoricalDataRegionTsFileAndDeletionSource.java |   6 +-
 .../realtime/assigner/PipeDataRegionAssigner.java  |   2 +-
 .../listener/PipeInsertionDataNodeListener.java    |  22 ++
 .../iotdb/db/storageengine/StorageEngine.java      |  10 +-
 .../db/storageengine/dataregion/DataRegion.java    |   7 +-
 .../DeletionResourceManagerCleanupTest.java        | 338 +++++++++++++++++++++
 .../persist/PageCacheDeletionBufferTest.java       | 187 ++++++++++++
 .../PipeInsertionDataNodeListenerTest.java         |  31 ++
 13 files changed, 824 insertions(+), 82 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 7a6b4ac32e1..3153e962e11 100644
--- 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -3184,4 +3184,8 @@ public final class DataNodePipeMessages {
       
EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68
 =
           "Failed to pin the TsFile %s by creating a hardlink, and the TsFile 
can not be copied"
               + " because it is not closed yet";
+  public static final String 
EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622
 =
+      "Deletion resource manager for DataRegion %s is closed or deleted.";
+  public static final String 
LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D
 =
+      "Deletion persist-{}: timed out waiting for {} deletions to flush before 
shutdown.";
 }
diff --git 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index b4872119644..550323119fd 100644
--- 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2958,4 +2958,8 @@ public final class DataNodePipeMessages {
   public static final String
       
EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68
 =
           "通过创建硬链接固化 TsFile %s 失败,且该 TsFile 尚未关闭,无法改为拷贝";
+  public static final String 
EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622
 =
+      "DataRegion %s 的删除资源管理器已关闭或已删除。";
+  public static final String 
LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D
 =
+      "Deletion persist-{}:关闭前等待 {} 条删除记录持久化超时。";
 }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
index 051f224890c..18010f067cb 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
@@ -70,7 +70,10 @@ public class DeletionResource implements PersistentResource {
             
DataRegionConsensusImpl.getInstance().getReplicationNum(consensusGroupId) - 1);
   }
 
-  public synchronized void decreaseReference() {
+  // The atomic counter selects one release callback. Do not hold this 
resource's monitor while
+  // the callback acquires the region lifecycle lock: close waits for 
synchronized persist
+  // callbacks.
+  public void decreaseReference() {
     if (pipeTaskReferenceCount.decrementAndGet() == 0) {
       removeSelf();
     }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
index 98f447a4b12..597177b39da 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
@@ -34,6 +34,7 @@ import 
org.apache.iotdb.db.pipe.consensus.deletion.recover.DeletionReader;
 import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Striped;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,6 +45,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
@@ -62,6 +64,11 @@ public class DeletionResourceManager implements 
AutoCloseable {
       String.format(
           "^_(?<%s>\\d+)-(?<%s>\\d+)\\%s$",
           REBOOT_TIME, MEM_TABLE_FLUSH_ORDER, DELETION_FILE_SUFFIX);
+  // Stable, bounded locks are shared by creation, ordinary close and region 
deletion. Never remove
+  // a lock while a waiter may still hold its reference.
+  private static final Striped<Lock> REGION_LOCKS = Striped.lock(1024);
+  private static final Set<Integer> DELETED_REGIONS = 
ConcurrentHashMap.newKeySet();
+  private boolean closed;
   private final int dataRegionId;
   private final DeletionBuffer deletionBuffer;
   private final File storageDir;
@@ -132,25 +139,50 @@ public class DeletionResourceManager implements 
AutoCloseable {
 
   @Override
   public void close() {
-    LOGGER.info(DataNodePipeMessages.CLOSING_DELETION_RESOURCE_MANAGER_FOR, 
dataRegionId);
-    this.deleteNode2ResourcesMap.clear();
-    this.deletionBuffer.close();
-    if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP != 
null) {
-      
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.remove(dataRegionId,
 this);
+    final Lock lock = REGION_LOCKS.get(dataRegionId);
+    lock.lock();
+    try {
+      if (closed) {
+        return;
+      }
+      closed = true;
+      LOGGER.info(DataNodePipeMessages.CLOSING_DELETION_RESOURCE_MANAGER_FOR, 
dataRegionId);
+      this.deletionBuffer.close();
+      this.deleteNode2ResourcesMap.clear();
+      if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP != 
null) {
+        
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.remove(dataRegionId,
 this);
+      }
+      LOGGER.info(
+          
DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY, 
dataRegionId);
+    } finally {
+      lock.unlock();
     }
-    LOGGER.info(
-        
DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY, 
dataRegionId);
   }
 
   public DeletionResource registerDeletionResource(AbstractDeleteDataNode 
deleteDataNode) {
-    DeletionResource deletionResource =
-        deleteNode2ResourcesMap.computeIfAbsent(
-            deleteDataNode,
-            key ->
-                new DeletionResource(deleteDataNode, 
this::removeDeletionResource, dataRegionId));
-    // register a persist task for current deletionResource
-    deletionBuffer.registerDeletionResource(deletionResource);
-    return deletionResource;
+    final Lock lock = REGION_LOCKS.get(dataRegionId);
+    lock.lock();
+    try {
+      final DeletionResource deletionResource =
+          new DeletionResource(deleteDataNode, this::removeDeletionResource, 
dataRegionId);
+      // A producer may have obtained this manager before the region deletion 
barrier.
+      // Complete its result instead of leaving it waiting on an already 
closed buffer.
+      if (closed || DELETED_REGIONS.contains(dataRegionId)) {
+        deletionResource.onPersistFailed(
+            new IOException(
+                String.format(
+                    DataNodePipeMessages
+                        
.EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622,
+                    dataRegionId)));
+        return deletionResource;
+      }
+      final DeletionResource resource =
+          deleteNode2ResourcesMap.computeIfAbsent(deleteDataNode, key -> 
deletionResource);
+      deletionBuffer.registerDeletionResource(resource);
+      return resource;
+    } finally {
+      lock.unlock();
+    }
   }
 
   public DeletionResource getDeletionResource(AbstractDeleteDataNode 
deleteDataNode) {
@@ -176,7 +208,55 @@ public class DeletionResourceManager implements 
AutoCloseable {
     }
   }
 
-  public void removeDAL() {
+  /** Block new DAL creation before stopping region producers. Does not wait 
for producer tasks. */
+  public static void markRegionDeleted(int groupId) {
+    final Lock lock = REGION_LOCKS.get(groupId);
+    lock.lock();
+    try {
+      DELETED_REGIONS.add(groupId);
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  /**
+   * Only the storage lifecycle may permit DAL creation again, after 
old-region cleanup finishes.
+   */
+  public static void onRegionCreated(int groupId) {
+    final Lock lock = REGION_LOCKS.get(groupId);
+    lock.lock();
+    try {
+      DELETED_REGIONS.remove(groupId);
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  /** Close and remove DAL atomically with respect to creation, without 
constructing a manager. */
+  public static void closeAndRemoveDAL(int groupId) {
+    final Lock lock = REGION_LOCKS.get(groupId);
+    lock.lock();
+    try {
+      DELETED_REGIONS.add(groupId);
+      if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == 
null) {
+        return;
+      }
+      final DeletionResourceManager manager =
+          
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId);
+      if (manager != null) {
+        manager.close();
+      }
+      removeDAL(
+          groupId,
+          new File(
+              
IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir(),
+              Integer.toString(groupId)));
+    } finally {
+      lock.unlock();
+    }
+  }
+
+  private static void removeDAL(int dataRegionId, File storageDir) {
     if (storageDir.exists()) {
       FileUtils.deleteFileOrDirectory(storageDir);
       LOGGER.info(
@@ -195,43 +275,53 @@ public class DeletionResourceManager implements 
AutoCloseable {
    * This is a hook function, which will be automatically invoked when 
deletionResource's reference
    * count returns to 0.
    */
-  private synchronized void removeDeletionResource(DeletionResource 
deletionResource) {
-    // Clean memory
-    deleteNode2ResourcesMap.remove(deletionResource.getDeleteDataNode());
-    // Clean disk
-    ProgressIndex currentProgressIndex =
-        ReplicateProgressDataNodeManager.extractLocalSimpleProgressIndex(
-            deletionResource.getProgressIndex());
+  private void removeDeletionResource(DeletionResource deletionResource) {
+    final Lock lock = REGION_LOCKS.get(dataRegionId);
+    lock.lock();
+    try {
+      // Late reference releases from an old manager must not touch a new 
lifecycle's DAL.
+      if (closed || DELETED_REGIONS.contains(dataRegionId)) {
+        return;
+      }
+      // Clean memory
+      deleteNode2ResourcesMap.remove(deletionResource.getDeleteDataNode());
+      // Clean disk
+      ProgressIndex currentProgressIndex =
+          ReplicateProgressDataNodeManager.extractLocalSimpleProgressIndex(
+              deletionResource.getProgressIndex());
 
-    try (Stream<Path> pathStream = Files.walk(Paths.get(storageDir.getPath()), 
1)) {
-      Path[] deletionPaths =
-          pathStream
-              .filter(Files::isRegularFile)
-              .filter(path -> 
path.getFileName().toString().matches(DELETION_FILE_NAME_PATTERN))
-              .filter(
-                  path ->
-                      isFileProgressCoveredByGivenProgress(
-                          path.getFileName().toString(), currentProgressIndex))
-              .sorted(this::compareFileProgressIndex)
-              .toArray(Path[]::new);
-      // File name represents the max progressIndex in its previous file. If 
currentProgressIndex is
-      // larger than a fileName's progressIndex, it means that the file before 
this file has been
-      // fully synchronized and can be deleted.
-      // So here we cannot guarantee that the last file can be deleted, we can 
only guarantee that
-      // the first n-1 files can be deleted (if the length of deletionPaths is 
n)
-      for (int i = 0; i < deletionPaths.length - 1; i++) {
-        File fileToDelete = deletionPaths[i].toFile();
-        FileUtils.deleteFileOrDirectory(fileToDelete);
-        LOGGER.info(
-            DataNodePipeMessages.DELETIONMANAGER_DELETE_DELETION_FILE_IN_DIR,
+      try (Stream<Path> pathStream = 
Files.walk(Paths.get(storageDir.getPath()), 1)) {
+        Path[] deletionPaths =
+            pathStream
+                .filter(Files::isRegularFile)
+                .filter(path -> 
path.getFileName().toString().matches(DELETION_FILE_NAME_PATTERN))
+                .filter(
+                    path ->
+                        isFileProgressCoveredByGivenProgress(
+                            path.getFileName().toString(), 
currentProgressIndex))
+                .sorted(this::compareFileProgressIndex)
+                .toArray(Path[]::new);
+        // File names describe the previous file's max progressIndex. A larger 
currentProgressIndex
+        // means that the file before this file has been
+        // fully synchronized and can be deleted.
+        // So here we cannot guarantee that the last file can be deleted, we 
can only guarantee that
+        // the first n-1 files can be deleted (if the length of deletionPaths 
is n)
+        for (int i = 0; i < deletionPaths.length - 1; i++) {
+          File fileToDelete = deletionPaths[i].toFile();
+          FileUtils.deleteFileOrDirectory(fileToDelete);
+          LOGGER.info(
+              DataNodePipeMessages.DELETIONMANAGER_DELETE_DELETION_FILE_IN_DIR,
+              dataRegionId,
+              fileToDelete);
+        }
+      } catch (IOException e) {
+        LOGGER.warn(
+            DataNodePipeMessages.DELETIONMANAGER_FAILED_TO_DELETE_FILE_IN_DIR,
             dataRegionId,
-            fileToDelete);
+            storageDir);
       }
-    } catch (IOException e) {
-      LOGGER.warn(
-          DataNodePipeMessages.DELETIONMANAGER_FAILED_TO_DELETE_FILE_IN_DIR,
-          dataRegionId,
-          storageDir);
+    } finally {
+      lock.unlock();
     }
   }
 
@@ -291,21 +381,41 @@ public class DeletionResourceManager implements 
AutoCloseable {
     }
   }
 
+  /** Obtain an existing manager without creating DAL resources for a 
lookup-only caller. */
+  public static DeletionResourceManager getInstanceIfPresent(int groupId) {
+    final Lock lock = REGION_LOCKS.get(groupId);
+    lock.lock();
+    try {
+      return DELETED_REGIONS.contains(groupId)
+              || 
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null
+          ? null
+          : 
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId);
+    } finally {
+      lock.unlock();
+    }
+  }
+
   public static DeletionResourceManager getInstance(int groupId) {
-    // If consensusImpl is not IoTConsensusV2.
-    if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == 
null) {
-      return null;
+    final Lock lock = REGION_LOCKS.get(groupId);
+    lock.lock();
+    try {
+      if (DELETED_REGIONS.contains(groupId)
+          || DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP 
== null) {
+        return null;
+      }
+      return 
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent(
+          groupId,
+          key -> {
+            try {
+              return new DeletionResourceManager(groupId);
+            } catch (IOException e) {
+              
LOGGER.error(DataNodePipeMessages.FAILED_TO_INITIALIZE_DELETIONRESOURCEMANAGER, 
e);
+              throw new RuntimeException(e);
+            }
+          });
+    } finally {
+      lock.unlock();
     }
-    return 
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent(
-        groupId,
-        key -> {
-          try {
-            return new DeletionResourceManager(groupId);
-          } catch (IOException e) {
-            
LOGGER.error(DataNodePipeMessages.FAILED_TO_INITIALIZE_DELETIONRESOURCEMANAGER, 
e);
-            throw new RuntimeException(e);
-          }
-        });
   }
 
   // Only when consensus protocol is IoTConsensusV2, will this class be 
initialized.
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
index 6fe2246627f..956edd80ef0 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
@@ -62,6 +62,7 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
   private static final double FSYNC_BUFFER_RATIO = 0.95;
   private static final int QUEUE_CAPACITY = 
config.getDeletionAheadLogBufferQueueCapacity();
   private static final long MAX_WAIT_CLOSE_TIME_IN_MS = 10000;
+  private static final long CLOSE_CHECK_INTERVAL_MS = 100;
 
   // Buffer config keep consistent with WAL.
   public static int DAL_BUFFER_SIZE = config.getWalBufferSize() / 3;
@@ -282,11 +283,20 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
     }
     // Force sync existing data in memory to disk.
     // first waiting serialize and sync tasks finished, then release all 
resources
-    waitUntilFlushAllDeletionsOrTimeOut();
+    final boolean flushed = waitUntilFlushAllDeletionsOrTimeOut();
     if (persistThread != null) {
       lifecycleLock.lock();
       try {
-        persistThread.shutdownNow();
+        if (flushed) {
+          persistThread.shutdown();
+        } else {
+          LOGGER.warn(
+              DataNodePipeMessages
+                  
.LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D,
+              dataRegionId,
+              unflushedDeletionCount.get());
+          persistThread.shutdownNow();
+        }
       } finally {
         lifecycleLock.unlock();
       }
@@ -310,7 +320,7 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
     serializeBuffer = null;
   }
 
-  private void waitUntilFlushAllDeletionsOrTimeOut() {
+  private boolean waitUntilFlushAllDeletionsOrTimeOut() {
     long currentTime = System.currentTimeMillis();
     while (!isAllDeletionFlushed()
         && System.currentTimeMillis() - currentTime < 
MAX_WAIT_CLOSE_TIME_IN_MS) {
@@ -319,8 +329,10 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
       } catch (InterruptedException e) {
         
LOGGER.error(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_ALL_DELETIONS_FLUSHED);
         Thread.currentThread().interrupt();
+        return false;
       }
     }
+    return isAllDeletionFlushed();
   }
 
   private class PersistTask implements Runnable {
@@ -374,9 +386,13 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
     }
 
     private void persistDeletion() throws IOException {
-      // For first deletion we use blocking take() method.
+      // Bounded waiting lets an idle task observe close without using 
interruption as control flow.
       try {
-        DeletionResource firstDeletionResource = deletionResources.take();
+        DeletionResource firstDeletionResource =
+            deletionResources.poll(CLOSE_CHECK_INTERVAL_MS, 
TimeUnit.MILLISECONDS);
+        if (firstDeletionResource == null) {
+          return;
+        }
         pendingDeletionsInOneTask.add(firstDeletionResource);
         currentTaskDeletionCount++;
         // A deletion can exceed the DAL buffer even when it is the first 
entry in a batch.
@@ -390,7 +406,11 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
             
maxProgressIndexInCurrentFile.updateToMinimumEqualOrIsAfterProgressIndex(
                 firstDeletionResource.getProgressIndex());
       } catch (InterruptedException e) {
-        
LOGGER.warn(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM);
+        // Forced shutdown after a flush timeout may still interrupt the queue 
wait.
+        if (!isClosed) {
+          LOGGER.warn(
+              
DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM);
+        }
         Thread.currentThread().interrupt();
         return;
       }
@@ -401,10 +421,26 @@ public class PageCacheDeletionBuffer implements 
DeletionBuffer {
         DeletionResource deletionResource = null;
         try {
           // Timeout config keep consistent with WAL async mode.
-          deletionResource =
-              deletionResources.poll(config.getWalAsyncModeFsyncDelayInMs(), 
TimeUnit.MILLISECONDS);
+          final long deadline =
+              System.nanoTime()
+                  + 
TimeUnit.MILLISECONDS.toNanos(config.getWalAsyncModeFsyncDelayInMs());
+          do {
+            deletionResource = deletionResources.poll();
+            if (deletionResource != null || isClosed) {
+              break;
+            }
+            final long remaining = deadline - System.nanoTime();
+            if (remaining <= 0) {
+              break;
+            }
+            deletionResource =
+                deletionResources.poll(
+                    Math.min(remaining, 
TimeUnit.MILLISECONDS.toNanos(CLOSE_CHECK_INTERVAL_MS)),
+                    TimeUnit.NANOSECONDS);
+          } while (deletionResource == null);
         } catch (InterruptedException e) {
-          
LOGGER.warn(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_WALENTRY_FROM);
+          LOGGER.warn(
+              
DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM);
           Thread.currentThread().interrupt();
         }
         // If timeout, flush deletions to disk.
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
index 19c78fa993b..bc34b4acd01 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
@@ -568,6 +568,10 @@ public class 
PipeHistoricalDataRegionTsFileAndDeletionSource
         DataNodePipeMessages
             
.MESSAGE_PIPE_START_TO_EXTRACT_HISTORICAL_TSFILE_AND_DELETION_IF_USES_IOTCONSENSUSV2_C1603968);
     try {
+      if (dataRegion.isDeleted()) {
+        pendingQueue = new ArrayDeque<>();
+        return;
+      }
       List<PersistentResource> originalResourceList = new ArrayList<>();
 
       if (shouldExtractInsertion) {
@@ -1427,7 +1431,7 @@ public class 
PipeHistoricalDataRegionTsFileAndDeletionSource
           dataRegionId,
           event);
     } else {
-      Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId))
+      
Optional.ofNullable(DeletionResourceManager.getInstanceIfPresent(dataRegionId))
           .ifPresent(
               manager ->
                   event.setDeletionResource(
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
index 6e308b5f159..bdf0fd652f2 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
@@ -387,7 +387,7 @@ public class PipeDataRegionAssigner implements Closeable {
                 final PipeDeleteDataNodeEvent deleteDataNodeEvent =
                     (PipeDeleteDataNodeEvent) innerEvent;
                 final DeletionResourceManager manager =
-                    
DeletionResourceManager.getInstance(source.getDataRegionId());
+                    
DeletionResourceManager.getInstanceIfPresent(source.getDataRegionId());
                 // increase deletion resource's reference and bind real 
deleteEvent
                 if (Objects.nonNull(manager)
                     && DeletionResource.isDeleteNodeGeneratedInLocalByIoTV2(
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
index 9c2a4bee090..effbc014f9d 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
@@ -33,6 +33,7 @@ import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDe
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
@@ -52,10 +53,15 @@ public class PipeInsertionDataNodeListener {
   private final ConcurrentMap<Integer, PipeDataRegionAssigner> 
dataRegionId2Assigner =
       new ConcurrentHashMap<>();
 
+  private final Set<Integer> deletedRegions = ConcurrentHashMap.newKeySet();
+
   //////////////////////////// start & stop ////////////////////////////
 
   public synchronized void startListenAndAssign(
       final int dataRegionId, final PipeRealtimeDataRegionSource source) {
+    if (deletedRegions.contains(dataRegionId)) {
+      return;
+    }
     // Keep registration inside compute so the assigner is fully started 
before it becomes visible
     // to concurrent listeners.
     dataRegionId2Assigner.compute(
@@ -68,6 +74,22 @@ public class PipeInsertionDataNodeListener {
         });
   }
 
+  /** Stop admission and drain the region assigner before its DAL is closed. */
+  public void releaseDataRegion(final int dataRegionId) {
+    final PipeDataRegionAssigner assigner;
+    synchronized (this) {
+      deletedRegions.add(dataRegionId);
+      assigner = dataRegionId2Assigner.remove(dataRegionId);
+    }
+    if (assigner != null) {
+      assigner.close();
+    }
+  }
+
+  public synchronized void onRegionCreated(final int dataRegionId) {
+    deletedRegions.remove(dataRegionId);
+  }
+
   public void stopListenAndAssign(
       final int dataRegionId, final PipeRealtimeDataRegionSource source) {
     PipeDataRegionAssigner assignerToClose = null;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 6c128c027a4..75f1834dab0 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -57,6 +57,7 @@ import 
org.apache.iotdb.db.exception.load.LoadReadOnlyException;
 import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException;
 import org.apache.iotdb.db.i18n.StorageEngineMessages;
 import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner.CompletionToken;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
 import 
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
@@ -542,6 +543,9 @@ public class StorageEngine implements IService {
     WRITING_METRICS.createActiveMemtableCounterMetrics(dataRegionId);
     dataRegion.setCustomFlushListeners(customFlushListeners);
     dataRegion.setCustomCloseFileListeners(customCloseFileListeners);
+    // The previous region's cleanup has completed before a new storage 
lifecycle starts.
+    DeletionResourceManager.onRegionCreated(dataRegionId.getId());
+    
PipeInsertionDataNodeListener.getInstance().onRegionCreated(dataRegionId.getId());
     return dataRegion;
   }
 
@@ -892,11 +896,15 @@ public class StorageEngine implements IService {
       LOGGER.info(StorageEngineMessages.REMOVING_DATA_REGION, regionId);
       region.markDeleted();
       try {
+        // Install the creation barrier before draining tasks. Do not hold the 
DAL lock while
+        // waiting for receivers/assigners, whose in-flight work may access 
the manager.
+        DeletionResourceManager.markRegionDeleted(regionId.getId());
+        
PipeDataNodeAgent.receiver().iotConsensusV2().releaseReceiverResource(regionId);
+        
PipeInsertionDataNodeListener.getInstance().releaseDataRegion(regionId.getId());
         region.abortCompaction();
         region.syncDeleteDataFiles();
         region.deleteFolder(systemDir);
         region.deleteDALFolderAndClose();
-        
PipeDataNodeAgent.receiver().iotConsensusV2().releaseReceiverResource(regionId);
         switch (CONFIG.getDataRegionConsensusProtocolClass()) {
           case ConsensusFactory.IOT_CONSENSUS:
           case ConsensusFactory.IOT_CONSENSUS_V2:
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 8af7efdb5cf..e4bf8bddd4c 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2456,12 +2456,7 @@ public class DataRegion implements IDataRegionForQuery {
   }
 
   public void deleteDALFolderAndClose() {
-    
Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId.getId()))
-        .ifPresent(
-            manager -> {
-              manager.close();
-              manager.removeDAL();
-            });
+    DeletionResourceManager.closeAndRemoveDAL(dataRegionId.getId());
   }
 
   /** close all tsfile resource */
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java
new file mode 100644
index 00000000000..3c433e1c97a
--- /dev/null
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.consensus.deletion;
+
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import 
org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer;
+import 
org.apache.iotdb.db.pipe.receiver.protocol.iotconsensusv2.IoTConsensusV2ReceiverAgent;
+import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner;
+import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
+import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode;
+import org.apache.iotdb.db.storageengine.StorageEngine;
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class DeletionResourceManagerCleanupTest {
+  private static final int REGION_ID = 91821;
+  @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+  private String previousDirectory;
+  private int previousBufferSize;
+  private Logger bufferLogger;
+  private Logger managerLogger;
+  private Appender<ILoggingEvent> appender;
+
+  @Before
+  @SuppressWarnings("unchecked")
+  public void setUp() throws Exception {
+    previousDirectory =
+        
IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir();
+    previousBufferSize = PageCacheDeletionBuffer.DAL_BUFFER_SIZE;
+    IoTDBDescriptor.getInstance()
+        .getConfig()
+        
.setIotConsensusV2DeletionFileDir(temporaryFolder.getRoot().getAbsolutePath());
+    PageCacheDeletionBuffer.setDalBufferSize(1024);
+    DeletionResourceManager.buildForTest();
+    DeletionResourceManager.onRegionCreated(REGION_ID);
+    bufferLogger = (Logger) 
LoggerFactory.getLogger(PageCacheDeletionBuffer.class);
+    appender = mock(Appender.class);
+    bufferLogger.addAppender(appender);
+    managerLogger = (Logger) 
LoggerFactory.getLogger(DeletionResourceManager.class);
+  }
+
+  @After
+  public void tearDown() {
+    try {
+      DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+    } finally {
+      bufferLogger.detachAppender(appender);
+      managerLogger.detachAppender(appender);
+      DeletionResourceManager.onRegionCreated(REGION_ID);
+      
IoTDBDescriptor.getInstance().getConfig().setIotConsensusV2DeletionFileDir(previousDirectory);
+      PageCacheDeletionBuffer.setDalBufferSize(previousBufferSize);
+    }
+  }
+
+  @Test
+  public void testCleanupWithoutManagerDoesNotStartBuffer() {
+    DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+
+    Assert.assertFalse(regionDirectory().exists());
+    verify(appender, never()).doAppend(any(ILoggingEvent.class));
+  }
+
+  @Test
+  public void testCleanupRemovesOrphanedDALWithoutStartingBuffer() throws 
Exception {
+    final File directory = regionDirectory();
+    Assert.assertTrue(directory.mkdir());
+    Files.write(new File(directory, "_0-0.deletion").toPath(), new byte[] {1, 
2, 3});
+
+    DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+
+    Assert.assertFalse(directory.exists());
+    verify(appender, never()).doAppend(any(ILoggingEvent.class));
+  }
+
+  @Test
+  public void testCleanupBlocksRecreationUntilNewRegionLifecycle() {
+    final DeletionResourceManager manager = 
DeletionResourceManager.getInstance(REGION_ID);
+    Assert.assertTrue(regionDirectory().exists());
+
+    DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+
+    Assert.assertFalse(regionDirectory().exists());
+    Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+    Assert.assertNull(DeletionResourceManager.getInstanceIfPresent(REGION_ID));
+    DeletionResourceManager.onRegionCreated(REGION_ID);
+    Assert.assertNotSame(manager, 
DeletionResourceManager.getInstance(REGION_ID));
+    Assert.assertTrue(regionDirectory().exists());
+  }
+
+  @Test
+  public void testLookupDoesNotCreateManager() {
+    Assert.assertNull(DeletionResourceManager.getInstanceIfPresent(REGION_ID));
+    Assert.assertFalse(regionDirectory().exists());
+    verify(appender, never()).doAppend(any(ILoggingEvent.class));
+  }
+
+  @Test
+  public void testDeletionBarrierRejectsRetainedManagerRegistration() {
+    final DeletionResourceManager manager = 
DeletionResourceManager.getInstance(REGION_ID);
+    DeletionResourceManager.markRegionDeleted(REGION_ID);
+    Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+    final DeletionResource resource =
+        manager.registerDeletionResource(mock(AbstractDeleteDataNode.class));
+    Assert.assertSame(DeletionResource.Status.FAILURE, 
resource.waitForResult());
+  }
+
+  @Test
+  public void testOldManagerCannotWriteOrRemoveNewLifecycleDAL() throws 
Exception {
+    final DeletionResourceManager old = 
DeletionResourceManager.getInstance(REGION_ID);
+    DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+    DeletionResourceManager.onRegionCreated(REGION_ID);
+    DeletionResourceManager.getInstance(REGION_ID);
+    final File marker = new File(regionDirectory(), "_0-1.deletion");
+    Files.write(marker.toPath(), new byte[] {1});
+    final DeletionResource rejected =
+        old.registerDeletionResource(mock(AbstractDeleteDataNode.class));
+    Assert.assertSame(DeletionResource.Status.FAILURE, 
rejected.waitForResult());
+    rejected.removeSelf();
+    Assert.assertTrue(marker.exists());
+  }
+
+  @Test
+  public void testReferenceReleaseDoesNotHoldPersistCallbackMonitor() {
+    final AtomicBoolean released = new AtomicBoolean();
+    final DeletionResource resource =
+        new DeletionResource(
+            mock(AbstractDeleteDataNode.class),
+            deletion -> {
+              Assert.assertFalse(Thread.holdsLock(deletion));
+              released.set(true);
+            },
+            REGION_ID);
+    resource.setPipeTaskReferenceCount(1);
+    resource.decreaseReference();
+    Assert.assertTrue(released.get());
+  }
+
+  @Test(timeout = 30000)
+  public void testCleanupWaitsForInFlightInitialization() throws Exception {
+    final CountDownLatch initializing = new CountDownLatch(1);
+    final CountDownLatch finishInitialization = new CountDownLatch(1);
+    doAnswer(
+            invocation -> {
+              final ILoggingEvent event = invocation.getArgument(0);
+              if 
(DataNodePipeMessages.DELETION_PERSIST_STARTING_TO_PERSIST_CURRENT_WRITING.equals(
+                  event.getMessage())) {
+                initializing.countDown();
+                Assert.assertTrue(finishInitialization.await(10, 
TimeUnit.SECONDS));
+              }
+              return null;
+            })
+        .when(appender)
+        .doAppend(any(ILoggingEvent.class));
+    final ExecutorService executor = Executors.newFixedThreadPool(2);
+    try {
+      final Future<DeletionResourceManager> creation =
+          executor.submit(() -> 
DeletionResourceManager.getInstance(REGION_ID));
+      Assert.assertTrue(initializing.await(10, TimeUnit.SECONDS));
+      final CountDownLatch cleanupStarted = new CountDownLatch(1);
+      final Future<?> cleanup =
+          executor.submit(
+              () -> {
+                cleanupStarted.countDown();
+                DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+              });
+      Assert.assertTrue(cleanupStarted.await(10, TimeUnit.SECONDS));
+      assertWaiting(cleanup);
+      Assert.assertTrue(regionDirectory().exists());
+      finishInitialization.countDown();
+      creation.get(10, TimeUnit.SECONDS);
+      cleanup.get(10, TimeUnit.SECONDS);
+      Assert.assertFalse(regionDirectory().exists());
+      Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+    } finally {
+      finishInitialization.countDown();
+      executor.shutdownNow();
+      Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+    }
+  }
+
+  @Test(timeout = 30000)
+  public void testCreationCannotEnterBetweenCloseAndDirectoryRemoval() throws 
Exception {
+    DeletionResourceManager.getInstance(REGION_ID);
+    final CountDownLatch managerClosed = new CountDownLatch(1);
+    final CountDownLatch removeDirectory = new CountDownLatch(1);
+    managerLogger.addAppender(appender);
+    doAnswer(
+            invocation -> {
+              final ILoggingEvent event = invocation.getArgument(0);
+              if 
(DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY.equals(
+                  event.getMessage())) {
+                managerClosed.countDown();
+                Assert.assertTrue(removeDirectory.await(10, TimeUnit.SECONDS));
+              }
+              return null;
+            })
+        .when(appender)
+        .doAppend(any(ILoggingEvent.class));
+    final ExecutorService executor = Executors.newFixedThreadPool(2);
+    try {
+      final Future<?> cleanup =
+          executor.submit(() -> 
DeletionResourceManager.closeAndRemoveDAL(REGION_ID));
+      Assert.assertTrue(managerClosed.await(10, TimeUnit.SECONDS));
+      final CountDownLatch creationStarted = new CountDownLatch(1);
+      final Future<DeletionResourceManager> creation =
+          executor.submit(
+              () -> {
+                creationStarted.countDown();
+                return DeletionResourceManager.getInstance(REGION_ID);
+              });
+      Assert.assertTrue(creationStarted.await(10, TimeUnit.SECONDS));
+      assertWaiting(creation);
+      removeDirectory.countDown();
+      cleanup.get(10, TimeUnit.SECONDS);
+      Assert.assertNull(creation.get(10, TimeUnit.SECONDS));
+      Assert.assertFalse(regionDirectory().exists());
+    } finally {
+      removeDirectory.countDown();
+      executor.shutdownNow();
+      Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+    }
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testStorageDeletionStopsReceiverAndAssignerBeforeDALRemoval() 
throws Exception {
+    final DataRegionId regionId = new DataRegionId(REGION_ID);
+    final StorageEngine engine = StorageEngine.getInstance();
+    final Field regionsField = 
StorageEngine.class.getDeclaredField("dataRegionMap");
+    regionsField.setAccessible(true);
+    final Map<DataRegionId, DataRegion> regions =
+        (Map<DataRegionId, DataRegion>) regionsField.get(engine);
+    final DataRegion region = mock(DataRegion.class);
+    when(region.getDatabaseName()).thenReturn("root.dal_cleanup");
+    
when(region.getDataRegionIdString()).thenReturn(Integer.toString(REGION_ID));
+    final PipeInsertionDataNodeListener listener = 
PipeInsertionDataNodeListener.getInstance();
+    final Field assignerField =
+        
PipeInsertionDataNodeListener.class.getDeclaredField("dataRegionId2Assigner");
+    assignerField.setAccessible(true);
+    final Map<Integer, PipeDataRegionAssigner> assigners =
+        (Map<Integer, PipeDataRegionAssigner>) assignerField.get(listener);
+    final PipeDataRegionAssigner assigner = mock(PipeDataRegionAssigner.class);
+    final IoTConsensusV2ReceiverAgent receiver = 
PipeDataNodeAgent.receiver().iotConsensusV2();
+    final Field staleField = 
IoTConsensusV2ReceiverAgent.class.getDeclaredField("staleRegions");
+    staleField.setAccessible(true);
+    final Set<Integer> stale = (Set<Integer>) staleField.get(receiver);
+    DeletionResourceManager.getInstance(REGION_ID);
+    doAnswer(
+            invocation -> {
+              verify(assigner).close();
+              Assert.assertFalse(assigners.containsKey(REGION_ID));
+              Assert.assertTrue(stale.contains(REGION_ID));
+              
Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+              Assert.assertTrue(regionDirectory().exists());
+              DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+              return null;
+            })
+        .when(region)
+        .deleteDALFolderAndClose();
+    regions.put(regionId, region);
+    assigners.put(REGION_ID, assigner);
+    try {
+      Assert.assertEquals(
+          TSStatusCode.SUCCESS_STATUS.getStatusCode(), 
engine.deleteDataRegion(regionId).getCode());
+      verify(region).deleteDALFolderAndClose();
+      Assert.assertFalse(regionDirectory().exists());
+    } finally {
+      regions.remove(regionId, region);
+      assigners.remove(REGION_ID, assigner);
+      listener.onRegionCreated(REGION_ID);
+      stale.remove(REGION_ID);
+    }
+  }
+
+  private void assertWaiting(Future<?> operation) throws Exception {
+    try {
+      operation.get(200, TimeUnit.MILLISECONDS);
+      Assert.fail("Operation must wait for the concurrent lifecycle 
transition");
+    } catch (TimeoutException expected) {
+      // The other operation is deliberately paused while holding the 
per-region lock.
+    }
+  }
+
+  private File regionDirectory() {
+    return new File(temporaryFolder.getRoot(), Integer.toString(REGION_ID));
+  }
+}
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
index 59d0d2862bd..fdc11c91d1d 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
@@ -24,11 +24,16 @@ import 
org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
 import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource;
 import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
 import org.apache.iotdb.db.pipe.consensus.deletion.recover.DeletionReader;
 import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
 
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
 import org.awaitility.Awaitility;
 import org.junit.After;
 import org.junit.Assert;
@@ -37,6 +42,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.mockito.InOrder;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -45,11 +51,20 @@ import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
+import static org.mockito.AdditionalAnswers.delegatesTo;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
@@ -75,6 +90,178 @@ public class PageCacheDeletionBufferTest {
     PageCacheDeletionBuffer.setDalBufferSize(previousDalBufferSize);
   }
 
+  @Test
+  public void testIdleCloseDoesNotWarnAboutQueueInterruption() throws 
Exception {
+    assertQueueInterruptionLogging(true);
+  }
+
+  @Test
+  public void testUnexpectedQueueInterruptionStillWarns() throws Exception {
+    assertQueueInterruptionLogging(false);
+  }
+
+  @SuppressWarnings("unchecked")
+  private void assertQueueInterruptionLogging(boolean closeNormally) throws 
Exception {
+    final PageCacheDeletionBuffer deletionBuffer =
+        new PageCacheDeletionBuffer(1, 
temporaryFolder.newFolder("interrupt").getAbsolutePath());
+    final CountDownLatch taking = new CountDownLatch(1);
+    final AtomicBoolean queueInterrupted = new AtomicBoolean();
+    final AtomicReference<Thread> worker = new AtomicReference<>();
+    final Field queueField = 
PageCacheDeletionBuffer.class.getDeclaredField("deletionResources");
+    queueField.setAccessible(true);
+    final BlockingQueue<DeletionResource> originalQueue =
+        (BlockingQueue<DeletionResource>) queueField.get(deletionBuffer);
+    final BlockingQueue<DeletionResource> queue =
+        mock(BlockingQueue.class, delegatesTo(originalQueue));
+    doAnswer(
+            invocation -> {
+              worker.set(Thread.currentThread());
+              taking.countDown();
+              try {
+                return originalQueue.poll(invocation.getArgument(0), 
invocation.getArgument(1));
+              } catch (InterruptedException e) {
+                queueInterrupted.set(true);
+                throw e;
+              }
+            })
+        .when(queue)
+        .poll(anyLong(), any(TimeUnit.class));
+    queueField.set(deletionBuffer, queue);
+    final Logger logger = (Logger) 
LoggerFactory.getLogger(PageCacheDeletionBuffer.class);
+    final Appender<ILoggingEvent> appender = mock(Appender.class);
+    logger.addAppender(appender);
+    deletionBuffer.start();
+    try {
+      Assert.assertTrue(taking.await(10, TimeUnit.SECONDS));
+      if (!closeNormally) {
+        worker.get().interrupt();
+        Awaitility.await()
+            .atMost(10, TimeUnit.SECONDS)
+            .untilAsserted(
+                () ->
+                    verify(appender, atLeastOnce())
+                        .doAppend(argThat(this::isQueueInterruptionWarning)));
+      }
+    } finally {
+      try {
+        deletionBuffer.close();
+        if (closeNormally) {
+          verify(appender, 
never()).doAppend(argThat(this::isQueueInterruptionWarning));
+          Assert.assertFalse(queueInterrupted.get());
+        }
+      } finally {
+        logger.detachAppender(appender);
+      }
+    }
+  }
+
+  @Test(timeout = 30000)
+  @SuppressWarnings("unchecked")
+  public void testCloseWhileBatchPollingFlushesWithoutInterruptingWorker() 
throws Exception {
+    final PageCacheDeletionBuffer buffer =
+        new PageCacheDeletionBuffer(1, 
temporaryFolder.newFolder("batch-close").getAbsolutePath());
+    final Field queueField = 
PageCacheDeletionBuffer.class.getDeclaredField("deletionResources");
+    queueField.setAccessible(true);
+    final BlockingQueue<DeletionResource> original =
+        (BlockingQueue<DeletionResource>) queueField.get(buffer);
+    final BlockingQueue<DeletionResource> queue = mock(BlockingQueue.class, 
delegatesTo(original));
+    final CountDownLatch collectingBatch = new CountDownLatch(1);
+    final CountDownLatch batchCanContinue = new CountDownLatch(1);
+    final AtomicBoolean queueInterrupted = new AtomicBoolean();
+    doAnswer(
+            invocation -> {
+              try {
+                if (invocation.getArgument(1) == TimeUnit.NANOSECONDS) {
+                  collectingBatch.countDown();
+                  Assert.assertTrue(batchCanContinue.await(10, 
TimeUnit.SECONDS));
+                }
+                return original.poll(invocation.getArgument(0), 
invocation.getArgument(1));
+              } catch (InterruptedException e) {
+                queueInterrupted.set(true);
+                throw e;
+              }
+            })
+        .when(queue)
+        .poll(anyLong(), any(TimeUnit.class));
+    queueField.set(buffer, queue);
+    final DeletionResource deletion = createDeletion(1, 1);
+    final ExecutorService closer = Executors.newSingleThreadExecutor();
+    buffer.start();
+    try {
+      buffer.registerDeletionResource(deletion);
+      Assert.assertTrue(collectingBatch.await(10, TimeUnit.SECONDS));
+      final Future<?> close = closer.submit(buffer::close);
+      final Field closed = 
PageCacheDeletionBuffer.class.getDeclaredField("isClosed");
+      closed.setAccessible(true);
+      Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> 
closed.getBoolean(buffer));
+      batchCanContinue.countDown();
+      close.get(10, TimeUnit.SECONDS);
+    } finally {
+      batchCanContinue.countDown();
+      closer.shutdownNow();
+      Assert.assertTrue(closer.awaitTermination(10, TimeUnit.SECONDS));
+    }
+    Assert.assertSame(DeletionResource.Status.SUCCESS, 
deletion.waitForResult());
+    // ThreadPoolExecutor may interrupt an idle worker during shutdown; assert 
the DAL wait itself.
+    Assert.assertFalse(queueInterrupted.get());
+    Assert.assertTrue(buffer.isAllDeletionFlushed());
+  }
+
+  @Test(timeout = 30000)
+  @SuppressWarnings("unchecked")
+  public void testFlushTimeoutWarnsAndForceInterruptsBatchPoll() throws 
Exception {
+    final PageCacheDeletionBuffer buffer =
+        new PageCacheDeletionBuffer(
+            1, temporaryFolder.newFolder("timeout-close").getAbsolutePath());
+    final Field queueField = 
PageCacheDeletionBuffer.class.getDeclaredField("deletionResources");
+    queueField.setAccessible(true);
+    final BlockingQueue<DeletionResource> original =
+        (BlockingQueue<DeletionResource>) queueField.get(buffer);
+    final BlockingQueue<DeletionResource> queue = mock(BlockingQueue.class, 
delegatesTo(original));
+    final CountDownLatch batchPollStarted = new CountDownLatch(1);
+    final CountDownLatch blockBatch = new CountDownLatch(1);
+    doAnswer(
+            invocation -> {
+              if (invocation.getArgument(1) == TimeUnit.NANOSECONDS) {
+                batchPollStarted.countDown();
+                blockBatch.await();
+              }
+              return original.poll(invocation.getArgument(0), 
invocation.getArgument(1));
+            })
+        .when(queue)
+        .poll(anyLong(), any(TimeUnit.class));
+    queueField.set(buffer, queue);
+    final Logger logger = (Logger) 
LoggerFactory.getLogger(PageCacheDeletionBuffer.class);
+    final Appender<ILoggingEvent> appender = mock(Appender.class);
+    logger.addAppender(appender);
+    final DeletionResource deletion = createDeletion(1, 1);
+    buffer.start();
+    try {
+      buffer.registerDeletionResource(deletion);
+      Assert.assertTrue(batchPollStarted.await(10, TimeUnit.SECONDS));
+      buffer.close();
+      verify(appender)
+          .doAppend(
+              argThat(
+                  event ->
+                      event.getLevel() == Level.WARN
+                          && DataNodePipeMessages
+                              
.LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D
+                              .equals(event.getMessage())));
+      Assert.assertSame(DeletionResource.Status.FAILURE, 
deletion.waitForResult());
+      Assert.assertNotNull(deletion.getCause());
+    } finally {
+      blockBatch.countDown();
+      logger.detachAppender(appender);
+    }
+  }
+
+  private boolean isQueueInterruptionWarning(ILoggingEvent event) {
+    return event.getLevel() == Level.WARN
+        && 
DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM.equals(
+            event.getMessage());
+  }
+
   @Test
   public void testDequeuedDeletionIsNotReportedAsFlushed() throws Exception {
     final CountDownLatch serializationStarted = new CountDownLatch(1);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
index 08240ae24f7..f084c63129f 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
@@ -19,8 +19,10 @@
 
 package org.apache.iotdb.db.pipe.source.dataregion.realtime.listener;
 
+import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner;
 
+import org.junit.Assert;
 import org.junit.Test;
 
 import java.lang.reflect.Field;
@@ -35,6 +37,35 @@ import static org.mockito.Mockito.when;
 
 public class PipeInsertionDataNodeListenerTest {
 
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testRegionDeletionDrainsAssignerAndBlocksLateRegistration() 
throws Exception {
+    final PipeInsertionDataNodeListener listener = 
PipeInsertionDataNodeListener.getInstance();
+    final Field field =
+        
PipeInsertionDataNodeListener.class.getDeclaredField("dataRegionId2Assigner");
+    field.setAccessible(true);
+    final ConcurrentMap<Integer, PipeDataRegionAssigner> assigners =
+        (ConcurrentMap<Integer, PipeDataRegionAssigner>) field.get(listener);
+    final int id = Integer.MIN_VALUE + 2;
+    final PipeDataRegionAssigner assigner = mock(PipeDataRegionAssigner.class);
+    assigners.put(id, assigner);
+    try {
+      listener.releaseDataRegion(id);
+      verify(assigner).close();
+      Assert.assertFalse(assigners.containsKey(id));
+      listener.startListenAndAssign(id, 
mock(PipeRealtimeDataRegionSource.class));
+      Assert.assertFalse(assigners.containsKey(id));
+      listener.onRegionCreated(id);
+      assigners.put(id, assigner);
+      final PipeRealtimeDataRegionSource source = 
mock(PipeRealtimeDataRegionSource.class);
+      listener.startListenAndAssign(id, source);
+      verify(assigner).startAssignTo(source);
+    } finally {
+      assigners.remove(id);
+      listener.onRegionCreated(id);
+    }
+  }
+
   @Test
   @SuppressWarnings("unchecked")
   public void testInsertUsesBarrierInvalidatingPublicationPath() throws 
Exception {

Reply via email to