This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d20ef864a80 [Pipe] Guard DAL lifecycle and drain producers before
region cleanup (#19321)
d20ef864a80 is described below
commit d20ef864a809e870f5213bf323876e47921035e9
Author: Peng Junzhi <[email protected]>
AuthorDate: Fri Nov 20 11:06:27 2026 +0800
[Pipe] Guard DAL lifecycle and drain producers before region cleanup
(#19321)
* [Pipe] Avoid creating DAL managers during region cleanup and suppress
expected close warnings
* [Pipe] Serialize DAL lifecycle and drain region producers before cleanup
---
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 4 +
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 4 +
.../pipe/consensus/deletion/DeletionResource.java | 5 +-
.../deletion/DeletionResourceManager.java | 236 ++++++++++----
.../deletion/persist/PageCacheDeletionBuffer.java | 54 +++-
...istoricalDataRegionTsFileAndDeletionSource.java | 6 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 2 +-
.../listener/PipeInsertionDataNodeListener.java | 22 ++
.../iotdb/db/storageengine/StorageEngine.java | 10 +-
.../db/storageengine/dataregion/DataRegion.java | 7 +-
.../DeletionResourceManagerCleanupTest.java | 338 +++++++++++++++++++++
.../persist/PageCacheDeletionBufferTest.java | 187 ++++++++++++
.../PipeInsertionDataNodeListenerTest.java | 31 ++
13 files changed, 824 insertions(+), 82 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 7a6b4ac32e1..3153e962e11 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -3184,4 +3184,8 @@ public final class DataNodePipeMessages {
EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68
=
"Failed to pin the TsFile %s by creating a hardlink, and the TsFile
can not be copied"
+ " because it is not closed yet";
+ public static final String
EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622
=
+ "Deletion resource manager for DataRegion %s is closed or deleted.";
+ public static final String
LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D
=
+ "Deletion persist-{}: timed out waiting for {} deletions to flush before
shutdown.";
}
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index b4872119644..550323119fd 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2958,4 +2958,8 @@ public final class DataNodePipeMessages {
public static final String
EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68
=
"通过创建硬链接固化 TsFile %s 失败,且该 TsFile 尚未关闭,无法改为拷贝";
+ public static final String
EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622
=
+ "DataRegion %s 的删除资源管理器已关闭或已删除。";
+ public static final String
LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D
=
+ "Deletion persist-{}:关闭前等待 {} 条删除记录持久化超时。";
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
index 051f224890c..18010f067cb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResource.java
@@ -70,7 +70,10 @@ public class DeletionResource implements PersistentResource {
DataRegionConsensusImpl.getInstance().getReplicationNum(consensusGroupId) - 1);
}
- public synchronized void decreaseReference() {
+ // The atomic counter selects one release callback. Do not hold this
resource's monitor while
+ // the callback acquires the region lifecycle lock: close waits for
synchronized persist
+ // callbacks.
+ public void decreaseReference() {
if (pipeTaskReferenceCount.decrementAndGet() == 0) {
removeSelf();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
index 98f447a4b12..597177b39da 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManager.java
@@ -34,6 +34,7 @@ import
org.apache.iotdb.db.pipe.consensus.deletion.recover.DeletionReader;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode;
import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Striped;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +45,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
@@ -62,6 +64,11 @@ public class DeletionResourceManager implements
AutoCloseable {
String.format(
"^_(?<%s>\\d+)-(?<%s>\\d+)\\%s$",
REBOOT_TIME, MEM_TABLE_FLUSH_ORDER, DELETION_FILE_SUFFIX);
+ // Stable, bounded locks are shared by creation, ordinary close and region
deletion. Never remove
+ // a lock while a waiter may still hold its reference.
+ private static final Striped<Lock> REGION_LOCKS = Striped.lock(1024);
+ private static final Set<Integer> DELETED_REGIONS =
ConcurrentHashMap.newKeySet();
+ private boolean closed;
private final int dataRegionId;
private final DeletionBuffer deletionBuffer;
private final File storageDir;
@@ -132,25 +139,50 @@ public class DeletionResourceManager implements
AutoCloseable {
@Override
public void close() {
- LOGGER.info(DataNodePipeMessages.CLOSING_DELETION_RESOURCE_MANAGER_FOR,
dataRegionId);
- this.deleteNode2ResourcesMap.clear();
- this.deletionBuffer.close();
- if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP !=
null) {
-
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.remove(dataRegionId,
this);
+ final Lock lock = REGION_LOCKS.get(dataRegionId);
+ lock.lock();
+ try {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ LOGGER.info(DataNodePipeMessages.CLOSING_DELETION_RESOURCE_MANAGER_FOR,
dataRegionId);
+ this.deletionBuffer.close();
+ this.deleteNode2ResourcesMap.clear();
+ if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP !=
null) {
+
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.remove(dataRegionId,
this);
+ }
+ LOGGER.info(
+
DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY,
dataRegionId);
+ } finally {
+ lock.unlock();
}
- LOGGER.info(
-
DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY,
dataRegionId);
}
public DeletionResource registerDeletionResource(AbstractDeleteDataNode
deleteDataNode) {
- DeletionResource deletionResource =
- deleteNode2ResourcesMap.computeIfAbsent(
- deleteDataNode,
- key ->
- new DeletionResource(deleteDataNode,
this::removeDeletionResource, dataRegionId));
- // register a persist task for current deletionResource
- deletionBuffer.registerDeletionResource(deletionResource);
- return deletionResource;
+ final Lock lock = REGION_LOCKS.get(dataRegionId);
+ lock.lock();
+ try {
+ final DeletionResource deletionResource =
+ new DeletionResource(deleteDataNode, this::removeDeletionResource,
dataRegionId);
+ // A producer may have obtained this manager before the region deletion
barrier.
+ // Complete its result instead of leaving it waiting on an already
closed buffer.
+ if (closed || DELETED_REGIONS.contains(dataRegionId)) {
+ deletionResource.onPersistFailed(
+ new IOException(
+ String.format(
+ DataNodePipeMessages
+
.EXCEPTION_DELETION_RESOURCE_MANAGER_FOR_DATAREGION_ARG_IS_CLOSED_OR_DELETED_951C0622,
+ dataRegionId)));
+ return deletionResource;
+ }
+ final DeletionResource resource =
+ deleteNode2ResourcesMap.computeIfAbsent(deleteDataNode, key ->
deletionResource);
+ deletionBuffer.registerDeletionResource(resource);
+ return resource;
+ } finally {
+ lock.unlock();
+ }
}
public DeletionResource getDeletionResource(AbstractDeleteDataNode
deleteDataNode) {
@@ -176,7 +208,55 @@ public class DeletionResourceManager implements
AutoCloseable {
}
}
- public void removeDAL() {
+ /** Block new DAL creation before stopping region producers. Does not wait
for producer tasks. */
+ public static void markRegionDeleted(int groupId) {
+ final Lock lock = REGION_LOCKS.get(groupId);
+ lock.lock();
+ try {
+ DELETED_REGIONS.add(groupId);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * Only the storage lifecycle may permit DAL creation again, after
old-region cleanup finishes.
+ */
+ public static void onRegionCreated(int groupId) {
+ final Lock lock = REGION_LOCKS.get(groupId);
+ lock.lock();
+ try {
+ DELETED_REGIONS.remove(groupId);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ /** Close and remove DAL atomically with respect to creation, without
constructing a manager. */
+ public static void closeAndRemoveDAL(int groupId) {
+ final Lock lock = REGION_LOCKS.get(groupId);
+ lock.lock();
+ try {
+ DELETED_REGIONS.add(groupId);
+ if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP ==
null) {
+ return;
+ }
+ final DeletionResourceManager manager =
+
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId);
+ if (manager != null) {
+ manager.close();
+ }
+ removeDAL(
+ groupId,
+ new File(
+
IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir(),
+ Integer.toString(groupId)));
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private static void removeDAL(int dataRegionId, File storageDir) {
if (storageDir.exists()) {
FileUtils.deleteFileOrDirectory(storageDir);
LOGGER.info(
@@ -195,43 +275,53 @@ public class DeletionResourceManager implements
AutoCloseable {
* This is a hook function, which will be automatically invoked when
deletionResource's reference
* count returns to 0.
*/
- private synchronized void removeDeletionResource(DeletionResource
deletionResource) {
- // Clean memory
- deleteNode2ResourcesMap.remove(deletionResource.getDeleteDataNode());
- // Clean disk
- ProgressIndex currentProgressIndex =
- ReplicateProgressDataNodeManager.extractLocalSimpleProgressIndex(
- deletionResource.getProgressIndex());
+ private void removeDeletionResource(DeletionResource deletionResource) {
+ final Lock lock = REGION_LOCKS.get(dataRegionId);
+ lock.lock();
+ try {
+ // Late reference releases from an old manager must not touch a new
lifecycle's DAL.
+ if (closed || DELETED_REGIONS.contains(dataRegionId)) {
+ return;
+ }
+ // Clean memory
+ deleteNode2ResourcesMap.remove(deletionResource.getDeleteDataNode());
+ // Clean disk
+ ProgressIndex currentProgressIndex =
+ ReplicateProgressDataNodeManager.extractLocalSimpleProgressIndex(
+ deletionResource.getProgressIndex());
- try (Stream<Path> pathStream = Files.walk(Paths.get(storageDir.getPath()),
1)) {
- Path[] deletionPaths =
- pathStream
- .filter(Files::isRegularFile)
- .filter(path ->
path.getFileName().toString().matches(DELETION_FILE_NAME_PATTERN))
- .filter(
- path ->
- isFileProgressCoveredByGivenProgress(
- path.getFileName().toString(), currentProgressIndex))
- .sorted(this::compareFileProgressIndex)
- .toArray(Path[]::new);
- // File name represents the max progressIndex in its previous file. If
currentProgressIndex is
- // larger than a fileName's progressIndex, it means that the file before
this file has been
- // fully synchronized and can be deleted.
- // So here we cannot guarantee that the last file can be deleted, we can
only guarantee that
- // the first n-1 files can be deleted (if the length of deletionPaths is
n)
- for (int i = 0; i < deletionPaths.length - 1; i++) {
- File fileToDelete = deletionPaths[i].toFile();
- FileUtils.deleteFileOrDirectory(fileToDelete);
- LOGGER.info(
- DataNodePipeMessages.DELETIONMANAGER_DELETE_DELETION_FILE_IN_DIR,
+ try (Stream<Path> pathStream =
Files.walk(Paths.get(storageDir.getPath()), 1)) {
+ Path[] deletionPaths =
+ pathStream
+ .filter(Files::isRegularFile)
+ .filter(path ->
path.getFileName().toString().matches(DELETION_FILE_NAME_PATTERN))
+ .filter(
+ path ->
+ isFileProgressCoveredByGivenProgress(
+ path.getFileName().toString(),
currentProgressIndex))
+ .sorted(this::compareFileProgressIndex)
+ .toArray(Path[]::new);
+ // File names describe the previous file's max progressIndex. A larger
currentProgressIndex
+ // means that the file before this file has been
+ // fully synchronized and can be deleted.
+ // So here we cannot guarantee that the last file can be deleted, we
can only guarantee that
+ // the first n-1 files can be deleted (if the length of deletionPaths
is n)
+ for (int i = 0; i < deletionPaths.length - 1; i++) {
+ File fileToDelete = deletionPaths[i].toFile();
+ FileUtils.deleteFileOrDirectory(fileToDelete);
+ LOGGER.info(
+ DataNodePipeMessages.DELETIONMANAGER_DELETE_DELETION_FILE_IN_DIR,
+ dataRegionId,
+ fileToDelete);
+ }
+ } catch (IOException e) {
+ LOGGER.warn(
+ DataNodePipeMessages.DELETIONMANAGER_FAILED_TO_DELETE_FILE_IN_DIR,
dataRegionId,
- fileToDelete);
+ storageDir);
}
- } catch (IOException e) {
- LOGGER.warn(
- DataNodePipeMessages.DELETIONMANAGER_FAILED_TO_DELETE_FILE_IN_DIR,
- dataRegionId,
- storageDir);
+ } finally {
+ lock.unlock();
}
}
@@ -291,21 +381,41 @@ public class DeletionResourceManager implements
AutoCloseable {
}
}
+ /** Obtain an existing manager without creating DAL resources for a
lookup-only caller. */
+ public static DeletionResourceManager getInstanceIfPresent(int groupId) {
+ final Lock lock = REGION_LOCKS.get(groupId);
+ lock.lock();
+ try {
+ return DELETED_REGIONS.contains(groupId)
+ ||
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP == null
+ ? null
+ :
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.get(groupId);
+ } finally {
+ lock.unlock();
+ }
+ }
+
public static DeletionResourceManager getInstance(int groupId) {
- // If consensusImpl is not IoTConsensusV2.
- if (DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP ==
null) {
- return null;
+ final Lock lock = REGION_LOCKS.get(groupId);
+ lock.lock();
+ try {
+ if (DELETED_REGIONS.contains(groupId)
+ || DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP
== null) {
+ return null;
+ }
+ return
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent(
+ groupId,
+ key -> {
+ try {
+ return new DeletionResourceManager(groupId);
+ } catch (IOException e) {
+
LOGGER.error(DataNodePipeMessages.FAILED_TO_INITIALIZE_DELETIONRESOURCEMANAGER,
e);
+ throw new RuntimeException(e);
+ }
+ });
+ } finally {
+ lock.unlock();
}
- return
DeletionResourceManagerHolder.CONSENSUS_GROUP_ID_2_INSTANCE_MAP.computeIfAbsent(
- groupId,
- key -> {
- try {
- return new DeletionResourceManager(groupId);
- } catch (IOException e) {
-
LOGGER.error(DataNodePipeMessages.FAILED_TO_INITIALIZE_DELETIONRESOURCEMANAGER,
e);
- throw new RuntimeException(e);
- }
- });
}
// Only when consensus protocol is IoTConsensusV2, will this class be
initialized.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
index 6fe2246627f..956edd80ef0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBuffer.java
@@ -62,6 +62,7 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
private static final double FSYNC_BUFFER_RATIO = 0.95;
private static final int QUEUE_CAPACITY =
config.getDeletionAheadLogBufferQueueCapacity();
private static final long MAX_WAIT_CLOSE_TIME_IN_MS = 10000;
+ private static final long CLOSE_CHECK_INTERVAL_MS = 100;
// Buffer config keep consistent with WAL.
public static int DAL_BUFFER_SIZE = config.getWalBufferSize() / 3;
@@ -282,11 +283,20 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
}
// Force sync existing data in memory to disk.
// first waiting serialize and sync tasks finished, then release all
resources
- waitUntilFlushAllDeletionsOrTimeOut();
+ final boolean flushed = waitUntilFlushAllDeletionsOrTimeOut();
if (persistThread != null) {
lifecycleLock.lock();
try {
- persistThread.shutdownNow();
+ if (flushed) {
+ persistThread.shutdown();
+ } else {
+ LOGGER.warn(
+ DataNodePipeMessages
+
.LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D,
+ dataRegionId,
+ unflushedDeletionCount.get());
+ persistThread.shutdownNow();
+ }
} finally {
lifecycleLock.unlock();
}
@@ -310,7 +320,7 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
serializeBuffer = null;
}
- private void waitUntilFlushAllDeletionsOrTimeOut() {
+ private boolean waitUntilFlushAllDeletionsOrTimeOut() {
long currentTime = System.currentTimeMillis();
while (!isAllDeletionFlushed()
&& System.currentTimeMillis() - currentTime <
MAX_WAIT_CLOSE_TIME_IN_MS) {
@@ -319,8 +329,10 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
} catch (InterruptedException e) {
LOGGER.error(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_ALL_DELETIONS_FLUSHED);
Thread.currentThread().interrupt();
+ return false;
}
}
+ return isAllDeletionFlushed();
}
private class PersistTask implements Runnable {
@@ -374,9 +386,13 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
}
private void persistDeletion() throws IOException {
- // For first deletion we use blocking take() method.
+ // Bounded waiting lets an idle task observe close without using
interruption as control flow.
try {
- DeletionResource firstDeletionResource = deletionResources.take();
+ DeletionResource firstDeletionResource =
+ deletionResources.poll(CLOSE_CHECK_INTERVAL_MS,
TimeUnit.MILLISECONDS);
+ if (firstDeletionResource == null) {
+ return;
+ }
pendingDeletionsInOneTask.add(firstDeletionResource);
currentTaskDeletionCount++;
// A deletion can exceed the DAL buffer even when it is the first
entry in a batch.
@@ -390,7 +406,11 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
maxProgressIndexInCurrentFile.updateToMinimumEqualOrIsAfterProgressIndex(
firstDeletionResource.getProgressIndex());
} catch (InterruptedException e) {
-
LOGGER.warn(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM);
+ // Forced shutdown after a flush timeout may still interrupt the queue
wait.
+ if (!isClosed) {
+ LOGGER.warn(
+
DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM);
+ }
Thread.currentThread().interrupt();
return;
}
@@ -401,10 +421,26 @@ public class PageCacheDeletionBuffer implements
DeletionBuffer {
DeletionResource deletionResource = null;
try {
// Timeout config keep consistent with WAL async mode.
- deletionResource =
- deletionResources.poll(config.getWalAsyncModeFsyncDelayInMs(),
TimeUnit.MILLISECONDS);
+ final long deadline =
+ System.nanoTime()
+ +
TimeUnit.MILLISECONDS.toNanos(config.getWalAsyncModeFsyncDelayInMs());
+ do {
+ deletionResource = deletionResources.poll();
+ if (deletionResource != null || isClosed) {
+ break;
+ }
+ final long remaining = deadline - System.nanoTime();
+ if (remaining <= 0) {
+ break;
+ }
+ deletionResource =
+ deletionResources.poll(
+ Math.min(remaining,
TimeUnit.MILLISECONDS.toNanos(CLOSE_CHECK_INTERVAL_MS)),
+ TimeUnit.NANOSECONDS);
+ } while (deletionResource == null);
} catch (InterruptedException e) {
-
LOGGER.warn(DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_WALENTRY_FROM);
+ LOGGER.warn(
+
DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM);
Thread.currentThread().interrupt();
}
// If timeout, flush deletions to disk.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
index 19c78fa993b..bc34b4acd01 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java
@@ -568,6 +568,10 @@ public class
PipeHistoricalDataRegionTsFileAndDeletionSource
DataNodePipeMessages
.MESSAGE_PIPE_START_TO_EXTRACT_HISTORICAL_TSFILE_AND_DELETION_IF_USES_IOTCONSENSUSV2_C1603968);
try {
+ if (dataRegion.isDeleted()) {
+ pendingQueue = new ArrayDeque<>();
+ return;
+ }
List<PersistentResource> originalResourceList = new ArrayList<>();
if (shouldExtractInsertion) {
@@ -1427,7 +1431,7 @@ public class
PipeHistoricalDataRegionTsFileAndDeletionSource
dataRegionId,
event);
} else {
- Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId))
+
Optional.ofNullable(DeletionResourceManager.getInstanceIfPresent(dataRegionId))
.ifPresent(
manager ->
event.setDeletionResource(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
index 6e308b5f159..bdf0fd652f2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java
@@ -387,7 +387,7 @@ public class PipeDataRegionAssigner implements Closeable {
final PipeDeleteDataNodeEvent deleteDataNodeEvent =
(PipeDeleteDataNodeEvent) innerEvent;
final DeletionResourceManager manager =
-
DeletionResourceManager.getInstance(source.getDataRegionId());
+
DeletionResourceManager.getInstanceIfPresent(source.getDataRegionId());
// increase deletion resource's reference and bind real
deleteEvent
if (Objects.nonNull(manager)
&& DeletionResource.isDeleteNodeGeneratedInLocalByIoTV2(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
index 9c2a4bee090..effbc014f9d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java
@@ -33,6 +33,7 @@ import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDe
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
@@ -52,10 +53,15 @@ public class PipeInsertionDataNodeListener {
private final ConcurrentMap<Integer, PipeDataRegionAssigner>
dataRegionId2Assigner =
new ConcurrentHashMap<>();
+ private final Set<Integer> deletedRegions = ConcurrentHashMap.newKeySet();
+
//////////////////////////// start & stop ////////////////////////////
public synchronized void startListenAndAssign(
final int dataRegionId, final PipeRealtimeDataRegionSource source) {
+ if (deletedRegions.contains(dataRegionId)) {
+ return;
+ }
// Keep registration inside compute so the assigner is fully started
before it becomes visible
// to concurrent listeners.
dataRegionId2Assigner.compute(
@@ -68,6 +74,22 @@ public class PipeInsertionDataNodeListener {
});
}
+ /** Stop admission and drain the region assigner before its DAL is closed. */
+ public void releaseDataRegion(final int dataRegionId) {
+ final PipeDataRegionAssigner assigner;
+ synchronized (this) {
+ deletedRegions.add(dataRegionId);
+ assigner = dataRegionId2Assigner.remove(dataRegionId);
+ }
+ if (assigner != null) {
+ assigner.close();
+ }
+ }
+
+ public synchronized void onRegionCreated(final int dataRegionId) {
+ deletedRegions.remove(dataRegionId);
+ }
+
public void stopListenAndAssign(
final int dataRegionId, final PipeRealtimeDataRegionSource source) {
PipeDataRegionAssigner assignerToClose = null;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 6c128c027a4..75f1834dab0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -57,6 +57,7 @@ import
org.apache.iotdb.db.exception.load.LoadReadOnlyException;
import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner.CompletionToken;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
@@ -542,6 +543,9 @@ public class StorageEngine implements IService {
WRITING_METRICS.createActiveMemtableCounterMetrics(dataRegionId);
dataRegion.setCustomFlushListeners(customFlushListeners);
dataRegion.setCustomCloseFileListeners(customCloseFileListeners);
+ // The previous region's cleanup has completed before a new storage
lifecycle starts.
+ DeletionResourceManager.onRegionCreated(dataRegionId.getId());
+
PipeInsertionDataNodeListener.getInstance().onRegionCreated(dataRegionId.getId());
return dataRegion;
}
@@ -892,11 +896,15 @@ public class StorageEngine implements IService {
LOGGER.info(StorageEngineMessages.REMOVING_DATA_REGION, regionId);
region.markDeleted();
try {
+ // Install the creation barrier before draining tasks. Do not hold the
DAL lock while
+ // waiting for receivers/assigners, whose in-flight work may access
the manager.
+ DeletionResourceManager.markRegionDeleted(regionId.getId());
+
PipeDataNodeAgent.receiver().iotConsensusV2().releaseReceiverResource(regionId);
+
PipeInsertionDataNodeListener.getInstance().releaseDataRegion(regionId.getId());
region.abortCompaction();
region.syncDeleteDataFiles();
region.deleteFolder(systemDir);
region.deleteDALFolderAndClose();
-
PipeDataNodeAgent.receiver().iotConsensusV2().releaseReceiverResource(regionId);
switch (CONFIG.getDataRegionConsensusProtocolClass()) {
case ConsensusFactory.IOT_CONSENSUS:
case ConsensusFactory.IOT_CONSENSUS_V2:
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 8af7efdb5cf..e4bf8bddd4c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2456,12 +2456,7 @@ public class DataRegion implements IDataRegionForQuery {
}
public void deleteDALFolderAndClose() {
-
Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId.getId()))
- .ifPresent(
- manager -> {
- manager.close();
- manager.removeDAL();
- });
+ DeletionResourceManager.closeAndRemoveDAL(dataRegionId.getId());
}
/** close all tsfile resource */
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java
new file mode 100644
index 00000000000..3c433e1c97a
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/DeletionResourceManagerCleanupTest.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.consensus.deletion;
+
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import
org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer;
+import
org.apache.iotdb.db.pipe.receiver.protocol.iotconsensusv2.IoTConsensusV2ReceiverAgent;
+import
org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner;
+import
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode;
+import org.apache.iotdb.db.storageengine.StorageEngine;
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class DeletionResourceManagerCleanupTest {
+ private static final int REGION_ID = 91821;
+ @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private String previousDirectory;
+ private int previousBufferSize;
+ private Logger bufferLogger;
+ private Logger managerLogger;
+ private Appender<ILoggingEvent> appender;
+
+ @Before
+ @SuppressWarnings("unchecked")
+ public void setUp() throws Exception {
+ previousDirectory =
+
IoTDBDescriptor.getInstance().getConfig().getIotConsensusV2DeletionFileDir();
+ previousBufferSize = PageCacheDeletionBuffer.DAL_BUFFER_SIZE;
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setIotConsensusV2DeletionFileDir(temporaryFolder.getRoot().getAbsolutePath());
+ PageCacheDeletionBuffer.setDalBufferSize(1024);
+ DeletionResourceManager.buildForTest();
+ DeletionResourceManager.onRegionCreated(REGION_ID);
+ bufferLogger = (Logger)
LoggerFactory.getLogger(PageCacheDeletionBuffer.class);
+ appender = mock(Appender.class);
+ bufferLogger.addAppender(appender);
+ managerLogger = (Logger)
LoggerFactory.getLogger(DeletionResourceManager.class);
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+ } finally {
+ bufferLogger.detachAppender(appender);
+ managerLogger.detachAppender(appender);
+ DeletionResourceManager.onRegionCreated(REGION_ID);
+
IoTDBDescriptor.getInstance().getConfig().setIotConsensusV2DeletionFileDir(previousDirectory);
+ PageCacheDeletionBuffer.setDalBufferSize(previousBufferSize);
+ }
+ }
+
+ @Test
+ public void testCleanupWithoutManagerDoesNotStartBuffer() {
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+
+ Assert.assertFalse(regionDirectory().exists());
+ verify(appender, never()).doAppend(any(ILoggingEvent.class));
+ }
+
+ @Test
+ public void testCleanupRemovesOrphanedDALWithoutStartingBuffer() throws
Exception {
+ final File directory = regionDirectory();
+ Assert.assertTrue(directory.mkdir());
+ Files.write(new File(directory, "_0-0.deletion").toPath(), new byte[] {1,
2, 3});
+
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+
+ Assert.assertFalse(directory.exists());
+ verify(appender, never()).doAppend(any(ILoggingEvent.class));
+ }
+
+ @Test
+ public void testCleanupBlocksRecreationUntilNewRegionLifecycle() {
+ final DeletionResourceManager manager =
DeletionResourceManager.getInstance(REGION_ID);
+ Assert.assertTrue(regionDirectory().exists());
+
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+
+ Assert.assertFalse(regionDirectory().exists());
+ Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+ Assert.assertNull(DeletionResourceManager.getInstanceIfPresent(REGION_ID));
+ DeletionResourceManager.onRegionCreated(REGION_ID);
+ Assert.assertNotSame(manager,
DeletionResourceManager.getInstance(REGION_ID));
+ Assert.assertTrue(regionDirectory().exists());
+ }
+
+ @Test
+ public void testLookupDoesNotCreateManager() {
+ Assert.assertNull(DeletionResourceManager.getInstanceIfPresent(REGION_ID));
+ Assert.assertFalse(regionDirectory().exists());
+ verify(appender, never()).doAppend(any(ILoggingEvent.class));
+ }
+
+ @Test
+ public void testDeletionBarrierRejectsRetainedManagerRegistration() {
+ final DeletionResourceManager manager =
DeletionResourceManager.getInstance(REGION_ID);
+ DeletionResourceManager.markRegionDeleted(REGION_ID);
+ Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+ final DeletionResource resource =
+ manager.registerDeletionResource(mock(AbstractDeleteDataNode.class));
+ Assert.assertSame(DeletionResource.Status.FAILURE,
resource.waitForResult());
+ }
+
+ @Test
+ public void testOldManagerCannotWriteOrRemoveNewLifecycleDAL() throws
Exception {
+ final DeletionResourceManager old =
DeletionResourceManager.getInstance(REGION_ID);
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+ DeletionResourceManager.onRegionCreated(REGION_ID);
+ DeletionResourceManager.getInstance(REGION_ID);
+ final File marker = new File(regionDirectory(), "_0-1.deletion");
+ Files.write(marker.toPath(), new byte[] {1});
+ final DeletionResource rejected =
+ old.registerDeletionResource(mock(AbstractDeleteDataNode.class));
+ Assert.assertSame(DeletionResource.Status.FAILURE,
rejected.waitForResult());
+ rejected.removeSelf();
+ Assert.assertTrue(marker.exists());
+ }
+
+ @Test
+ public void testReferenceReleaseDoesNotHoldPersistCallbackMonitor() {
+ final AtomicBoolean released = new AtomicBoolean();
+ final DeletionResource resource =
+ new DeletionResource(
+ mock(AbstractDeleteDataNode.class),
+ deletion -> {
+ Assert.assertFalse(Thread.holdsLock(deletion));
+ released.set(true);
+ },
+ REGION_ID);
+ resource.setPipeTaskReferenceCount(1);
+ resource.decreaseReference();
+ Assert.assertTrue(released.get());
+ }
+
+ @Test(timeout = 30000)
+ public void testCleanupWaitsForInFlightInitialization() throws Exception {
+ final CountDownLatch initializing = new CountDownLatch(1);
+ final CountDownLatch finishInitialization = new CountDownLatch(1);
+ doAnswer(
+ invocation -> {
+ final ILoggingEvent event = invocation.getArgument(0);
+ if
(DataNodePipeMessages.DELETION_PERSIST_STARTING_TO_PERSIST_CURRENT_WRITING.equals(
+ event.getMessage())) {
+ initializing.countDown();
+ Assert.assertTrue(finishInitialization.await(10,
TimeUnit.SECONDS));
+ }
+ return null;
+ })
+ .when(appender)
+ .doAppend(any(ILoggingEvent.class));
+ final ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ final Future<DeletionResourceManager> creation =
+ executor.submit(() ->
DeletionResourceManager.getInstance(REGION_ID));
+ Assert.assertTrue(initializing.await(10, TimeUnit.SECONDS));
+ final CountDownLatch cleanupStarted = new CountDownLatch(1);
+ final Future<?> cleanup =
+ executor.submit(
+ () -> {
+ cleanupStarted.countDown();
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+ });
+ Assert.assertTrue(cleanupStarted.await(10, TimeUnit.SECONDS));
+ assertWaiting(cleanup);
+ Assert.assertTrue(regionDirectory().exists());
+ finishInitialization.countDown();
+ creation.get(10, TimeUnit.SECONDS);
+ cleanup.get(10, TimeUnit.SECONDS);
+ Assert.assertFalse(regionDirectory().exists());
+ Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+ } finally {
+ finishInitialization.countDown();
+ executor.shutdownNow();
+ Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test(timeout = 30000)
+ public void testCreationCannotEnterBetweenCloseAndDirectoryRemoval() throws
Exception {
+ DeletionResourceManager.getInstance(REGION_ID);
+ final CountDownLatch managerClosed = new CountDownLatch(1);
+ final CountDownLatch removeDirectory = new CountDownLatch(1);
+ managerLogger.addAppender(appender);
+ doAnswer(
+ invocation -> {
+ final ILoggingEvent event = invocation.getArgument(0);
+ if
(DataNodePipeMessages.DELETION_RESOURCE_MANAGER_FOR_HAS_BEEN_SUCCESSFULLY.equals(
+ event.getMessage())) {
+ managerClosed.countDown();
+ Assert.assertTrue(removeDirectory.await(10, TimeUnit.SECONDS));
+ }
+ return null;
+ })
+ .when(appender)
+ .doAppend(any(ILoggingEvent.class));
+ final ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ final Future<?> cleanup =
+ executor.submit(() ->
DeletionResourceManager.closeAndRemoveDAL(REGION_ID));
+ Assert.assertTrue(managerClosed.await(10, TimeUnit.SECONDS));
+ final CountDownLatch creationStarted = new CountDownLatch(1);
+ final Future<DeletionResourceManager> creation =
+ executor.submit(
+ () -> {
+ creationStarted.countDown();
+ return DeletionResourceManager.getInstance(REGION_ID);
+ });
+ Assert.assertTrue(creationStarted.await(10, TimeUnit.SECONDS));
+ assertWaiting(creation);
+ removeDirectory.countDown();
+ cleanup.get(10, TimeUnit.SECONDS);
+ Assert.assertNull(creation.get(10, TimeUnit.SECONDS));
+ Assert.assertFalse(regionDirectory().exists());
+ } finally {
+ removeDirectory.countDown();
+ executor.shutdownNow();
+ Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testStorageDeletionStopsReceiverAndAssignerBeforeDALRemoval()
throws Exception {
+ final DataRegionId regionId = new DataRegionId(REGION_ID);
+ final StorageEngine engine = StorageEngine.getInstance();
+ final Field regionsField =
StorageEngine.class.getDeclaredField("dataRegionMap");
+ regionsField.setAccessible(true);
+ final Map<DataRegionId, DataRegion> regions =
+ (Map<DataRegionId, DataRegion>) regionsField.get(engine);
+ final DataRegion region = mock(DataRegion.class);
+ when(region.getDatabaseName()).thenReturn("root.dal_cleanup");
+
when(region.getDataRegionIdString()).thenReturn(Integer.toString(REGION_ID));
+ final PipeInsertionDataNodeListener listener =
PipeInsertionDataNodeListener.getInstance();
+ final Field assignerField =
+
PipeInsertionDataNodeListener.class.getDeclaredField("dataRegionId2Assigner");
+ assignerField.setAccessible(true);
+ final Map<Integer, PipeDataRegionAssigner> assigners =
+ (Map<Integer, PipeDataRegionAssigner>) assignerField.get(listener);
+ final PipeDataRegionAssigner assigner = mock(PipeDataRegionAssigner.class);
+ final IoTConsensusV2ReceiverAgent receiver =
PipeDataNodeAgent.receiver().iotConsensusV2();
+ final Field staleField =
IoTConsensusV2ReceiverAgent.class.getDeclaredField("staleRegions");
+ staleField.setAccessible(true);
+ final Set<Integer> stale = (Set<Integer>) staleField.get(receiver);
+ DeletionResourceManager.getInstance(REGION_ID);
+ doAnswer(
+ invocation -> {
+ verify(assigner).close();
+ Assert.assertFalse(assigners.containsKey(REGION_ID));
+ Assert.assertTrue(stale.contains(REGION_ID));
+
Assert.assertNull(DeletionResourceManager.getInstance(REGION_ID));
+ Assert.assertTrue(regionDirectory().exists());
+ DeletionResourceManager.closeAndRemoveDAL(REGION_ID);
+ return null;
+ })
+ .when(region)
+ .deleteDALFolderAndClose();
+ regions.put(regionId, region);
+ assigners.put(REGION_ID, assigner);
+ try {
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
engine.deleteDataRegion(regionId).getCode());
+ verify(region).deleteDALFolderAndClose();
+ Assert.assertFalse(regionDirectory().exists());
+ } finally {
+ regions.remove(regionId, region);
+ assigners.remove(REGION_ID, assigner);
+ listener.onRegionCreated(REGION_ID);
+ stale.remove(REGION_ID);
+ }
+ }
+
+ private void assertWaiting(Future<?> operation) throws Exception {
+ try {
+ operation.get(200, TimeUnit.MILLISECONDS);
+ Assert.fail("Operation must wait for the concurrent lifecycle
transition");
+ } catch (TimeoutException expected) {
+ // The other operation is deliberately paused while holding the
per-region lock.
+ }
+ }
+
+ private File regionDirectory() {
+ return new File(temporaryFolder.getRoot(), Integer.toString(REGION_ID));
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
index 59d0d2862bd..fdc11c91d1d 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/deletion/persist/PageCacheDeletionBufferTest.java
@@ -24,11 +24,16 @@ import
org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource;
import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
import org.apache.iotdb.db.pipe.consensus.deletion.recover.DeletionReader;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Assert;
@@ -37,6 +42,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.InOrder;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -45,11 +51,20 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import static org.mockito.AdditionalAnswers.delegatesTo;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
@@ -75,6 +90,178 @@ public class PageCacheDeletionBufferTest {
PageCacheDeletionBuffer.setDalBufferSize(previousDalBufferSize);
}
+ @Test
+ public void testIdleCloseDoesNotWarnAboutQueueInterruption() throws
Exception {
+ assertQueueInterruptionLogging(true);
+ }
+
+ @Test
+ public void testUnexpectedQueueInterruptionStillWarns() throws Exception {
+ assertQueueInterruptionLogging(false);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void assertQueueInterruptionLogging(boolean closeNormally) throws
Exception {
+ final PageCacheDeletionBuffer deletionBuffer =
+ new PageCacheDeletionBuffer(1,
temporaryFolder.newFolder("interrupt").getAbsolutePath());
+ final CountDownLatch taking = new CountDownLatch(1);
+ final AtomicBoolean queueInterrupted = new AtomicBoolean();
+ final AtomicReference<Thread> worker = new AtomicReference<>();
+ final Field queueField =
PageCacheDeletionBuffer.class.getDeclaredField("deletionResources");
+ queueField.setAccessible(true);
+ final BlockingQueue<DeletionResource> originalQueue =
+ (BlockingQueue<DeletionResource>) queueField.get(deletionBuffer);
+ final BlockingQueue<DeletionResource> queue =
+ mock(BlockingQueue.class, delegatesTo(originalQueue));
+ doAnswer(
+ invocation -> {
+ worker.set(Thread.currentThread());
+ taking.countDown();
+ try {
+ return originalQueue.poll(invocation.getArgument(0),
invocation.getArgument(1));
+ } catch (InterruptedException e) {
+ queueInterrupted.set(true);
+ throw e;
+ }
+ })
+ .when(queue)
+ .poll(anyLong(), any(TimeUnit.class));
+ queueField.set(deletionBuffer, queue);
+ final Logger logger = (Logger)
LoggerFactory.getLogger(PageCacheDeletionBuffer.class);
+ final Appender<ILoggingEvent> appender = mock(Appender.class);
+ logger.addAppender(appender);
+ deletionBuffer.start();
+ try {
+ Assert.assertTrue(taking.await(10, TimeUnit.SECONDS));
+ if (!closeNormally) {
+ worker.get().interrupt();
+ Awaitility.await()
+ .atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ verify(appender, atLeastOnce())
+ .doAppend(argThat(this::isQueueInterruptionWarning)));
+ }
+ } finally {
+ try {
+ deletionBuffer.close();
+ if (closeNormally) {
+ verify(appender,
never()).doAppend(argThat(this::isQueueInterruptionWarning));
+ Assert.assertFalse(queueInterrupted.get());
+ }
+ } finally {
+ logger.detachAppender(appender);
+ }
+ }
+ }
+
+ @Test(timeout = 30000)
+ @SuppressWarnings("unchecked")
+ public void testCloseWhileBatchPollingFlushesWithoutInterruptingWorker()
throws Exception {
+ final PageCacheDeletionBuffer buffer =
+ new PageCacheDeletionBuffer(1,
temporaryFolder.newFolder("batch-close").getAbsolutePath());
+ final Field queueField =
PageCacheDeletionBuffer.class.getDeclaredField("deletionResources");
+ queueField.setAccessible(true);
+ final BlockingQueue<DeletionResource> original =
+ (BlockingQueue<DeletionResource>) queueField.get(buffer);
+ final BlockingQueue<DeletionResource> queue = mock(BlockingQueue.class,
delegatesTo(original));
+ final CountDownLatch collectingBatch = new CountDownLatch(1);
+ final CountDownLatch batchCanContinue = new CountDownLatch(1);
+ final AtomicBoolean queueInterrupted = new AtomicBoolean();
+ doAnswer(
+ invocation -> {
+ try {
+ if (invocation.getArgument(1) == TimeUnit.NANOSECONDS) {
+ collectingBatch.countDown();
+ Assert.assertTrue(batchCanContinue.await(10,
TimeUnit.SECONDS));
+ }
+ return original.poll(invocation.getArgument(0),
invocation.getArgument(1));
+ } catch (InterruptedException e) {
+ queueInterrupted.set(true);
+ throw e;
+ }
+ })
+ .when(queue)
+ .poll(anyLong(), any(TimeUnit.class));
+ queueField.set(buffer, queue);
+ final DeletionResource deletion = createDeletion(1, 1);
+ final ExecutorService closer = Executors.newSingleThreadExecutor();
+ buffer.start();
+ try {
+ buffer.registerDeletionResource(deletion);
+ Assert.assertTrue(collectingBatch.await(10, TimeUnit.SECONDS));
+ final Future<?> close = closer.submit(buffer::close);
+ final Field closed =
PageCacheDeletionBuffer.class.getDeclaredField("isClosed");
+ closed.setAccessible(true);
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() ->
closed.getBoolean(buffer));
+ batchCanContinue.countDown();
+ close.get(10, TimeUnit.SECONDS);
+ } finally {
+ batchCanContinue.countDown();
+ closer.shutdownNow();
+ Assert.assertTrue(closer.awaitTermination(10, TimeUnit.SECONDS));
+ }
+ Assert.assertSame(DeletionResource.Status.SUCCESS,
deletion.waitForResult());
+ // ThreadPoolExecutor may interrupt an idle worker during shutdown; assert
the DAL wait itself.
+ Assert.assertFalse(queueInterrupted.get());
+ Assert.assertTrue(buffer.isAllDeletionFlushed());
+ }
+
+ @Test(timeout = 30000)
+ @SuppressWarnings("unchecked")
+ public void testFlushTimeoutWarnsAndForceInterruptsBatchPoll() throws
Exception {
+ final PageCacheDeletionBuffer buffer =
+ new PageCacheDeletionBuffer(
+ 1, temporaryFolder.newFolder("timeout-close").getAbsolutePath());
+ final Field queueField =
PageCacheDeletionBuffer.class.getDeclaredField("deletionResources");
+ queueField.setAccessible(true);
+ final BlockingQueue<DeletionResource> original =
+ (BlockingQueue<DeletionResource>) queueField.get(buffer);
+ final BlockingQueue<DeletionResource> queue = mock(BlockingQueue.class,
delegatesTo(original));
+ final CountDownLatch batchPollStarted = new CountDownLatch(1);
+ final CountDownLatch blockBatch = new CountDownLatch(1);
+ doAnswer(
+ invocation -> {
+ if (invocation.getArgument(1) == TimeUnit.NANOSECONDS) {
+ batchPollStarted.countDown();
+ blockBatch.await();
+ }
+ return original.poll(invocation.getArgument(0),
invocation.getArgument(1));
+ })
+ .when(queue)
+ .poll(anyLong(), any(TimeUnit.class));
+ queueField.set(buffer, queue);
+ final Logger logger = (Logger)
LoggerFactory.getLogger(PageCacheDeletionBuffer.class);
+ final Appender<ILoggingEvent> appender = mock(Appender.class);
+ logger.addAppender(appender);
+ final DeletionResource deletion = createDeletion(1, 1);
+ buffer.start();
+ try {
+ buffer.registerDeletionResource(deletion);
+ Assert.assertTrue(batchPollStarted.await(10, TimeUnit.SECONDS));
+ buffer.close();
+ verify(appender)
+ .doAppend(
+ argThat(
+ event ->
+ event.getLevel() == Level.WARN
+ && DataNodePipeMessages
+
.LOG_DELETION_PERSIST_ARG_TIMED_OUT_WAITING_FOR_ARG_DELETIONS_TO_FLUSH_BEFORE_SHUTDOWN_52469B0D
+ .equals(event.getMessage())));
+ Assert.assertSame(DeletionResource.Status.FAILURE,
deletion.waitForResult());
+ Assert.assertNotNull(deletion.getCause());
+ } finally {
+ blockBatch.countDown();
+ logger.detachAppender(appender);
+ }
+ }
+
+ private boolean isQueueInterruptionWarning(ILoggingEvent event) {
+ return event.getLevel() == Level.WARN
+ &&
DataNodePipeMessages.INTERRUPTED_WHEN_WAITING_FOR_TAKING_DELETIONRESOURCE_FROM.equals(
+ event.getMessage());
+ }
+
@Test
public void testDequeuedDeletionIsNotReportedAsFlushed() throws Exception {
final CountDownLatch serializationStarted = new CountDownLatch(1);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
index 08240ae24f7..f084c63129f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListenerTest.java
@@ -19,8 +19,10 @@
package org.apache.iotdb.db.pipe.source.dataregion.realtime.listener;
+import
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner.PipeDataRegionAssigner;
+import org.junit.Assert;
import org.junit.Test;
import java.lang.reflect.Field;
@@ -35,6 +37,35 @@ import static org.mockito.Mockito.when;
public class PipeInsertionDataNodeListenerTest {
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRegionDeletionDrainsAssignerAndBlocksLateRegistration()
throws Exception {
+ final PipeInsertionDataNodeListener listener =
PipeInsertionDataNodeListener.getInstance();
+ final Field field =
+
PipeInsertionDataNodeListener.class.getDeclaredField("dataRegionId2Assigner");
+ field.setAccessible(true);
+ final ConcurrentMap<Integer, PipeDataRegionAssigner> assigners =
+ (ConcurrentMap<Integer, PipeDataRegionAssigner>) field.get(listener);
+ final int id = Integer.MIN_VALUE + 2;
+ final PipeDataRegionAssigner assigner = mock(PipeDataRegionAssigner.class);
+ assigners.put(id, assigner);
+ try {
+ listener.releaseDataRegion(id);
+ verify(assigner).close();
+ Assert.assertFalse(assigners.containsKey(id));
+ listener.startListenAndAssign(id,
mock(PipeRealtimeDataRegionSource.class));
+ Assert.assertFalse(assigners.containsKey(id));
+ listener.onRegionCreated(id);
+ assigners.put(id, assigner);
+ final PipeRealtimeDataRegionSource source =
mock(PipeRealtimeDataRegionSource.class);
+ listener.startListenAndAssign(id, source);
+ verify(assigner).startAssignTo(source);
+ } finally {
+ assigners.remove(id);
+ listener.onRegionCreated(id);
+ }
+ }
+
@Test
@SuppressWarnings("unchecked")
public void testInsertUsesBarrierInvalidatingPublicationPath() throws
Exception {