This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b2a6c642e53 Pipe: Fix potential resource leaks in pipe lifecycle
cleanup (#17666)
b2a6c642e53 is described below
commit b2a6c642e533fef9e0c6e73f5cc81334db8ed03f
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 19 09:54:10 2026 +0800
Pipe: Fix potential resource leaks in pipe lifecycle cleanup (#17666)
* Fix
* leak_fix
* spotless
* Restore schema pipe queue from snapshots
* Pipe: Avoid recreating committer state for late commits
* Pipe: Avoid stale metrics state after drop
* Pipe: Serialize late metric creation with drop
* Declare Logback test dependency in node-commons
* Pipe: prevent stale state after pipe drop
* Pipe: balance late memory accounting and avoid passive listener recreation
* Pipe: clear shared runtime state on pipe drop
* Fix late Pipe cleanup state recreation
* Pipe: reset stale schema queues when loading snapshots
* Pipe: add schema snapshot recovery IT
* Pipe: clean unused schema listener after snapshot load
* Move snapshot recovery test to DataNode suite
---
.../schemaregion/SchemaExecutionVisitor.java | 21 +-
.../schemaregion/SchemaRegionStateMachine.java | 49 ++-
.../agent/runtime/PipeDataNodeRuntimeAgent.java | 18 +
.../runtime/PipeSchemaRegionListenerManager.java | 101 +++--
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 13 +-
.../overview/PipeDataNodeSinglePipeMetrics.java | 157 +++++---
.../overview/PipeTsFileToTabletsMetrics.java | 18 +-
.../processor/aggregate/AggregateProcessor.java | 20 +-
.../thrift/async/IoTDBDataRegionAsyncSink.java | 1 +
.../thrift/sync/IoTDBDataRegionSyncSink.java | 1 +
.../schemaregion/IoTDBSchemaRegionSource.java | 10 +
.../schemaregion/SchemaRegionListeningQueue.java | 4 +
.../thrift/impl/DataNodeRegionManager.java | 8 +-
.../PipeSchemaRegionListenerManagerTest.java | 247 ++++++++++++
.../PipeSchemaRegionSnapshotRecoveryTest.java | 281 ++++++++++++++
.../db/pipe/metric/PipeMetricsRestartTest.java | 24 ++
.../PipeDataNodeSinglePipeMetricsTest.java | 245 ++++++++++++
iotdb-core/node-commons/pom.xml | 5 +
.../commons/pipe/agent/task/PipeTaskAgent.java | 122 ++++--
.../task/progress/PipeEventCommitManager.java | 59 ++-
.../pipe/sink/limiter/PipeEndPointRateLimiter.java | 9 +-
.../commons/pipe/sink/protocol/IoTDBSink.java | 24 +-
.../pipe/source/IoTDBNonDataRegionSource.java | 12 +-
.../commons/pipe/agent/task/PipeTaskAgentTest.java | 421 +++++++++++++++++++++
.../task/progress/PipeEventCommitManagerTest.java | 277 ++++++++++++++
.../sink/protocol/IoTDBSinkRateLimiterTest.java | 99 +++++
26 files changed, 2057 insertions(+), 189 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
index 734ef457063..f1f4c8c9e50 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
@@ -1060,13 +1060,20 @@ public class SchemaExecutionVisitor implements
PlanVisitor<TSStatus, ISchemaRegi
public TSStatus visitPipeOperateSchemaQueueNode(
final PipeOperateSchemaQueueNode node, final ISchemaRegion schemaRegion)
{
final SchemaRegionId id = schemaRegion.getSchemaRegionId();
- final SchemaRegionListeningQueue queue =
PipeDataNodeAgent.runtime().schemaListener(id);
- if (node.isOpen() && !queue.isOpened()) {
- logger.info(DataNodeMiscMessages.OPENED_PIPE_LISTENING_QUEUE, id);
- queue.open();
- } else if (!node.isOpen() && queue.isOpened()) {
- logger.info(DataNodeMiscMessages.CLOSED_PIPE_LISTENING_QUEUE, id);
- queue.close();
+ if (node.isOpen()) {
+ final SchemaRegionListeningQueue queue =
PipeDataNodeAgent.runtime().schemaListener(id);
+ if (!queue.isOpened()) {
+ logger.info(DataNodeMiscMessages.OPENED_PIPE_LISTENING_QUEUE, id);
+ queue.open();
+ }
+ } else {
+ final SchemaRegionListeningQueue queue =
+ PipeDataNodeAgent.runtime().schemaListenerIfPresent(id);
+ if (queue != null && queue.isOpened()) {
+ logger.info(DataNodeMiscMessages.CLOSED_PIPE_LISTENING_QUEUE, id);
+ queue.close();
+ PipeDataNodeAgent.runtime().cleanupSchemaListenerIfUnused(id);
+ }
}
return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
index 947c64aeb26..65e58648236 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
@@ -128,14 +128,21 @@ public class SchemaRegionStateMachine extends
BaseStateMachine {
@Override
public boolean takeSnapshot(final File snapshotDir) {
- if (schemaRegion.createSnapshot(snapshotDir)
- && PipeDataNodeAgent.runtime()
- .schemaListener(schemaRegion.getSchemaRegionId())
- .createSnapshot(snapshotDir)) {
- listen2Snapshot4PipeListener(true);
+ if (!schemaRegion.createSnapshot(snapshotDir)) {
+ return false;
+ }
+
+ final SchemaRegionListeningQueue listener =
+
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId());
+ if (listener == null) {
return true;
}
- return false;
+
+ if (!listener.createSnapshot(snapshotDir)) {
+ return false;
+ }
+ listen2Snapshot4PipeListener(true);
+ return true;
}
@Override
@@ -145,12 +152,17 @@ public class SchemaRegionStateMachine extends
BaseStateMachine {
// callers (e.g. the AddPeer flow and the Ratis snapshot-install path)
can detect a real
// failure instead of treating a fallback-to-empty load as success.
final boolean loadSucceeded =
schemaRegion.loadSnapshot(latestSnapshotRootDir);
- PipeDataNodeAgent.runtime()
- .schemaListener(schemaRegion.getSchemaRegionId())
- .loadSnapshot(latestSnapshotRootDir);
- // We recompute the snapshot for pipe listener when loading snapshot
- // to recover the newest snapshot in cache
- listen2Snapshot4PipeListener(false);
+ final SchemaRegionListeningQueue listener =
+ PipeDataNodeAgent.runtime()
+ .schemaListenerForSnapshotLoad(
+ schemaRegion.getSchemaRegionId(), latestSnapshotRootDir);
+ if (listener != null) {
+ listener.loadSnapshot(latestSnapshotRootDir);
+
PipeDataNodeAgent.runtime().cleanupSchemaListenerIfUnused(schemaRegion.getSchemaRegionId());
+ // We recompute the snapshot for pipe listener when loading snapshot
+ // to recover the newest snapshot in cache
+ listen2Snapshot4PipeListener(false);
+ }
return loadSucceeded;
} catch (Exception e) {
logger.error(
@@ -169,7 +181,10 @@ public class SchemaRegionStateMachine extends
BaseStateMachine {
.toString(),
isTmp);
final SchemaRegionListeningQueue listener =
-
PipeDataNodeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId());
+
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId());
+ if (listener == null) {
+ return;
+ }
if (Objects.isNull(snapshotPaths) || Objects.isNull(snapshotPaths.get(0)))
{
if (listener.isOpened()) {
logger.warn(
@@ -197,9 +212,11 @@ public class SchemaRegionStateMachine extends
BaseStateMachine {
final TSStatus result =
((PlanNode) request).accept(new SchemaExecutionVisitor(),
schemaRegion);
if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- PipeDataNodeAgent.runtime()
- .schemaListener(schemaRegion.getSchemaRegionId())
- .tryListenToNode((PlanNode) request);
+ final SchemaRegionListeningQueue listener =
+
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegion.getSchemaRegionId());
+ if (listener != null) {
+ listener.tryListenToNode((PlanNode) request);
+ }
}
return result;
} catch (final IllegalArgumentException e) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
index 87fe67e75b7..1353605d722 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
@@ -58,6 +58,7 @@ import org.apache.tsfile.file.metadata.IDeviceID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -178,6 +179,15 @@ public class PipeDataNodeRuntimeAgent implements IService {
return regionListenerManager.listener(schemaRegionId);
}
+ public SchemaRegionListeningQueue schemaListenerIfPresent(SchemaRegionId
schemaRegionId) {
+ return regionListenerManager.listenerIfPresent(schemaRegionId);
+ }
+
+ public SchemaRegionListeningQueue schemaListenerForSnapshotLoad(
+ SchemaRegionId schemaRegionId, File snapshotDir) {
+ return regionListenerManager.listenerForSnapshotLoad(schemaRegionId,
snapshotDir);
+ }
+
public int increaseAndGetSchemaListenerReferenceCount(SchemaRegionId
schemaRegionId) {
return regionListenerManager.increaseAndGetReferenceCount(schemaRegionId);
}
@@ -186,6 +196,14 @@ public class PipeDataNodeRuntimeAgent implements IService {
return regionListenerManager.decreaseAndGetReferenceCount(schemaRegionId);
}
+ public void cleanupSchemaListenerIfUnused(SchemaRegionId schemaRegionId) {
+ regionListenerManager.cleanupListenerIfUnused(schemaRegionId);
+ }
+
+ public void clearSchemaRegionState(SchemaRegionId schemaRegionId) {
+ regionListenerManager.clearSchemaRegionState(schemaRegionId);
+ }
+
public void notifySchemaLeaderReady(SchemaRegionId schemaRegionId) {
regionListenerManager.notifyLeaderReady(schemaRegionId);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java
index d28929b36a2..61a904c339c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManager.java
@@ -20,10 +20,10 @@
package org.apache.iotdb.db.pipe.agent.runtime;
import org.apache.iotdb.commons.consensus.SchemaRegionId;
-import org.apache.iotdb.commons.pipe.agent.task.PipeTask;
import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics;
import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
+import java.io.File;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -34,6 +34,7 @@ public class PipeSchemaRegionListenerManager {
private final Map<SchemaRegionId, PipeSchemaRegionListener> id2ListenerMap =
new ConcurrentHashMap<>();
+ private final Map<SchemaRegionId, AtomicBoolean> id2LeaderReadyMap = new
ConcurrentHashMap<>();
public synchronized Set<SchemaRegionId> regionIds() {
return id2ListenerMap.keySet();
@@ -44,6 +45,27 @@ public class PipeSchemaRegionListenerManager {
.listeningQueue;
}
+ public synchronized SchemaRegionListeningQueue listenerIfPresent(
+ final SchemaRegionId schemaRegionId) {
+ final PipeSchemaRegionListener listener =
id2ListenerMap.get(schemaRegionId);
+ return listener == null ? null : listener.listeningQueue;
+ }
+
+ public synchronized SchemaRegionListeningQueue listenerForSnapshotLoad(
+ final SchemaRegionId schemaRegionId, final File snapshotDir) {
+ final PipeSchemaRegionListener listener =
id2ListenerMap.get(schemaRegionId);
+ if (!SchemaRegionListeningQueue.hasSnapshot(snapshotDir)) {
+ // The snapshot can replace a follower that missed the queue-close log
entry. Absence of a
+ // queue snapshot represents a closed queue, so discard the follower's
previous contents.
+ if (listener != null) {
+ listener.listeningQueue.close();
+ cleanupListenerIfUnused(schemaRegionId, listener);
+ }
+ return null;
+ }
+ return listener == null ? listener(schemaRegionId) :
listener.listeningQueue;
+ }
+
public synchronized int increaseAndGetReferenceCount(final SchemaRegionId
schemaRegionId) {
return id2ListenerMap
.computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
@@ -52,28 +74,55 @@ public class PipeSchemaRegionListenerManager {
}
public synchronized int decreaseAndGetReferenceCount(final SchemaRegionId
schemaRegionId) {
- return id2ListenerMap
- .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
- .listeningQueueReferenceCount
- .updateAndGet(v -> v > 0 ? v - 1 : 0);
+ final PipeSchemaRegionListener listener =
id2ListenerMap.get(schemaRegionId);
+ if (listener == null) {
+ return 0;
+ }
+
+ final int referenceCount =
+ listener.listeningQueueReferenceCount.updateAndGet(v -> v > 0 ? v - 1
: 0);
+ if (referenceCount == 0 && !listener.listeningQueue.isOpened()) {
+ cleanupListenerIfUnused(schemaRegionId, listener);
+ }
+ return referenceCount;
}
public synchronized void notifyLeaderReady(final SchemaRegionId
schemaRegionId) {
- id2ListenerMap
- .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
- .notifyLeaderReady();
+ id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new
AtomicBoolean()).set(true);
}
public synchronized void notifyLeaderUnavailable(final SchemaRegionId
schemaRegionId) {
- id2ListenerMap
- .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
- .notifyLeaderUnavailable();
+ id2LeaderReadyMap.computeIfAbsent(schemaRegionId, id -> new
AtomicBoolean()).set(false);
}
public synchronized boolean isLeaderReady(final SchemaRegionId
schemaRegionId) {
- return id2ListenerMap
- .computeIfAbsent(schemaRegionId, PipeSchemaRegionListener::new)
- .isLeaderReady();
+ final AtomicBoolean isLeaderReady = id2LeaderReadyMap.get(schemaRegionId);
+ return isLeaderReady != null && isLeaderReady.get();
+ }
+
+ public synchronized void cleanupListenerIfUnused(final SchemaRegionId
schemaRegionId) {
+ final PipeSchemaRegionListener listener =
id2ListenerMap.get(schemaRegionId);
+ if (listener != null) {
+ cleanupListenerIfUnused(schemaRegionId, listener);
+ }
+ }
+
+ public synchronized void clearSchemaRegionState(final SchemaRegionId
schemaRegionId) {
+ final PipeSchemaRegionListener listener =
id2ListenerMap.remove(schemaRegionId);
+ if (listener != null) {
+
PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId());
+ }
+ id2LeaderReadyMap.remove(schemaRegionId);
+ }
+
+ private void cleanupListenerIfUnused(
+ final SchemaRegionId schemaRegionId, final PipeSchemaRegionListener
listener) {
+ if (listener.listeningQueueReferenceCount.get() > 0 ||
listener.listeningQueue.isOpened()) {
+ return;
+ }
+ if (id2ListenerMap.remove(schemaRegionId, listener)) {
+
PipeSchemaRegionListenerMetrics.getInstance().deregister(schemaRegionId.getId());
+ }
}
private static class PipeSchemaRegionListener {
@@ -81,33 +130,9 @@ public class PipeSchemaRegionListenerManager {
private final SchemaRegionListeningQueue listeningQueue = new
SchemaRegionListeningQueue();
private final AtomicInteger listeningQueueReferenceCount = new
AtomicInteger(0);
- private final AtomicBoolean isLeaderReady = new AtomicBoolean(false);
-
protected PipeSchemaRegionListener(final SchemaRegionId schemaRegionId) {
PipeSchemaRegionListenerMetrics.getInstance()
.register(listeningQueue, schemaRegionId.getId());
}
-
- /**
- * Get leader ready state, DO NOT use consensus layer's leader ready flag
because
- * SimpleConsensus' ready flag is always {@code true}. Note that this flag
has nothing to do
- * with listening and a {@link PipeTask} starts only iff the current node
is a leader and ready.
- *
- * @return {@code true} iff the current node is a leader and ready
- */
- private boolean isLeaderReady() {
- return isLeaderReady.get();
- }
-
- // Leader ready flag has the following effect
- // 1. The linked list starts serving only after leader gets ready
- // 2. Config pipe task is only created after leader gets ready
- private void notifyLeaderReady() {
- isLeaderReady.set(true);
- }
-
- private void notifyLeaderUnavailable() {
- isLeaderReady.set(false);
- }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
index 4bd1fc9f762..68711ec855a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java
@@ -70,6 +70,7 @@ import
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegio
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource;
import
org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningFilter;
+import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
@@ -457,10 +458,14 @@ public class PipeDataNodeTaskAgent extends PipeTaskAgent {
}
schemaRegionId2ListeningQueueNewFirstIndex.forEach(
- (schemaRegionId, listeningQueueNewFirstIndex) ->
- PipeDataNodeAgent.runtime()
- .schemaListener(new SchemaRegionId(schemaRegionId))
- .removeBefore(listeningQueueNewFirstIndex));
+ (schemaRegionId, listeningQueueNewFirstIndex) -> {
+ final SchemaRegionListeningQueue listeningQueue =
+ PipeDataNodeAgent.runtime()
+ .schemaListenerIfPresent(new SchemaRegionId(schemaRegionId));
+ if (listeningQueue != null) {
+ listeningQueue.removeBefore(listeningQueueNewFirstIndex);
+ }
+ });
return schemaRegionId2ListeningQueueNewFirstIndex.keySet();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
index 10c48f7d9ea..92d8499fa05 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetrics.java
@@ -162,12 +162,14 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
}
private void removeMetrics(final String pipeID) {
- removeAutoGauge(pipeID);
- }
-
- private void removeAutoGauge(final String pipeID) {
final PipeDataNodeRemainingEventAndTimeOperator operator =
remainingEventAndTimeOperatorMap.get(pipeID);
+ if (Objects.nonNull(operator) && Objects.nonNull(metricService)) {
+ removeAutoGauge(operator);
+ }
+ }
+
+ private void removeAutoGauge(final PipeDataNodeRemainingEventAndTimeOperator
operator) {
metricService.remove(
MetricType.AUTO_GAUGE,
Metric.PIPE_DATANODE_REMAINING_EVENT_COUNT.toString(),
@@ -182,7 +184,8 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
operator.getPipeName(),
Tag.CREATION_TIME.toString(),
String.valueOf(operator.getCreationTime()));
- if (completionOperatorMap.containsKey(pipeID)) {
+ if (completionOperatorMap.containsKey(
+ generatePipeID(operator.getPipeName(), operator.getCreationTime()))) {
metricService.remove(
MetricType.AUTO_GAUGE,
Metric.PIPE_DATANODE_COMPLETION_READY.toString(),
@@ -226,9 +229,40 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
//////////////////////////// register & deregister (pipe integration)
////////////////////////////
+ private static String generatePipeID(final String pipeName, final long
creationTime) {
+ return pipeName + "_" + creationTime;
+ }
+
+ private boolean isPipeAlive(final String pipeName, final long creationTime) {
+ return PipeDataNodeAgent.task().isPipeAlive(pipeName, creationTime);
+ }
+
+ private PipeDataNodeRemainingEventAndTimeOperator
getOrCreateOperatorIfPipeAlive(
+ final String pipeName, final long creationTime) {
+ final String pipeID = generatePipeID(pipeName, creationTime);
+ PipeDataNodeRemainingEventAndTimeOperator operator =
+ remainingEventAndTimeOperatorMap.get(pipeID);
+ if (Objects.nonNull(operator)) {
+ return operator;
+ }
+
+ // Serialize creation with deregistration so a late update cannot restore
a removed operator.
+ synchronized (this) {
+ operator = remainingEventAndTimeOperatorMap.get(pipeID);
+ if (Objects.nonNull(operator)) {
+ return operator;
+ }
+ if (!isPipeAlive(pipeName, creationTime)) {
+ return null;
+ }
+ return remainingEventAndTimeOperatorMap.computeIfAbsent(
+ pipeID, key -> new
PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime));
+ }
+ }
+
public synchronized void register(final IoTDBDataRegionSource source) {
// The metric is global thus the regionId is omitted
- final String pipeID = source.getPipeName() + "_" +
source.getCreationTime();
+ final String pipeID = generatePipeID(source.getPipeName(),
source.getCreationTime());
final PipeDataNodeRemainingEventAndTimeOperator remainingOperator =
remainingEventAndTimeOperatorMap.computeIfAbsent(
pipeID,
@@ -318,7 +352,7 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
public synchronized void register(final IoTDBSchemaRegionSource source) {
// The metric is global thus the regionId is omitted
- final String pipeID = source.getPipeName() + "_" +
source.getCreationTime();
+ final String pipeID = generatePipeID(source.getPipeName(),
source.getCreationTime());
remainingEventAndTimeOperatorMap
.computeIfAbsent(
pipeID,
@@ -332,19 +366,20 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
}
public void increaseInsertNodeEventCount(final String pipeName, final long
creationTime) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .increaseInsertNodeEventCount();
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ getOrCreateOperatorIfPipeAlive(pipeName, creationTime);
+ if (Objects.nonNull(operator)) {
+ operator.increaseInsertNodeEventCount();
+ }
}
public void decreaseInsertNodeEventCount(
final String pipeName, final long creationTime, final long transferTime)
{
final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime));
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.isNull(operator)) {
+ return;
+ }
operator.decreaseInsertNodeEventCount();
@@ -355,46 +390,44 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
public void updateInsertNodeTransferTimer(
final String pipeName, final long creationTime, final long transferTime)
{
- if (transferTime > 0) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .getInsertNodeTransferTimer()
- .update(transferTime, TimeUnit.NANOSECONDS);
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.nonNull(operator) && transferTime > 0) {
+ operator.getInsertNodeTransferTimer().update(transferTime,
TimeUnit.NANOSECONDS);
}
}
public void increaseRawTabletEventCount(final String pipeName, final long
creationTime) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .increaseRawTabletEventCount();
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ getOrCreateOperatorIfPipeAlive(pipeName, creationTime);
+ if (Objects.nonNull(operator)) {
+ operator.increaseRawTabletEventCount();
+ }
}
public void decreaseRawTabletEventCount(final String pipeName, final long
creationTime) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .decreaseRawTabletEventCount();
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.nonNull(operator)) {
+ operator.decreaseRawTabletEventCount();
+ }
}
public void increaseTsFileEventCount(final String pipeName, final long
creationTime) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .increaseTsFileEventCount();
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ getOrCreateOperatorIfPipeAlive(pipeName, creationTime);
+ if (Objects.nonNull(operator)) {
+ operator.increaseTsFileEventCount();
+ }
}
public void decreaseTsFileEventCount(
final String pipeName, final long creationTime, final long transferTime)
{
final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime));
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.isNull(operator)) {
+ return;
+ }
operator.decreaseTsFileEventCount();
@@ -405,28 +438,25 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
public void updateTsFileTransferTimer(
final String pipeName, final long creationTime, final long transferTime)
{
- if (transferTime > 0) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .getTsFileTransferTimer()
- .update(transferTime, TimeUnit.NANOSECONDS);
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.nonNull(operator) && transferTime > 0) {
+ operator.getTsFileTransferTimer().update(transferTime,
TimeUnit.NANOSECONDS);
}
}
public void increaseHeartbeatEventCount(final String pipeName, final long
creationTime) {
- remainingEventAndTimeOperatorMap
- .computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime))
- .increaseHeartbeatEventCount();
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ getOrCreateOperatorIfPipeAlive(pipeName, creationTime);
+ if (Objects.nonNull(operator)) {
+ operator.increaseHeartbeatEventCount();
+ }
}
public void decreaseHeartbeatEventCount(final String pipeName, final long
creationTime) {
final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.get(pipeName + "_" + creationTime);
- if (operator != null) {
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.nonNull(operator)) {
operator.decreaseHeartbeatEventCount();
}
}
@@ -452,12 +482,16 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
}
public synchronized void deregister(final String pipeID) {
- if (!remainingEventAndTimeOperatorMap.containsKey(pipeID)) {
+ final PipeDataNodeRemainingEventAndTimeOperator operator =
+ remainingEventAndTimeOperatorMap.remove(pipeID);
+ if (Objects.isNull(operator)) {
LOGGER.warn(DataNodePipeMessages.FAILED_TO_DEREGISTER_PIPE_REMAINING_EVENT_AND,
pipeID);
- } else if (Objects.nonNull(metricService)) {
- removeMetrics(pipeID);
+ completionOperatorMap.remove(pipeID);
+ return;
+ }
+ if (Objects.nonNull(metricService)) {
+ removeAutoGauge(operator);
}
- remainingEventAndTimeOperatorMap.remove(pipeID);
completionOperatorMap.remove(pipeID);
}
@@ -500,9 +534,10 @@ public class PipeDataNodeSinglePipeMetrics implements
IMetricSet {
public Pair<Long, Double> getRemainingEventAndTime(
final String pipeName, final long creationTime) {
final PipeDataNodeRemainingEventAndTimeOperator operator =
- remainingEventAndTimeOperatorMap.computeIfAbsent(
- pipeName + "_" + creationTime,
- k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName,
creationTime));
+ remainingEventAndTimeOperatorMap.get(generatePipeID(pipeName,
creationTime));
+ if (Objects.isNull(operator)) {
+ return new Pair<>(0L, 0.0);
+ }
return new Pair<>(operator.getRemainingNonHeartbeatEvents(),
operator.getRemainingTime());
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
index 39aef7896eb..90878ab3c4d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeTsFileToTabletsMetrics.java
@@ -109,39 +109,47 @@ public class PipeTsFileToTabletsMetrics implements
IMetricSet {
}
private void removeMetrics(final String pipeID) {
+ pipeTimerMap.remove(pipeID);
metricService.remove(
MetricType.TIMER,
Metric.PIPE_TSFILE_TO_TABLETS_TIME.toString(),
Tag.NAME.toString(),
pipeID);
- pipeTimerMap.remove(pipeID);
+ pipeRateMap.remove(pipeID);
metricService.remove(
MetricType.RATE,
Metric.PIPE_TSFILE_TO_TABLETS_RATE.toString(),
Tag.NAME.toString(),
pipeID);
- pipeRateMap.remove(pipeID);
+ pipeTabletCountMap.remove(pipeID);
metricService.remove(
MetricType.COUNTER,
Metric.PIPE_TSFILE_TO_TABLETS_COUNT.toString(),
Tag.NAME.toString(),
pipeID);
- pipeTabletCountMap.remove(pipeID);
+ pipeTabletMemoryMap.remove(pipeID);
metricService.remove(
MetricType.COUNTER,
Metric.PIPE_TSFILE_TO_TABLETS_TOTAL_MEMORY.toString(),
Tag.NAME.toString(),
pipeID);
- pipeTabletMemoryMap.remove(pipeID);
+ pipeParseFileCountMap.remove(pipeID);
metricService.remove(
MetricType.COUNTER,
Metric.PIPE_TSFILE_PARSE_FILE_COUNT.toString(),
Tag.NAME.toString(),
pipeID);
+ }
+
+ private void removePipeStats(final String pipeID) {
+ pipeTimerMap.remove(pipeID);
+ pipeRateMap.remove(pipeID);
+ pipeTabletCountMap.remove(pipeID);
+ pipeTabletMemoryMap.remove(pipeID);
pipeParseFileCountMap.remove(pipeID);
}
@@ -163,6 +171,8 @@ public class PipeTsFileToTabletsMetrics implements
IMetricSet {
try {
if (Objects.nonNull(metricService)) {
removeMetrics(pipeID);
+ } else {
+ removePipeStats(pipeID);
}
} finally {
pipe.remove(pipeID);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
index ff5d34030f6..d451a57ae6c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
@@ -758,12 +758,20 @@ public class AggregateProcessor implements PipeProcessor {
@Override
public void close() throws Exception {
- if (Objects.nonNull(pipeName)
- && pipeName2referenceCountMap.compute(
- pipeName, (name, count) -> Objects.nonNull(count) ? count - 1
: 0)
- == 0) {
- pipeName2timeSeries2TimeSeriesRuntimeStateMap.get(pipeName).clear();
- pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName);
+ boolean isLastReference = false;
+ if (Objects.nonNull(pipeName)) {
+ isLastReference =
+ pipeName2referenceCountMap.computeIfPresent(
+ pipeName, (name, count) -> count > 1 ? count - 1 : null)
+ == null;
+ }
+ if (isLastReference) {
+ final ConcurrentMap<String, AtomicReference<TimeSeriesRuntimeState>>
+ timeSeries2RuntimeStateMap =
+ pipeName2timeSeries2TimeSeriesRuntimeStateMap.remove(pipeName);
+ if (timeSeries2RuntimeStateMap != null) {
+ timeSeries2RuntimeStateMap.clear();
+ }
pipeName2LastValueReceiveTimeMap.remove(pipeName);
}
if (Objects.nonNull(windowingProcessor)) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
index 9251616d0bf..9ee114deab4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
@@ -1371,6 +1371,7 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink
implements PipeSinkWithS
@Override
public synchronized void discardEventsOfPipe(
final String pipeNameToDrop, final long creationTimeToDrop, final int
regionId) {
+ super.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId);
discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop,
regionId, -1));
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java
index a94e3fa4766..52c9b71fbb9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java
@@ -818,6 +818,7 @@ public class IoTDBDataRegionSyncSink extends
IoTDBDataNodeSyncSink {
@Override
public synchronized void discardEventsOfPipe(
final String pipeNameToDrop, final long creationTimeToDrop, final int
regionId) {
+ super.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId);
discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop,
regionId, -1));
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java
index f98590ad66b..48153fc9789 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/IoTDBSchemaRegionSource.java
@@ -330,6 +330,16 @@ public class IoTDBSchemaRegionSource extends
IoTDBNonDataRegionSource {
return PipeDataNodeAgent.runtime().schemaListener(schemaRegionId);
}
+ @Override
+ public long getUnTransferredEventCount() {
+ if (hasBeenClosed.get() || Objects.isNull(pipeTaskMeta)) {
+ return 0L;
+ }
+ // Metric callbacks can outlive region cleanup and must not recreate a
listener.
+ return getUnTransferredEventCount(
+ PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ }
+
@Override
protected boolean isTypeListened(final PipeWritePlanEvent event) {
final PlanNode planNode = ((PipeSchemaRegionWritePlanEvent)
event).getPlanNode();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java
index 3ae8bff32bb..aec6565e6af 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/schemaregion/SchemaRegionListeningQueue.java
@@ -114,6 +114,10 @@ public class SchemaRegionListeningQueue extends
AbstractPipeListeningQueue {
}
}
+ public static boolean hasSnapshot(final File snapshotDir) {
+ return snapshotDir != null && new File(snapshotDir,
SNAPSHOT_FILE_NAME).isFile();
+ }
+
public synchronized void loadSnapshot(final File snapshotDir) {
try {
super.deserializeFromFile(new File(snapshotDir, SNAPSHOT_FILE_NAME));
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java
index 9501c8e677d..357044efe5c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java
@@ -38,6 +38,7 @@ import
org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
import org.apache.iotdb.db.exception.DataRegionException;
import org.apache.iotdb.db.i18n.DataNodeMiscMessages;
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.rpc.RpcUtils;
@@ -220,7 +221,12 @@ public class DataNodeRegionManager {
if (!schemaEngine.deleteSchemaRegion(schemaRegionId)) {
return RpcUtils.getStatus(TSStatusCode.REGION_NOT_EXIST);
}
- PipeDataNodeAgent.runtime().schemaListener(schemaRegionId).close();
+ final SchemaRegionListeningQueue listener =
+ PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId);
+ if (listener != null) {
+ listener.close();
+ }
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
schemaRegionLockMap.remove(schemaRegionId);
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
} catch (MetadataException e) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java
new file mode 100644
index 00000000000..c195fedf0fc
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionListenerManagerTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.agent.runtime;
+
+import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNode;
+import
org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType;
+import
org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionListenerMetrics;
+import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource;
+import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
+import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
+
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class PipeSchemaRegionListenerManagerTest {
+
+ @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Test
+ public void testLeaderStateDoesNotCreateListener() {
+ final PipeSchemaRegionListenerManager manager = new
PipeSchemaRegionListenerManager();
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(1);
+
+ manager.notifyLeaderReady(schemaRegionId);
+
+ Assert.assertTrue(manager.isLeaderReady(schemaRegionId));
+ Assert.assertTrue(manager.regionIds().isEmpty());
+ Assert.assertNull(manager.listenerIfPresent(schemaRegionId));
+ }
+
+ @Test
+ public void testCleanupUnusedClosedListener() {
+ final PipeSchemaRegionListenerManager manager = new
PipeSchemaRegionListenerManager();
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(2);
+
+ final SchemaRegionListeningQueue listeningQueue =
manager.listener(schemaRegionId);
+ Assert.assertEquals(1,
manager.increaseAndGetReferenceCount(schemaRegionId));
+
+ listeningQueue.open();
+ Assert.assertEquals(0,
manager.decreaseAndGetReferenceCount(schemaRegionId));
+ Assert.assertNotNull(manager.listenerIfPresent(schemaRegionId));
+
+ listeningQueue.close();
+ manager.cleanupListenerIfUnused(schemaRegionId);
+
+ Assert.assertNull(manager.listenerIfPresent(schemaRegionId));
+ Assert.assertTrue(manager.regionIds().isEmpty());
+ }
+
+ @Test
+ public void testAutoCleanupAfterLastReferenceReleased() {
+ final PipeSchemaRegionListenerManager manager = new
PipeSchemaRegionListenerManager();
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(3);
+
+ manager.increaseAndGetReferenceCount(schemaRegionId);
+
+ Assert.assertEquals(0,
manager.decreaseAndGetReferenceCount(schemaRegionId));
+ Assert.assertNull(manager.listenerIfPresent(schemaRegionId));
+ Assert.assertTrue(manager.regionIds().isEmpty());
+ }
+
+ @Test
+ public void testMetricsDoNotRecreateListenerAfterRegionCleanup() throws
Exception {
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE
- 10);
+ final TestSchemaRegionSource source = new TestSchemaRegionSource();
+ final Field schemaRegionIdField =
+ IoTDBSchemaRegionSource.class.getDeclaredField("schemaRegionId");
+ schemaRegionIdField.setAccessible(true);
+ schemaRegionIdField.set(source, schemaRegionId);
+
+ try {
+ final SchemaRegionListeningQueue listener =
+ PipeDataNodeAgent.runtime().schemaListener(schemaRegionId);
+ listener.open();
+ listener.close();
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
+
+ Assert.assertEquals(0L, source.getUnTransferredEventCount());
+
Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+
+ source.close();
+ Assert.assertEquals(0L, source.getUnTransferredEventCount());
+
Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ } finally {
+ final SchemaRegionListeningQueue listener =
+ PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId);
+ if (listener != null) {
+ listener.close();
+ }
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
+ }
+ }
+
+ @Test
+ public void testCreateListenerForSnapshotLoadOnlyWhenSnapshotExists() throws
IOException {
+ final PipeSchemaRegionListenerManager manager = new
PipeSchemaRegionListenerManager();
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(4);
+ final File snapshotDir = temporaryFolder.newFolder();
+
+ Assert.assertNull(manager.listenerForSnapshotLoad(schemaRegionId,
snapshotDir));
+ Assert.assertTrue(manager.regionIds().isEmpty());
+
+ final SchemaRegionListeningQueue originalListener =
manager.listener(schemaRegionId);
+ originalListener.open();
+ Assert.assertTrue(originalListener.createSnapshot(snapshotDir));
+ originalListener.close();
+ manager.clearSchemaRegionState(schemaRegionId);
+
+ final SchemaRegionListeningQueue restoredListener =
+ manager.listenerForSnapshotLoad(schemaRegionId, snapshotDir);
+ Assert.assertNotNull(restoredListener);
+ restoredListener.loadSnapshot(snapshotDir);
+ Assert.assertTrue(restoredListener.isOpened());
+
+ restoredListener.close();
+ manager.clearSchemaRegionState(schemaRegionId);
+ }
+
+ @Test
+ public void testLoadClosedListenerSnapshotCleansUnusedListener() throws
IOException {
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE
- 13);
+ final File snapshotDir = temporaryFolder.newFolder();
+ final ISchemaRegion schemaRegion = mock(ISchemaRegion.class);
+ when(schemaRegion.getSchemaRegionId()).thenReturn(schemaRegionId);
+ when(schemaRegion.loadSnapshot(snapshotDir)).thenReturn(true);
+ final SchemaRegionStateMachine stateMachine = new
SchemaRegionStateMachine(schemaRegion);
+ final SchemaRegionListeningQueue originalListener =
+ PipeDataNodeAgent.runtime().schemaListener(schemaRegionId);
+ originalListener.close();
+ Assert.assertTrue(originalListener.createSnapshot(snapshotDir));
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
+
+ try {
+ Assert.assertTrue(stateMachine.loadSnapshot(snapshotDir));
+
Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ } finally {
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
+ }
+ }
+
+ @Test
+ public void testSnapshotWithoutListenerClearsStaleFollowerQueue() throws
Exception {
+ assertSnapshotWithoutListenerClearsStaleQueue(false);
+ }
+
+ @Test
+ public void testSnapshotWithoutListenerRetainsReferencedClosedQueue() throws
Exception {
+ assertSnapshotWithoutListenerClearsStaleQueue(true);
+ }
+
+ private void assertSnapshotWithoutListenerClearsStaleQueue(final boolean
hasLocalReference)
+ throws Exception {
+ final SchemaRegionId schemaRegionId =
+ new SchemaRegionId(Integer.MAX_VALUE - (hasLocalReference ? 12 : 11));
+ final File snapshotDir = temporaryFolder.newFolder();
+ final ISchemaRegion schemaRegion = mock(ISchemaRegion.class);
+ when(schemaRegion.getSchemaRegionId()).thenReturn(schemaRegionId);
+ when(schemaRegion.createSnapshot(snapshotDir)).thenReturn(true);
+ when(schemaRegion.loadSnapshot(snapshotDir)).thenReturn(true);
+ final SchemaRegionStateMachine stateMachine = new
SchemaRegionStateMachine(schemaRegion);
+ final Field queueMapField =
+
PipeSchemaRegionListenerMetrics.class.getDeclaredField("listeningQueueMap");
+ queueMapField.setAccessible(true);
+ final Map<?, ?> metricQueues =
+ (Map<?, ?>)
queueMapField.get(PipeSchemaRegionListenerMetrics.getInstance());
+
+ try {
+ // The leader has applied queue-close and cleaned its listener before
taking this snapshot.
+ Assert.assertTrue(stateMachine.takeSnapshot(snapshotDir));
+ Assert.assertFalse(SchemaRegionListeningQueue.hasSnapshot(snapshotDir));
+
Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+
+ // A lagging follower can still have the queue from before the close log
entry.
+ final SchemaRegionListeningQueue staleQueue =
+ PipeDataNodeAgent.runtime().schemaListener(schemaRegionId);
+ if (hasLocalReference) {
+
PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId);
+ }
+ staleQueue.open();
+ final PlanNode node = mock(PlanNode.class);
+ when(node.getType()).thenReturn(PlanNodeType.CREATE_TIME_SERIES);
+ staleQueue.tryListenToNode(node);
+ Assert.assertEquals(1L, staleQueue.getSize());
+
+ Assert.assertTrue(stateMachine.loadSnapshot(snapshotDir));
+
+ Assert.assertFalse(staleQueue.isOpened());
+ Assert.assertEquals(0L, staleQueue.getSize());
+ if (hasLocalReference) {
+ Assert.assertSame(
+ staleQueue,
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ Assert.assertTrue(metricQueues.containsKey(schemaRegionId.getId()));
+ Assert.assertEquals(
+ 0,
+
PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId));
+ }
+
Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ Assert.assertFalse(metricQueues.containsKey(schemaRegionId.getId()));
+ } finally {
+ final SchemaRegionListeningQueue listener =
+ PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId);
+ if (listener != null) {
+ listener.close();
+ }
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
+ }
+ }
+
+ private static class TestSchemaRegionSource extends IoTDBSchemaRegionSource {
+
+ private TestSchemaRegionSource() {
+ pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 0);
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionSnapshotRecoveryTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionSnapshotRecoveryTest.java
new file mode 100644
index 00000000000..5ea9c288006
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/runtime/PipeSchemaRegionSnapshotRecoveryTest.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.db.pipe.agent.runtime;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.consensus.ConsensusGroupId;
+import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.consensus.IConsensus;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.config.ConsensusConfig;
+import org.apache.iotdb.consensus.config.RatisConfig;
+import
org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaRegionStateMachine;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode;
+import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
+
+import org.apache.ratis.util.ExitUtils;
+import org.apache.ratis.util.TimeDuration;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Proxy;
+import java.net.ServerSocket;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class PipeSchemaRegionSnapshotRecoveryTest {
+
+ @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @After
+ public void clearRatisExitState() {
+ ExitUtils.clear();
+ }
+
+ @Test
+ public void testInstallingSnapshotWithoutListenerClosesStaleQueue() throws
Exception {
+ ExitUtils.disableSystemExit();
+
+ final SchemaRegionId schemaRegionId = new SchemaRegionId(Integer.MAX_VALUE
- 17666);
+ final int leaderPort = freePort();
+ int followerPort = freePort();
+ while (followerPort == leaderPort) {
+ followerPort = freePort();
+ }
+ final Peer leaderPeer = new Peer(schemaRegionId, 0, new
TEndPoint("127.0.0.1", leaderPort));
+ final Peer followerPeer = new Peer(schemaRegionId, 1, new
TEndPoint("127.0.0.1", followerPort));
+ final CountDownLatch leaderSnapshotCreated = new CountDownLatch(1);
+ final CountDownLatch followerSnapshotLoaded = new CountDownLatch(1);
+ final AtomicReference<File> leaderSnapshotDir = new AtomicReference<>();
+ final AtomicBoolean followerSnapshotLoadSucceeded = new AtomicBoolean();
+ final AtomicReference<SchemaRegionListeningQueue> staleQueueRef = new
AtomicReference<>();
+ final ISchemaRegion schemaRegion =
+ newSchemaRegion(schemaRegionId, leaderSnapshotDir,
leaderSnapshotCreated);
+
+ final RatisConfig ratisConfig =
+ RatisConfig.newBuilder()
+ .setRpc(
+ RatisConfig.Rpc.newBuilder()
+ .setFirstElectionTimeoutMin(TimeDuration.valueOf(1,
TimeUnit.SECONDS))
+ .setFirstElectionTimeoutMax(TimeDuration.valueOf(4,
TimeUnit.SECONDS))
+ .setTimeoutMin(TimeDuration.valueOf(1, TimeUnit.SECONDS))
+ .setTimeoutMax(TimeDuration.valueOf(4, TimeUnit.SECONDS))
+ .build())
+
.setLog(RatisConfig.Log.newBuilder().setPurgeUptoSnapshotIndex(true).build())
+ .build();
+ final IConsensus leader =
+ newConsensus(
+ leaderPeer,
+ ratisConfig,
+ schemaRegion,
+ new File(temporaryFolder.getRoot(), "leader"),
+ null,
+ null);
+ final IConsensus follower =
+ newConsensus(
+ followerPeer,
+ ratisConfig,
+ schemaRegion,
+ new File(temporaryFolder.getRoot(), "follower"),
+ followerSnapshotLoaded,
+ followerSnapshotLoadSucceeded);
+ boolean leaderStarted = false;
+ boolean followerStarted = false;
+
+ try {
+ leader.start();
+ leaderStarted = true;
+ follower.start();
+ followerStarted = true;
+
+ leader.createLocalPeer(schemaRegionId,
Collections.singletonList(leaderPeer));
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .until(() -> leader.isLeaderReady(schemaRegionId));
+
+ for (int i = 0; i < 10; i++) {
+ Assert.assertEquals(
+ 200,
+ leader
+ .write(
+ schemaRegionId,
+ new PipeOperateSchemaQueueNode(new PlanNodeId("close" +
i), false))
+ .getCode());
+ }
+ leader.triggerSnapshot(schemaRegionId, true);
+ Assert.assertTrue(leaderSnapshotCreated.await(30, TimeUnit.SECONDS));
+
Assert.assertFalse(SchemaRegionListeningQueue.hasSnapshot(leaderSnapshotDir.get()));
+
+ final SchemaRegionListeningQueue staleQueue =
+ PipeDataNodeAgent.runtime().schemaListener(schemaRegionId);
+ staleQueueRef.set(staleQueue);
+
PipeDataNodeAgent.runtime().increaseAndGetSchemaListenerReferenceCount(schemaRegionId);
+ staleQueue.open();
+ staleQueue.tryListenToNode(
+ new CreateTimeSeriesNode(
+ new PlanNodeId("stale"),
+ new MeasurementPath("root.db.d1.s1"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null,
+ null,
+ null,
+ "alias"));
+ Assert.assertEquals(1, staleQueue.getSize());
+
+ follower.createLocalPeer(schemaRegionId, Collections.emptyList());
+ leader.addRemotePeer(schemaRegionId, followerPeer);
+
+ Assert.assertTrue(followerSnapshotLoaded.await(30, TimeUnit.SECONDS));
+ Assert.assertTrue(followerSnapshotLoadSucceeded.get());
+ Assert.assertFalse(staleQueue.isOpened());
+ Assert.assertEquals(0, staleQueue.getSize());
+ Assert.assertSame(
+ staleQueue,
PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ Assert.assertEquals(
+ 0,
+
PipeDataNodeAgent.runtime().decreaseAndGetSchemaListenerReferenceCount(schemaRegionId));
+
Assert.assertNull(PipeDataNodeAgent.runtime().schemaListenerIfPresent(schemaRegionId));
+ } finally {
+ if (followerStarted) {
+ follower.stop();
+ }
+ if (leaderStarted) {
+ leader.stop();
+ }
+ if (staleQueueRef.get() != null) {
+ staleQueueRef.get().close();
+ }
+ PipeDataNodeAgent.runtime().clearSchemaRegionState(schemaRegionId);
+ }
+ }
+
+ private ISchemaRegion newSchemaRegion(
+ final SchemaRegionId schemaRegionId,
+ final AtomicReference<File> snapshotDir,
+ final CountDownLatch snapshotCreated) {
+ return (ISchemaRegion)
+ Proxy.newProxyInstance(
+ ISchemaRegion.class.getClassLoader(),
+ new Class<?>[] {ISchemaRegion.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "getSchemaRegionId":
+ return schemaRegionId;
+ case "createSnapshot":
+ final File dir = (File) args[0];
+ Files.createDirectories(dir.toPath());
+ Files.writeString(new File(dir, "schema.snapshot").toPath(),
"schema");
+ snapshotDir.set(dir);
+ snapshotCreated.countDown();
+ return true;
+ case "loadSnapshot":
+ return new File((File) args[0], "schema.snapshot").isFile();
+ default:
+ throw new UnsupportedOperationException(method.toString());
+ }
+ });
+ }
+
+ private IConsensus newConsensus(
+ final Peer peer,
+ final RatisConfig ratisConfig,
+ final ISchemaRegion schemaRegion,
+ final File storageDir,
+ final CountDownLatch snapshotLoaded,
+ final AtomicBoolean snapshotLoadSucceeded) {
+ return ConsensusFactory.getConsensusImpl(
+ ConsensusFactory.RATIS_CONSENSUS,
+ ConsensusConfig.newBuilder()
+ .setThisNodeId(peer.getNodeId())
+ .setThisNode(peer.getEndpoint())
+ .setRatisConfig(ratisConfig)
+ .setStorageDir(storageDir.getAbsolutePath())
+ .setConsensusGroupType(TConsensusGroupType.SchemaRegion)
+ .build(),
+ ignored ->
+ new QuietSchemaRegionStateMachine(
+ schemaRegion, snapshotLoaded, snapshotLoadSucceeded))
+ .orElseThrow(IllegalStateException::new);
+ }
+
+ private int freePort() throws IOException {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ return socket.getLocalPort();
+ }
+ }
+
+ private static class QuietSchemaRegionStateMachine extends
SchemaRegionStateMachine {
+
+ private final CountDownLatch snapshotLoaded;
+ private final AtomicBoolean snapshotLoadSucceeded;
+
+ private QuietSchemaRegionStateMachine(
+ final ISchemaRegion schemaRegion,
+ final CountDownLatch snapshotLoaded,
+ final AtomicBoolean snapshotLoadSucceeded) {
+ super(schemaRegion);
+ this.snapshotLoaded = snapshotLoaded;
+ this.snapshotLoadSucceeded = snapshotLoadSucceeded;
+ }
+
+ @Override
+ public boolean loadSnapshot(final File snapshotDir) {
+ final boolean succeeded = super.loadSnapshot(snapshotDir);
+ if (snapshotLoaded != null) {
+ snapshotLoadSucceeded.set(succeeded);
+ snapshotLoaded.countDown();
+ }
+ return succeeded;
+ }
+
+ @Override
+ public void notifyLeaderChanged(final ConsensusGroupId groupId, final int
newLeaderId) {}
+
+ @Override
+ public void notifyLeaderReady() {}
+
+ @Override
+ public void notifyNotLeader() {}
+
+ @Override
+ public void stop() {}
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java
index ab982c78af0..79277b651af 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/PipeMetricsRestartTest.java
@@ -131,6 +131,30 @@ public class PipeMetricsRestartTest {
assertTrue(metrics.remainingEventAndTimeOperatorMap.isEmpty());
}
+ @Test
+ public void testSinglePipeMetricsIgnoreLateEventsAfterDeregister() {
+ final PipeDataNodeSinglePipeMetrics metrics =
PipeDataNodeSinglePipeMetrics.getInstance();
+ bind(metrics);
+ metrics.register(mockDataRegionSource());
+ metrics.deregister(PIPE_ID);
+
+ metrics.increaseInsertNodeEventCount(PIPE, CREATION_TIME);
+ metrics.decreaseInsertNodeEventCount(PIPE, CREATION_TIME, 10);
+ metrics.updateInsertNodeTransferTimer(PIPE, CREATION_TIME, 10);
+ metrics.increaseRawTabletEventCount(PIPE, CREATION_TIME);
+ metrics.decreaseRawTabletEventCount(PIPE, CREATION_TIME);
+ metrics.increaseTsFileEventCount(PIPE, CREATION_TIME);
+ metrics.decreaseTsFileEventCount(PIPE, CREATION_TIME, 10);
+ metrics.updateTsFileTransferTimer(PIPE, CREATION_TIME, 10);
+ metrics.increaseHeartbeatEventCount(PIPE, CREATION_TIME);
+ metrics.decreaseHeartbeatEventCount(PIPE, CREATION_TIME);
+ assertEquals(0L, (long) metrics.getRemainingEventAndTime(PIPE,
CREATION_TIME).getLeft());
+
+ assertTrue(metrics.remainingEventAndTimeOperatorMap.isEmpty());
+ service.restartService();
+ assertEquals(0, count(Metric.PIPE_DATANODE_REMAINING_EVENT_COUNT));
+ }
+
@Test
public void testDataRegionSourceMetrics() {
final PipeDataRegionSourceMetrics metrics =
PipeDataRegionSourceMetrics.getInstance();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java
index 8246f9080d3..702d2e1dc20 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeSinglePipeMetricsTest.java
@@ -20,10 +20,18 @@
package org.apache.iotdb.db.pipe.metric.overview;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
+import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
import org.apache.iotdb.db.pipe.source.dataregion.IoTDBDataRegionSource;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource;
@@ -37,10 +45,20 @@ import org.junit.Test;
import org.mockito.invocation.Invocation;
import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockingDetails;
@@ -154,6 +172,177 @@ public class PipeDataNodeSinglePipeMetricsTest {
}
}
+ @Test
+ public void testMetricCreationUsesExactPipeInstanceAcrossModels() throws
Exception {
+ final PipeDataNodeSinglePipeMetrics metrics =
PipeDataNodeSinglePipeMetrics.getInstance();
+ final Field metricServiceField =
+ PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService");
+ metricServiceField.setAccessible(true);
+ reset(metrics, metricServiceField);
+
+ final String pipeName = "shared_model_metric_" + System.nanoTime();
+ final long treeCreationTime = System.nanoTime();
+ final long tableCreationTime = treeCreationTime + 1;
+ final PipeMeta treePipe =
+ createPipeMeta(pipeName, treeCreationTime,
SystemConstant.SQL_DIALECT_TREE_VALUE);
+ final PipeMeta tablePipe =
+ createPipeMeta(pipeName, tableCreationTime,
SystemConstant.SQL_DIALECT_TABLE_VALUE);
+ final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper();
+
+ try {
+ pipeMetaKeeper.addPipeMeta(treePipe);
+ pipeMetaKeeper.addPipeMeta(tablePipe);
+
+ assertEquals(treeCreationTime,
PipeDataNodeAgent.task().getPipeCreationTime(pipeName));
+ assertTrue(PipeDataNodeAgent.task().isPipeExisted(pipeName,
tableCreationTime));
+
+ metrics.increaseTsFileEventCount(pipeName, tableCreationTime);
+
+ assertTrue(
+ metrics.remainingEventAndTimeOperatorMap.containsKey(pipeName + "_"
+ tableCreationTime));
+ } finally {
+ metrics.deregister(pipeName + "_" + tableCreationTime);
+ pipeMetaKeeper.removePipeMeta(treePipe.getStaticMeta());
+ pipeMetaKeeper.removePipeMeta(tablePipe.getStaticMeta());
+ reset(metrics, metricServiceField);
+ }
+ }
+
+ @Test
+ public void testMetricCreationSerializesWithDeregister() throws Exception {
+ final PipeDataNodeSinglePipeMetrics metrics =
PipeDataNodeSinglePipeMetrics.getInstance();
+ final Field metricServiceField =
+ PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService");
+ metricServiceField.setAccessible(true);
+ reset(metrics, metricServiceField);
+
+ final String pipeName = "metric_drop_race_" + System.nanoTime();
+ final long creationTime = System.nanoTime();
+ final String pipeID = pipeName + "_" + creationTime;
+ final PipeMeta pipeMeta =
+ createPipeMeta(pipeName, creationTime,
SystemConstant.SQL_DIALECT_TREE_VALUE);
+ final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper();
+ final Field pipeMetaMapField =
PipeMetaKeeper.class.getDeclaredField("pipeNameToPipeMetaMap");
+ pipeMetaMapField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ final Map<String, PipeMeta> originalPipeMetaMap =
+ (Map<String, PipeMeta>) pipeMetaMapField.get(pipeMetaKeeper);
+ final BlockingPipeMetaMap blockingPipeMetaMap =
+ new BlockingPipeMetaMap(originalPipeMetaMap, pipeName, creationTime);
+ pipeMetaMapField.set(pipeMetaKeeper, blockingPipeMetaMap);
+ pipeMetaKeeper.addPipeMeta(pipeMeta);
+
+ final CountDownLatch metadataRemoved = new CountDownLatch(1);
+ final AtomicReference<Throwable> backgroundFailure = new
AtomicReference<>();
+ final Thread updateThread =
+ new Thread(
+ () -> {
+ try {
+ metrics.increaseTsFileEventCount(pipeName, creationTime);
+ } catch (final Throwable e) {
+ backgroundFailure.compareAndSet(null, e);
+ }
+ });
+ final Thread deregisterThread =
+ new Thread(
+ () -> {
+ try {
+ pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta());
+ metadataRemoved.countDown();
+ metrics.deregister(pipeID);
+ } catch (final Throwable e) {
+ backgroundFailure.compareAndSet(null, e);
+ }
+ });
+
+ try {
+ try {
+ updateThread.start();
+ assertTrue(blockingPipeMetaMap.lookupPaused.await(10,
TimeUnit.SECONDS));
+ deregisterThread.start();
+ assertTrue(metadataRemoved.await(10, TimeUnit.SECONDS));
+ assertEquals(Thread.State.BLOCKED,
waitForBlockedOrTerminated(deregisterThread));
+ } finally {
+ blockingPipeMetaMap.resumeLookup.countDown();
+ updateThread.join(TimeUnit.SECONDS.toMillis(10));
+ deregisterThread.join(TimeUnit.SECONDS.toMillis(10));
+ }
+
+ assertFalse(updateThread.isAlive());
+ assertFalse(deregisterThread.isAlive());
+ assertNull(backgroundFailure.get());
+
assertFalse(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID));
+ } finally {
+ pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta());
+ pipeMetaMapField.set(pipeMetaKeeper, originalPipeMetaMap);
+ metrics.deregister(pipeID);
+ reset(metrics, metricServiceField);
+ }
+ }
+
+ @Test
+ public void testLateMetricUpdateAfterDropDeregisterDoesNotRecreateOperator()
throws Exception {
+ final PipeDataNodeSinglePipeMetrics metrics =
PipeDataNodeSinglePipeMetrics.getInstance();
+ final Field metricServiceField =
+ PipeDataNodeSinglePipeMetrics.class.getDeclaredField("metricService");
+ metricServiceField.setAccessible(true);
+ reset(metrics, metricServiceField);
+
+ final String pipeName = "metric_late_update_after_drop_" +
System.nanoTime();
+ final long creationTime = System.nanoTime();
+ final String pipeID = pipeName + "_" + creationTime;
+ final PipeMeta pipeMeta =
+ createPipeMeta(pipeName, creationTime,
SystemConstant.SQL_DIALECT_TREE_VALUE);
+ final PipeMetaKeeper pipeMetaKeeper = getPipeMetaKeeper();
+
+ try {
+ pipeMetaKeeper.addPipeMeta(pipeMeta);
+ metrics.increaseTsFileEventCount(pipeName, creationTime);
+ assertTrue(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID));
+
+ // dropPipe marks the metadata dropped before task deregistration and
removes it afterward.
+ pipeMeta.getRuntimeMeta().getStatus().set(PipeStatus.DROPPED);
+ metrics.deregister(pipeID);
+
+ metrics.increaseTsFileEventCount(pipeName, creationTime);
+
+
assertFalse(metrics.remainingEventAndTimeOperatorMap.containsKey(pipeID));
+ } finally {
+ pipeMetaKeeper.removePipeMeta(pipeMeta.getStaticMeta());
+ metrics.deregister(pipeID);
+ reset(metrics, metricServiceField);
+ }
+ }
+
+ private static PipeMeta createPipeMeta(
+ final String pipeName, final long creationTime, final String sqlDialect)
{
+ final Map<String, String> sourceAttributes = new HashMap<>();
+ sourceAttributes.put(SystemConstant.SQL_DIALECT_KEY, sqlDialect);
+ return new PipeMeta(
+ new PipeStaticMeta(
+ pipeName, creationTime, sourceAttributes, new HashMap<>(), new
HashMap<>()),
+ new PipeRuntimeMeta(new ConcurrentHashMap<>()));
+ }
+
+ private static PipeMetaKeeper getPipeMetaKeeper() throws Exception {
+ final Field pipeMetaKeeperField =
PipeTaskAgent.class.getDeclaredField("pipeMetaKeeper");
+ pipeMetaKeeperField.setAccessible(true);
+ return (PipeMetaKeeper) pipeMetaKeeperField.get(PipeDataNodeAgent.task());
+ }
+
+ private static Thread.State waitForBlockedOrTerminated(final Thread thread)
+ throws InterruptedException {
+ final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ Thread.State state = thread.getState();
+ while (state != Thread.State.BLOCKED
+ && state != Thread.State.TERMINATED
+ && System.nanoTime() < deadline) {
+ Thread.sleep(1);
+ state = thread.getState();
+ }
+ return state;
+ }
+
private static boolean hasCompletionGaugeCreation(
final AbstractMetricService metricService, final String pipeName, final
long creationTime) {
for (final Invocation invocation :
mockingDetails(metricService).getInvocations()) {
@@ -185,6 +374,62 @@ public class PipeDataNodeSinglePipeMetricsTest {
metricServiceField.set(metrics, null);
}
+ private static class BlockingPipeMetaMap extends ConcurrentHashMap<String,
PipeMeta> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String pipeName;
+ private final long creationTime;
+ private final AtomicBoolean pauseNextLookup = new AtomicBoolean(true);
+ private final CountDownLatch lookupPaused = new CountDownLatch(1);
+ private final CountDownLatch resumeLookup = new CountDownLatch(1);
+
+ private BlockingPipeMetaMap(
+ final Map<String, PipeMeta> originalMap, final String pipeName, final
long creationTime) {
+ putAll(originalMap);
+ this.pipeName = pipeName;
+ this.creationTime = creationTime;
+ }
+
+ @Override
+ public PipeMeta get(final Object key) {
+ final PipeMeta pipeMeta = super.get(key);
+ pauseIfMatching(pipeMeta);
+ return pipeMeta;
+ }
+
+ @Override
+ public Collection<PipeMeta> values() {
+ final Collection<PipeMeta> values = super.values();
+ final ArrayList<PipeMeta> snapshot = new ArrayList<>(values);
+ if (snapshot.stream().anyMatch(this::matches)) {
+
pauseIfMatching(snapshot.stream().filter(this::matches).findFirst().orElse(null));
+ return snapshot;
+ }
+ return values;
+ }
+
+ private boolean matches(final PipeMeta pipeMeta) {
+ return pipeMeta != null
+ && pipeName.equals(pipeMeta.getStaticMeta().getPipeName())
+ && creationTime == pipeMeta.getStaticMeta().getCreationTime();
+ }
+
+ private void pauseIfMatching(final PipeMeta pipeMeta) {
+ if (matches(pipeMeta) && pauseNextLookup.compareAndSet(true, false)) {
+ lookupPaused.countDown();
+ try {
+ if (!resumeLookup.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting to resume pipe
metadata lookup");
+ }
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+ }
+
private static class TestSource extends PipeRealtimeDataRegionSource {
private TestSource(
diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml
index 8c6ea6ed534..a3a58dc351a 100644
--- a/iotdb-core/node-commons/pom.xml
+++ b/iotdb-core/node-commons/pom.xml
@@ -187,6 +187,11 @@
<artifactId>reflections</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
index 2edf20190c6..b62f7b0db70 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.resource.PipeResourceFailureType;
import org.apache.iotdb.commons.pipe.resource.log.PipeLogger;
import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter;
+import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq;
import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage;
@@ -61,6 +62,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
@@ -101,10 +103,14 @@ public abstract class PipeTaskAgent {
protected final PipeMetaKeeper pipeMetaKeeper;
protected final PipeTaskManager pipeTaskManager;
+ private final Map<String, AtomicLong>
pipeNameWithCreationTime2FloatingMemoryUsageInByteMap;
+ private final Map<String, Set<Long>> pipeName2CreationTimeSetMap;
protected PipeTaskAgent() {
pipeMetaKeeper = new PipeMetaKeeper();
pipeTaskManager = new PipeTaskManager();
+ pipeNameWithCreationTime2FloatingMemoryUsageInByteMap = new
ConcurrentHashMap<>();
+ pipeName2CreationTimeSetMap = new ConcurrentHashMap<>();
// Help PipeEndPointRateLimiter to check if the pipe is still alive
PipeEndPointRateLimiter.setTaskAgent(this);
@@ -669,6 +675,8 @@ public abstract class PipeTaskAgent {
// Remove pipe meta from pipe meta keeper
pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta());
+ cleanupFloatingMemoryUsageCounterIfNecessary(pipeName, creationTime);
+ clearPipeRuntimeState(pipeName, creationTime);
return true;
}
@@ -711,6 +719,9 @@ public abstract class PipeTaskAgent {
// Remove pipe meta from pipe meta keeper
pipeMetaKeeper.removePipeMeta(existedPipeMeta.getStaticMeta());
+ cleanupFloatingMemoryUsageCounterIfNecessary(
+ pipeName, existedPipeMeta.getStaticMeta().getCreationTime());
+ clearPipeRuntimeState(pipeName,
existedPipeMeta.getStaticMeta().getCreationTime());
return true;
}
@@ -1328,6 +1339,41 @@ public abstract class PipeTaskAgent {
///////////////////////// Maintain meta info /////////////////////////
+ private static String generatePipeNameWithCreationTime(
+ final String pipeName, final long creationTime) {
+ return pipeName + "_" + creationTime;
+ }
+
+ private AtomicLong cleanupFloatingMemoryUsageCounterIfUnused(
+ final String pipeName, final long creationTime, final AtomicLong
counter) {
+ if (counter.get() != 0) {
+ return counter;
+ }
+
+ // A missing counter also reports zero; a later acquisition recreates it
without scanning all
+ // pipe metadata on the event-release path.
+ pipeName2CreationTimeSetMap.computeIfPresent(
+ pipeName,
+ (key, creationTimes) -> {
+ creationTimes.remove(creationTime);
+ return creationTimes.isEmpty() ? null : creationTimes;
+ });
+ return null;
+ }
+
+ private void cleanupFloatingMemoryUsageCounterIfNecessary(
+ final String pipeName, final long creationTime) {
+ pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfPresent(
+ generatePipeNameWithCreationTime(pipeName, creationTime),
+ (key, counter) ->
+ cleanupFloatingMemoryUsageCounterIfUnused(pipeName, creationTime,
counter));
+ }
+
+ private void clearPipeRuntimeState(final String pipeName, final long
creationTime) {
+ PipeEventCommitManager.getInstance().clear(pipeName, creationTime);
+ IoTDBSink.clearPipeRateLimiter(pipeName, creationTime);
+ }
+
public long getPipeCreationTime(final String pipeName) {
final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName);
return pipeMeta == null ? 0 : pipeMeta.getStaticMeta().getCreationTime();
@@ -1337,10 +1383,15 @@ public abstract class PipeTaskAgent {
return pipeMetaKeeper.getPipeMeta(pipeName, creationTime) != null;
}
+ public boolean isPipeAlive(final String pipeName, final long creationTime) {
+ final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName,
creationTime);
+ return pipeMeta != null && pipeMeta.getRuntimeMeta().getStatus().get() !=
PipeStatus.DROPPED;
+ }
+
public String getPipeNameWithCreationTime(final String pipeName, final long
creationTime) {
final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName,
creationTime);
return pipeMeta == null
- ? pipeName + "_" + creationTime
+ ? generatePipeNameWithCreationTime(pipeName, creationTime)
: ((PipeTemporaryMetaInAgent)
pipeMeta.getTemporaryMeta()).getPipeNameWithCreationTime();
}
@@ -1354,49 +1405,60 @@ public abstract class PipeTaskAgent {
}
public long getAllFloatingMemoryUsageInByte() {
- final AtomicLong bytes = new AtomicLong(0);
- pipeMetaKeeper
- .getPipeMetaList()
- .forEach(
- pipeMeta ->
- bytes.addAndGet(
- ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta())
- .getFloatingMemoryUsageInByte()));
- return bytes.get();
+ return
pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.values().stream()
+ .mapToLong(AtomicLong::get)
+ .sum();
}
public long getFloatingMemoryUsageInByte(final String pipeName) {
- final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName);
- return pipeMeta == null
- ? 0
- : ((PipeTemporaryMetaInAgent)
pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte();
+ final Set<Long> creationTimes = pipeName2CreationTimeSetMap.get(pipeName);
+ if (Objects.isNull(creationTimes)) {
+ return 0;
+ }
+
+ return creationTimes.stream()
+ .map(creationTime -> generatePipeNameWithCreationTime(pipeName,
creationTime))
+ .map(pipeNameWithCreationTime2FloatingMemoryUsageInByteMap::get)
+ .filter(Objects::nonNull)
+ .mapToLong(AtomicLong::get)
+ .sum();
}
public long getFloatingMemoryUsageInByte(final String pipeName, final long
creationTime) {
- final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName,
creationTime);
- return pipeMeta == null
- ? 0
- : ((PipeTemporaryMetaInAgent)
pipeMeta.getTemporaryMeta()).getFloatingMemoryUsageInByte();
+ final AtomicLong counter =
+ pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.get(
+ generatePipeNameWithCreationTime(pipeName, creationTime));
+ return Objects.isNull(counter) ? 0 : counter.get();
}
public void addFloatingMemoryUsageInByte(
final String pipeName, final long creationTime, final long sizeInByte) {
- final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName,
creationTime);
- // To avoid stale pipe before alter
- if (Objects.nonNull(pipeMeta) &&
pipeMeta.getStaticMeta().getCreationTime() == creationTime) {
- ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta())
- .addFloatingMemoryUsageInByte(sizeInByte);
- }
+ // Late events still own memory after drop and must be paired with their
eventual release.
+ pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.compute(
+ generatePipeNameWithCreationTime(pipeName, creationTime),
+ (key, counter) -> {
+ final AtomicLong updatedCounter = Objects.isNull(counter) ? new
AtomicLong(0) : counter;
+ updatedCounter.addAndGet(sizeInByte);
+ pipeName2CreationTimeSetMap.compute(
+ pipeName,
+ (name, creationTimes) -> {
+ final Set<Long> updatedCreationTimes =
+ Objects.isNull(creationTimes) ?
ConcurrentHashMap.newKeySet() : creationTimes;
+ updatedCreationTimes.add(creationTime);
+ return updatedCreationTimes;
+ });
+ return updatedCounter;
+ });
}
public void decreaseFloatingMemoryUsageInByte(
final String pipeName, final long creationTime, final long sizeInByte) {
- final PipeMeta pipeMeta = pipeMetaKeeper.getPipeMeta(pipeName,
creationTime);
- // To avoid stale pipe before alter
- if (Objects.nonNull(pipeMeta) &&
pipeMeta.getStaticMeta().getCreationTime() == creationTime) {
- ((PipeTemporaryMetaInAgent) pipeMeta.getTemporaryMeta())
- .decreaseFloatingMemoryUsageInByte(sizeInByte);
- }
+ pipeNameWithCreationTime2FloatingMemoryUsageInByteMap.computeIfPresent(
+ generatePipeNameWithCreationTime(pipeName, creationTime),
+ (key, counter) -> {
+ counter.addAndGet(-sizeInByte);
+ return cleanupFloatingMemoryUsageCounterIfUnused(pipeName,
creationTime, counter);
+ });
}
public void recordPipeResourceFailure(
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
index b2654d1e8a7..a04f7519551 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java
@@ -77,6 +77,13 @@ public class PipeEventCommitManager {
LOGGER.info(PipeMessages.PIPE_COMMITTER_DEREGISTERED, committerKey);
}
+ public void clear(final String pipeName, final long creationTime) {
+ eventCommitterMap.keySet().removeIf(key -> clearIfMatches(key, pipeName,
creationTime, true));
+ eventCommitterRestartTimesMap
+ .keySet()
+ .removeIf(key -> clearIfMatches(key, pipeName, creationTime, false));
+ }
+
/**
* Assign a commit id and a key for commit. Make sure {@code
EnrichedEvent.pipeName} is set before
* calling this.
@@ -87,8 +94,15 @@ public class PipeEventCommitManager {
return;
}
+ final String pipeName = event.getPipeName();
+ final CommitterKey restartTimesKey =
+ generateCommitterRestartTimesKey(pipeName, creationTime, regionId);
final CommitterKey committerKey =
- generateCommitterKey(event.getPipeName(), creationTime, regionId);
+ taskAgent.getCommitterKey(
+ pipeName,
+ creationTime,
+ regionId,
+ eventCommitterRestartTimesMap.getOrDefault(restartTimesKey, 0));
final PipeEventCommitter committer = eventCommitterMap.get(committerKey);
if (committer == null) {
return;
@@ -143,10 +157,11 @@ public class PipeEventCommitManager {
final PipeEventCommitter committer = eventCommitterMap.get(committerKey);
if (committer == null) {
- final int currentRestartTimes =
- eventCommitterRestartTimesMap.computeIfAbsent(
- generateCommitterRestartTimesKey(committerKey), k -> 0);
if (LOGGER.isDebugEnabled()) {
+ // Late commits must not recreate restart state after the pipe has
been cleared.
+ final int currentRestartTimes =
+ eventCommitterRestartTimesMap.getOrDefault(
+ generateCommitterRestartTimesKey(committerKey), 0);
if (committerKey.getRestartTimes() < currentRestartTimes) {
LOGGER.debug(
PipeMessages.STALE_PIPE_EVENT_COMMITTER,
@@ -179,13 +194,25 @@ public class PipeEventCommitManager {
* older committer after the same Pipe/DataRegion task has been restarted.
*/
public boolean isCurrentCommitterKey(final CommitterKey committerKey) {
- return committerKey != null
- && eventCommitterMap.containsKey(committerKey)
- && committerKey.equals(
- generateCommitterKey(
+ if (committerKey == null) {
+ return false;
+ }
+
+ final PipeEventCommitter committer = eventCommitterMap.get(committerKey);
+ if (committer == null) {
+ return false;
+ }
+
+ final int restartTimes =
+ eventCommitterRestartTimesMap.getOrDefault(
+ generateCommitterRestartTimesKey(committerKey), 0);
+ return committerKey.equals(
+ taskAgent.getCommitterKey(
committerKey.getPipeName(),
committerKey.getCreationTime(),
- committerKey.getRegionId()));
+ committerKey.getRegionId(),
+ restartTimes))
+ && eventCommitterMap.get(committerKey) == committer;
}
private CommitterKey generateCommitterKey(
@@ -216,6 +243,20 @@ public class PipeEventCommitManager {
this.commitRateMarker = commitRateMarker;
}
+ private boolean clearIfMatches(
+ final CommitterKey key,
+ final String pipeName,
+ final long creationTime,
+ final boolean shouldClearMetrics) {
+ if (!Objects.equals(key.getPipeName(), pipeName) || key.getCreationTime()
!= creationTime) {
+ return false;
+ }
+ if (shouldClearMetrics) {
+ PipeEventCommitMetrics.getInstance().deregister(key.stringify());
+ }
+ return true;
+ }
+
//////////////////////////// singleton ////////////////////////////
private PipeEventCommitManager() {
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java
index 2701e5bb237..fbbe95a7034 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/limiter/PipeEndPointRateLimiter.java
@@ -55,6 +55,11 @@ public class PipeEndPointRateLimiter {
PipeEndPointRateLimiter.taskAgent = taskAgent;
}
+ public static boolean isPipeAlive(final String pipeName, final long
creationTime) {
+ final PipeTaskAgent finalTaskAgent = taskAgent;
+ return Objects.isNull(finalTaskAgent) ||
finalTaskAgent.isPipeAlive(pipeName, creationTime);
+ }
+
public void acquire(final TEndPoint endPoint, long bytes) {
if (endPoint == null) {
return;
@@ -82,9 +87,7 @@ public class PipeEndPointRateLimiter {
bytes,
PipeConfig.getInstance().getRateLimiterHotReloadCheckIntervalMs(),
TimeUnit.MILLISECONDS)) {
- final PipeTaskAgent finalTaskAgent = taskAgent;
- if (Objects.nonNull(finalTaskAgent)
- && !finalTaskAgent.isPipeExisted(pipeName, creationTime)) {
+ if (!isPipeAlive(pipeName, creationTime)) {
return false;
}
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
index ab5fba5bbf8..06878bb7d49 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
@@ -685,25 +685,33 @@ public abstract class IoTDBSink implements PipeConnector,
PipeConnectorWithEvent
final TEndPoint endPoint,
final long bytesLength) {
if (pipeName != null && endPointRateLimitBytesPerSecond > 0) {
- PIPE_END_POINT_RATE_LIMITER_MAP
- .computeIfAbsent(
+ final PipeEndPointRateLimiter limiter =
+ PIPE_END_POINT_RATE_LIMITER_MAP.computeIfAbsent(
new Pair<>(pipeName, creationTime),
endpoint ->
- new PipeEndPointRateLimiter(
- pipeName, creationTime, endPointRateLimitBytesPerSecond))
- .acquire(endPoint, bytesLength);
+ PipeEndPointRateLimiter.isPipeAlive(pipeName, creationTime)
+ ? new PipeEndPointRateLimiter(
+ pipeName, creationTime,
endPointRateLimitBytesPerSecond)
+ : null);
+ if (limiter != null) {
+ limiter.acquire(endPoint, bytesLength);
+ }
}
GLOBAL_RATE_LIMITER.acquire(bytesLength);
}
+ public static void clearPipeRateLimiter(final String pipeName, final long
creationTime) {
+ PIPE_END_POINT_RATE_LIMITER_MAP.remove(new Pair<>(pipeName, creationTime));
+ }
+
/**
- * When a pipe is dropped, the connector maybe reused and will not be
closed. We need to discard
- * its batched or queued events in the output pipe connector.
+ * When a region task is dropped, the connector may be reused and will not
be closed. Discard its
+ * batched or queued events in the output pipe connector.
*/
public synchronized void discardEventsOfPipe(
final String pipeName, final long creationTime, final int regionId) {
- // Do nothing by default
+ // Rate limiter state is pipe-wide and is cleared only after the whole
pipe is dropped.
}
public PipeReceiverStatusHandler statusHandler() {
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
index 8551696fd6d..90a74621333 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBNonDataRegionSource.java
@@ -328,12 +328,20 @@ public abstract class IoTDBNonDataRegionSource extends
IoTDBSource {
if (Objects.isNull(pipeTaskMeta)) {
return 0L;
}
+ return getUnTransferredEventCount(getListeningQueue());
+ }
+
+ /** Calculate metrics from a captured queue without looking it up again
during cleanup. */
+ protected long getUnTransferredEventCount(final AbstractPipeListeningQueue
listeningQueue) {
+ if (Objects.isNull(pipeTaskMeta) || Objects.isNull(listeningQueue)) {
+ return 0L;
+ }
final ProgressIndex progressIndex = pipeTaskMeta.getProgressIndex();
warnIfUnexpectedHybridProgressIndex(progressIndex);
final MetaProgressIndex metaProgressIndex =
extractMetaProgressIndex(progressIndex);
return Objects.nonNull(metaProgressIndex)
- ? getListeningQueue().getTailIndex() - metaProgressIndex.getIndex() - 1
- : getListeningQueue().getSize() + historicalEventsCount;
+ ? listeningQueue.getTailIndex() - metaProgressIndex.getIndex() - 1
+ : listeningQueue.getSize() + historicalEventsCount;
}
private static MetaProgressIndex extractMetaProgressIndex(final
ProgressIndex progressIndex) {
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java
new file mode 100644
index 00000000000..1f54a0d6d15
--- /dev/null
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgentTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.agent.task;
+
+import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
+import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey;
+import
org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager;
+import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter;
+import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink;
+import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq;
+
+import org.apache.thrift.TException;
+import org.apache.tsfile.utils.Pair;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+public class PipeTaskAgentTest {
+
+ @Test
+ public void testFloatingMemoryUsageSurvivesDropUntilLateRelease() throws
IllegalPathException {
+ final DummyPipeTaskAgent agent = new DummyPipeTaskAgent();
+
+ agent.createPipeForTest(generatePipeMeta("pipe", 1L));
+ agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L);
+ Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe"));
+ Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L));
+
+ agent.dropPipeForTest("pipe", 1L);
+ Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe"));
+ Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L));
+
+ agent.createPipeForTest(generatePipeMeta("pipe", 2L));
+ agent.addFloatingMemoryUsageInByte("pipe", 2L, 20L);
+ Assert.assertEquals(120L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(120L, agent.getFloatingMemoryUsageInByte("pipe"));
+ Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L));
+ Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 2L));
+
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L);
+ Assert.assertEquals(20L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe"));
+ Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("pipe", 1L));
+ Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 2L));
+
+ agent.dropPipeForTest("pipe", 2L);
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 2L, 20L);
+ Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(0L, agent.getFloatingMemoryUsageInByte("pipe"));
+ }
+
+ @Test
+ public void testZeroFloatingMemoryUsageCounterIsCleanedAfterDrop() throws
Exception {
+ final DummyPipeTaskAgent agent = new DummyPipeTaskAgent();
+
+ agent.createPipeForTest(generatePipeMeta("pipe", 1L));
+ agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L);
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L);
+
+ Assert.assertTrue(
+ getMapField(agent,
"pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty());
+ Assert.assertTrue(getMapField(agent,
"pipeName2CreationTimeSetMap").isEmpty());
+
+ agent.dropPipeForTest("pipe", 1L);
+
+ Assert.assertTrue(
+ getMapField(agent,
"pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty());
+ Assert.assertTrue(getMapField(agent,
"pipeName2CreationTimeSetMap").isEmpty());
+ }
+
+ @Test
+ public void
testLateFloatingMemoryAcquisitionDoesNotConsumeOutstandingUsage() throws
Exception {
+ final DummyPipeTaskAgent agent = new DummyPipeTaskAgent();
+ agent.createPipeForTest(generatePipeMeta("pipe", 1L));
+ agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L);
+ agent.dropPipeForTest("pipe", 1L);
+
+ // An assigner can retain a matched source across drop while other events
still own memory.
+ agent.addFloatingMemoryUsageInByte("pipe", 1L, 20L);
+ Assert.assertEquals(120L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(120L, agent.getFloatingMemoryUsageInByte("pipe"));
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 20L);
+ Assert.assertEquals(100L, agent.getFloatingMemoryUsageInByte("pipe", 1L));
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L);
+
+ Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertTrue(
+ getMapField(agent,
"pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty());
+ Assert.assertTrue(getMapField(agent,
"pipeName2CreationTimeSetMap").isEmpty());
+ }
+
+ @Test
+ public void testLateFloatingMemoryAcquisitionAfterCounterCleanup() throws
Exception {
+ final DummyPipeTaskAgent agent = new DummyPipeTaskAgent();
+ agent.createPipeForTest(generatePipeMeta("pipe", 1L));
+ agent.addFloatingMemoryUsageInByte("pipe", 1L, 100L);
+ agent.dropPipeForTest("pipe", 1L);
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 100L);
+
+ // Late acquisitions must also be tracked after the previous events
released the last byte.
+ agent.addFloatingMemoryUsageInByte("pipe", 1L, 20L);
+ Assert.assertEquals(20L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe"));
+ Assert.assertEquals(20L, agent.getFloatingMemoryUsageInByte("pipe", 1L));
+ agent.decreaseFloatingMemoryUsageInByte("pipe", 1L, 20L);
+
+ Assert.assertTrue(
+ getMapField(agent,
"pipeNameWithCreationTime2FloatingMemoryUsageInByteMap").isEmpty());
+ Assert.assertTrue(getMapField(agent,
"pipeName2CreationTimeSetMap").isEmpty());
+ }
+
+ @Test
+ public void testFloatingMemoryIncrementIsNotLostDuringDropCleanup() throws
Exception {
+ final DummyPipeTaskAgent agent = new DummyPipeTaskAgent();
+ final PipeMeta pipeMeta = generatePipeMeta("concurrent-pipe", 1L);
+ agent.createPipeForTest(pipeMeta);
+ agent.addFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L);
+ agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L);
+
+ final Field counterMapField =
+ PipeTaskAgent.class.getDeclaredField(
+ "pipeNameWithCreationTime2FloatingMemoryUsageInByteMap");
+ counterMapField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ final Map<String, AtomicLong> originalCounterMap =
+ (Map<String, AtomicLong>) counterMapField.get(agent);
+ final BlockingFloatingMemoryMap blockingCounterMap =
+ new BlockingFloatingMemoryMap(originalCounterMap, "concurrent-pipe_1");
+ counterMapField.set(agent, blockingCounterMap);
+
+ final AtomicReference<Throwable> backgroundFailure = new
AtomicReference<>();
+ final Thread increaseThread =
+ new Thread(
+ () ->
+ runAndRecordFailure(
+ () ->
agent.addFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L),
+ backgroundFailure));
+ final Thread dropThread =
+ new Thread(
+ () ->
+ runAndRecordFailure(
+ () -> agent.dropPipeForTest("concurrent-pipe", 1L),
backgroundFailure));
+
+ try {
+ try {
+ blockingCounterMap.pauseNextUpdate();
+ increaseThread.start();
+ Assert.assertTrue(blockingCounterMap.updatePaused.await(10,
TimeUnit.SECONDS));
+ dropThread.start();
+ Assert.assertTrue(waitForDrop(agent, "concurrent-pipe", 1L));
+ } finally {
+ blockingCounterMap.resumeUpdate.countDown();
+ increaseThread.join(TimeUnit.SECONDS.toMillis(10));
+ dropThread.join(TimeUnit.SECONDS.toMillis(10));
+ }
+
+ Assert.assertFalse(increaseThread.isAlive());
+ Assert.assertFalse(dropThread.isAlive());
+ Assert.assertNull(backgroundFailure.get());
+ Assert.assertEquals(100L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(100L,
agent.getFloatingMemoryUsageInByte("concurrent-pipe"));
+
+ agent.decreaseFloatingMemoryUsageInByte("concurrent-pipe", 1L, 100L);
+ Assert.assertEquals(0L, agent.getAllFloatingMemoryUsageInByte());
+ Assert.assertEquals(0L,
agent.getFloatingMemoryUsageInByte("concurrent-pipe"));
+ } finally {
+ counterMapField.set(agent, originalCounterMap);
+ }
+ }
+
+ @Test
+ public void testPipeDropClearsSharedRuntimeState() throws Exception {
+ assertPipeDropClearsSharedRuntimeState(false);
+ assertPipeDropClearsSharedRuntimeState(true);
+ }
+
+ private static void assertPipeDropClearsSharedRuntimeState(final boolean
dropByName)
+ throws Exception {
+ final String pipeName = "runtime_state_cleanup_" + System.nanoTime();
+ final long creationTime = System.nanoTime();
+ final int regionId = 17;
+ final DummyPipeTaskAgent agent = new DummyPipeTaskAgent();
+ final PipeEventCommitManager commitManager =
PipeEventCommitManager.getInstance();
+ final Map<Pair<String, Long>, PipeEndPointRateLimiter> rateLimiters =
getRateLimiterMap();
+ final Pair<String, Long> pipeKey = new Pair<>(pipeName, creationTime);
+
+ try {
+ commitManager.register(pipeName, creationTime, regionId, "test-sink");
+ final CommitterKey committerKey =
+ commitManager.getCommitterKey(pipeName, creationTime, regionId);
+ final CommitterKey restartTimesKey = new CommitterKey(pipeName,
creationTime, regionId, -1);
+ rateLimiters.put(pipeKey, new PipeEndPointRateLimiter(pipeName,
creationTime, 1.0));
+ agent.createPipeForTest(generatePipeMeta(pipeName, creationTime));
+
+ if (dropByName) {
+ Assert.assertTrue(agent.dropPipeByNameForTest(pipeName));
+ } else {
+ Assert.assertTrue(agent.dropPipeForTest(pipeName, creationTime));
+ }
+
+ Assert.assertFalse(
+ getCommitManagerMap(commitManager,
"eventCommitterMap").containsKey(committerKey));
+ Assert.assertFalse(
+ getCommitManagerMap(commitManager, "eventCommitterRestartTimesMap")
+ .containsKey(restartTimesKey));
+ Assert.assertFalse(rateLimiters.containsKey(pipeKey));
+ } finally {
+ commitManager.clear(pipeName, creationTime);
+ IoTDBSink.clearPipeRateLimiter(pipeName, creationTime);
+ final PipeMeta pipeMeta = agent.getPipeMetaForTest(pipeName,
creationTime);
+ if (pipeMeta != null) {
+ agent.dropPipeForTest(pipeName, creationTime);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<Pair<String, Long>, PipeEndPointRateLimiter>
getRateLimiterMap()
+ throws ReflectiveOperationException {
+ final Field field =
IoTDBSink.class.getDeclaredField("PIPE_END_POINT_RATE_LIMITER_MAP");
+ field.setAccessible(true);
+ return (Map<Pair<String, Long>, PipeEndPointRateLimiter>) field.get(null);
+ }
+
+ private static Map<?, ?> getCommitManagerMap(
+ final PipeEventCommitManager commitManager, final String fieldName)
+ throws ReflectiveOperationException {
+ final Field field =
PipeEventCommitManager.class.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ return (Map<?, ?>) field.get(commitManager);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<?, ?> getMapField(final PipeTaskAgent agent, final String
fieldName)
+ throws NoSuchFieldException, IllegalAccessException {
+ final Field field = PipeTaskAgent.class.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ return (Map<?, ?>) field.get(agent);
+ }
+
+ private static PipeMeta generatePipeMeta(final String pipeName, final long
creationTime) {
+ return new PipeMeta(
+ new PipeStaticMeta(
+ pipeName, creationTime, new HashMap<>(), new HashMap<>(), new
HashMap<>()),
+ new PipeRuntimeMeta(new ConcurrentHashMap<>()));
+ }
+
+ private static void runAndRecordFailure(
+ final Runnable operation, final AtomicReference<Throwable> failure) {
+ try {
+ operation.run();
+ } catch (final Throwable e) {
+ failure.compareAndSet(null, e);
+ }
+ }
+
+ private static boolean waitForDrop(
+ final DummyPipeTaskAgent agent, final String pipeName, final long
creationTime)
+ throws InterruptedException {
+ final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ while (System.nanoTime() < deadline) {
+ if (agent.getPipeMetaForTest(pipeName, creationTime) == null) {
+ return true;
+ }
+ Thread.sleep(1);
+ }
+ return false;
+ }
+
+ private static class DummyPipeTaskAgent extends PipeTaskAgent {
+
+ private boolean createPipeForTest(final PipeMeta pipeMeta) throws
IllegalPathException {
+ return createPipe(pipeMeta);
+ }
+
+ private boolean dropPipeForTest(final String pipeName, final long
creationTime) {
+ return dropPipe(pipeName, creationTime);
+ }
+
+ private boolean dropPipeByNameForTest(final String pipeName) {
+ return dropPipe(pipeName);
+ }
+
+ private PipeMeta getPipeMetaForTest(final String pipeName, final long
creationTime) {
+ return pipeMetaKeeper.getPipeMeta(pipeName, creationTime);
+ }
+
+ @Override
+ protected boolean isShutdown() {
+ return false;
+ }
+
+ @Override
+ protected void thawRate(final String pipeName, final long creationTime) {
+ // Do nothing
+ }
+
+ @Override
+ protected void freezeRate(final String pipeName, final long creationTime) {
+ // Do nothing
+ }
+
+ @Override
+ protected Map<Integer, PipeTask> buildPipeTasks(final PipeMeta
pipeMetaFromCoordinator) {
+ return new HashMap<>();
+ }
+
+ @Override
+ protected void createPipeTask(
+ final int consensusGroupId,
+ final PipeStaticMeta pipeStaticMeta,
+ final org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta
pipeTaskMeta) {
+ // Do nothing
+ }
+
+ @Override
+ protected void collectPipeMetaListInternal(
+ final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws
TException {
+ // Do nothing
+ }
+
+ @Override
+ public void runPipeTasks(
+ final Collection<PipeTask> pipeTasks, final Consumer<PipeTask>
runSingle) {
+ pipeTasks.forEach(runSingle);
+ }
+ }
+
+ private static class BlockingFloatingMemoryMap extends
ConcurrentHashMap<String, AtomicLong> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String pipeID;
+ private final AtomicBoolean shouldPause = new AtomicBoolean();
+ private final CountDownLatch updatePaused = new CountDownLatch(1);
+ private final CountDownLatch resumeUpdate = new CountDownLatch(1);
+
+ private BlockingFloatingMemoryMap(
+ final Map<String, AtomicLong> originalMap, final String pipeID) {
+ putAll(originalMap);
+ this.pipeID = pipeID;
+ }
+
+ private void pauseNextUpdate() {
+ shouldPause.set(true);
+ }
+
+ @Override
+ public AtomicLong compute(
+ final String key,
+ final BiFunction<? super String, ? super AtomicLong, ? extends
AtomicLong>
+ remappingFunction) {
+ return super.compute(
+ key,
+ (currentKey, currentValue) -> {
+ final AtomicLong updatedValue =
remappingFunction.apply(currentKey, currentValue);
+ pauseIfNecessary(currentKey);
+ return updatedValue;
+ });
+ }
+
+ @Override
+ public AtomicLong computeIfAbsent(
+ final String key, final Function<? super String, ? extends AtomicLong>
mappingFunction) {
+ final AtomicLong value = super.computeIfAbsent(key, mappingFunction);
+ pauseIfNecessary(key);
+ return value;
+ }
+
+ private void pauseIfNecessary(final String key) {
+ if (!pipeID.equals(key) || !shouldPause.compareAndSet(true, false)) {
+ return;
+ }
+ updatePaused.countDown();
+ try {
+ if (!resumeUpdate.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting to resume
floating memory update");
+ }
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+}
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java
new file mode 100644
index 00000000000..2d81059d623
--- /dev/null
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManagerTest.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.agent.task.progress;
+
+import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
+import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+@RunWith(Parameterized.class)
+public class PipeEventCommitManagerTest {
+
+ private static final String PIPE_NAME = "late-commit-test";
+ private static final long CREATION_TIME = 1L;
+ private static final int REGION_ID = 1;
+
+ private final Level logLevel;
+ private final Logger logger = (Logger)
LoggerFactory.getLogger(PipeEventCommitManager.class);
+ private Level originalLogLevel;
+ private PipeEventCommitManager manager;
+
+ public PipeEventCommitManagerTest(final Level logLevel) {
+ this.logLevel = logLevel;
+ }
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Iterable<Level> logLevels() {
+ return Arrays.asList(Level.INFO, Level.DEBUG);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ originalLogLevel = logger.getLevel();
+ logger.setLevel(logLevel);
+ final Constructor<PipeEventCommitManager> constructor =
+ PipeEventCommitManager.class.getDeclaredConstructor();
+ constructor.setAccessible(true);
+ manager = constructor.newInstance();
+
+ final PipeTaskAgent taskAgent = mock(PipeTaskAgent.class);
+ when(taskAgent.getCommitterKey(eq(PIPE_NAME), eq(CREATION_TIME),
eq(REGION_ID), anyInt()))
+ .thenAnswer(
+ invocation ->
+ new CommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID,
invocation.getArgument(3)));
+ manager.setTaskAgent(taskAgent);
+ manager.register(PIPE_NAME, CREATION_TIME, REGION_ID, "iotdb-thrift-sink");
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ if (manager != null) {
+ manager.clear(PIPE_NAME, CREATION_TIME);
+ }
+ } finally {
+ logger.setLevel(originalLogLevel);
+ }
+ }
+
+ @Test
+ public void testLateSingleCommitAfterClearDoesNotRestoreState() throws
Exception {
+ final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME,
REGION_ID);
+ final Runnable onCommitted = mock(Runnable.class);
+ final EnrichedEvent event = createEvent(onCommitted);
+ manager.clear(PIPE_NAME, CREATION_TIME);
+
+ manager.commit(event, key);
+
+ assertTrue(getManagerMap("eventCommitterMap").isEmpty());
+ assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty());
+ verify(onCommitted, never()).run();
+ }
+
+ @Test
+ public void testLateMultipleCommitsAfterClearDoNotRestoreState() throws
Exception {
+ final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME,
REGION_ID);
+ final Runnable onCommitted = mock(Runnable.class);
+ final EnrichedEvent event = createEvent(onCommitted);
+ when(event.hasMultipleCommitIds()).thenReturn(true);
+ when(event.getCommitIds()).thenReturn(Arrays.asList(1L, 2L, 3L));
+ manager.clear(PIPE_NAME, CREATION_TIME);
+
+ manager.commit(event, key);
+
+ assertTrue(getManagerMap("eventCommitterMap").isEmpty());
+ assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty());
+ verify(onCommitted, never()).run();
+ }
+
+ @Test
+ public void testLateEventEnrichmentAfterClearDoesNotRestoreState() throws
Exception {
+ final EnrichedEvent event = createEvent(mock(Runnable.class));
+ manager.clear(PIPE_NAME, CREATION_TIME);
+
+ manager.enrichWithCommitterKeyAndCommitId(event, CREATION_TIME, REGION_ID);
+
+ assertTrue(getManagerMap("eventCommitterMap").isEmpty());
+ assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty());
+ }
+
+ @Test
+ public void testLateCommitterCheckAfterClearDoesNotRestoreRestartState()
throws Exception {
+ final CommitterKey key = manager.getCommitterKey(PIPE_NAME, CREATION_TIME,
REGION_ID);
+ final Field eventCommitterMapField =
+ PipeEventCommitManager.class.getDeclaredField("eventCommitterMap");
+ eventCommitterMapField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ final Map<CommitterKey, PipeEventCommitter> originalEventCommitterMap =
+ (Map<CommitterKey, PipeEventCommitter>)
eventCommitterMapField.get(manager);
+ final BlockingCommitterMap blockingMap = new
BlockingCommitterMap(originalEventCommitterMap);
+ eventCommitterMapField.set(manager, blockingMap);
+
+ final AtomicReference<Boolean> result = new AtomicReference<>();
+ final AtomicReference<Throwable> backgroundFailure = new
AtomicReference<>();
+ final Thread checkThread =
+ new Thread(
+ () -> {
+ try {
+ result.set(manager.isCurrentCommitterKey(key));
+ } catch (final Throwable e) {
+ backgroundFailure.compareAndSet(null, e);
+ }
+ });
+
+ try {
+ blockingMap.pauseNextLookup();
+ checkThread.start();
+ assertTrue(blockingMap.lookupPaused.await(10, TimeUnit.SECONDS));
+
+ manager.clear(PIPE_NAME, CREATION_TIME);
+ } finally {
+ blockingMap.resumeLookup.countDown();
+ checkThread.join(TimeUnit.SECONDS.toMillis(10));
+ eventCommitterMapField.set(manager, originalEventCommitterMap);
+ }
+
+ assertFalse(checkThread.isAlive());
+ assertTrue(backgroundFailure.get() == null);
+ assertFalse(result.get());
+ assertTrue(getManagerMap("eventCommitterRestartTimesMap").isEmpty());
+ }
+
+ @Test
+ public void testRestartRejectsStaleCommitAndAcceptsCurrentCommit() throws
Exception {
+ final CommitterKey oldKey = manager.getCommitterKey(PIPE_NAME,
CREATION_TIME, REGION_ID);
+ final Runnable oldOnCommitted = mock(Runnable.class);
+ manager.deregister(PIPE_NAME, CREATION_TIME, REGION_ID);
+
+ manager.commit(createEvent(oldOnCommitted), oldKey);
+ assertEquals(
+ Integer.valueOf(1),
+ getManagerMap("eventCommitterRestartTimesMap")
+ .get(new CommitterKey(PIPE_NAME, CREATION_TIME, REGION_ID)));
+
+ manager.register(PIPE_NAME, CREATION_TIME, REGION_ID, "iotdb-thrift-sink");
+ final CommitterKey newKey = manager.getCommitterKey(PIPE_NAME,
CREATION_TIME, REGION_ID);
+ assertEquals(1, newKey.getRestartTimes());
+ final Runnable newOnCommitted = mock(Runnable.class);
+ manager.commit(createEvent(oldOnCommitted), oldKey);
+ manager.commit(createEvent(newOnCommitted), newKey);
+
+ verify(oldOnCommitted, never()).run();
+ verify(newOnCommitted).run();
+ assertEquals(1, getManagerMap("eventCommitterMap").size());
+ assertEquals(1, getManagerMap("eventCommitterRestartTimesMap").size());
+ }
+
+ private static EnrichedEvent createEvent(final Runnable onCommitted) {
+ final EnrichedEvent event =
+ mock(
+ EnrichedEvent.class,
+ withSettings()
+ .useConstructor(
+ PIPE_NAME,
+ CREATION_TIME,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ false,
+ Long.MIN_VALUE,
+ Long.MAX_VALUE));
+ when(event.needToCommit()).thenReturn(true);
+ when(event.getCommitId()).thenReturn(1L);
+
when(event.getOnCommittedHooks()).thenReturn(Collections.singletonList(onCommitted));
+ return event;
+ }
+
+ private Map<?, ?> getManagerMap(final String fieldName) throws Exception {
+ final Field field =
PipeEventCommitManager.class.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ return (Map<?, ?>) field.get(manager);
+ }
+
+ private static class BlockingCommitterMap
+ extends ConcurrentHashMap<CommitterKey, PipeEventCommitter> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AtomicBoolean shouldPause = new AtomicBoolean();
+ private final CountDownLatch lookupPaused = new CountDownLatch(1);
+ private final CountDownLatch resumeLookup = new CountDownLatch(1);
+
+ private BlockingCommitterMap(final Map<CommitterKey, PipeEventCommitter>
originalMap) {
+ putAll(originalMap);
+ }
+
+ private void pauseNextLookup() {
+ shouldPause.set(true);
+ }
+
+ @Override
+ public PipeEventCommitter get(final Object key) {
+ final PipeEventCommitter committer = super.get(key);
+ if (committer != null && shouldPause.compareAndSet(true, false)) {
+ lookupPaused.countDown();
+ try {
+ if (!resumeLookup.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("Timed out waiting to resume
committer lookup");
+ }
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+ return committer;
+ }
+ }
+}
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java
new file mode 100644
index 00000000000..a98a3a59b3c
--- /dev/null
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSinkRateLimiterTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.pipe.sink.protocol;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
+import org.apache.iotdb.commons.pipe.sink.limiter.PipeEndPointRateLimiter;
+
+import org.apache.tsfile.utils.Pair;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IoTDBSinkRateLimiterTest {
+
+ @Test
+ public void testRegionTaskDiscardKeepsPipeWideRateLimiter() throws Exception
{
+ final String pipeName = "rate_limiter_lifecycle_test";
+ final long creationTime = System.nanoTime();
+ final Pair<String, Long> pipeKey = new Pair<>(pipeName, creationTime);
+ final Map<Pair<String, Long>, PipeEndPointRateLimiter> limiters =
getRateLimiterMap();
+ final PipeEndPointRateLimiter limiter =
mock(PipeEndPointRateLimiter.class);
+ final IoTDBSink sink = mock(IoTDBSink.class, CALLS_REAL_METHODS);
+ limiters.put(pipeKey, limiter);
+
+ try {
+ sink.discardEventsOfPipe(pipeName, creationTime, 1);
+
+ assertSame(limiter, limiters.get(pipeKey));
+
+ IoTDBSink.clearPipeRateLimiter(pipeName, creationTime);
+
+ assertFalse(limiters.containsKey(pipeKey));
+ } finally {
+ IoTDBSink.clearPipeRateLimiter(pipeName, creationTime);
+ }
+ }
+
+ @Test
+ public void testLateTransferDoesNotRecreateRateLimiterAfterDrop() throws
Exception {
+ final String pipeName = "late_rate_limiter_lifecycle_test";
+ final long creationTime = System.nanoTime();
+ final Pair<String, Long> pipeKey = new Pair<>(pipeName, creationTime);
+ final Map<Pair<String, Long>, PipeEndPointRateLimiter> limiters =
getRateLimiterMap();
+ final IoTDBSink sink = mock(IoTDBSink.class, CALLS_REAL_METHODS);
+ final PipeTaskAgent taskAgent = mock(PipeTaskAgent.class);
+ when(taskAgent.isPipeAlive(pipeName, creationTime)).thenReturn(false);
+
+ final Field taskAgentField =
PipeEndPointRateLimiter.class.getDeclaredField("taskAgent");
+ taskAgentField.setAccessible(true);
+ final PipeTaskAgent originalTaskAgent = (PipeTaskAgent)
taskAgentField.get(null);
+ final Field rateLimitField =
+ IoTDBSink.class.getDeclaredField("endPointRateLimitBytesPerSecond");
+ rateLimitField.setAccessible(true);
+ rateLimitField.setDouble(sink, 1.0);
+
+ PipeEndPointRateLimiter.setTaskAgent(taskAgent);
+ try {
+ sink.rateLimitIfNeeded(pipeName, creationTime, new
TEndPoint("localhost", 1), 0);
+
+ assertFalse(limiters.containsKey(pipeKey));
+ } finally {
+ IoTDBSink.clearPipeRateLimiter(pipeName, creationTime);
+ PipeEndPointRateLimiter.setTaskAgent(originalTaskAgent);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<Pair<String, Long>, PipeEndPointRateLimiter>
getRateLimiterMap()
+ throws ReflectiveOperationException {
+ final Field field =
IoTDBSink.class.getDeclaredField("PIPE_END_POINT_RATE_LIMITER_MAP");
+ field.setAccessible(true);
+ return (Map<Pair<String, Long>, PipeEndPointRateLimiter>) field.get(null);
+ }
+}