This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e9267062671 Fix subscription timeout cleanup during DataNode removal
(#19197)
e9267062671 is described below
commit e92670626716bae9215aac0259a554aae3e59274
Author: Caideyipi <[email protected]>
AuthorDate: Tue Nov 10 17:02:42 2026 +0800
Fix subscription timeout cleanup during DataNode removal (#19197)
* Avoid subscription timeout cleanup on retiring DataNodes
* Preserve subscription consumers across DataNode removal rollback
* Latch subscription timeout state on DataNode removal
---
.../impl/DataNodeInternalRPCServiceImpl.java | 4 +
.../receiver/SubscriptionReceiverV1.java | 41 ++++
.../receiver/SubscriptionReceiverV1Test.java | 251 +++++++++++++++++++++
3 files changed, 296 insertions(+)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 20f5eb888cd..f03366065cd 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -226,6 +226,7 @@ import
org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaMana
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
import
org.apache.iotdb.db.subscription.broker.consensus.ConsensusRegionRuntimeState;
import
org.apache.iotdb.db.subscription.broker.consensus.ConsensusSubscriptionSetupHandler;
+import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiverV1;
import org.apache.iotdb.db.trigger.executor.TriggerExecutor;
import org.apache.iotdb.db.trigger.executor.TriggerFireResult;
import org.apache.iotdb.db.trigger.service.TriggerManagementService;
@@ -2837,6 +2838,9 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
} else {
commonConfig.setNodeStatus(nodeStatus);
}
+ if (nodeStatus == NodeStatus.Removing) {
+ SubscriptionReceiverV1.handleNodeRetirement();
+ }
} catch (Exception e) {
return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR,
e.getMessage());
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
index c7812bbb0bf..1d361056a50 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.ConfigRegionId;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
@@ -152,6 +154,8 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
PipeSubscribeResponseVersion.VERSION_1.getVersion(),
PipeSubscribeResponseType.ACK.getType());
+ private static final AtomicLong NODE_RETIREMENT_VERSION = new AtomicLong();
+
private final ThreadLocal<ConsumerConfig> consumerConfigThreadLocal = new
ThreadLocal<>();
private final ThreadLocal<PollTimer> pollTimerThreadLocal = new
ThreadLocal<>();
private volatile String authenticatedUsername;
@@ -165,6 +169,9 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
private final PipeSubscribeSliceReqHandler sliceReqHandler =
new PipeSubscribeSliceReqHandler(maxReassembledRequestBodySize);
private long consumerStateVersion;
+ private boolean connectionExited;
+ private boolean timeoutCleanupSuspended;
+ private long observedNodeRetirementVersion = NODE_RETIREMENT_VERSION.get();
private static final String SQL_DIALECT_TABLE_VALUE = "table";
@@ -308,6 +315,12 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
}
}
} finally {
+ connectionExited = true;
+ if (timeoutCleanupSuspended || hasNodeRetired()) {
+ // Retiring this transport transfers timeout responsibility to the
other DataNodes.
+ // Discard only the local receiver state, including when removal has
already rolled back.
+ clearSharedConsumerState();
+ }
sliceReqHandler.clear();
consumerConfigThreadLocal.remove();
pollTimerThreadLocal.remove();
@@ -370,6 +383,16 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
@Override
public void handleTimeout() {
synchronized (this) {
+ timeoutCleanupSuspended |= hasNodeRetired();
+ if (timeoutCleanupSuspended) {
+ // The SDK retires this node's connection while the consumer remains
active elsewhere.
+ // Rollback alone does not restore that connection: wait for a new
request before allowing
+ // this receiver to close the cluster consumer again.
+ if (connectionExited) {
+ clearSharedConsumerState();
+ }
+ return;
+ }
final ConsumerConfig consumerConfig = sharedConsumerConfig;
if (Objects.isNull(consumerConfig) || inFlightRequestCount.get() > 0) {
return;
@@ -405,6 +428,20 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
}
}
+ private static boolean isNodeRetiring() {
+ final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ return commonConfig.getNodeStatus() == NodeStatus.Removing ||
commonConfig.isStopping();
+ }
+
+ public static void handleNodeRetirement() {
+ // A removal and rollback can both occur between timeout checks, requests,
and exit callbacks.
+ NODE_RETIREMENT_VERSION.incrementAndGet();
+ }
+
+ private boolean hasNodeRetired() {
+ return observedNodeRetirementVersion != NODE_RETIREMENT_VERSION.get() ||
isNodeRetiring();
+ }
+
@Override
public long remainingMs() {
final PollTimer pollTimer = pollTimerThreadLocal.get();
@@ -1579,6 +1616,10 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
private boolean beforeHandle(final short reqType) {
synchronized (this) {
+ // A request on a restored node starts a fresh local inactivity deadline.
+ observedNodeRetirementVersion = NODE_RETIREMENT_VERSION.get();
+ timeoutCleanupSuspended = isNodeRetiring();
+ connectionExited = false;
final boolean isHandshake = PipeSubscribeRequestType.HANDSHAKE.getType()
== reqType;
// Replaced connections are terminal, even when the consumer instance
itself is still valid.
// An old connection must not handshake again and replace the current
connection in turn.
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
index 2b901cda9b8..9e2865649f2 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
@@ -20,11 +20,17 @@
package org.apache.iotdb.db.subscription.receiver;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta;
import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta;
import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.protocol.thrift.impl.DataNodeInternalRPCServiceImpl;
+import org.apache.iotdb.db.service.DataNode.DataNodeContext;
import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.rpc.subscription.config.ConsumerConfig;
@@ -42,6 +48,7 @@ import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
@@ -68,6 +75,250 @@ public class SubscriptionReceiverV1Test {
Assert.assertFalse((boolean) getField(receiver, "consumerInvalidated"));
}
+ @Test
+ public void testHandleTimeoutKeepsConsumerOnRemovingNode() throws Exception {
+ assertRetiringNodeKeepsConsumer(NodeStatus.Removing, false);
+ }
+
+ @Test
+ public void testHandleTimeoutKeepsConsumerOnStoppingNode() throws Exception {
+ assertRetiringNodeKeepsConsumer(NodeStatus.ReadOnly, true);
+ }
+
+ private void assertRetiringNodeKeepsConsumer(final NodeStatus status, final
boolean stopping)
+ throws Exception {
+ final AtomicLong closeAttemptCount = new AtomicLong();
+ final SubscriptionReceiverV1 receiver =
+ new SubscriptionReceiverV1() {
+ @Override
+ void closeConsumer(final ConsumerConfig consumerConfig) {
+ closeAttemptCount.incrementAndGet();
+ }
+ };
+ final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+ setField(receiver, "sharedConsumerConfig", consumerConfig);
+ final long timeoutMs =
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+ setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() -
timeoutMs - 1L);
+
+ final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ final NodeStatus originalStatus = commonConfig.getNodeStatus();
+ final String originalReason = commonConfig.getStatusReason();
+ final boolean originalStopping = commonConfig.isStopping();
+ try {
+ commonConfig.setNodeStatus(status);
+ commonConfig.setStopping(stopping);
+ receiver.handleTimeout();
+
+ Assert.assertEquals(
+ "Retiring a node must not unsubscribe a live consumer", 0L,
closeAttemptCount.get());
+ Assert.assertSame(consumerConfig, getField(receiver,
"sharedConsumerConfig"));
+ Assert.assertTrue(receiver.hasActiveConsumer());
+
+ // A failed removal can restore this node. Normal inactivity cleanup
must still work then,
+ // including on a read-only node that is not stopping.
+ commonConfig.setNodeStatus(NodeStatus.ReadOnly);
+ commonConfig.setStopping(false);
+ receiver.handleTimeout();
+ Assert.assertEquals(0L, closeAttemptCount.get());
+ Assert.assertTrue(receiver.hasActiveConsumer());
+
+ // Endpoint discovery may lag behind rollback. Only a new request can
establish a fresh
+ // inactivity deadline on this node.
+ final ConsumerGroupMeta consumerGroupMeta =
+ new ConsumerGroupMeta(
+ consumerConfig.getConsumerGroupId(),
+ System.currentTimeMillis(),
+ new ConsumerMeta(
+ consumerConfig.getConsumerId(),
+ System.currentTimeMillis(),
+ consumerConfig.getAttribute()));
+ Assert.assertNull(
+
SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(consumerGroupMeta));
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ receiver
+
.handle(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig))
+ .getStatus()
+ .getCode());
+ receiver.handleTimeout();
+ Assert.assertEquals(0L, closeAttemptCount.get());
+
+ setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() -
timeoutMs - 1L);
+ receiver.handleTimeout();
+ Assert.assertEquals(1L, closeAttemptCount.get());
+ Assert.assertFalse(receiver.hasActiveConsumer());
+ } finally {
+
SubscriptionAgent.consumer().handleDropConsumerGroup(consumerConfig.getConsumerGroupId());
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ commonConfig.setStopping(originalStopping);
+ }
+ }
+
+ @Test
+ public void testHandleExitRetiresTimeoutStateOnRetiringNode() throws
Exception {
+ assertExitRetiresTimeoutState(false, false);
+ }
+
+ @Test
+ public void testHandleTimeoutRetiresPreviouslyDisconnectedConsumer() throws
Exception {
+ assertExitRetiresTimeoutState(true, false);
+ }
+
+ @Test
+ public void testHandleExitRetiresTimeoutStateAfterRemovalRollback() throws
Exception {
+ assertExitRetiresTimeoutState(false, true);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRemovalRollbackRetiresConsumerWithoutTimeoutTick() throws
Exception {
+ final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ final NodeStatus originalStatus = commonConfig.getNodeStatus();
+ final String originalReason = commonConfig.getStatusReason();
+ final boolean originalStopping = commonConfig.isStopping();
+ final int originalDataNodeId =
IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
+ try {
+ // Coordinator requires a registered DataNode ID when the internal RPC
service initializes.
+ IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
+ final DataNodeInternalRPCServiceImpl service =
+ new
DataNodeInternalRPCServiceImpl(Mockito.mock(DataNodeContext.class));
+ for (final boolean exitBeforeRemoval : new boolean[] {false, true}) {
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setStopping(false);
+ final AtomicLong closeAttemptCount = new AtomicLong();
+ final SubscriptionReceiverV1 receiver =
+ new SubscriptionReceiverV1() {
+ @Override
+ void closeConsumer(final ConsumerConfig consumerConfig) {
+ closeAttemptCount.incrementAndGet();
+ }
+ };
+ final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+ setField(receiver, "sharedConsumerConfig", consumerConfig);
+ ((ThreadLocal<ConsumerConfig>) getField(receiver,
"consumerConfigThreadLocal"))
+ .set(consumerConfig);
+ final long timeoutMs =
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+ setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() -
timeoutMs - 1L);
+ if (exitBeforeRemoval) {
+ receiver.handleExit();
+ }
+ // The entire Removing interval can fall between two scheduled timeout
checks and between
+ // requests. A delayed transport-exit callback may only run after the
removal rolls back.
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+
service.setSystemStatus(NodeStatus.Removing.getStatus()).getCode());
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ service.setSystemStatus(NodeStatus.Running.getStatus()).getCode());
+ if (!exitBeforeRemoval) {
+ receiver.handleExit();
+ }
+ receiver.handleTimeout();
+
+ Assert.assertEquals(0L, closeAttemptCount.get());
+ Assert.assertFalse(receiver.hasActiveConsumer());
+ }
+ } finally {
+
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(originalDataNodeId);
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ commonConfig.setStopping(originalStopping);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void assertExitRetiresTimeoutState(
+ final boolean exitBeforeRetirement, final boolean restoreBeforeExit)
throws Exception {
+ final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ final NodeStatus originalStatus = commonConfig.getNodeStatus();
+ final String originalReason = commonConfig.getStatusReason();
+ final boolean originalStopping = commonConfig.isStopping();
+ try {
+ for (final boolean stopping : new boolean[] {false, true}) {
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setStopping(false);
+ final AtomicLong closeAttemptCount = new AtomicLong();
+ final SubscriptionReceiverV1 receiver =
+ new SubscriptionReceiverV1() {
+ @Override
+ void closeConsumer(final ConsumerConfig consumerConfig) {
+ closeAttemptCount.incrementAndGet();
+ }
+ };
+ final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+ setField(receiver, "sharedConsumerConfig", consumerConfig);
+ ((ThreadLocal<ConsumerConfig>) getField(receiver,
"consumerConfigThreadLocal"))
+ .set(consumerConfig);
+ final long timeoutMs =
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+ setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() -
timeoutMs - 1L);
+
+ if (exitBeforeRetirement) {
+ receiver.handleExit();
+ Assert.assertTrue(receiver.hasActiveConsumer());
+ }
+ commonConfig.setNodeStatus(stopping ? NodeStatus.ReadOnly :
NodeStatus.Removing);
+ commonConfig.setStopping(stopping);
+ receiver.handleTimeout();
+ if (restoreBeforeExit) {
+ commonConfig.setNodeStatus(NodeStatus.ReadOnly);
+ commonConfig.setStopping(false);
+ }
+ if (!exitBeforeRetirement) {
+ receiver.handleExit();
+ }
+
+ Assert.assertFalse(receiver.hasActiveConsumer());
+ Assert.assertNull(
+ ((ThreadLocal<ConsumerConfig>) getField(receiver,
"consumerConfigThreadLocal")).get());
+ commonConfig.setNodeStatus(NodeStatus.ReadOnly);
+ commonConfig.setStopping(false);
+ receiver.handleTimeout();
+ Assert.assertEquals(0L, closeAttemptCount.get());
+ }
+ } finally {
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ commonConfig.setStopping(originalStopping);
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testExplicitCloseStillWorksOnRetiringNode() throws Exception {
+ final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ final NodeStatus originalStatus = commonConfig.getNodeStatus();
+ final String originalReason = commonConfig.getStatusReason();
+ final boolean originalStopping = commonConfig.isStopping();
+ try {
+ for (final boolean stopping : new boolean[] {false, true}) {
+ final AtomicLong closeAttemptCount = new AtomicLong();
+ final SubscriptionReceiverV1 receiver =
+ new SubscriptionReceiverV1() {
+ @Override
+ void closeConsumer(final ConsumerConfig consumerConfig) {
+ closeAttemptCount.incrementAndGet();
+ }
+ };
+ final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+ setField(receiver, "sharedConsumerConfig", consumerConfig);
+ ((ThreadLocal<ConsumerConfig>) getField(receiver,
"consumerConfigThreadLocal"))
+ .set(consumerConfig);
+ final long timeoutMs =
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+ setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() -
timeoutMs - 1L);
+ commonConfig.setNodeStatus(stopping ? NodeStatus.ReadOnly :
NodeStatus.Removing);
+ commonConfig.setStopping(stopping);
+ receiver.handleTimeout();
+
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+
receiver.handle(PipeSubscribeCloseReq.toTPipeSubscribeReq()).getStatus().getCode());
+ Assert.assertEquals(1L, closeAttemptCount.get());
+ Assert.assertFalse(receiver.hasActiveConsumer());
+ }
+ } finally {
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ commonConfig.setStopping(originalStopping);
+ }
+ }
+
@Test
public void testHandleTimeoutSkipsConsumerWithInFlightRequests() throws
Exception {
final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1();