This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix/subscription-removal-timeout in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit beff6be031bdaba3523fe46bc1bec5e13db70256 Author: Caideyipi <[email protected]> AuthorDate: Tue Nov 10 10:32:21 2026 +0800 Latch subscription timeout state on DataNode removal --- .../impl/DataNodeInternalRPCServiceImpl.java | 4 ++ .../receiver/SubscriptionReceiverV1.java | 17 +++++- .../receiver/SubscriptionReceiverV1Test.java | 60 ++++++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 836f8d47f29..df0344a4341 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -226,6 +226,7 @@ import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaMana import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.db.subscription.broker.consensus.ConsensusRegionRuntimeState; import org.apache.iotdb.db.subscription.broker.consensus.ConsensusSubscriptionSetupHandler; +import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiverV1; import org.apache.iotdb.db.trigger.executor.TriggerExecutor; import org.apache.iotdb.db.trigger.executor.TriggerFireResult; import org.apache.iotdb.db.trigger.service.TriggerManagementService; @@ -2837,6 +2838,9 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface } else { commonConfig.setNodeStatus(nodeStatus); } + if (nodeStatus == NodeStatus.Removing) { + SubscriptionReceiverV1.handleNodeRetirement(); + } } catch (Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 7a7f91e476d..910fc0709f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -152,6 +152,8 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { PipeSubscribeResponseVersion.VERSION_1.getVersion(), PipeSubscribeResponseType.ACK.getType()); + private static final AtomicLong NODE_RETIREMENT_VERSION = new AtomicLong(); + private final ThreadLocal<ConsumerConfig> consumerConfigThreadLocal = new ThreadLocal<>(); private final ThreadLocal<PollTimer> pollTimerThreadLocal = new ThreadLocal<>(); private volatile String authenticatedUsername; @@ -167,6 +169,7 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { private long consumerStateVersion; private boolean connectionExited; private boolean timeoutCleanupSuspended; + private long observedNodeRetirementVersion = NODE_RETIREMENT_VERSION.get(); private static final String SQL_DIALECT_TABLE_VALUE = "table"; @@ -311,7 +314,7 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { } } finally { connectionExited = true; - if (timeoutCleanupSuspended || isNodeRetiring()) { + if (timeoutCleanupSuspended || hasNodeRetired()) { // Retiring this transport transfers timeout responsibility to the other DataNodes. // Discard only the local receiver state, including when removal has already rolled back. clearSharedConsumerState(); @@ -378,7 +381,7 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { @Override public void handleTimeout() { synchronized (this) { - timeoutCleanupSuspended |= isNodeRetiring(); + timeoutCleanupSuspended |= hasNodeRetired(); if (timeoutCleanupSuspended) { // The SDK retires this node's connection while the consumer remains active elsewhere. // Rollback alone does not restore that connection: wait for a new request before allowing @@ -428,6 +431,15 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { return commonConfig.getNodeStatus() == NodeStatus.Removing || commonConfig.isStopping(); } + public static void handleNodeRetirement() { + // A removal and rollback can both occur between timeout checks, requests, and exit callbacks. + NODE_RETIREMENT_VERSION.incrementAndGet(); + } + + private boolean hasNodeRetired() { + return observedNodeRetirementVersion != NODE_RETIREMENT_VERSION.get() || isNodeRetiring(); + } + @Override public long remainingMs() { final PollTimer pollTimer = pollTimerThreadLocal.get(); @@ -1575,6 +1587,7 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { private boolean beforeHandle(final short reqType) { synchronized (this) { // A request on a restored node starts a fresh local inactivity deadline. + observedNodeRetirementVersion = NODE_RETIREMENT_VERSION.get(); timeoutCleanupSuspended = isNodeRetiring(); connectionExited = false; final boolean isHandshake = PipeSubscribeRequestType.HANDSHAKE.getType() == reqType; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java index 5e35322938e..9e2865649f2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java @@ -27,7 +27,10 @@ import org.apache.iotdb.commons.subscription.config.SubscriptionConfig; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodePipeMessages; +import org.apache.iotdb.db.protocol.thrift.impl.DataNodeInternalRPCServiceImpl; +import org.apache.iotdb.db.service.DataNode.DataNodeContext; import org.apache.iotdb.db.subscription.agent.SubscriptionAgent; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.rpc.subscription.config.ConsumerConfig; @@ -45,6 +48,7 @@ import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -165,6 +169,62 @@ public class SubscriptionReceiverV1Test { assertExitRetiresTimeoutState(false, true); } + @Test + @SuppressWarnings("unchecked") + public void testRemovalRollbackRetiresConsumerWithoutTimeoutTick() throws Exception { + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final NodeStatus originalStatus = commonConfig.getNodeStatus(); + final String originalReason = commonConfig.getStatusReason(); + final boolean originalStopping = commonConfig.isStopping(); + final int originalDataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + try { + // Coordinator requires a registered DataNode ID when the internal RPC service initializes. + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0); + final DataNodeInternalRPCServiceImpl service = + new DataNodeInternalRPCServiceImpl(Mockito.mock(DataNodeContext.class)); + for (final boolean exitBeforeRemoval : new boolean[] {false, true}) { + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setStopping(false); + final AtomicLong closeAttemptCount = new AtomicLong(); + final SubscriptionReceiverV1 receiver = + new SubscriptionReceiverV1() { + @Override + void closeConsumer(final ConsumerConfig consumerConfig) { + closeAttemptCount.incrementAndGet(); + } + }; + final ConsumerConfig consumerConfig = createConsumerConfig(1_000L); + setField(receiver, "sharedConsumerConfig", consumerConfig); + ((ThreadLocal<ConsumerConfig>) getField(receiver, "consumerConfigThreadLocal")) + .set(consumerConfig); + final long timeoutMs = invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig); + setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - timeoutMs - 1L); + if (exitBeforeRemoval) { + receiver.handleExit(); + } + // The entire Removing interval can fall between two scheduled timeout checks and between + // requests. A delayed transport-exit callback may only run after the removal rolls back. + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + service.setSystemStatus(NodeStatus.Removing.getStatus()).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + service.setSystemStatus(NodeStatus.Running.getStatus()).getCode()); + if (!exitBeforeRemoval) { + receiver.handleExit(); + } + receiver.handleTimeout(); + + Assert.assertEquals(0L, closeAttemptCount.get()); + Assert.assertFalse(receiver.hasActiveConsumer()); + } + } finally { + IoTDBDescriptor.getInstance().getConfig().setDataNodeId(originalDataNodeId); + commonConfig.setNodeStatusWithReason(originalStatus, originalReason); + commonConfig.setStopping(originalStopping); + } + } + @SuppressWarnings("unchecked") private void assertExitRetiresTimeoutState( final boolean exitBeforeRetirement, final boolean restoreBeforeExit) throws Exception {
