This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch fix/subscription-removal-timeout in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 521f8f8dc15f5dd143cfe52fc2947711afe46257 Author: Caideyipi <[email protected]> AuthorDate: Tue Nov 10 09:59:03 2026 +0800 Preserve subscription consumers across DataNode removal rollback --- .../receiver/SubscriptionReceiverV1.java | 27 +++- .../receiver/SubscriptionReceiverV1Test.java | 136 +++++++++++++++++++++ 2 files changed, 159 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 13b52c685cf..7a7f91e476d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -165,6 +165,8 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { private final PipeSubscribeSliceReqHandler sliceReqHandler = new PipeSubscribeSliceReqHandler(maxReassembledRequestBodySize); private long consumerStateVersion; + private boolean connectionExited; + private boolean timeoutCleanupSuspended; private static final String SQL_DIALECT_TABLE_VALUE = "table"; @@ -308,6 +310,12 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { } } } finally { + connectionExited = true; + if (timeoutCleanupSuspended || isNodeRetiring()) { + // Retiring this transport transfers timeout responsibility to the other DataNodes. + // Discard only the local receiver state, including when removal has already rolled back. + clearSharedConsumerState(); + } sliceReqHandler.clear(); consumerConfigThreadLocal.remove(); pollTimerThreadLocal.remove(); @@ -370,11 +378,14 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { @Override public void handleTimeout() { synchronized (this) { - final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); - if (commonConfig.getNodeStatus() == NodeStatus.Removing || commonConfig.isStopping()) { + timeoutCleanupSuspended |= isNodeRetiring(); + if (timeoutCleanupSuspended) { // The SDK retires this node's connection while the consumer remains active elsewhere. - // Missing heartbeats here must not unsubscribe that consumer across the cluster. Keep - // the timeout state so ordinary cleanup can resume if the removal is rolled back. + // Rollback alone does not restore that connection: wait for a new request before allowing + // this receiver to close the cluster consumer again. + if (connectionExited) { + clearSharedConsumerState(); + } return; } final ConsumerConfig consumerConfig = sharedConsumerConfig; @@ -412,6 +423,11 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { } } + private static boolean isNodeRetiring() { + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + return commonConfig.getNodeStatus() == NodeStatus.Removing || commonConfig.isStopping(); + } + @Override public long remainingMs() { final PollTimer pollTimer = pollTimerThreadLocal.get(); @@ -1558,6 +1574,9 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { private boolean beforeHandle(final short reqType) { synchronized (this) { + // A request on a restored node starts a fresh local inactivity deadline. + timeoutCleanupSuspended = isNodeRetiring(); + connectionExited = false; final boolean isHandshake = PipeSubscribeRequestType.HANDSHAKE.getType() == reqType; // Replaced connections are terminal, even when the consumer instance itself is still valid. // An old connection must not handshake again and replace the current connection in turn. diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java index 6f52da42e4b..5e35322938e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java @@ -115,8 +115,144 @@ public class SubscriptionReceiverV1Test { commonConfig.setNodeStatus(NodeStatus.ReadOnly); commonConfig.setStopping(false); receiver.handleTimeout(); + Assert.assertEquals(0L, closeAttemptCount.get()); + Assert.assertTrue(receiver.hasActiveConsumer()); + + // Endpoint discovery may lag behind rollback. Only a new request can establish a fresh + // inactivity deadline on this node. + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + consumerConfig.getConsumerGroupId(), + System.currentTimeMillis(), + new ConsumerMeta( + consumerConfig.getConsumerId(), + System.currentTimeMillis(), + consumerConfig.getAttribute())); + Assert.assertNull( + SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(consumerGroupMeta)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + receiver + .handle(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig)) + .getStatus() + .getCode()); + receiver.handleTimeout(); + Assert.assertEquals(0L, closeAttemptCount.get()); + + setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - timeoutMs - 1L); + receiver.handleTimeout(); Assert.assertEquals(1L, closeAttemptCount.get()); Assert.assertFalse(receiver.hasActiveConsumer()); + } finally { + SubscriptionAgent.consumer().handleDropConsumerGroup(consumerConfig.getConsumerGroupId()); + commonConfig.setNodeStatusWithReason(originalStatus, originalReason); + commonConfig.setStopping(originalStopping); + } + } + + @Test + public void testHandleExitRetiresTimeoutStateOnRetiringNode() throws Exception { + assertExitRetiresTimeoutState(false, false); + } + + @Test + public void testHandleTimeoutRetiresPreviouslyDisconnectedConsumer() throws Exception { + assertExitRetiresTimeoutState(true, false); + } + + @Test + public void testHandleExitRetiresTimeoutStateAfterRemovalRollback() throws Exception { + assertExitRetiresTimeoutState(false, true); + } + + @SuppressWarnings("unchecked") + private void assertExitRetiresTimeoutState( + final boolean exitBeforeRetirement, final boolean restoreBeforeExit) throws Exception { + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final NodeStatus originalStatus = commonConfig.getNodeStatus(); + final String originalReason = commonConfig.getStatusReason(); + final boolean originalStopping = commonConfig.isStopping(); + try { + for (final boolean stopping : new boolean[] {false, true}) { + commonConfig.setNodeStatus(NodeStatus.Running); + commonConfig.setStopping(false); + final AtomicLong closeAttemptCount = new AtomicLong(); + final SubscriptionReceiverV1 receiver = + new SubscriptionReceiverV1() { + @Override + void closeConsumer(final ConsumerConfig consumerConfig) { + closeAttemptCount.incrementAndGet(); + } + }; + final ConsumerConfig consumerConfig = createConsumerConfig(1_000L); + setField(receiver, "sharedConsumerConfig", consumerConfig); + ((ThreadLocal<ConsumerConfig>) getField(receiver, "consumerConfigThreadLocal")) + .set(consumerConfig); + final long timeoutMs = invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig); + setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - timeoutMs - 1L); + + if (exitBeforeRetirement) { + receiver.handleExit(); + Assert.assertTrue(receiver.hasActiveConsumer()); + } + commonConfig.setNodeStatus(stopping ? NodeStatus.ReadOnly : NodeStatus.Removing); + commonConfig.setStopping(stopping); + receiver.handleTimeout(); + if (restoreBeforeExit) { + commonConfig.setNodeStatus(NodeStatus.ReadOnly); + commonConfig.setStopping(false); + } + if (!exitBeforeRetirement) { + receiver.handleExit(); + } + + Assert.assertFalse(receiver.hasActiveConsumer()); + Assert.assertNull( + ((ThreadLocal<ConsumerConfig>) getField(receiver, "consumerConfigThreadLocal")).get()); + commonConfig.setNodeStatus(NodeStatus.ReadOnly); + commonConfig.setStopping(false); + receiver.handleTimeout(); + Assert.assertEquals(0L, closeAttemptCount.get()); + } + } finally { + commonConfig.setNodeStatusWithReason(originalStatus, originalReason); + commonConfig.setStopping(originalStopping); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testExplicitCloseStillWorksOnRetiringNode() throws Exception { + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final NodeStatus originalStatus = commonConfig.getNodeStatus(); + final String originalReason = commonConfig.getStatusReason(); + final boolean originalStopping = commonConfig.isStopping(); + try { + for (final boolean stopping : new boolean[] {false, true}) { + final AtomicLong closeAttemptCount = new AtomicLong(); + final SubscriptionReceiverV1 receiver = + new SubscriptionReceiverV1() { + @Override + void closeConsumer(final ConsumerConfig consumerConfig) { + closeAttemptCount.incrementAndGet(); + } + }; + final ConsumerConfig consumerConfig = createConsumerConfig(1_000L); + setField(receiver, "sharedConsumerConfig", consumerConfig); + ((ThreadLocal<ConsumerConfig>) getField(receiver, "consumerConfigThreadLocal")) + .set(consumerConfig); + final long timeoutMs = invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig); + setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - timeoutMs - 1L); + commonConfig.setNodeStatus(stopping ? NodeStatus.ReadOnly : NodeStatus.Removing); + commonConfig.setStopping(stopping); + receiver.handleTimeout(); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + receiver.handle(PipeSubscribeCloseReq.toTPipeSubscribeReq()).getStatus().getCode()); + Assert.assertEquals(1L, closeAttemptCount.get()); + Assert.assertFalse(receiver.hasActiveConsumer()); + } } finally { commonConfig.setNodeStatusWithReason(originalStatus, originalReason); commonConfig.setStopping(originalStopping);
