This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new e9267062671 Fix subscription timeout cleanup during DataNode removal 
(#19197)
e9267062671 is described below

commit e92670626716bae9215aac0259a554aae3e59274
Author: Caideyipi <[email protected]>
AuthorDate: Tue Nov 10 17:02:42 2026 +0800

    Fix subscription timeout cleanup during DataNode removal (#19197)
    
    * Avoid subscription timeout cleanup on retiring DataNodes
    
    * Preserve subscription consumers across DataNode removal rollback
    
    * Latch subscription timeout state on DataNode removal
---
 .../impl/DataNodeInternalRPCServiceImpl.java       |   4 +
 .../receiver/SubscriptionReceiverV1.java           |  41 ++++
 .../receiver/SubscriptionReceiverV1Test.java       | 251 +++++++++++++++++++++
 3 files changed, 296 insertions(+)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 20f5eb888cd..f03366065cd 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -226,6 +226,7 @@ import 
org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaMana
 import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
 import 
org.apache.iotdb.db.subscription.broker.consensus.ConsensusRegionRuntimeState;
 import 
org.apache.iotdb.db.subscription.broker.consensus.ConsensusSubscriptionSetupHandler;
+import org.apache.iotdb.db.subscription.receiver.SubscriptionReceiverV1;
 import org.apache.iotdb.db.trigger.executor.TriggerExecutor;
 import org.apache.iotdb.db.trigger.executor.TriggerFireResult;
 import org.apache.iotdb.db.trigger.service.TriggerManagementService;
@@ -2837,6 +2838,9 @@ public class DataNodeInternalRPCServiceImpl implements 
IDataNodeRPCService.Iface
       } else {
         commonConfig.setNodeStatus(nodeStatus);
       }
+      if (nodeStatus == NodeStatus.Removing) {
+        SubscriptionReceiverV1.handleNodeRetirement();
+      }
     } catch (Exception e) {
       return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, 
e.getMessage());
     }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
index c7812bbb0bf..1d361056a50 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.commons.client.IClientManager;
 import org.apache.iotdb.commons.client.exception.ClientManagerException;
 import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.consensus.ConfigRegionId;
 import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
 import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
@@ -152,6 +154,8 @@ public class SubscriptionReceiverV1 implements 
SubscriptionReceiver {
           PipeSubscribeResponseVersion.VERSION_1.getVersion(),
           PipeSubscribeResponseType.ACK.getType());
 
+  private static final AtomicLong NODE_RETIREMENT_VERSION = new AtomicLong();
+
   private final ThreadLocal<ConsumerConfig> consumerConfigThreadLocal = new 
ThreadLocal<>();
   private final ThreadLocal<PollTimer> pollTimerThreadLocal = new 
ThreadLocal<>();
   private volatile String authenticatedUsername;
@@ -165,6 +169,9 @@ public class SubscriptionReceiverV1 implements 
SubscriptionReceiver {
   private final PipeSubscribeSliceReqHandler sliceReqHandler =
       new PipeSubscribeSliceReqHandler(maxReassembledRequestBodySize);
   private long consumerStateVersion;
+  private boolean connectionExited;
+  private boolean timeoutCleanupSuspended;
+  private long observedNodeRetirementVersion = NODE_RETIREMENT_VERSION.get();
 
   private static final String SQL_DIALECT_TABLE_VALUE = "table";
 
@@ -308,6 +315,12 @@ public class SubscriptionReceiverV1 implements 
SubscriptionReceiver {
           }
         }
       } finally {
+        connectionExited = true;
+        if (timeoutCleanupSuspended || hasNodeRetired()) {
+          // Retiring this transport transfers timeout responsibility to the 
other DataNodes.
+          // Discard only the local receiver state, including when removal has 
already rolled back.
+          clearSharedConsumerState();
+        }
         sliceReqHandler.clear();
         consumerConfigThreadLocal.remove();
         pollTimerThreadLocal.remove();
@@ -370,6 +383,16 @@ public class SubscriptionReceiverV1 implements 
SubscriptionReceiver {
   @Override
   public void handleTimeout() {
     synchronized (this) {
+      timeoutCleanupSuspended |= hasNodeRetired();
+      if (timeoutCleanupSuspended) {
+        // The SDK retires this node's connection while the consumer remains 
active elsewhere.
+        // Rollback alone does not restore that connection: wait for a new 
request before allowing
+        // this receiver to close the cluster consumer again.
+        if (connectionExited) {
+          clearSharedConsumerState();
+        }
+        return;
+      }
       final ConsumerConfig consumerConfig = sharedConsumerConfig;
       if (Objects.isNull(consumerConfig) || inFlightRequestCount.get() > 0) {
         return;
@@ -405,6 +428,20 @@ public class SubscriptionReceiverV1 implements 
SubscriptionReceiver {
     }
   }
 
+  private static boolean isNodeRetiring() {
+    final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
+    return commonConfig.getNodeStatus() == NodeStatus.Removing || 
commonConfig.isStopping();
+  }
+
+  public static void handleNodeRetirement() {
+    // A removal and rollback can both occur between timeout checks, requests, 
and exit callbacks.
+    NODE_RETIREMENT_VERSION.incrementAndGet();
+  }
+
+  private boolean hasNodeRetired() {
+    return observedNodeRetirementVersion != NODE_RETIREMENT_VERSION.get() || 
isNodeRetiring();
+  }
+
   @Override
   public long remainingMs() {
     final PollTimer pollTimer = pollTimerThreadLocal.get();
@@ -1579,6 +1616,10 @@ public class SubscriptionReceiverV1 implements 
SubscriptionReceiver {
 
   private boolean beforeHandle(final short reqType) {
     synchronized (this) {
+      // A request on a restored node starts a fresh local inactivity deadline.
+      observedNodeRetirementVersion = NODE_RETIREMENT_VERSION.get();
+      timeoutCleanupSuspended = isNodeRetiring();
+      connectionExited = false;
       final boolean isHandshake = PipeSubscribeRequestType.HANDSHAKE.getType() 
== reqType;
       // Replaced connections are terminal, even when the consumer instance 
itself is still valid.
       // An old connection must not handshake again and replace the current 
connection in turn.
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
index 2b901cda9b8..9e2865649f2 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
@@ -20,11 +20,17 @@
 package org.apache.iotdb.db.subscription.receiver;
 
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.subscription.config.SubscriptionConfig;
 import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta;
 import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta;
 import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+import org.apache.iotdb.db.protocol.thrift.impl.DataNodeInternalRPCServiceImpl;
+import org.apache.iotdb.db.service.DataNode.DataNodeContext;
 import org.apache.iotdb.db.subscription.agent.SubscriptionAgent;
 import org.apache.iotdb.rpc.TSStatusCode;
 import org.apache.iotdb.rpc.subscription.config.ConsumerConfig;
@@ -42,6 +48,7 @@ import org.apache.iotdb.service.rpc.thrift.TPipeSubscribeResp;
 
 import org.junit.Assert;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
@@ -68,6 +75,250 @@ public class SubscriptionReceiverV1Test {
     Assert.assertFalse((boolean) getField(receiver, "consumerInvalidated"));
   }
 
+  @Test
+  public void testHandleTimeoutKeepsConsumerOnRemovingNode() throws Exception {
+    assertRetiringNodeKeepsConsumer(NodeStatus.Removing, false);
+  }
+
+  @Test
+  public void testHandleTimeoutKeepsConsumerOnStoppingNode() throws Exception {
+    assertRetiringNodeKeepsConsumer(NodeStatus.ReadOnly, true);
+  }
+
+  private void assertRetiringNodeKeepsConsumer(final NodeStatus status, final 
boolean stopping)
+      throws Exception {
+    final AtomicLong closeAttemptCount = new AtomicLong();
+    final SubscriptionReceiverV1 receiver =
+        new SubscriptionReceiverV1() {
+          @Override
+          void closeConsumer(final ConsumerConfig consumerConfig) {
+            closeAttemptCount.incrementAndGet();
+          }
+        };
+    final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+    setField(receiver, "sharedConsumerConfig", consumerConfig);
+    final long timeoutMs = 
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+    setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - 
timeoutMs - 1L);
+
+    final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
+    final NodeStatus originalStatus = commonConfig.getNodeStatus();
+    final String originalReason = commonConfig.getStatusReason();
+    final boolean originalStopping = commonConfig.isStopping();
+    try {
+      commonConfig.setNodeStatus(status);
+      commonConfig.setStopping(stopping);
+      receiver.handleTimeout();
+
+      Assert.assertEquals(
+          "Retiring a node must not unsubscribe a live consumer", 0L, 
closeAttemptCount.get());
+      Assert.assertSame(consumerConfig, getField(receiver, 
"sharedConsumerConfig"));
+      Assert.assertTrue(receiver.hasActiveConsumer());
+
+      // A failed removal can restore this node. Normal inactivity cleanup 
must still work then,
+      // including on a read-only node that is not stopping.
+      commonConfig.setNodeStatus(NodeStatus.ReadOnly);
+      commonConfig.setStopping(false);
+      receiver.handleTimeout();
+      Assert.assertEquals(0L, closeAttemptCount.get());
+      Assert.assertTrue(receiver.hasActiveConsumer());
+
+      // Endpoint discovery may lag behind rollback. Only a new request can 
establish a fresh
+      // inactivity deadline on this node.
+      final ConsumerGroupMeta consumerGroupMeta =
+          new ConsumerGroupMeta(
+              consumerConfig.getConsumerGroupId(),
+              System.currentTimeMillis(),
+              new ConsumerMeta(
+                  consumerConfig.getConsumerId(),
+                  System.currentTimeMillis(),
+                  consumerConfig.getAttribute()));
+      Assert.assertNull(
+          
SubscriptionAgent.consumer().handleSingleConsumerGroupMetaChanges(consumerGroupMeta));
+      Assert.assertEquals(
+          TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+          receiver
+              
.handle(PipeSubscribeHandshakeReq.toTPipeSubscribeReq(consumerConfig))
+              .getStatus()
+              .getCode());
+      receiver.handleTimeout();
+      Assert.assertEquals(0L, closeAttemptCount.get());
+
+      setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - 
timeoutMs - 1L);
+      receiver.handleTimeout();
+      Assert.assertEquals(1L, closeAttemptCount.get());
+      Assert.assertFalse(receiver.hasActiveConsumer());
+    } finally {
+      
SubscriptionAgent.consumer().handleDropConsumerGroup(consumerConfig.getConsumerGroupId());
+      commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+      commonConfig.setStopping(originalStopping);
+    }
+  }
+
+  @Test
+  public void testHandleExitRetiresTimeoutStateOnRetiringNode() throws 
Exception {
+    assertExitRetiresTimeoutState(false, false);
+  }
+
+  @Test
+  public void testHandleTimeoutRetiresPreviouslyDisconnectedConsumer() throws 
Exception {
+    assertExitRetiresTimeoutState(true, false);
+  }
+
+  @Test
+  public void testHandleExitRetiresTimeoutStateAfterRemovalRollback() throws 
Exception {
+    assertExitRetiresTimeoutState(false, true);
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testRemovalRollbackRetiresConsumerWithoutTimeoutTick() throws 
Exception {
+    final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
+    final NodeStatus originalStatus = commonConfig.getNodeStatus();
+    final String originalReason = commonConfig.getStatusReason();
+    final boolean originalStopping = commonConfig.isStopping();
+    final int originalDataNodeId = 
IoTDBDescriptor.getInstance().getConfig().getDataNodeId();
+    try {
+      // Coordinator requires a registered DataNode ID when the internal RPC 
service initializes.
+      IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
+      final DataNodeInternalRPCServiceImpl service =
+          new 
DataNodeInternalRPCServiceImpl(Mockito.mock(DataNodeContext.class));
+      for (final boolean exitBeforeRemoval : new boolean[] {false, true}) {
+        commonConfig.setNodeStatus(NodeStatus.Running);
+        commonConfig.setStopping(false);
+        final AtomicLong closeAttemptCount = new AtomicLong();
+        final SubscriptionReceiverV1 receiver =
+            new SubscriptionReceiverV1() {
+              @Override
+              void closeConsumer(final ConsumerConfig consumerConfig) {
+                closeAttemptCount.incrementAndGet();
+              }
+            };
+        final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+        setField(receiver, "sharedConsumerConfig", consumerConfig);
+        ((ThreadLocal<ConsumerConfig>) getField(receiver, 
"consumerConfigThreadLocal"))
+            .set(consumerConfig);
+        final long timeoutMs = 
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+        setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - 
timeoutMs - 1L);
+        if (exitBeforeRemoval) {
+          receiver.handleExit();
+        }
+        // The entire Removing interval can fall between two scheduled timeout 
checks and between
+        // requests. A delayed transport-exit callback may only run after the 
removal rolls back.
+        Assert.assertEquals(
+            TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+            
service.setSystemStatus(NodeStatus.Removing.getStatus()).getCode());
+        Assert.assertEquals(
+            TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+            service.setSystemStatus(NodeStatus.Running.getStatus()).getCode());
+        if (!exitBeforeRemoval) {
+          receiver.handleExit();
+        }
+        receiver.handleTimeout();
+
+        Assert.assertEquals(0L, closeAttemptCount.get());
+        Assert.assertFalse(receiver.hasActiveConsumer());
+      }
+    } finally {
+      
IoTDBDescriptor.getInstance().getConfig().setDataNodeId(originalDataNodeId);
+      commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+      commonConfig.setStopping(originalStopping);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void assertExitRetiresTimeoutState(
+      final boolean exitBeforeRetirement, final boolean restoreBeforeExit) 
throws Exception {
+    final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
+    final NodeStatus originalStatus = commonConfig.getNodeStatus();
+    final String originalReason = commonConfig.getStatusReason();
+    final boolean originalStopping = commonConfig.isStopping();
+    try {
+      for (final boolean stopping : new boolean[] {false, true}) {
+        commonConfig.setNodeStatus(NodeStatus.Running);
+        commonConfig.setStopping(false);
+        final AtomicLong closeAttemptCount = new AtomicLong();
+        final SubscriptionReceiverV1 receiver =
+            new SubscriptionReceiverV1() {
+              @Override
+              void closeConsumer(final ConsumerConfig consumerConfig) {
+                closeAttemptCount.incrementAndGet();
+              }
+            };
+        final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+        setField(receiver, "sharedConsumerConfig", consumerConfig);
+        ((ThreadLocal<ConsumerConfig>) getField(receiver, 
"consumerConfigThreadLocal"))
+            .set(consumerConfig);
+        final long timeoutMs = 
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+        setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - 
timeoutMs - 1L);
+
+        if (exitBeforeRetirement) {
+          receiver.handleExit();
+          Assert.assertTrue(receiver.hasActiveConsumer());
+        }
+        commonConfig.setNodeStatus(stopping ? NodeStatus.ReadOnly : 
NodeStatus.Removing);
+        commonConfig.setStopping(stopping);
+        receiver.handleTimeout();
+        if (restoreBeforeExit) {
+          commonConfig.setNodeStatus(NodeStatus.ReadOnly);
+          commonConfig.setStopping(false);
+        }
+        if (!exitBeforeRetirement) {
+          receiver.handleExit();
+        }
+
+        Assert.assertFalse(receiver.hasActiveConsumer());
+        Assert.assertNull(
+            ((ThreadLocal<ConsumerConfig>) getField(receiver, 
"consumerConfigThreadLocal")).get());
+        commonConfig.setNodeStatus(NodeStatus.ReadOnly);
+        commonConfig.setStopping(false);
+        receiver.handleTimeout();
+        Assert.assertEquals(0L, closeAttemptCount.get());
+      }
+    } finally {
+      commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+      commonConfig.setStopping(originalStopping);
+    }
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testExplicitCloseStillWorksOnRetiringNode() throws Exception {
+    final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
+    final NodeStatus originalStatus = commonConfig.getNodeStatus();
+    final String originalReason = commonConfig.getStatusReason();
+    final boolean originalStopping = commonConfig.isStopping();
+    try {
+      for (final boolean stopping : new boolean[] {false, true}) {
+        final AtomicLong closeAttemptCount = new AtomicLong();
+        final SubscriptionReceiverV1 receiver =
+            new SubscriptionReceiverV1() {
+              @Override
+              void closeConsumer(final ConsumerConfig consumerConfig) {
+                closeAttemptCount.incrementAndGet();
+              }
+            };
+        final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+        setField(receiver, "sharedConsumerConfig", consumerConfig);
+        ((ThreadLocal<ConsumerConfig>) getField(receiver, 
"consumerConfigThreadLocal"))
+            .set(consumerConfig);
+        final long timeoutMs = 
invokeCalculateConsumerInactivityTimeoutMs(receiver, consumerConfig);
+        setField(receiver, "lastActivityTimeMs", System.currentTimeMillis() - 
timeoutMs - 1L);
+        commonConfig.setNodeStatus(stopping ? NodeStatus.ReadOnly : 
NodeStatus.Removing);
+        commonConfig.setStopping(stopping);
+        receiver.handleTimeout();
+
+        Assert.assertEquals(
+            TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+            
receiver.handle(PipeSubscribeCloseReq.toTPipeSubscribeReq()).getStatus().getCode());
+        Assert.assertEquals(1L, closeAttemptCount.get());
+        Assert.assertFalse(receiver.hasActiveConsumer());
+      }
+    } finally {
+      commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+      commonConfig.setStopping(originalStopping);
+    }
+  }
+
   @Test
   public void testHandleTimeoutSkipsConsumerWithInFlightRequests() throws 
Exception {
     final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1();

Reply via email to