This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/fix-subscription-reconnect-fencing-20261105 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d91c128013fbaf492f94f91547f18431cccb075b Author: Caideyipi <[email protected]> AuthorDate: Thu Nov 5 11:20:22 2026 +0800 Fix subscription connection fencing during consumer reconnect --- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../base/AbstractSubscriptionConsumer.java | 18 ++++-- .../base/AbstractSubscriptionProvider.java | 10 ++- .../base/AbstractSubscriptionProviders.java | 2 +- .../SubscriptionConsumerMultiProviderPollTest.java | 73 ++++++++++++++++++++++ .../SubscriptionProviderConnectionTimeoutTest.java | 25 +++++++- .../base/SubscriptionProviderStatusTest.java | 12 ++++ .../apache/iotdb/db/i18n/DataNodePipeMessages.java | 3 + .../apache/iotdb/db/i18n/DataNodePipeMessages.java | 3 + .../agent/SubscriptionReceiverAgent.java | 19 ++++-- .../receiver/SubscriptionReceiver.java | 5 ++ .../receiver/SubscriptionReceiverV1.java | 38 ++++++++--- .../agent/SubscriptionReceiverAgentTest.java | 24 +++++++ .../receiver/SubscriptionReceiverV1Test.java | 23 +++++++ 14 files changed, 231 insertions(+), 25 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 36d5d2e8aff..0a274627860 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -340,6 +340,7 @@ public enum TSStatusCode { SUBSCRIPTION_OWNER_LEASE_EXPIRED(1917), SUBSCRIPTION_OWNER_EPOCH_CONFLICT(1918), SUBSCRIPTION_CONSUMER_FENCED(1919), + SUBSCRIPTION_CONNECTION_REPLACED(1920), // Topic CREATE_TOPIC_ERROR(2000), diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 7f60b82a0e2..bdbd1a6b9ad 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -1639,15 +1639,15 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { checkIfFenced(); AbstractSubscriptionProvider.CommitResult result = AbstractSubscriptionProvider.CommitResult.empty(); - SubscriptionConnectionException connectionFailure = null; + SubscriptionException connectionFailure = null; SubscriptionException failure = null; try { result = commitInternal(dataNodeId, remainingCommitContexts, nack); } catch (final SubscriptionException e) { failure = getCommitFailure(e); result = getCompletedCommitResult(e); - if (failure instanceof SubscriptionConnectionException) { - connectionFailure = (SubscriptionConnectionException) failure; + if (isRetryableCommitFailure(failure)) { + connectionFailure = failure; } } @@ -1703,6 +1703,11 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { : e; } + private boolean isRetryableCommitFailure(final SubscriptionException failure) { + return failure instanceof SubscriptionConnectionException + || (failure instanceof SubscriptionConsumerFencedException && !isFenced()); + } + private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult( final SubscriptionException e) { return e instanceof AbstractSubscriptionProvider.CommitException @@ -1763,7 +1768,7 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { final Set<SubscriptionCommitContext> acceptedBeforeFailure = new HashSet<>(completedResult.getAcceptedCommitContexts()); removableCommitContexts.addAll(acceptedBeforeFailure); - if (!(failure instanceof SubscriptionConnectionException)) { + if (!isRetryableCommitFailure(failure)) { // The caller cannot receive the removable set when an error propagates. Remove confirmed // contexts from every completed group before leaving the automatic/close-time ACK path. onAckedCommitContexts(removableCommitContexts); @@ -1858,9 +1863,10 @@ abstract class AbstractSubscriptionConsumer implements AutoCloseable { return provider.commit(subscriptionCommitContexts, nack); } catch (final SubscriptionException e) { final SubscriptionException failure = getCommitFailure(e); - if (failure instanceof SubscriptionConsumerFencedException) { + if (failure instanceof SubscriptionConsumerFencedException + && providers.containsProvider(provider)) { fence((SubscriptionConsumerFencedException) failure); - } else if (failure instanceof SubscriptionConnectionException) { + } else if (isRetryableCommitFailure(failure)) { providers.syncIfUnavailable(this); } throw e; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java index a70acb07616..ae41d537d44 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java @@ -185,7 +185,13 @@ public abstract class AbstractSubscriptionProvider { SubscriptionSessionConnection connection = null; try { connection = getSessionConnection(); - return pipeSubscribeWithOptionalSlicing(req, requestBodySizeLimit, connection::pipeSubscribe); + final TPipeSubscribeResp resp = + pipeSubscribeWithOptionalSlicing(req, requestBodySizeLimit, connection::pipeSubscribe); + if (resp.getStatus().getCode() + == TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode()) { + setUnavailable(); + } + return resp; } catch (final TException | IoTDBConnectionException e) { setUnavailable(); if (connection != null) { @@ -1001,6 +1007,8 @@ public abstract class AbstractSubscriptionProvider { case 1919: // SUBSCRIPTION_CONSUMER_FENCED LOGGER.warn(status.message); throw new SubscriptionConsumerFencedException(status.message); + case 1920: // SUBSCRIPTION_CONNECTION_REPLACED + throw new SubscriptionConnectionException(status.message); case 1900: // SUBSCRIPTION_VERSION_ERROR case 1901: // SUBSCRIPTION_TYPE_ERROR case 1909: // SUBSCRIPTION_MISSING_CONSUMER diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java index afb5285b5f5..955eec7050f 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java @@ -394,7 +394,7 @@ final class AbstractSubscriptionProviders { } } - private boolean containsProvider(final AbstractSubscriptionProvider provider) { + boolean containsProvider(final AbstractSubscriptionProvider provider) { return subscriptionProviders.get(provider.getDataNodeId()) == provider; } diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java index 49f5292694f..62f394c9fce 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java @@ -428,6 +428,79 @@ public class SubscriptionConsumerMultiProviderPollTest { } } + @Test(timeout = 20_000) + public void testSyncCommitRetriesObsoleteProviderFencingAndPreservesConfirmedBatches() + throws Exception { + assertObsoleteProviderFencingRecovery(false); + } + + @Test(timeout = 20_000) + public void testAutoCommitRetainsUnconfirmedBatchesAfterObsoleteProviderFencing() + throws Exception { + assertObsoleteProviderFencingRecovery(true); + } + + @SuppressWarnings("unchecked") + private void assertObsoleteProviderFencingRecovery(final boolean automatic) throws Exception { + final TestPullConsumer consumer = new TestPullConsumer(1, 0, false, false, 3, true); + TestSubscriptionProvider obsolete = null; + try { + consumer.open(); + obsolete = consumer.createdProviders.get(0); + final TestSubscriptionProvider oldProvider = obsolete; + final TestSubscriptionProvider replacement = + (TestSubscriptionProvider) + consumer.constructProviderAndHandshake(new TEndPoint(HOST, FIRST_PORT)); + final Field field = + AbstractSubscriptionProviders.class.getDeclaredField("subscriptionProviders"); + field.setAccessible(true); + final Map<Integer, AbstractSubscriptionProvider> providers = + (Map<Integer, AbstractSubscriptionProvider>) field.get(getProviders(consumer)); + final SubscriptionCommitContext first = batchCommitContext(1L, false); + final SubscriptionCommitContext second = batchCommitContext(2L, false); + enableCommitBatching(oldProvider, first); + replacement.committedProgressByTopic = Collections.singletonMap(TOPIC, batchProgress(second)); + oldProvider.commitAction = + () -> { + if (oldProvider.commitBatches.size() == 2) { + // Model the late response from a provider whose replacement is already registered. + providers.put(1, replacement); + throw new SubscriptionConsumerFencedException("obsolete connection fenced"); + } + }; + final Set<SubscriptionCommitContext> pending = ConcurrentHashMap.newKeySet(); + pending.addAll(Arrays.asList(first, second)); + getUncommittedCommitContexts(consumer).put(0L, pending); + + if (automatic) { + final Class<?> worker = + Class.forName(AbstractSubscriptionPullConsumer.class.getName() + "$AutoCommitWorker"); + final java.lang.reflect.Constructor<?> constructor = + worker.getDeclaredConstructor(AbstractSubscriptionPullConsumer.class); + constructor.setAccessible(true); + ((Runnable) constructor.newInstance(consumer)).run(); + Assert.assertFalse(consumer.isFenced()); + Assert.assertEquals(Collections.singleton(second), pending); + consumer.commitSync(new SubscriptionMessage(second, 2L)); + } else { + consumer.commitSync( + Arrays.asList(new SubscriptionMessage(first, 1L), new SubscriptionMessage(second, 2L))); + } + + Assert.assertFalse(consumer.isFenced()); + Assert.assertTrue(getUncommittedCommitContexts(consumer).isEmpty()); + Assert.assertEquals(2, oldProvider.commitBatches.size()); + Assert.assertEquals(1, replacement.commitCount.get()); + Assert.assertEquals(Collections.singletonList(second), replacement.committed); + Assert.assertEquals(2, consumer.committedPositions(TOPIC).getRegionProgress().size()); + } finally { + consumer.close(); + if (obsolete != null) { + obsolete.closeSession(); + } + } + } + @Test(timeout = 20_000) public void testSyncCommitRecoversAlreadyUnavailableProvider() throws Exception { final TestPullConsumer consumer = new TestPullConsumer(1, 0); diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java index 46dbd1f8e9f..bfc2b2ffa14 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java @@ -131,6 +131,11 @@ public class SubscriptionProviderConnectionTimeoutTest { } } + @Test + public void testReplacedConnectionIsRetryableForPollAndCommit() throws Exception { + assertPollError(TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED, "connection replaced", true); + } + @Test public void testMissingBrokerPollErrorMarksProviderUnavailable() throws Exception { assertPollError( @@ -165,6 +170,12 @@ public class SubscriptionProviderConnectionTimeoutTest { } private void assertPollError(final String message, final boolean brokerMissing) throws Exception { + assertPollError(TSStatusCode.SUBSCRIPTION_POLL_ERROR, message, brokerMissing); + } + + private void assertPollError( + final TSStatusCode errorStatus, final String message, final boolean brokerMissing) + throws Exception { final AtomicInteger subscriptionCloseCount = new AtomicInteger(); final AtomicInteger sessionCloseCount = new AtomicInteger(); final ServerSocket socket = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1")); @@ -191,15 +202,16 @@ public class SubscriptionProviderConnectionTimeoutTest { return PipeSubscribeHandshakeResp.toTPipeSubscribeResp( successStatus(), 1, "consumer", "group"); } - if (request.getType() != PipeSubscribeRequestType.POLL.getType()) { + if (request.getType() != PipeSubscribeRequestType.POLL.getType() + && !(errorStatus == TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED + && request.getType() == PipeSubscribeRequestType.COMMIT.getType())) { if (request.getType() == PipeSubscribeRequestType.CLOSE.getType()) { subscriptionCloseCount.incrementAndGet(); } return new TPipeSubscribeResp(successStatus(), (byte) 1, (short) 6); } return new TPipeSubscribeResp( - new TSStatus(TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode()) - .setMessage(message), + new TSStatus(errorStatus.getStatusCode()).setMessage(message), (byte) 1, (short) 6); case "closeSession": @@ -226,6 +238,13 @@ public class SubscriptionProviderConnectionTimeoutTest { final SubscriptionCommitContext context = new SubscriptionCommitContext(1, 0, "topic", "group", 1L); + if (errorStatus == TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED) { + Assert.assertTrue(provider.isAvailable()); + Assert.assertThrows( + SubscriptionConnectionException.class, + () -> provider.commit(Collections.singletonList(context), false)); + Assert.assertFalse(provider.isAvailable()); + } if (brokerMissing) { Assert.assertThrows( SubscriptionConnectionException.class, diff --git a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java index 5da15cadcb4..e2c44275fd0 100644 --- a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java +++ b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java @@ -21,6 +21,7 @@ package org.apache.iotdb.session.subscription.consumer.base; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionConsumerFencedException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException; @@ -44,6 +45,17 @@ public class SubscriptionProviderStatusTest { Assert.assertEquals("consumer fenced", exception.getMessage()); } + @Test + public void testReplacedConnectionStatusIsRetryable() throws Exception { + final SubscriptionException exception = + invokeVerifyPipeSubscribeSuccess( + new TSStatus(TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode()) + .setMessage("connection replaced")); + + Assert.assertEquals(SubscriptionConnectionException.class, exception.getClass()); + Assert.assertEquals("connection replaced", exception.getMessage()); + } + @Test public void testMissingConsumerStatusRemainsCriticalException() throws Exception { final SubscriptionException exception = diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 8e7243d2c15..4e47212d81d 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -3087,6 +3087,9 @@ public final class DataNodePipeMessages { public static final String MESSAGE_MISSING_CONSUMER_CONFIG_PLEASE_HANDSHAKE_FIRST_CE588329 = "Missing consumer config, please handshake first."; + public static final String + MESSAGE_SUBSCRIPTION_CONNECTION_WAS_REPLACED_BY_A_RECONNECTION_FROM_THE_SAME_CONSUMER_INSTANCE_RETRY_ON_THE_CURRENT_CONNECTION_687C78B4 = + "Subscription connection was replaced by a reconnection from the same consumer instance. Retry on the current connection."; public static final String MESSAGE_UNKNOWN_PIPESUBSCRIBEREQUESTTYPE_ARG_7EBB93F8 = "Unknown PipeSubscribeRequestType %s."; public static final String diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index ab67d7ab46e..45f877afc61 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -2861,6 +2861,9 @@ public final class DataNodePipeMessages { MESSAGE_SUBSCRIPTION_REQUEST_BODY_SIZE_ARG_BYTES_EXCEEDS_THE_RECEIVER_LIMIT_OF_ARG_BYTES_SPLIT_THE_OPERATION_INTO_SMALLER_REQUESTS_C9513027 = "Subscription:请求体大小为 %d 字节,超过接收端上限 %d 字节。请将操作拆分为更小的请求。"; + public static final String + MESSAGE_SUBSCRIPTION_CONNECTION_WAS_REPLACED_BY_A_RECONNECTION_FROM_THE_SAME_CONSUMER_INSTANCE_RETRY_ON_THE_CURRENT_CONNECTION_687C78B4 = + "订阅连接已被同一 consumer 实例的重连替换,请在当前连接上重试。"; public static final String MESSAGE_MISSING_CONSUMER_CONFIG_PLEASE_HANDSHAKE_FIRST_CE588329 = "缺少 consumer 配置,请先完成 handshake。"; public static final String MESSAGE_UNKNOWN_PIPESUBSCRIBEREQUESTTYPE_ARG_7EBB93F8 = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java index 828b1887204..4f15a58a970 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java @@ -182,7 +182,8 @@ public class SubscriptionReceiverAgent { if (currentReceiver != null && currentReceiver != receiver) { releaseInFlightEventsOnNewInstanceTakeover( currentReceiver, receiver.getConsumerInstanceId()); - invalidateReplacedReceiver(currentReceiver, identity); + invalidateReplacedReceiver( + currentReceiver, identity, receiver.getConsumerInstanceId()); } return receiver; } @@ -321,7 +322,8 @@ public class SubscriptionReceiverAgent { final TPipeSubscribeReq req, final SubscriptionReceiver currentReceiver) { if (!isHandshake(req) && currentReceiver != null && currentReceiver != receiver) { - receiver.invalidateConsumer(); + invalidateReplacedReceiver( + receiver, getConsumerIdentity(receiver), currentReceiver.getConsumerInstanceId()); } return receiver.handle(req); } @@ -344,14 +346,14 @@ public class SubscriptionReceiverAgent { && !shouldKeepCurrentReceiver(currentReceiver, receiver.getConsumerInstanceId())) { releaseInFlightEventsOnNewInstanceTakeover( currentReceiver, receiver.getConsumerInstanceId()); - invalidateReplacedReceiver(currentReceiver, key); + invalidateReplacedReceiver(currentReceiver, key, receiver.getConsumerInstanceId()); registered.set(true); return receiver; } // The receiver completed its handshake after another receiver had already claimed the // identity. Keep the current owner and fence this late receiver instead of allowing an // old connection to take the consumer back. - invalidateReplacedReceiver(receiver, key); + invalidateReplacedReceiver(receiver, key, currentReceiver.getConsumerInstanceId()); return currentReceiver; }); if (registered.get()) { @@ -377,7 +379,14 @@ public class SubscriptionReceiverAgent { } private void invalidateReplacedReceiver( - final SubscriptionReceiver receiver, final ConsumerIdentity identity) { + final SubscriptionReceiver receiver, + final ConsumerIdentity identity, + final String incomingConsumerInstanceId) { + if (incomingConsumerInstanceId != null + && incomingConsumerInstanceId.equals(receiver.getConsumerInstanceId())) { + receiver.invalidateConnection(); + return; + } LOGGER.info( DataNodePipeMessages .LOG_SUBSCRIPTION_CONSUMER_ARG_IN_CONSUMER_GROUP_ARG_WAS_TAKEN_OVER_BY_A_NEWER_CONNECTION_FENCED_THE_PREVIOUS_CONNECTION_4E72DBD9, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java index 01c736b29dc..b20d25ec800 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java @@ -66,6 +66,11 @@ public interface SubscriptionReceiver { */ void invalidateConsumer(); + /** Invalidates a connection replaced by a reconnection from the same consumer instance. */ + default void invalidateConnection() { + invalidateConsumer(); + } + /** Returns whether this receiver still owns an active consumer timeout state. */ boolean hasActiveConsumer(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java index 93d309cc5db..052dcbd0fc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java @@ -135,6 +135,15 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { PipeSubscribeResponseVersion.VERSION_1.getVersion(), PipeSubscribeResponseType.ACK.getType()); + private static final TPipeSubscribeResp SUBSCRIPTION_CONNECTION_REPLACED_RESP = + new TPipeSubscribeResp( + RpcUtils.getStatus( + TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED, + DataNodePipeMessages + .MESSAGE_SUBSCRIPTION_CONNECTION_WAS_REPLACED_BY_A_RECONNECTION_FROM_THE_SAME_CONSUMER_INSTANCE_RETRY_ON_THE_CURRENT_CONNECTION_687C78B4), + PipeSubscribeResponseVersion.VERSION_1.getVersion(), + PipeSubscribeResponseType.ACK.getType()); + private static final TPipeSubscribeResp SUBSCRIPTION_SLICE_ACK_RESP = new TPipeSubscribeResp( new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), @@ -147,6 +156,7 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { private volatile ConsumerConfig sharedConsumerConfig; private volatile boolean consumerInvalidated; private volatile boolean consumerFenced; + private volatile boolean connectionReplaced; private volatile long lastActivityTimeMs = System.currentTimeMillis(); private final AtomicLong inFlightRequestCount = new AtomicLong(0); private final int maxReassembledRequestBodySize = getMaxReassembledRequestBodySize(); @@ -162,10 +172,12 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { if (PipeSubscribeRequestType.SLICE.getType() != reqType) { sliceReqHandler.clear(); } - final boolean isFencedRequest = beforeHandle(reqType); + final boolean isObsoleteRequest = beforeHandle(reqType); try { - if (isFencedRequest) { - return SUBSCRIPTION_CONSUMER_FENCED_RESP; + if (isObsoleteRequest) { + return consumerFenced + ? SUBSCRIPTION_CONSUMER_FENCED_RESP + : SUBSCRIPTION_CONNECTION_REPLACED_RESP; } if (PipeSubscribeRequestType.isValidatedRequestType(reqType)) { switch (PipeSubscribeRequestType.valueOf(reqType)) { @@ -253,6 +265,15 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { } } + @Override + public void invalidateConnection() { + synchronized (this) { + connectionReplaced = true; + sliceReqHandler.clear(); + clearSharedConsumerState(); + } + } + @Override public void releaseInFlightEvents() { synchronized (this) { @@ -1529,20 +1550,19 @@ public class SubscriptionReceiverV1 implements SubscriptionReceiver { private boolean beforeHandle(final short reqType) { synchronized (this) { final boolean isHandshake = PipeSubscribeRequestType.HANDSHAKE.getType() == reqType; - // A receiver fenced by a newer connection is terminal. In particular, do not allow the old - // connection to handshake again and reclaim the consumer identity. A normal disconnected - // receiver remains recoverable through the existing consumerInvalidated handshake path. - final boolean isFencedRequest = consumerFenced; + // Replaced connections are terminal, even when the consumer instance itself is still valid. + // An old connection must not handshake again and replace the current connection in turn. + final boolean isObsoleteRequest = consumerFenced || connectionReplaced; if (consumerInvalidated) { consumerConfigThreadLocal.remove(); pollTimerThreadLocal.remove(); - if (isHandshake && !consumerFenced) { + if (isHandshake && !isObsoleteRequest) { consumerInvalidated = false; } } inFlightRequestCount.incrementAndGet(); lastActivityTimeMs = System.currentTimeMillis(); - return isFencedRequest; + return isObsoleteRequest; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java index fc68f97c6e8..73e67e1e426 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java @@ -404,6 +404,20 @@ public class SubscriptionReceiverAgentTest { Assert.assertEquals(2, receivers.size()); Assert.assertEquals(0, receivers.get(0).releaseCount.get()); Assert.assertTrue(receivers.get(0).invalidated); + Assert.assertTrue(receivers.get(0).connectionReplaced); + Assert.assertEquals( + TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode(), + agent.handle(SubscriptionHeartbeatReq.toThriftReq(), "root").getStatus().getCode()); + Assert.assertEquals( + TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode(), + agent + .handle(createHandshakeRequest("group", "consumer", "instance-1"), "root") + .getStatus() + .getCode()); + Assert.assertFalse(receivers.get(1).invalidated); + agent.checkReceiverTimeouts(); + Assert.assertEquals(0, receivers.get(0).timeoutCount.get()); + Assert.assertEquals(1, receivers.get(1).timeoutCount.get()); } @Test @@ -643,6 +657,7 @@ public class SubscriptionReceiverAgentTest { private final AtomicInteger releaseCount = new AtomicInteger(); private volatile ConsumerConfig consumerConfig; private volatile boolean invalidated; + private volatile boolean connectionReplaced; private FakeSubscriptionReceiver(final boolean closeOnTimeout) { this(closeOnTimeout, false, null, null); @@ -661,6 +676,9 @@ public class SubscriptionReceiverAgentTest { @Override public TPipeSubscribeResp handle(final TPipeSubscribeReq req) { + if (connectionReplaced) { + return response(TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED); + } if (req.getType() == PipeSubscribeRequestType.HANDSHAKE.getType()) { consumerConfig = ConsumerConfig.deserialize(req.bufferForBody()); if (assignDefaultIdentity) { @@ -738,6 +756,12 @@ public class SubscriptionReceiverAgentTest { invalidated = true; } + @Override + public void invalidateConnection() { + invalidateConsumer(); + connectionReplaced = true; + } + @Override public boolean hasActiveConsumer() { return consumerConfig != null; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java index 05eccb978f9..2b901cda9b8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java @@ -204,6 +204,29 @@ public class SubscriptionReceiverV1Test { receiver.handle(SubscriptionHeartbeatReq.toThriftReq()).getStatus().getCode()); } + @Test + public void testReplacedConnectionReturnsRetryableStatusForAllRequests() throws Exception { + final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1(); + final ConsumerConfig consumerConfig = createConsumerConfig(1_000L); + setField(receiver, "sharedConsumerConfig", consumerConfig); + + receiver.invalidateConnection(); + + for (final PipeSubscribeRequestType requestType : PipeSubscribeRequestType.values()) { + Assert.assertEquals( + requestType.name(), + TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode(), + receiver + .handle(new TPipeSubscribeReq((byte) 1, requestType.getType())) + .getStatus() + .getCode()); + } + Assert.assertFalse(receiver.hasActiveConsumer()); + Assert.assertFalse((boolean) getField(receiver, "consumerFenced")); + Assert.assertTrue((boolean) getField(receiver, "consumerInvalidated")); + Assert.assertEquals(0L, ((AtomicLong) getField(receiver, "inFlightRequestCount")).get()); + } + @Test public void testFencedConsumerCannotHandshakeAgain() throws Exception { final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1();
