This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 51748ae1fde Fix subscription connection fencing during consumer
reconnect (#19130)
51748ae1fde is described below
commit 51748ae1fdec5548f93800fd169597a8b177f531
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 17:22:43 2026 +0800
Fix subscription connection fencing during consumer reconnect (#19130)
---
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../base/AbstractSubscriptionConsumer.java | 18 ++++--
.../base/AbstractSubscriptionProvider.java | 10 ++-
.../base/AbstractSubscriptionProviders.java | 2 +-
.../SubscriptionConsumerMultiProviderPollTest.java | 73 ++++++++++++++++++++++
.../SubscriptionProviderConnectionTimeoutTest.java | 25 +++++++-
.../base/SubscriptionProviderStatusTest.java | 12 ++++
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 3 +
.../apache/iotdb/db/i18n/DataNodePipeMessages.java | 3 +
.../agent/SubscriptionReceiverAgent.java | 19 ++++--
.../receiver/SubscriptionReceiver.java | 5 ++
.../receiver/SubscriptionReceiverV1.java | 38 ++++++++---
.../agent/SubscriptionReceiverAgentTest.java | 24 +++++++
.../receiver/SubscriptionReceiverV1Test.java | 23 +++++++
14 files changed, 231 insertions(+), 25 deletions(-)
diff --git
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 36d5d2e8aff..0a274627860 100644
---
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -340,6 +340,7 @@ public enum TSStatusCode {
SUBSCRIPTION_OWNER_LEASE_EXPIRED(1917),
SUBSCRIPTION_OWNER_EPOCH_CONFLICT(1918),
SUBSCRIPTION_CONSUMER_FENCED(1919),
+ SUBSCRIPTION_CONNECTION_REPLACED(1920),
// Topic
CREATE_TOPIC_ERROR(2000),
diff --git
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
index 7f60b82a0e2..bdbd1a6b9ad 100644
---
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
+++
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
@@ -1639,15 +1639,15 @@ abstract class AbstractSubscriptionConsumer implements
AutoCloseable {
checkIfFenced();
AbstractSubscriptionProvider.CommitResult result =
AbstractSubscriptionProvider.CommitResult.empty();
- SubscriptionConnectionException connectionFailure = null;
+ SubscriptionException connectionFailure = null;
SubscriptionException failure = null;
try {
result = commitInternal(dataNodeId, remainingCommitContexts, nack);
} catch (final SubscriptionException e) {
failure = getCommitFailure(e);
result = getCompletedCommitResult(e);
- if (failure instanceof SubscriptionConnectionException) {
- connectionFailure = (SubscriptionConnectionException) failure;
+ if (isRetryableCommitFailure(failure)) {
+ connectionFailure = failure;
}
}
@@ -1703,6 +1703,11 @@ abstract class AbstractSubscriptionConsumer implements
AutoCloseable {
: e;
}
+ private boolean isRetryableCommitFailure(final SubscriptionException
failure) {
+ return failure instanceof SubscriptionConnectionException
+ || (failure instanceof SubscriptionConsumerFencedException &&
!isFenced());
+ }
+
private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult(
final SubscriptionException e) {
return e instanceof AbstractSubscriptionProvider.CommitException
@@ -1763,7 +1768,7 @@ abstract class AbstractSubscriptionConsumer implements
AutoCloseable {
final Set<SubscriptionCommitContext> acceptedBeforeFailure =
new HashSet<>(completedResult.getAcceptedCommitContexts());
removableCommitContexts.addAll(acceptedBeforeFailure);
- if (!(failure instanceof SubscriptionConnectionException)) {
+ if (!isRetryableCommitFailure(failure)) {
// The caller cannot receive the removable set when an error
propagates. Remove confirmed
// contexts from every completed group before leaving the
automatic/close-time ACK path.
onAckedCommitContexts(removableCommitContexts);
@@ -1858,9 +1863,10 @@ abstract class AbstractSubscriptionConsumer implements
AutoCloseable {
return provider.commit(subscriptionCommitContexts, nack);
} catch (final SubscriptionException e) {
final SubscriptionException failure = getCommitFailure(e);
- if (failure instanceof SubscriptionConsumerFencedException) {
+ if (failure instanceof SubscriptionConsumerFencedException
+ && providers.containsProvider(provider)) {
fence((SubscriptionConsumerFencedException) failure);
- } else if (failure instanceof SubscriptionConnectionException) {
+ } else if (isRetryableCommitFailure(failure)) {
providers.syncIfUnavailable(this);
}
throw e;
diff --git
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
index a70acb07616..ae41d537d44 100644
---
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
+++
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
@@ -185,7 +185,13 @@ public abstract class AbstractSubscriptionProvider {
SubscriptionSessionConnection connection = null;
try {
connection = getSessionConnection();
- return pipeSubscribeWithOptionalSlicing(req, requestBodySizeLimit,
connection::pipeSubscribe);
+ final TPipeSubscribeResp resp =
+ pipeSubscribeWithOptionalSlicing(req, requestBodySizeLimit,
connection::pipeSubscribe);
+ if (resp.getStatus().getCode()
+ == TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode()) {
+ setUnavailable();
+ }
+ return resp;
} catch (final TException | IoTDBConnectionException e) {
setUnavailable();
if (connection != null) {
@@ -1001,6 +1007,8 @@ public abstract class AbstractSubscriptionProvider {
case 1919: // SUBSCRIPTION_CONSUMER_FENCED
LOGGER.warn(status.message);
throw new SubscriptionConsumerFencedException(status.message);
+ case 1920: // SUBSCRIPTION_CONNECTION_REPLACED
+ throw new SubscriptionConnectionException(status.message);
case 1900: // SUBSCRIPTION_VERSION_ERROR
case 1901: // SUBSCRIPTION_TYPE_ERROR
case 1909: // SUBSCRIPTION_MISSING_CONSUMER
diff --git
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java
index afb5285b5f5..955eec7050f 100644
---
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java
+++
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java
@@ -394,7 +394,7 @@ final class AbstractSubscriptionProviders {
}
}
- private boolean containsProvider(final AbstractSubscriptionProvider
provider) {
+ boolean containsProvider(final AbstractSubscriptionProvider provider) {
return subscriptionProviders.get(provider.getDataNodeId()) == provider;
}
diff --git
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
index 49f5292694f..62f394c9fce 100644
---
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
+++
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
@@ -428,6 +428,79 @@ public class SubscriptionConsumerMultiProviderPollTest {
}
}
+ @Test(timeout = 20_000)
+ public void
testSyncCommitRetriesObsoleteProviderFencingAndPreservesConfirmedBatches()
+ throws Exception {
+ assertObsoleteProviderFencingRecovery(false);
+ }
+
+ @Test(timeout = 20_000)
+ public void
testAutoCommitRetainsUnconfirmedBatchesAfterObsoleteProviderFencing()
+ throws Exception {
+ assertObsoleteProviderFencingRecovery(true);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void assertObsoleteProviderFencingRecovery(final boolean automatic)
throws Exception {
+ final TestPullConsumer consumer = new TestPullConsumer(1, 0, false, false,
3, true);
+ TestSubscriptionProvider obsolete = null;
+ try {
+ consumer.open();
+ obsolete = consumer.createdProviders.get(0);
+ final TestSubscriptionProvider oldProvider = obsolete;
+ final TestSubscriptionProvider replacement =
+ (TestSubscriptionProvider)
+ consumer.constructProviderAndHandshake(new TEndPoint(HOST,
FIRST_PORT));
+ final Field field =
+
AbstractSubscriptionProviders.class.getDeclaredField("subscriptionProviders");
+ field.setAccessible(true);
+ final Map<Integer, AbstractSubscriptionProvider> providers =
+ (Map<Integer, AbstractSubscriptionProvider>)
field.get(getProviders(consumer));
+ final SubscriptionCommitContext first = batchCommitContext(1L, false);
+ final SubscriptionCommitContext second = batchCommitContext(2L, false);
+ enableCommitBatching(oldProvider, first);
+ replacement.committedProgressByTopic = Collections.singletonMap(TOPIC,
batchProgress(second));
+ oldProvider.commitAction =
+ () -> {
+ if (oldProvider.commitBatches.size() == 2) {
+ // Model the late response from a provider whose replacement is
already registered.
+ providers.put(1, replacement);
+ throw new SubscriptionConsumerFencedException("obsolete
connection fenced");
+ }
+ };
+ final Set<SubscriptionCommitContext> pending =
ConcurrentHashMap.newKeySet();
+ pending.addAll(Arrays.asList(first, second));
+ getUncommittedCommitContexts(consumer).put(0L, pending);
+
+ if (automatic) {
+ final Class<?> worker =
+ Class.forName(AbstractSubscriptionPullConsumer.class.getName() +
"$AutoCommitWorker");
+ final java.lang.reflect.Constructor<?> constructor =
+
worker.getDeclaredConstructor(AbstractSubscriptionPullConsumer.class);
+ constructor.setAccessible(true);
+ ((Runnable) constructor.newInstance(consumer)).run();
+ Assert.assertFalse(consumer.isFenced());
+ Assert.assertEquals(Collections.singleton(second), pending);
+ consumer.commitSync(new SubscriptionMessage(second, 2L));
+ } else {
+ consumer.commitSync(
+ Arrays.asList(new SubscriptionMessage(first, 1L), new
SubscriptionMessage(second, 2L)));
+ }
+
+ Assert.assertFalse(consumer.isFenced());
+ Assert.assertTrue(getUncommittedCommitContexts(consumer).isEmpty());
+ Assert.assertEquals(2, oldProvider.commitBatches.size());
+ Assert.assertEquals(1, replacement.commitCount.get());
+ Assert.assertEquals(Collections.singletonList(second),
replacement.committed);
+ Assert.assertEquals(2,
consumer.committedPositions(TOPIC).getRegionProgress().size());
+ } finally {
+ consumer.close();
+ if (obsolete != null) {
+ obsolete.closeSession();
+ }
+ }
+ }
+
@Test(timeout = 20_000)
public void testSyncCommitRecoversAlreadyUnavailableProvider() throws
Exception {
final TestPullConsumer consumer = new TestPullConsumer(1, 0);
diff --git
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java
index 46dbd1f8e9f..bfc2b2ffa14 100644
---
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java
+++
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderConnectionTimeoutTest.java
@@ -131,6 +131,11 @@ public class SubscriptionProviderConnectionTimeoutTest {
}
}
+ @Test
+ public void testReplacedConnectionIsRetryableForPollAndCommit() throws
Exception {
+ assertPollError(TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED, "connection
replaced", true);
+ }
+
@Test
public void testMissingBrokerPollErrorMarksProviderUnavailable() throws
Exception {
assertPollError(
@@ -165,6 +170,12 @@ public class SubscriptionProviderConnectionTimeoutTest {
}
private void assertPollError(final String message, final boolean
brokerMissing) throws Exception {
+ assertPollError(TSStatusCode.SUBSCRIPTION_POLL_ERROR, message,
brokerMissing);
+ }
+
+ private void assertPollError(
+ final TSStatusCode errorStatus, final String message, final boolean
brokerMissing)
+ throws Exception {
final AtomicInteger subscriptionCloseCount = new AtomicInteger();
final AtomicInteger sessionCloseCount = new AtomicInteger();
final ServerSocket socket = new ServerSocket(0, 50,
InetAddress.getByName("127.0.0.1"));
@@ -191,15 +202,16 @@ public class SubscriptionProviderConnectionTimeoutTest {
return PipeSubscribeHandshakeResp.toTPipeSubscribeResp(
successStatus(), 1, "consumer", "group");
}
- if (request.getType() !=
PipeSubscribeRequestType.POLL.getType()) {
+ if (request.getType() !=
PipeSubscribeRequestType.POLL.getType()
+ && !(errorStatus ==
TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED
+ && request.getType() ==
PipeSubscribeRequestType.COMMIT.getType())) {
if (request.getType() ==
PipeSubscribeRequestType.CLOSE.getType()) {
subscriptionCloseCount.incrementAndGet();
}
return new TPipeSubscribeResp(successStatus(), (byte)
1, (short) 6);
}
return new TPipeSubscribeResp(
- new
TSStatus(TSStatusCode.SUBSCRIPTION_POLL_ERROR.getStatusCode())
- .setMessage(message),
+ new
TSStatus(errorStatus.getStatusCode()).setMessage(message),
(byte) 1,
(short) 6);
case "closeSession":
@@ -226,6 +238,13 @@ public class SubscriptionProviderConnectionTimeoutTest {
final SubscriptionCommitContext context =
new SubscriptionCommitContext(1, 0, "topic", "group", 1L);
+ if (errorStatus == TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED) {
+ Assert.assertTrue(provider.isAvailable());
+ Assert.assertThrows(
+ SubscriptionConnectionException.class,
+ () -> provider.commit(Collections.singletonList(context), false));
+ Assert.assertFalse(provider.isAvailable());
+ }
if (brokerMissing) {
Assert.assertThrows(
SubscriptionConnectionException.class,
diff --git
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java
index 5da15cadcb4..e2c44275fd0 100644
---
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java
+++
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionProviderStatusTest.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.session.subscription.consumer.base;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.rpc.TSStatusCode;
+import
org.apache.iotdb.rpc.subscription.exception.SubscriptionConnectionException;
import
org.apache.iotdb.rpc.subscription.exception.SubscriptionConsumerFencedException;
import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;
import
org.apache.iotdb.rpc.subscription.exception.SubscriptionRuntimeCriticalException;
@@ -44,6 +45,17 @@ public class SubscriptionProviderStatusTest {
Assert.assertEquals("consumer fenced", exception.getMessage());
}
+ @Test
+ public void testReplacedConnectionStatusIsRetryable() throws Exception {
+ final SubscriptionException exception =
+ invokeVerifyPipeSubscribeSuccess(
+ new
TSStatus(TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode())
+ .setMessage("connection replaced"));
+
+ Assert.assertEquals(SubscriptionConnectionException.class,
exception.getClass());
+ Assert.assertEquals("connection replaced", exception.getMessage());
+ }
+
@Test
public void testMissingConsumerStatusRemainsCriticalException() throws
Exception {
final SubscriptionException exception =
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 8e7243d2c15..4e47212d81d 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -3087,6 +3087,9 @@ public final class DataNodePipeMessages {
public static final String
MESSAGE_MISSING_CONSUMER_CONFIG_PLEASE_HANDSHAKE_FIRST_CE588329 =
"Missing consumer config, please handshake first.";
+ public static final String
+
MESSAGE_SUBSCRIPTION_CONNECTION_WAS_REPLACED_BY_A_RECONNECTION_FROM_THE_SAME_CONSUMER_INSTANCE_RETRY_ON_THE_CURRENT_CONNECTION_687C78B4
=
+ "Subscription connection was replaced by a reconnection from the
same consumer instance. Retry on the current connection.";
public static final String
MESSAGE_UNKNOWN_PIPESUBSCRIBEREQUESTTYPE_ARG_7EBB93F8 =
"Unknown PipeSubscribeRequestType %s.";
public static final String
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index ab67d7ab46e..45f877afc61 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2861,6 +2861,9 @@ public final class DataNodePipeMessages {
MESSAGE_SUBSCRIPTION_REQUEST_BODY_SIZE_ARG_BYTES_EXCEEDS_THE_RECEIVER_LIMIT_OF_ARG_BYTES_SPLIT_THE_OPERATION_INTO_SMALLER_REQUESTS_C9513027
=
"Subscription:请求体大小为 %d 字节,超过接收端上限 %d 字节。请将操作拆分为更小的请求。";
+ public static final String
+
MESSAGE_SUBSCRIPTION_CONNECTION_WAS_REPLACED_BY_A_RECONNECTION_FROM_THE_SAME_CONSUMER_INSTANCE_RETRY_ON_THE_CURRENT_CONNECTION_687C78B4
=
+ "订阅连接已被同一 consumer 实例的重连替换,请在当前连接上重试。";
public static final String
MESSAGE_MISSING_CONSUMER_CONFIG_PLEASE_HANDSHAKE_FIRST_CE588329 =
"缺少 consumer 配置,请先完成 handshake。";
public static final String
MESSAGE_UNKNOWN_PIPESUBSCRIBEREQUESTTYPE_ARG_7EBB93F8 =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
index 828b1887204..4f15a58a970 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgent.java
@@ -182,7 +182,8 @@ public class SubscriptionReceiverAgent {
if (currentReceiver != null && currentReceiver != receiver) {
releaseInFlightEventsOnNewInstanceTakeover(
currentReceiver, receiver.getConsumerInstanceId());
- invalidateReplacedReceiver(currentReceiver, identity);
+ invalidateReplacedReceiver(
+ currentReceiver, identity,
receiver.getConsumerInstanceId());
}
return receiver;
}
@@ -321,7 +322,8 @@ public class SubscriptionReceiverAgent {
final TPipeSubscribeReq req,
final SubscriptionReceiver currentReceiver) {
if (!isHandshake(req) && currentReceiver != null && currentReceiver !=
receiver) {
- receiver.invalidateConsumer();
+ invalidateReplacedReceiver(
+ receiver, getConsumerIdentity(receiver),
currentReceiver.getConsumerInstanceId());
}
return receiver.handle(req);
}
@@ -344,14 +346,14 @@ public class SubscriptionReceiverAgent {
&& !shouldKeepCurrentReceiver(currentReceiver,
receiver.getConsumerInstanceId())) {
releaseInFlightEventsOnNewInstanceTakeover(
currentReceiver, receiver.getConsumerInstanceId());
- invalidateReplacedReceiver(currentReceiver, key);
+ invalidateReplacedReceiver(currentReceiver, key,
receiver.getConsumerInstanceId());
registered.set(true);
return receiver;
}
// The receiver completed its handshake after another receiver had
already claimed the
// identity. Keep the current owner and fence this late receiver
instead of allowing an
// old connection to take the consumer back.
- invalidateReplacedReceiver(receiver, key);
+ invalidateReplacedReceiver(receiver, key,
currentReceiver.getConsumerInstanceId());
return currentReceiver;
});
if (registered.get()) {
@@ -377,7 +379,14 @@ public class SubscriptionReceiverAgent {
}
private void invalidateReplacedReceiver(
- final SubscriptionReceiver receiver, final ConsumerIdentity identity) {
+ final SubscriptionReceiver receiver,
+ final ConsumerIdentity identity,
+ final String incomingConsumerInstanceId) {
+ if (incomingConsumerInstanceId != null
+ &&
incomingConsumerInstanceId.equals(receiver.getConsumerInstanceId())) {
+ receiver.invalidateConnection();
+ return;
+ }
LOGGER.info(
DataNodePipeMessages
.LOG_SUBSCRIPTION_CONSUMER_ARG_IN_CONSUMER_GROUP_ARG_WAS_TAKEN_OVER_BY_A_NEWER_CONNECTION_FENCED_THE_PREVIOUS_CONNECTION_4E72DBD9,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java
index 01c736b29dc..b20d25ec800 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiver.java
@@ -66,6 +66,11 @@ public interface SubscriptionReceiver {
*/
void invalidateConsumer();
+ /** Invalidates a connection replaced by a reconnection from the same
consumer instance. */
+ default void invalidateConnection() {
+ invalidateConsumer();
+ }
+
/** Returns whether this receiver still owns an active consumer timeout
state. */
boolean hasActiveConsumer();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
index 93d309cc5db..052dcbd0fc5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1.java
@@ -135,6 +135,15 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
PipeSubscribeResponseVersion.VERSION_1.getVersion(),
PipeSubscribeResponseType.ACK.getType());
+ private static final TPipeSubscribeResp
SUBSCRIPTION_CONNECTION_REPLACED_RESP =
+ new TPipeSubscribeResp(
+ RpcUtils.getStatus(
+ TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED,
+ DataNodePipeMessages
+
.MESSAGE_SUBSCRIPTION_CONNECTION_WAS_REPLACED_BY_A_RECONNECTION_FROM_THE_SAME_CONSUMER_INSTANCE_RETRY_ON_THE_CURRENT_CONNECTION_687C78B4),
+ PipeSubscribeResponseVersion.VERSION_1.getVersion(),
+ PipeSubscribeResponseType.ACK.getType());
+
private static final TPipeSubscribeResp SUBSCRIPTION_SLICE_ACK_RESP =
new TPipeSubscribeResp(
new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()),
@@ -147,6 +156,7 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
private volatile ConsumerConfig sharedConsumerConfig;
private volatile boolean consumerInvalidated;
private volatile boolean consumerFenced;
+ private volatile boolean connectionReplaced;
private volatile long lastActivityTimeMs = System.currentTimeMillis();
private final AtomicLong inFlightRequestCount = new AtomicLong(0);
private final int maxReassembledRequestBodySize =
getMaxReassembledRequestBodySize();
@@ -162,10 +172,12 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
if (PipeSubscribeRequestType.SLICE.getType() != reqType) {
sliceReqHandler.clear();
}
- final boolean isFencedRequest = beforeHandle(reqType);
+ final boolean isObsoleteRequest = beforeHandle(reqType);
try {
- if (isFencedRequest) {
- return SUBSCRIPTION_CONSUMER_FENCED_RESP;
+ if (isObsoleteRequest) {
+ return consumerFenced
+ ? SUBSCRIPTION_CONSUMER_FENCED_RESP
+ : SUBSCRIPTION_CONNECTION_REPLACED_RESP;
}
if (PipeSubscribeRequestType.isValidatedRequestType(reqType)) {
switch (PipeSubscribeRequestType.valueOf(reqType)) {
@@ -253,6 +265,15 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
}
}
+ @Override
+ public void invalidateConnection() {
+ synchronized (this) {
+ connectionReplaced = true;
+ sliceReqHandler.clear();
+ clearSharedConsumerState();
+ }
+ }
+
@Override
public void releaseInFlightEvents() {
synchronized (this) {
@@ -1529,20 +1550,19 @@ public class SubscriptionReceiverV1 implements
SubscriptionReceiver {
private boolean beforeHandle(final short reqType) {
synchronized (this) {
final boolean isHandshake = PipeSubscribeRequestType.HANDSHAKE.getType()
== reqType;
- // A receiver fenced by a newer connection is terminal. In particular,
do not allow the old
- // connection to handshake again and reclaim the consumer identity. A
normal disconnected
- // receiver remains recoverable through the existing consumerInvalidated
handshake path.
- final boolean isFencedRequest = consumerFenced;
+ // Replaced connections are terminal, even when the consumer instance
itself is still valid.
+ // An old connection must not handshake again and replace the current
connection in turn.
+ final boolean isObsoleteRequest = consumerFenced || connectionReplaced;
if (consumerInvalidated) {
consumerConfigThreadLocal.remove();
pollTimerThreadLocal.remove();
- if (isHandshake && !consumerFenced) {
+ if (isHandshake && !isObsoleteRequest) {
consumerInvalidated = false;
}
}
inFlightRequestCount.incrementAndGet();
lastActivityTimeMs = System.currentTimeMillis();
- return isFencedRequest;
+ return isObsoleteRequest;
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
index fc68f97c6e8..73e67e1e426 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/agent/SubscriptionReceiverAgentTest.java
@@ -404,6 +404,20 @@ public class SubscriptionReceiverAgentTest {
Assert.assertEquals(2, receivers.size());
Assert.assertEquals(0, receivers.get(0).releaseCount.get());
Assert.assertTrue(receivers.get(0).invalidated);
+ Assert.assertTrue(receivers.get(0).connectionReplaced);
+ Assert.assertEquals(
+ TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode(),
+ agent.handle(SubscriptionHeartbeatReq.toThriftReq(),
"root").getStatus().getCode());
+ Assert.assertEquals(
+ TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode(),
+ agent
+ .handle(createHandshakeRequest("group", "consumer", "instance-1"),
"root")
+ .getStatus()
+ .getCode());
+ Assert.assertFalse(receivers.get(1).invalidated);
+ agent.checkReceiverTimeouts();
+ Assert.assertEquals(0, receivers.get(0).timeoutCount.get());
+ Assert.assertEquals(1, receivers.get(1).timeoutCount.get());
}
@Test
@@ -643,6 +657,7 @@ public class SubscriptionReceiverAgentTest {
private final AtomicInteger releaseCount = new AtomicInteger();
private volatile ConsumerConfig consumerConfig;
private volatile boolean invalidated;
+ private volatile boolean connectionReplaced;
private FakeSubscriptionReceiver(final boolean closeOnTimeout) {
this(closeOnTimeout, false, null, null);
@@ -661,6 +676,9 @@ public class SubscriptionReceiverAgentTest {
@Override
public TPipeSubscribeResp handle(final TPipeSubscribeReq req) {
+ if (connectionReplaced) {
+ return response(TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED);
+ }
if (req.getType() == PipeSubscribeRequestType.HANDSHAKE.getType()) {
consumerConfig = ConsumerConfig.deserialize(req.bufferForBody());
if (assignDefaultIdentity) {
@@ -738,6 +756,12 @@ public class SubscriptionReceiverAgentTest {
invalidated = true;
}
+ @Override
+ public void invalidateConnection() {
+ invalidateConsumer();
+ connectionReplaced = true;
+ }
+
@Override
public boolean hasActiveConsumer() {
return consumerConfig != null;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
index 05eccb978f9..2b901cda9b8 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/receiver/SubscriptionReceiverV1Test.java
@@ -204,6 +204,29 @@ public class SubscriptionReceiverV1Test {
receiver.handle(SubscriptionHeartbeatReq.toThriftReq()).getStatus().getCode());
}
+ @Test
+ public void testReplacedConnectionReturnsRetryableStatusForAllRequests()
throws Exception {
+ final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1();
+ final ConsumerConfig consumerConfig = createConsumerConfig(1_000L);
+ setField(receiver, "sharedConsumerConfig", consumerConfig);
+
+ receiver.invalidateConnection();
+
+ for (final PipeSubscribeRequestType requestType :
PipeSubscribeRequestType.values()) {
+ Assert.assertEquals(
+ requestType.name(),
+ TSStatusCode.SUBSCRIPTION_CONNECTION_REPLACED.getStatusCode(),
+ receiver
+ .handle(new TPipeSubscribeReq((byte) 1, requestType.getType()))
+ .getStatus()
+ .getCode());
+ }
+ Assert.assertFalse(receiver.hasActiveConsumer());
+ Assert.assertFalse((boolean) getField(receiver, "consumerFenced"));
+ Assert.assertTrue((boolean) getField(receiver, "consumerInvalidated"));
+ Assert.assertEquals(0L, ((AtomicLong) getField(receiver,
"inFlightRequestCount")).get());
+ }
+
@Test
public void testFencedConsumerCannotHandshakeAgain() throws Exception {
final SubscriptionReceiverV1 receiver = new SubscriptionReceiverV1();