This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/fix-consensus-subscription-replay in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6216488c3e0465533d341411ff65a7784d680c04 Author: Caideyipi <[email protected]> AuthorDate: Fri Nov 6 10:03:35 2026 +0800 Prevent subscription progress rollback on leader activation --- .../ConsensusSubscriptionCommitManager.java | 49 ++++++++++++++++++---- .../ConsensusSubscriptionCommitStateTest.java | 33 +++++++++++++++ 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java index b183ca745f9..dec1c0b9cd9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java @@ -525,7 +525,7 @@ public class ConsensusSubscriptionCommitManager { } } - /** Refreshes an exact authoritative frontier for preferred-writer activation. */ + /** Refreshes the activation frontier without regressing newer locally replicated progress. */ public ConfigNodeProgressQueryResult refreshAuthoritativeProgress( final String consumerGroupId, final String topicName, final ConsensusGroupId regionId) { final CommitStateKey stateKey = @@ -545,17 +545,19 @@ public class ConsensusSubscriptionCommitManager { if (!queryResult.isAvailable() && !startNewRegionFromBeginning) { return queryResult; } - final RegionProgress authoritativeProgress = - startNewRegionFromBeginning - ? new RegionProgress(Collections.emptyMap()) - : queryResult.getRegionProgress(); final ConsensusSubscriptionCommitState state = commitStates.computeIfAbsent( stateKey.encodedStateKey, ignored -> new ConsensusSubscriptionCommitState( stateKey.regionIdStr, new SubscriptionConsensusProgress())); - state.resetForSeek(authoritativeProgress); + final RegionProgress refreshedProgress = + mergeRegionProgress( + state.getCommittedRegionProgress(), + startNewRegionFromBeginning + ? new RegionProgress(Collections.emptyMap()) + : queryResult.getRegionProgress()); + state.resetForSeek(refreshedProgress); persistRegionProgress(stateKey, state); pendingInitialProgressKeys.remove(stateKey.encodedStateKey); if (!startNewRegionFromBeginning) { @@ -563,7 +565,7 @@ public class ConsensusSubscriptionCommitManager { } configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); tailProposalInitializedKeys.add(stateKey.encodedStateKey); - return ConfigNodeProgressQueryResult.available(authoritativeProgress); + return ConfigNodeProgressQueryResult.available(refreshedProgress); } } @@ -1578,6 +1580,39 @@ public class ConsensusSubscriptionCommitManager { : Collections.emptyMap()); } + private static RegionProgress mergeRegionProgress( + final RegionProgress currentProgress, final RegionProgress refreshedProgress) { + final Map<WriterId, WriterProgress> mergedWriterPositions = new LinkedHashMap<>(); + mergeWriterPositions(mergedWriterPositions, currentProgress); + mergeWriterPositions(mergedWriterPositions, refreshedProgress); + return new RegionProgress(mergedWriterPositions); + } + + private static void mergeWriterPositions( + final Map<WriterId, WriterProgress> mergedWriterPositions, + final RegionProgress regionProgress) { + if (Objects.isNull(regionProgress)) { + return; + } + regionProgress + .getWriterPositions() + .forEach( + (writerId, writerProgress) -> { + if (Objects.isNull(writerId) || Objects.isNull(writerProgress)) { + return; + } + mergedWriterPositions.merge( + writerId, + writerProgress, + (current, refreshed) -> + new ProgressKey(writerId, refreshed) + .compareTo(new ProgressKey(writerId, current)) + > 0 + ? refreshed + : current); + }); + } + @FunctionalInterface interface ConfigNodeProgressFetcher { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java index 9734e19593f..68e450fbd96 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java @@ -330,6 +330,39 @@ public class ConsensusSubscriptionCommitStateTest { } } + @Test + public void testAuthoritativeRefreshPreservesReplicatedProgressWhenConfigNodeIsEmpty() + throws Exception { + final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); + final File systemDir = temporaryFolder.newFolder("refreshDoesNotRegressReplicatedProgress"); + try { + final DataRegionId regionId = new DataRegionId(27); + final WriterId writerId = new WriterId(regionId.toString(), 7); + final WriterProgress replicatedProgress = new WriterProgress(200L, 2L); + final ConsensusSubscriptionCommitManager manager = + newCommitManager( + systemDir, + (consumerGroupId, topicName, ignoredRegionId) -> + ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available( + new RegionProgress(Collections.emptyMap()))); + manager.receiveProgressBroadcast( + "cg", "topic", regionId.toString(), writerId, replicatedProgress); + + final RegionProgress refreshedProgress = + manager.refreshAuthoritativeProgress("cg", "topic", regionId).getRegionProgress(); + + assertEquals(replicatedProgress, refreshedProgress.getWriterPositions().get(writerId)); + assertEquals( + replicatedProgress, + manager + .getCommittedRegionProgress("cg", "topic", regionId) + .getWriterPositions() + .get(writerId)); + } finally { + IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); + } + } + @Test public void testNewRegionWaitsThroughUnavailableConfigNodeThenAcceptsExplicitAbsence() throws Exception {
