This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/fix-consensus-subscription-replay in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d16f351791fc5bdff0e6db97297e57db3606a2bd Author: Caideyipi <[email protected]> AuthorDate: Fri Nov 6 16:26:28 2026 +0800 Preserve subscription progress across activation races --- .../consensus/ConsensusPrefetchingQueue.java | 10 +- .../ConsensusSubscriptionCommitManager.java | 255 ++++++++++++++++----- .../consensus/SubscriptionConsensusProgress.java | 29 ++- .../ConsensusPrefetchingQueueSeekTest.java | 87 ++++++- .../ConsensusSubscriptionCommitStateTest.java | 85 ++++++- 5 files changed, 392 insertions(+), 74 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index c4df0d2b2ce..bd1a270f22f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -4695,10 +4695,12 @@ public class ConsensusPrefetchingQueue { topicName, consensusGroupId, request.committedRegionProgress, - request.explicitSeek); - if (!committedRegionProgress.getWriterPositions().isEmpty()) { - installRecoveryWriterProgress(committedRegionProgress); - } + request.explicitSeek, + progress -> { + if (!progress.getWriterPositions().isEmpty()) { + installRecoveryWriterProgress(progress); + } + }); // 4. Reset WAL read position. The target has been normalized by replay lookup: if the // requested progress points before the first retained replayable WAL entry, it is moved to that diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java index 29a4fad25d4..fa79c98a76a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java @@ -81,6 +81,7 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.function.LongSupplier; /** @@ -160,6 +161,10 @@ public class ConsensusSubscriptionCommitManager { /** New regions that may start from the beginning once ConfigNode explicitly reports no entry. */ private final Set<String> pendingNewRegionProgressKeys = ConcurrentHashMap.newKeySet(); + /** Broadcast progress received while a fresh local-tail proposal awaits ConfigNode resolution. */ + private final Map<String, Map<WriterId, WriterProgress>> pendingInitialBroadcastProgress = + new ConcurrentHashMap<>(); + /** Explicit seeks take precedence over stale, higher progress retained by ConfigNode. */ private final Set<String> explicitSeekResetKeys = ConcurrentHashMap.newKeySet(); @@ -279,12 +284,14 @@ public class ConsensusSubscriptionCommitManager { new StateSetupSnapshot( new SubscriptionConsensusProgress( copyRegionProgress(state.getCommittedRegionProgress()), - state.getProgress().getPersistenceThrottleCounter()), + state.getProgress().getPersistenceThrottleCounter(), + state.getProgress().isExplicitSeekReset()), pendingInitialProgressKeys.contains(entry.getKey()), pendingNewRegionProgressKeys.contains(entry.getKey()), configNodeProgressAvailableKeys.contains(entry.getKey()), tailProposalInitializedKeys.contains(entry.getKey()), - explicitSeekResetKeys.contains(entry.getKey()))); + explicitSeekResetKeys.contains(entry.getKey()), + copyPendingInitialBroadcastProgress(entry.getKey()))); } topicSnapshots.put(topicName, new TopicSetupSnapshot(topicFileKey, stateSnapshots)); } @@ -334,7 +341,8 @@ public class ConsensusSubscriptionCommitManager { stateKey.regionIdStr, new SubscriptionConsensusProgress( copyRegionProgress(stateSnapshot.progress.getCommittedRegionProgress()), - stateSnapshot.progress.getPersistenceThrottleCounter()))); + stateSnapshot.progress.getPersistenceThrottleCounter(), + stateSnapshot.explicitSeekReset))); restoreSetupMarker( pendingInitialProgressKeys, stateKey.encodedStateKey, @@ -353,6 +361,8 @@ public class ConsensusSubscriptionCommitManager { stateSnapshot.tailProposalInitialized); restoreSetupMarker( explicitSeekResetKeys, stateKey.encodedStateKey, stateSnapshot.explicitSeekReset); + restorePendingInitialBroadcastProgress( + stateKey.encodedStateKey, stateSnapshot.pendingInitialBroadcastProgress); stateKeyForRewrite = stateKey; } @@ -392,6 +402,7 @@ public class ConsensusSubscriptionCommitManager { configNodeProgressAvailableKeys.remove(entry.getKey()); tailProposalInitializedKeys.remove(entry.getKey()); explicitSeekResetKeys.remove(entry.getKey()); + pendingInitialBroadcastProgress.remove(entry.getKey()); keyIterator.remove(); } } @@ -405,6 +416,71 @@ public class ConsensusSubscriptionCommitManager { } } + private Map<WriterId, WriterProgress> copyPendingInitialBroadcastProgress(final String stateKey) { + final Map<WriterId, WriterProgress> progress = pendingInitialBroadcastProgress.get(stateKey); + return Objects.nonNull(progress) ? new LinkedHashMap<>(progress) : Collections.emptyMap(); + } + + private void restorePendingInitialBroadcastProgress( + final String stateKey, final Map<WriterId, WriterProgress> progress) { + if (Objects.isNull(progress) || progress.isEmpty()) { + pendingInitialBroadcastProgress.remove(stateKey); + } else { + pendingInitialBroadcastProgress.put(stateKey, new ConcurrentHashMap<>(progress)); + } + } + + private RegionProgress takePendingInitialBroadcastProgress(final String stateKey) { + final Map<WriterId, WriterProgress> progress = pendingInitialBroadcastProgress.remove(stateKey); + return new RegionProgress( + Objects.nonNull(progress) ? new LinkedHashMap<>(progress) : Collections.emptyMap()); + } + + private void recordPendingInitialBroadcastProgress( + final String stateKey, final WriterId writerId, final WriterProgress writerProgress) { + if (!pendingInitialProgressKeys.contains(stateKey)) { + return; + } + pendingInitialBroadcastProgress + .computeIfAbsent(stateKey, ignored -> new ConcurrentHashMap<>()) + .merge( + writerId, + writerProgress, + (current, candidate) -> + new ProgressKey(writerId, candidate).compareTo(new ProgressKey(writerId, current)) + > 0 + ? candidate + : current); + } + + private void setExplicitSeekReset( + final String stateKey, + final ConsensusSubscriptionCommitState state, + final boolean explicitSeekReset) { + if (explicitSeekReset) { + explicitSeekResetKeys.add(stateKey); + } else { + explicitSeekResetKeys.remove(stateKey); + } + state.getProgress().setExplicitSeekReset(explicitSeekReset); + } + + private RegionProgress resolveInitialProgress( + final String stateKey, + final ConsensusSubscriptionCommitState state, + final RegionProgress authoritativeProgress) { + final RegionProgress pendingBroadcastProgress = takePendingInitialBroadcastProgress(stateKey); + if (explicitSeekResetKeys.contains(stateKey)) { + final RegionProgress refreshedProgress = + mergeRegionProgress(state.getCommittedRegionProgress(), pendingBroadcastProgress); + setExplicitSeekReset( + stateKey, state, !isRegionProgressAtLeast(refreshedProgress, authoritativeProgress)); + return refreshedProgress; + } + setExplicitSeekReset(stateKey, state, false); + return mergeRegionProgress(authoritativeProgress, pendingBroadcastProgress); + } + /** * Initializes an already-bound existing-region queue from its local WAL-tail proposal. * @@ -432,18 +508,25 @@ public class ConsensusSubscriptionCommitManager { ignored -> new ConsensusSubscriptionCommitState( stateKey.regionIdStr, new SubscriptionConsensusProgress())); - final ConfigNodeProgressQueryResult queryResult = - configNodeProgressFetcher.fetch(consumerGroupId, topicName, regionId); - if (queryResult.isAvailable()) { - localState.resetForSeek(queryResult.getRegionProgress()); - pendingInitialProgressKeys.remove(stateKey.encodedStateKey); - configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); - } else { - if (!hasLocalPersistedState) { - localState.resetForSeek(copyRegionProgress(tailProposal)); + synchronized (localState) { + if (!pendingInitialProgressKeys.contains(stateKey.encodedStateKey)) { + pendingInitialBroadcastProgress.remove(stateKey.encodedStateKey); } pendingInitialProgressKeys.add(stateKey.encodedStateKey); - configNodeProgressAvailableKeys.remove(stateKey.encodedStateKey); + final ConfigNodeProgressQueryResult queryResult = + configNodeProgressFetcher.fetch(consumerGroupId, topicName, regionId); + if (queryResult.isAvailable()) { + localState.resetForSeek( + resolveInitialProgress( + stateKey.encodedStateKey, localState, queryResult.getRegionProgress())); + pendingInitialProgressKeys.remove(stateKey.encodedStateKey); + configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); + } else { + if (!hasLocalPersistedState) { + localState.resetForSeek(copyRegionProgress(tailProposal)); + } + configNodeProgressAvailableKeys.remove(stateKey.encodedStateKey); + } } persistRegionProgress(stateKey, localState); } @@ -463,28 +546,39 @@ public class ConsensusSubscriptionCommitManager { ignored -> new ConsensusSubscriptionCommitState( stateKey.regionIdStr, new SubscriptionConsensusProgress())); - final ConfigNodeProgressQueryResult queryResult = - configNodeProgressFetcher.fetch(consumerGroupId, topicName, regionId); - if (queryResult.isAvailable()) { - localState.resetForSeek(queryResult.getRegionProgress()); - pendingInitialProgressKeys.remove(stateKey.encodedStateKey); - pendingNewRegionProgressKeys.remove(stateKey.encodedStateKey); - configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); - persistRegionProgress(stateKey, localState); - } else if (queryResult.getAvailability() == ConfigNodeProgressAvailability.ABSENT - && (!hasLocalPersistedState - || pendingNewRegionProgressKeys.contains(stateKey.encodedStateKey))) { - localState.resetForSeek(new RegionProgress(Collections.emptyMap())); - pendingInitialProgressKeys.remove(stateKey.encodedStateKey); - pendingNewRegionProgressKeys.add(stateKey.encodedStateKey); - configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); - persistRegionProgress(stateKey, localState); - } else { + synchronized (localState) { + if (!pendingInitialProgressKeys.contains(stateKey.encodedStateKey)) { + pendingInitialBroadcastProgress.remove(stateKey.encodedStateKey); + } pendingInitialProgressKeys.add(stateKey.encodedStateKey); - if (!hasLocalPersistedState) { + final ConfigNodeProgressQueryResult queryResult = + configNodeProgressFetcher.fetch(consumerGroupId, topicName, regionId); + if (queryResult.isAvailable()) { + localState.resetForSeek( + resolveInitialProgress( + stateKey.encodedStateKey, localState, queryResult.getRegionProgress())); + pendingInitialProgressKeys.remove(stateKey.encodedStateKey); + pendingNewRegionProgressKeys.remove(stateKey.encodedStateKey); + configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); + persistRegionProgress(stateKey, localState); + } else if (queryResult.getAvailability() == ConfigNodeProgressAvailability.ABSENT + && (!hasLocalPersistedState + || pendingNewRegionProgressKeys.contains(stateKey.encodedStateKey))) { + localState.resetForSeek( + resolveInitialProgress( + stateKey.encodedStateKey, + localState, + new RegionProgress(Collections.emptyMap()))); + pendingInitialProgressKeys.remove(stateKey.encodedStateKey); pendingNewRegionProgressKeys.add(stateKey.encodedStateKey); + configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); + persistRegionProgress(stateKey, localState); + } else { + if (!hasLocalPersistedState) { + pendingNewRegionProgressKeys.add(stateKey.encodedStateKey); + } + configNodeProgressAvailableKeys.remove(stateKey.encodedStateKey); } - configNodeProgressAvailableKeys.remove(stateKey.encodedStateKey); } } } @@ -504,30 +598,34 @@ public class ConsensusSubscriptionCommitManager { if (!pendingInitialProgressKeys.contains(stateKey.encodedStateKey)) { return true; } - final ConfigNodeProgressQueryResult queryResult = - configNodeProgressFetcher.fetch(consumerGroupId, topicName, regionId); - final boolean startNewRegionFromBeginning = - queryResult.getAvailability() == ConfigNodeProgressAvailability.ABSENT - && pendingNewRegionProgressKeys.contains(stateKey.encodedStateKey); - if (!queryResult.isAvailable() && !startNewRegionFromBeginning) { - return false; - } final ConsensusSubscriptionCommitState state = commitStates.computeIfAbsent( stateKey.encodedStateKey, ignored -> new ConsensusSubscriptionCommitState( stateKey.regionIdStr, new SubscriptionConsensusProgress())); - state.resetForSeek( - startNewRegionFromBeginning - ? new RegionProgress(Collections.emptyMap()) - : queryResult.getRegionProgress()); - persistRegionProgress(stateKey, state); - pendingInitialProgressKeys.remove(stateKey.encodedStateKey); - if (!startNewRegionFromBeginning) { - pendingNewRegionProgressKeys.remove(stateKey.encodedStateKey); + synchronized (state) { + final ConfigNodeProgressQueryResult queryResult = + configNodeProgressFetcher.fetch(consumerGroupId, topicName, regionId); + final boolean startNewRegionFromBeginning = + queryResult.getAvailability() == ConfigNodeProgressAvailability.ABSENT + && pendingNewRegionProgressKeys.contains(stateKey.encodedStateKey); + if (!queryResult.isAvailable() && !startNewRegionFromBeginning) { + return false; + } + final RegionProgress authoritativeProgress = + startNewRegionFromBeginning + ? new RegionProgress(Collections.emptyMap()) + : queryResult.getRegionProgress(); + state.resetForSeek( + resolveInitialProgress(stateKey.encodedStateKey, state, authoritativeProgress)); + pendingInitialProgressKeys.remove(stateKey.encodedStateKey); + if (!startNewRegionFromBeginning) { + pendingNewRegionProgressKeys.remove(stateKey.encodedStateKey); + } + configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); } - configNodeProgressAvailableKeys.add(stateKey.encodedStateKey); + persistRegionProgress(stateKey, state); return true; } } @@ -568,22 +666,31 @@ public class ConsensusSubscriptionCommitManager { if (explicitSeekResetKeys.contains(stateKey.encodedStateKey)) { // Explicit seek is an intentional reset; ConfigNode can retain a higher pre-seek // checkpoint until every DataNode has reported the replayed progress. - refreshedProgress = localProgress; - if (isRegionProgressAtLeast(localProgress, authoritativeProgress)) { - explicitSeekResetKeys.remove(stateKey.encodedStateKey); - } + refreshedProgress = + mergeRegionProgress( + localProgress, takePendingInitialBroadcastProgress(stateKey.encodedStateKey)); + setExplicitSeekReset( + stateKey.encodedStateKey, + state, + !isRegionProgressAtLeast(refreshedProgress, authoritativeProgress)); } else if (pendingInitialProgressKeys.contains(stateKey.encodedStateKey)) { // A pending local-tail proposal is not a committed checkpoint. ConfigNode must resolve - // it exactly before activation can preserve subsequent replicated progress. - refreshedProgress = authoritativeProgress; + // it exactly, while broadcasts received during the wait remain committed progress. + refreshedProgress = + mergeRegionProgress( + authoritativeProgress, + takePendingInitialBroadcastProgress(stateKey.encodedStateKey)); } else { refreshedProgress = mergeRegionProgress(localProgress, authoritativeProgress); } // Keep the snapshot, merge, and reset atomic with respect to progress broadcasts and ACKs. state.resetForSeek(refreshedProgress); + if (!explicitSeekResetKeys.contains(stateKey.encodedStateKey)) { + setExplicitSeekReset(stateKey.encodedStateKey, state, false); + } + pendingInitialProgressKeys.remove(stateKey.encodedStateKey); } persistRegionProgress(stateKey, state); - pendingInitialProgressKeys.remove(stateKey.encodedStateKey); if (!startNewRegionFromBeginning) { pendingNewRegionProgressKeys.remove(stateKey.encodedStateKey); } @@ -759,6 +866,7 @@ public class ConsensusSubscriptionCommitManager { configNodeProgressAvailableKeys.remove(stateKey.encodedStateKey); tailProposalInitializedKeys.remove(stateKey.encodedStateKey); explicitSeekResetKeys.remove(stateKey.encodedStateKey); + pendingInitialBroadcastProgress.remove(stateKey.encodedStateKey); rewriteTopicProgressFiles(stateKey); } @@ -782,6 +890,7 @@ public class ConsensusSubscriptionCommitManager { configNodeProgressAvailableKeys.remove(entry.getKey()); tailProposalInitializedKeys.remove(entry.getKey()); explicitSeekResetKeys.remove(entry.getKey()); + pendingInitialBroadcastProgress.remove(entry.getKey()); keyIterator.remove(); } } @@ -846,6 +955,17 @@ public class ConsensusSubscriptionCommitManager { final ConsensusGroupId regionId, final RegionProgress regionProgress, final boolean explicitSeek) { + return resetState( + consumerGroupId, topicName, regionId, regionProgress, explicitSeek, ignored -> {}); + } + + RegionProgress resetState( + final String consumerGroupId, + final String topicName, + final ConsensusGroupId regionId, + final RegionProgress regionProgress, + final boolean explicitSeek, + final Consumer<RegionProgress> afterReset) { final CommitStateKey stateKey = getCommitStateKey(consumerGroupId, topicName, regionId.toString()); synchronized (getTopicPersistLock(stateKey.topicFileKey)) { @@ -857,7 +977,9 @@ public class ConsensusSubscriptionCommitManager { consumerGroupId, topicName, regionId); - return copyRegionProgress(regionProgress); + final RegionProgress resetProgress = copyRegionProgress(regionProgress); + afterReset.accept(resetProgress); + return resetProgress; } final RegionProgress resetProgress; synchronized (state) { @@ -869,8 +991,9 @@ public class ConsensusSubscriptionCommitManager { : mergeRegionProgress(state.getCommittedRegionProgress(), regionProgress); state.resetForSeek(resetProgress); if (explicitSeek) { - explicitSeekResetKeys.add(stateKey.encodedStateKey); + setExplicitSeekReset(stateKey.encodedStateKey, state, true); } + afterReset.accept(resetProgress); } persistRegionProgress(stateKey, state); return resetProgress; @@ -1129,7 +1252,10 @@ public class ConsensusSubscriptionCommitManager { final ConsensusSubscriptionCommitState state = commitStates.get(stateKey.encodedStateKey); if (state != null) { // Update only if broadcast is ahead - state.updateFromBroadcast(writerId, writerProgress); + synchronized (state) { + state.updateFromBroadcast(writerId, writerProgress); + recordPendingInitialBroadcastProgress(stateKey.encodedStateKey, writerId, writerProgress); + } persistProgressIfNeeded(stateKey, state); } else { // Create a new state from the broadcast progress @@ -1290,6 +1416,11 @@ public class ConsensusSubscriptionCommitManager { final CommitStateKey recoveredStateKey = getCommitStateKey(consumerGroupId, topicName, entry.getKey()); commitStates.putIfAbsent(recoveredStateKey.encodedStateKey, entry.getValue()); + final ConsensusSubscriptionCommitState recoveredState = + commitStates.get(recoveredStateKey.encodedStateKey); + if (recoveredState.getProgress().isExplicitSeekReset()) { + explicitSeekResetKeys.add(recoveredStateKey.encodedStateKey); + } } topicProgressIndexes.put(topicFileKey, snapshot.indexEntries); } catch (final IOException e) { @@ -1794,19 +1925,23 @@ public class ConsensusSubscriptionCommitManager { private final boolean explicitSeekReset; + private final Map<WriterId, WriterProgress> pendingInitialBroadcastProgress; + private StateSetupSnapshot( final SubscriptionConsensusProgress progress, final boolean initialProgressPending, final boolean newRegionProgressPending, final boolean configNodeProgressAvailable, final boolean tailProposalInitialized, - final boolean explicitSeekReset) { + final boolean explicitSeekReset, + final Map<WriterId, WriterProgress> pendingInitialBroadcastProgress) { this.progress = progress; this.initialProgressPending = initialProgressPending; this.newRegionProgressPending = newRegionProgressPending; this.configNodeProgressAvailable = configNodeProgressAvailable; this.tailProposalInitialized = tailProposalInitialized; this.explicitSeekReset = explicitSeekReset; + this.pendingInitialBroadcastProgress = pendingInitialBroadcastProgress; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/SubscriptionConsensusProgress.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/SubscriptionConsensusProgress.java index 9448a416873..0b6a56389de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/SubscriptionConsensusProgress.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/SubscriptionConsensusProgress.java @@ -46,14 +46,24 @@ public class SubscriptionConsensusProgress { private final AtomicLong persistenceThrottleCounter; + private volatile boolean explicitSeekReset; + public SubscriptionConsensusProgress() { this(new RegionProgress(Collections.emptyMap()), 0L); } public SubscriptionConsensusProgress( final RegionProgress committedRegionProgress, final long persistenceThrottleCounter) { + this(committedRegionProgress, persistenceThrottleCounter, false); + } + + public SubscriptionConsensusProgress( + final RegionProgress committedRegionProgress, + final long persistenceThrottleCounter, + final boolean explicitSeekReset) { this.committedRegionProgress = normalize(committedRegionProgress); this.persistenceThrottleCounter = new AtomicLong(persistenceThrottleCounter); + this.explicitSeekReset = explicitSeekReset; } public RegionProgress getCommittedRegionProgress() { @@ -64,6 +74,14 @@ public class SubscriptionConsensusProgress { this.committedRegionProgress = normalize(committedRegionProgress); } + public boolean isExplicitSeekReset() { + return explicitSeekReset; + } + + public void setExplicitSeekReset(final boolean explicitSeekReset) { + this.explicitSeekReset = explicitSeekReset; + } + public WriterId getCommittedWriterId() { return getDerivedCommittedWriterState().writerId; } @@ -83,12 +101,15 @@ public class SubscriptionConsensusProgress { public void serialize(final DataOutputStream stream) throws IOException { committedRegionProgress.serialize(stream); ReadWriteIOUtils.write(persistenceThrottleCounter.get(), stream); + ReadWriteIOUtils.write(explicitSeekReset, stream); } public static SubscriptionConsensusProgress deserialize(final ByteBuffer buffer) { final RegionProgress committedRegionProgress = RegionProgress.deserialize(buffer); final long persistenceThrottleCounter = ReadWriteIOUtils.readLong(buffer); - return new SubscriptionConsensusProgress(committedRegionProgress, persistenceThrottleCounter); + final boolean explicitSeekReset = buffer.hasRemaining() && ReadWriteIOUtils.readBool(buffer); + return new SubscriptionConsensusProgress( + committedRegionProgress, persistenceThrottleCounter, explicitSeekReset); } @Override @@ -101,12 +122,14 @@ public class SubscriptionConsensusProgress { } final SubscriptionConsensusProgress that = (SubscriptionConsensusProgress) o; return persistenceThrottleCounter.get() == that.persistenceThrottleCounter.get() + && explicitSeekReset == that.explicitSeekReset && Objects.equals(committedRegionProgress, that.committedRegionProgress); } @Override public int hashCode() { - return Objects.hash(committedRegionProgress, persistenceThrottleCounter.get()); + return Objects.hash( + committedRegionProgress, persistenceThrottleCounter.get(), explicitSeekReset); } @Override @@ -116,6 +139,8 @@ public class SubscriptionConsensusProgress { + committedRegionProgress + ", persistenceThrottleCounter=" + persistenceThrottleCounter.get() + + ", explicitSeekReset=" + + explicitSeekReset + '}'; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java index 66da47a2a38..00f7c921f2a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.subscription.broker.consensus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; @@ -65,8 +66,11 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -157,9 +161,14 @@ public class ConsensusPrefetchingQueueSeekTest { assertActivationProgress(false, false, true); } + @Test(timeout = 20_000L) + public void testWorkerResetInstallsRecoveryFrontierAtomicallyWithBroadcast() throws Exception { + assertActivationProgress(false, false, false, true); + } + private void assertActivationProgress( final boolean explicitSeek, final boolean broadcastDuringActivation) throws Exception { - assertActivationProgress(explicitSeek, broadcastDuringActivation, false); + assertActivationProgress(explicitSeek, broadcastDuringActivation, false, false); } private void assertActivationProgress( @@ -167,6 +176,15 @@ public class ConsensusPrefetchingQueueSeekTest { final boolean broadcastDuringActivation, final boolean seekDuringActivation) throws Exception { + assertActivationProgress(explicitSeek, broadcastDuringActivation, seekDuringActivation, false); + } + + private void assertActivationProgress( + final boolean explicitSeek, + final boolean broadcastDuringActivation, + final boolean seekDuringActivation, + final boolean broadcastDuringWorkerReset) + throws Exception { final int originalPrefetchThreads = config.getSubscriptionConsensusPrefetchExecutorMaxThreadNum(); final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); @@ -185,16 +203,64 @@ public class ConsensusPrefetchingQueueSeekTest { final WriterId writerId = new WriterId(regionId.toString(), 7); final RegionProgress emptyProgress = new RegionProgress(Collections.emptyMap()); final RegionProgress initialProgress = - broadcastDuringActivation || explicitSeek || seekDuringActivation + broadcastDuringActivation + || explicitSeek + || seekDuringActivation + || broadcastDuringWorkerReset ? new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(100L, 1L))) : emptyProgress; final AtomicReference<RegionProgress> configNodeProgress = new AtomicReference<>(initialProgress); + final RegionProgress broadcastProgress = + new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(200L, 2L))); + final AtomicReference<ConsensusPrefetchingQueue> queueReference = new AtomicReference<>(); + final AtomicReference<RegionProgress> recoveryProgressAtReset = new AtomicReference<>(); + final AtomicReference<Thread> resetBroadcastThread = new AtomicReference<>(); final ConsensusSubscriptionCommitManager commitManager = new ConsensusSubscriptionCommitManager( (consumerGroupId, topicName, ignored) -> ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available( - configNodeProgress.get())); + configNodeProgress.get())) { + @Override + RegionProgress resetState( + final String consumerGroupId, + final String topicName, + final ConsensusGroupId regionId, + final RegionProgress regionProgress, + final boolean explicitSeek, + final Consumer<RegionProgress> afterReset) { + return super.resetState( + consumerGroupId, + topicName, + regionId, + regionProgress, + explicitSeek, + progress -> { + afterReset.accept(progress); + if (!explicitSeek && broadcastDuringWorkerReset) { + recoveryProgressAtReset.set(progress); + assertEquals( + "1", queueReference.get().coreReportMessage().get("recoveryWriterCount")); + final Thread broadcastThread = + new Thread( + () -> + receiveProgressBroadcast( + "consumerGroup", + "topic", + regionId.toString(), + writerId, + broadcastProgress.getWriterPositions().get(writerId))); + broadcastThread.setDaemon(true); + resetBroadcastThread.set(broadcastThread); + broadcastThread.start(); + await() + .pollInterval(1L, TimeUnit.MILLISECONDS) + .atMost(5L, TimeUnit.SECONDS) + .until(() -> broadcastThread.getState() == Thread.State.BLOCKED); + } + }); + } + }; commitManager.getOrCreateState("consumerGroup", "topic", regionId); final AtomicReference<Runnable> duringReplayLookup = new AtomicReference<>(); queue = @@ -241,17 +307,16 @@ public class ConsensusPrefetchingQueueSeekTest { }; } }; + queueReference.set(queue); queue.setSubscriptionMemoryManager(new SubscriptionMemoryManager(16L * 1024 * 1024)); queue.poll("consumer"); queue.setActive(false); // A seek while inactive initializes the cursor, so activation must apply a worker seek. - if (broadcastDuringActivation || seekDuringActivation) { + if (broadcastDuringActivation || seekDuringActivation || broadcastDuringWorkerReset) { queue.seekAfterRegionProgress(initialProgress); } else { queue.seekToBeginning(); } - final RegionProgress broadcastProgress = - new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(200L, 2L))); if (broadcastDuringActivation) { // Advance after the manager's activation refresh, before the worker applies the seek. duringReplayLookup.set( @@ -272,10 +337,18 @@ public class ConsensusPrefetchingQueueSeekTest { queue.applyRuntimeState( new ConsensusRegionRuntimeState(2L, 7, true, Collections.singleton(7))); assertNull(duringReplayLookup.get()); + if (broadcastDuringWorkerReset) { + final Thread broadcastThread = resetBroadcastThread.get(); + broadcastThread.join(TimeUnit.SECONDS.toMillis(5L)); + assertFalse(broadcastThread.isAlive()); + assertEquals(initialProgress, recoveryProgressAtReset.get()); + } assertEquals( explicitSeek || seekDuringActivation ? emptyProgress - : broadcastDuringActivation ? broadcastProgress : initialProgress, + : broadcastDuringActivation || broadcastDuringWorkerReset + ? broadcastProgress + : initialProgress, commitManager.getCommittedRegionProgress("consumerGroup", "topic", regionId)); if (seekDuringActivation) { assertEquals(1L, queue.getCurrentReadSearchIndex()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java index 0126655cc62..052b97717ef 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitStateTest.java @@ -244,6 +244,9 @@ public class ConsensusSubscriptionCommitStateTest { assertTrue(manager.hasPersistedState("cg", "topic", regionId)); assertTrue(manager.isInitialProgressPending("cg", "topic", regionId)); assertEquals(tailProposal, manager.getCommittedRegionProgress("cg", "topic", regionId)); + final WriterProgress broadcastProgress = new WriterProgress(300L, 3L); + manager.receiveProgressBroadcast( + "cg", "topic", regionId.toString(), writerId, broadcastProgress); assertFalse(manager.refreshPendingInitialProgress("cg", "topic", regionId)); assertFalse(manager.refreshAuthoritativeProgress("cg", "topic", regionId).isAvailable()); @@ -257,13 +260,54 @@ public class ConsensusSubscriptionCommitStateTest { manager .getCommittedRegionProgress("cg", "topic", regionId) .getWriterPositions() - .isEmpty()); + .get(writerId) + .equals(broadcastProgress)); assertTrue(manager.refreshAuthoritativeProgress("cg", "topic", regionId).isAvailable()); } finally { IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); } } + @Test + public void testAuthoritativeRefreshPreservesBroadcastDuringPendingTailProposal() + throws Exception { + final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); + final File systemDir = temporaryFolder.newFolder("pendingTailBroadcast"); + try { + final DataRegionId regionId = new DataRegionId(33); + final WriterId writerId = new WriterId(regionId.toString(), 7); + final AtomicReference<ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult> + queryResult = + new AtomicReference<>( + ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.absent()); + final ConsensusSubscriptionCommitManager manager = + newCommitManager(systemDir, (consumerGroupId, topicName, ignored) -> queryResult.get()); + manager.initializeStateFromTailProposal( + "cg", + "topic", + regionId, + new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(200L, 2L)))); + final WriterProgress broadcastProgress = new WriterProgress(300L, 3L); + manager.receiveProgressBroadcast( + "cg", "topic", regionId.toString(), writerId, broadcastProgress); + + queryResult.set( + ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available( + new RegionProgress( + Collections.singletonMap(writerId, new WriterProgress(100L, 1L))))); + + assertEquals( + broadcastProgress, + manager + .refreshAuthoritativeProgress("cg", "topic", regionId) + .getRegionProgress() + .getWriterPositions() + .get(writerId)); + } finally { + IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); + } + } + @Test public void testFreshTailProposalUsesAvailableConfigNodeProgressExactly() throws Exception { final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); @@ -500,6 +544,45 @@ public class ConsensusSubscriptionCommitStateTest { } } + @Test + public void testExplicitSeekResetSurvivesManagerRecovery() throws Exception { + final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); + final File systemDir = temporaryFolder.newFolder("seekResetRecovery"); + try { + final DataRegionId regionId = new DataRegionId(34); + final WriterId writerId = new WriterId(regionId.toString(), 7); + final RegionProgress staleConfigNodeProgress = + new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(200L, 2L))); + final RegionProgress seekProgress = + new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(100L, 1L))); + final ConsensusSubscriptionCommitManager manager = + newCommitManager( + systemDir, + (consumerGroupId, topicName, ignored) -> + ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available( + staleConfigNodeProgress)); + manager.getOrCreateState("cg", "topic", regionId); + manager.resetState("cg", "topic", regionId, seekProgress); + + final ConsensusSubscriptionCommitManager recoveredManager = + newCommitManager( + systemDir, + (consumerGroupId, topicName, ignored) -> + ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available( + staleConfigNodeProgress)); + + assertEquals( + seekProgress, + recoveredManager + .refreshAuthoritativeProgress("cg", "topic", regionId) + .getRegionProgress()); + assertEquals( + seekProgress, recoveredManager.getCommittedRegionProgress("cg", "topic", regionId)); + } finally { + IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); + } + } + @Test public void testNewRegionActivationPreservesReplicatedProgressWhenConfigNodeHasNoEntry() throws Exception {
