This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/fix-consensus-subscription-replay in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 51aa076fb5c0627da04f002103667c1d9a46bc46 Author: Caideyipi <[email protected]> AuthorDate: Fri Nov 6 12:08:57 2026 +0800 Preserve subscription progress during activation seeks --- .../consensus/ConsensusPrefetchingQueue.java | 61 ++++++--- .../ConsensusSubscriptionCommitManager.java | 27 +++- .../ConsensusPrefetchingQueueSeekTest.java | 147 +++++++++++++++++++++ 3 files changed, 214 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index 765202b74f2..2350c1a8870 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -602,6 +602,7 @@ public class ConsensusPrefetchingQueue { private final long targetSearchIndex; private final RegionProgress committedRegionProgress; private final String seekReason; + private final boolean explicitSeek; private final boolean previousPrefetchInitialized; private final long previousSeekGeneration; private final long targetSeekGeneration; @@ -613,12 +614,14 @@ public class ConsensusPrefetchingQueue { final long targetSearchIndex, final RegionProgress committedRegionProgress, final String seekReason, + final boolean explicitSeek, final boolean previousPrefetchInitialized, final long previousSeekGeneration, final long targetSeekGeneration) { this.targetSearchIndex = targetSearchIndex; this.committedRegionProgress = committedRegionProgress; this.seekReason = seekReason; + this.explicitSeek = explicitSeek; this.previousPrefetchInitialized = previousPrefetchInitialized; this.previousSeekGeneration = previousSeekGeneration; this.targetSeekGeneration = targetSeekGeneration; @@ -4384,15 +4387,27 @@ public class ConsensusPrefetchingQueue { * denotes an empty WAL. If old files have been reclaimed, replay may start at a later index. */ public void seekToBeginning() { + seekToBeginning(true); + } + + private void seekToBeginning(final boolean explicitSeek) { seekToResolvedPosition( - FIRST_CONSENSUS_SEARCH_INDEX, new RegionProgress(Collections.emptyMap()), "beginning"); + FIRST_CONSENSUS_SEARCH_INDEX, + new RegionProgress(Collections.emptyMap()), + "beginning", + explicitSeek); } /** * Seeks to the current WAL write position. After this, only newly written data will be consumed. */ public void seekToEnd() { - seekToResolvedPosition(nextSearchIndexAfterCurrent(), computeTailRegionProgress(), "end"); + seekToEnd(true); + } + + private void seekToEnd(final boolean explicitSeek) { + seekToResolvedPosition( + nextSearchIndexAfterCurrent(), computeTailRegionProgress(), "end", explicitSeek); } public void seekToRegionProgress(final RegionProgress regionProgress) { @@ -4422,7 +4437,8 @@ public class ConsensusPrefetchingQueue { seekToResolvedPosition( replayTarget.getStartSearchIndex(), replayTarget.getRecoveryRegionProgress(), - "regionProgress"); + "regionProgress", + true); return; case RETRY_LATER: case LOCATE_MISS: @@ -4438,12 +4454,17 @@ public class ConsensusPrefetchingQueue { } public void seekAfterRegionProgress(final RegionProgress regionProgress) { + seekAfterRegionProgress(regionProgress, true); + } + + private void seekAfterRegionProgress( + final RegionProgress regionProgress, final boolean explicitSeek) { if (!(consensusReqReader instanceof WALNode)) { LOGGER.warn( DataNodePipeMessages .PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_SEEKAFTERREGIONPROGRESS_NOT_SUPPORTED_55F36BE8, this); - seekToEnd(); + seekToEnd(explicitSeek); return; } final WALNode walNode = (WALNode) consensusReqReader; @@ -4464,7 +4485,8 @@ public class ConsensusPrefetchingQueue { seekToResolvedPosition( replayTarget.getStartSearchIndex(), replayTarget.getRecoveryRegionProgress(), - "regionProgressAfter"); + "regionProgressAfter", + explicitSeek); return; case RETRY_LATER: case LOCATE_MISS: @@ -4482,7 +4504,8 @@ public class ConsensusPrefetchingQueue { private synchronized void seekToResolvedPosition( final long targetSearchIndex, final RegionProgress committedRegionProgress, - final String seekReason) { + final String seekReason, + final boolean explicitSeek) { final PendingSeekRequest request; acquireWriteLock(); @@ -4500,6 +4523,7 @@ public class ConsensusPrefetchingQueue { targetSearchIndex, committedRegionProgress, seekReason, + explicitSeek, previousPrefetchInitialized, previousSeekGeneration, targetSeekGeneration); @@ -4650,9 +4674,15 @@ public class ConsensusPrefetchingQueue { writerStates.clear(); clearRecoveryWriterProgress(); materializedProgressByWriter.clear(); - if (Objects.nonNull(request.committedRegionProgress) - && !request.committedRegionProgress.getWriterPositions().isEmpty()) { - installRecoveryWriterProgress(request.committedRegionProgress); + final RegionProgress committedRegionProgress = + commitManager.resetState( + consumerGroupId, + topicName, + consensusGroupId, + request.committedRegionProgress, + request.explicitSeek); + if (!committedRegionProgress.getWriterPositions().isEmpty()) { + installRecoveryWriterProgress(committedRegionProgress); } // 4. Reset WAL read position. The target has been normalized by replay lookup: if the @@ -4668,10 +4698,7 @@ public class ConsensusPrefetchingQueue { walGapWaitStartTimeMs = 0L; lastWalGapWaitLogTimeMs = 0L; - // 5. Reset commit state to the writer progress immediately before the first re-delivered - // entry so seek/rebind resumes from the intended frontier. - commitManager.resetState( - consumerGroupId, topicName, consensusGroupId, request.committedRegionProgress); + // 5. Refresh WAL retention from the installed committed frontier. refreshCommittedWalRetentionBoundAndNotify(); LOGGER.info( @@ -4680,9 +4707,7 @@ public class ConsensusPrefetchingQueue { this, request.seekReason, request.targetSearchIndex, - Objects.nonNull(request.committedRegionProgress) - ? request.committedRegionProgress.getWriterPositions().size() - : 0, + committedRegionProgress.getWriterPositions().size(), seekGeneration.get()); } @@ -5873,9 +5898,9 @@ public class ConsensusPrefetchingQueue { final RegionProgress authoritativeProgress = queryResult.getRegionProgress(); if (prefetchInitialized) { if (authoritativeProgress.getWriterPositions().isEmpty()) { - seekToBeginning(); + seekToBeginning(false); } else { - seekAfterRegionProgress(authoritativeProgress); + seekAfterRegionProgress(authoritativeProgress, false); } } pendingActivationRuntimeState = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java index 9e926c0b490..29a4fad25d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java @@ -837,6 +837,15 @@ public class ConsensusSubscriptionCommitManager { final String topicName, final ConsensusGroupId regionId, final RegionProgress regionProgress) { + resetState(consumerGroupId, topicName, regionId, regionProgress, true); + } + + RegionProgress resetState( + final String consumerGroupId, + final String topicName, + final ConsensusGroupId regionId, + final RegionProgress regionProgress, + final boolean explicitSeek) { final CommitStateKey stateKey = getCommitStateKey(consumerGroupId, topicName, regionId.toString()); synchronized (getTopicPersistLock(stateKey.topicFileKey)) { @@ -848,11 +857,23 @@ public class ConsensusSubscriptionCommitManager { consumerGroupId, topicName, regionId); - return; + return copyRegionProgress(regionProgress); + } + final RegionProgress resetProgress; + synchronized (state) { + // Activation applies its seek asynchronously, after the authoritative refresh. Preserve + // committed progress received since that refresh; only a user seek may rewind it. + resetProgress = + explicitSeek + ? copyRegionProgress(regionProgress) + : mergeRegionProgress(state.getCommittedRegionProgress(), regionProgress); + state.resetForSeek(resetProgress); + if (explicitSeek) { + explicitSeekResetKeys.add(stateKey.encodedStateKey); + } } - state.resetForSeek(regionProgress); - explicitSeekResetKeys.add(stateKey.encodedStateKey); persistRegionProgress(stateKey, state); + return resetProgress; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java index d113e172ca3..51f7bf1eae2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java @@ -40,6 +40,8 @@ import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType; import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload; +import org.apache.iotdb.rpc.subscription.payload.poll.WriterId; +import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; @@ -62,8 +64,10 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; @@ -133,6 +137,149 @@ public class ConsensusPrefetchingQueueSeekTest { Arrays.asList(createRequest(1L, 1L, 7), createRequest(-1L, 10L, 8)), 2L, 0L, true); } + @Test(timeout = 20_000L) + public void testResolvedSeekDoesNotMaskNewerConfigNodeProgress() throws Exception { + assertActivationProgress(false, false); + } + + @Test(timeout = 20_000L) + public void testActivationSeekPreservesConcurrentProgressBroadcast() throws Exception { + assertActivationProgress(false, true); + } + + @Test(timeout = 20_000L) + public void testExplicitSeekSurvivesActivationQueueReset() throws Exception { + assertActivationProgress(true, false); + } + + private void assertActivationProgress( + final boolean explicitSeek, final boolean broadcastDuringActivation) throws Exception { + final int originalPrefetchThreads = + config.getSubscriptionConsensusPrefetchExecutorMaxThreadNum(); + final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); + final File systemDir = temporaryFolder.newFolder(); + ConsensusSubscriptionPrefetchExecutor executor = null; + ConsensusPrefetchingQueue queue = null; + try { + config.setSubscriptionConsensusPrefetchExecutorMaxThreadNum(1); + IoTDBDescriptor.getInstance().getConfig().setSystemDir(systemDir.getAbsolutePath()); + executor = new ConsensusSubscriptionPrefetchExecutor(); + when(manager.getExecutor()).thenReturn(executor); + when(walNode.getCurrentSearchIndex()).thenReturn(10L); + when(walNode.getLogDirectory()).thenReturn(systemDir); + when(server.getWriterSafeFrontierTracker()).thenReturn(new WriterSafeFrontierTracker()); + final DataRegionId regionId = new DataRegionId(1); + final WriterId writerId = new WriterId(regionId.toString(), 7); + final RegionProgress emptyProgress = new RegionProgress(Collections.emptyMap()); + final RegionProgress initialProgress = + broadcastDuringActivation || explicitSeek + ? new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(100L, 1L))) + : emptyProgress; + final AtomicReference<RegionProgress> configNodeProgress = + new AtomicReference<>(initialProgress); + final ConsensusSubscriptionCommitManager commitManager = + new ConsensusSubscriptionCommitManager( + (consumerGroupId, topicName, ignored) -> + ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available( + configNodeProgress.get())); + commitManager.getOrCreateState("consumerGroup", "topic", regionId); + final AtomicReference<Runnable> duringReplayLookup = new AtomicReference<>(); + queue = + new ConsensusPrefetchingQueue( + "consumerGroup", + "topic", + TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE, + regionId, + server, + new SubscriptionWalRetentionPolicy( + "topic", + SubscriptionWalRetentionPolicy.UNBOUNDED, + SubscriptionWalRetentionPolicy.UNBOUNDED), + converter, + commitManager, + initialProgress, + 1L, + 1L, + true) { + @Override + protected ReplayLocateDecision locateReplayStartForRegionProgress( + final RegionProgress progress, final boolean seekAfter) { + final Runnable callback = duringReplayLookup.getAndSet(null); + if (callback != null) { + callback.run(); + } + final long startSearchIndex = + progress.getWriterPositions().values().stream() + .mapToLong(WriterProgress::getLocalSeq) + .max() + .orElse(0L) + + 1L; + return ReplayLocateDecision.found(startSearchIndex, progress, "activation replay"); + } + + @Override + protected ProgressWALIterator createSubscriptionWALIterator( + final long startSearchIndex) { + return new ProgressWALIterator(systemDir, startSearchIndex) { + @Override + boolean hasNext(final ScanBudget scanBudget) { + return false; + } + }; + } + }; + queue.setSubscriptionMemoryManager(new SubscriptionMemoryManager(16L * 1024 * 1024)); + queue.poll("consumer"); + queue.setActive(false); + // A seek while inactive initializes the cursor, so activation must apply a worker seek. + if (broadcastDuringActivation) { + queue.seekAfterRegionProgress(initialProgress); + } else { + queue.seekToBeginning(); + } + final RegionProgress broadcastProgress = + new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(200L, 2L))); + if (broadcastDuringActivation) { + // Advance after the manager's activation refresh, before the worker applies the seek. + duringReplayLookup.set( + () -> + commitManager.receiveProgressBroadcast( + "consumerGroup", + "topic", + regionId.toString(), + writerId, + broadcastProgress.getWriterPositions().get(writerId))); + } + queue.applyRuntimeState( + new ConsensusRegionRuntimeState(2L, 7, true, Collections.singleton(7))); + assertNull(duringReplayLookup.get()); + assertEquals( + explicitSeek + ? emptyProgress + : broadcastDuringActivation ? broadcastProgress : initialProgress, + commitManager.getCommittedRegionProgress("consumerGroup", "topic", regionId)); + + final RegionProgress newerConfigNodeProgress = + new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(300L, 3L))); + configNodeProgress.set(newerConfigNodeProgress); + queue.setActive(false); + queue.applyRuntimeState( + new ConsensusRegionRuntimeState(3L, 7, true, Collections.singleton(7))); + assertEquals( + explicitSeek ? emptyProgress : newerConfigNodeProgress, + commitManager.getCommittedRegionProgress("consumerGroup", "topic", regionId)); + } finally { + if (queue != null) { + queue.close(); + } + if (executor != null) { + executor.shutdown(); + } + config.setSubscriptionConsensusPrefetchExecutorMaxThreadNum(originalPrefetchThreads); + IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir); + } + } + private void assertBeginningReplay( final List<IndexedConsensusRequest> requests, final long expectedNextIndex,
