This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/fix-consensus-subscription-replay in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 27ee0a51f2cc281688edf73c986796578bae54e9 Author: Caideyipi <[email protected]> AuthorDate: Fri Nov 6 12:53:34 2026 +0800 Preserve explicit seeks during activation lookup --- .../consensus/ConsensusPrefetchingQueue.java | 55 +++++++++++++++------- .../ConsensusPrefetchingQueueSeekTest.java | 30 ++++++++++-- 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java index 2350c1a8870..c4df0d2b2ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java @@ -4387,27 +4387,32 @@ public class ConsensusPrefetchingQueue { * denotes an empty WAL. If old files have been reclaimed, replay may start at a later index. */ public void seekToBeginning() { - seekToBeginning(true); + seekToBeginning(true, -1L); } - private void seekToBeginning(final boolean explicitSeek) { + private void seekToBeginning(final boolean explicitSeek, final long expectedSeekGeneration) { seekToResolvedPosition( FIRST_CONSENSUS_SEARCH_INDEX, new RegionProgress(Collections.emptyMap()), "beginning", - explicitSeek); + explicitSeek, + expectedSeekGeneration); } /** * Seeks to the current WAL write position. After this, only newly written data will be consumed. */ public void seekToEnd() { - seekToEnd(true); + seekToEnd(true, -1L); } - private void seekToEnd(final boolean explicitSeek) { + private void seekToEnd(final boolean explicitSeek, final long expectedSeekGeneration) { seekToResolvedPosition( - nextSearchIndexAfterCurrent(), computeTailRegionProgress(), "end", explicitSeek); + nextSearchIndexAfterCurrent(), + computeTailRegionProgress(), + "end", + explicitSeek, + expectedSeekGeneration); } public void seekToRegionProgress(final RegionProgress regionProgress) { @@ -4438,7 +4443,8 @@ public class ConsensusPrefetchingQueue { replayTarget.getStartSearchIndex(), replayTarget.getRecoveryRegionProgress(), "regionProgress", - true); + true, + -1L); return; case RETRY_LATER: case LOCATE_MISS: @@ -4454,17 +4460,19 @@ public class ConsensusPrefetchingQueue { } public void seekAfterRegionProgress(final RegionProgress regionProgress) { - seekAfterRegionProgress(regionProgress, true); + seekAfterRegionProgress(regionProgress, true, -1L); } private void seekAfterRegionProgress( - final RegionProgress regionProgress, final boolean explicitSeek) { + final RegionProgress regionProgress, + final boolean explicitSeek, + final long expectedSeekGeneration) { if (!(consensusReqReader instanceof WALNode)) { LOGGER.warn( DataNodePipeMessages .PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_SEEKAFTERREGIONPROGRESS_NOT_SUPPORTED_55F36BE8, this); - seekToEnd(explicitSeek); + seekToEnd(explicitSeek, expectedSeekGeneration); return; } final WALNode walNode = (WALNode) consensusReqReader; @@ -4486,7 +4494,8 @@ public class ConsensusPrefetchingQueue { replayTarget.getStartSearchIndex(), replayTarget.getRecoveryRegionProgress(), "regionProgressAfter", - explicitSeek); + explicitSeek, + expectedSeekGeneration); return; case RETRY_LATER: case LOCATE_MISS: @@ -4505,7 +4514,8 @@ public class ConsensusPrefetchingQueue { final long targetSearchIndex, final RegionProgress committedRegionProgress, final String seekReason, - final boolean explicitSeek) { + final boolean explicitSeek, + final long expectedSeekGeneration) { final PendingSeekRequest request; acquireWriteLock(); @@ -4513,6 +4523,11 @@ public class ConsensusPrefetchingQueue { if (isClosed || closeRequested) { return; } + // A user seek completed after activation captured its frontier. Its cursor and recovery + // progress supersede this older lookup, even if that lookup resolved to a higher checkpoint. + if (!explicitSeek && seekGeneration.get() != expectedSeekGeneration) { + return; + } // Fence old commit contexts immediately. The grouped reset itself is applied later by the // prefetch worker so WAL state and queue state still move under the queue's serial context. final boolean previousPrefetchInitialized = prefetchInitialized; @@ -5889,8 +5904,16 @@ public class ConsensusPrefetchingQueue { if (Objects.isNull(pendingActivationRuntimeState)) { return true; } - final ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult queryResult = - commitManager.refreshAuthoritativeProgress(consumerGroupId, topicName, consensusGroupId); + final long activationSeekGeneration; + final ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult queryResult; + synchronized (this) { + // Wait for an already submitted user seek to finish before capturing the activation + // snapshot. A later user seek is detected again when the resolved lookup is submitted. + activationSeekGeneration = seekGeneration.get(); + queryResult = + commitManager.refreshAuthoritativeProgress( + consumerGroupId, topicName, consensusGroupId); + } if (!queryResult.isAvailable()) { return false; } @@ -5898,9 +5921,9 @@ public class ConsensusPrefetchingQueue { final RegionProgress authoritativeProgress = queryResult.getRegionProgress(); if (prefetchInitialized) { if (authoritativeProgress.getWriterPositions().isEmpty()) { - seekToBeginning(false); + seekToBeginning(false, activationSeekGeneration); } else { - seekAfterRegionProgress(authoritativeProgress, false); + seekAfterRegionProgress(authoritativeProgress, false, activationSeekGeneration); } } pendingActivationRuntimeState = null; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java index 51f7bf1eae2..66da47a2a38 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java @@ -152,8 +152,21 @@ public class ConsensusPrefetchingQueueSeekTest { assertActivationProgress(true, false); } + @Test(timeout = 20_000L) + public void testExplicitSeekDuringActivationOverridesLocatedFrontier() throws Exception { + assertActivationProgress(false, false, true); + } + private void assertActivationProgress( final boolean explicitSeek, final boolean broadcastDuringActivation) throws Exception { + assertActivationProgress(explicitSeek, broadcastDuringActivation, false); + } + + private void assertActivationProgress( + final boolean explicitSeek, + final boolean broadcastDuringActivation, + final boolean seekDuringActivation) + throws Exception { final int originalPrefetchThreads = config.getSubscriptionConsensusPrefetchExecutorMaxThreadNum(); final String originalSystemDir = IoTDBDescriptor.getInstance().getConfig().getSystemDir(); @@ -172,7 +185,7 @@ public class ConsensusPrefetchingQueueSeekTest { final WriterId writerId = new WriterId(regionId.toString(), 7); final RegionProgress emptyProgress = new RegionProgress(Collections.emptyMap()); final RegionProgress initialProgress = - broadcastDuringActivation || explicitSeek + broadcastDuringActivation || explicitSeek || seekDuringActivation ? new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(100L, 1L))) : emptyProgress; final AtomicReference<RegionProgress> configNodeProgress = @@ -232,7 +245,7 @@ public class ConsensusPrefetchingQueueSeekTest { queue.poll("consumer"); queue.setActive(false); // A seek while inactive initializes the cursor, so activation must apply a worker seek. - if (broadcastDuringActivation) { + if (broadcastDuringActivation || seekDuringActivation) { queue.seekAfterRegionProgress(initialProgress); } else { queue.seekToBeginning(); @@ -250,14 +263,23 @@ public class ConsensusPrefetchingQueueSeekTest { writerId, broadcastProgress.getWriterPositions().get(writerId))); } + if (seekDuringActivation) { + // Complete a user seek after activation captured its frontier, before it queues the + // automatic worker reset. The older lookup must not supersede this cursor and progress. + final ConsensusPrefetchingQueue activatedQueue = queue; + duringReplayLookup.set(activatedQueue::seekToBeginning); + } queue.applyRuntimeState( new ConsensusRegionRuntimeState(2L, 7, true, Collections.singleton(7))); assertNull(duringReplayLookup.get()); assertEquals( - explicitSeek + explicitSeek || seekDuringActivation ? emptyProgress : broadcastDuringActivation ? broadcastProgress : initialProgress, commitManager.getCommittedRegionProgress("consumerGroup", "topic", regionId)); + if (seekDuringActivation) { + assertEquals(1L, queue.getCurrentReadSearchIndex()); + } final RegionProgress newerConfigNodeProgress = new RegionProgress(Collections.singletonMap(writerId, new WriterProgress(300L, 3L))); @@ -266,7 +288,7 @@ public class ConsensusPrefetchingQueueSeekTest { queue.applyRuntimeState( new ConsensusRegionRuntimeState(3L, 7, true, Collections.singleton(7))); assertEquals( - explicitSeek ? emptyProgress : newerConfigNodeProgress, + explicitSeek || seekDuringActivation ? emptyProgress : newerConfigNodeProgress, commitManager.getCommittedRegionProgress("consumerGroup", "topic", regionId)); } finally { if (queue != null) {
