This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch codex/fix-consensus-subscription-replay
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 51aa076fb5c0627da04f002103667c1d9a46bc46
Author: Caideyipi <[email protected]>
AuthorDate: Fri Nov 6 12:08:57 2026 +0800

    Preserve subscription progress during activation seeks
---
 .../consensus/ConsensusPrefetchingQueue.java       |  61 ++++++---
 .../ConsensusSubscriptionCommitManager.java        |  27 +++-
 .../ConsensusPrefetchingQueueSeekTest.java         | 147 +++++++++++++++++++++
 3 files changed, 214 insertions(+), 21 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
index 765202b74f2..2350c1a8870 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java
@@ -602,6 +602,7 @@ public class ConsensusPrefetchingQueue {
     private final long targetSearchIndex;
     private final RegionProgress committedRegionProgress;
     private final String seekReason;
+    private final boolean explicitSeek;
     private final boolean previousPrefetchInitialized;
     private final long previousSeekGeneration;
     private final long targetSeekGeneration;
@@ -613,12 +614,14 @@ public class ConsensusPrefetchingQueue {
         final long targetSearchIndex,
         final RegionProgress committedRegionProgress,
         final String seekReason,
+        final boolean explicitSeek,
         final boolean previousPrefetchInitialized,
         final long previousSeekGeneration,
         final long targetSeekGeneration) {
       this.targetSearchIndex = targetSearchIndex;
       this.committedRegionProgress = committedRegionProgress;
       this.seekReason = seekReason;
+      this.explicitSeek = explicitSeek;
       this.previousPrefetchInitialized = previousPrefetchInitialized;
       this.previousSeekGeneration = previousSeekGeneration;
       this.targetSeekGeneration = targetSeekGeneration;
@@ -4384,15 +4387,27 @@ public class ConsensusPrefetchingQueue {
    * denotes an empty WAL. If old files have been reclaimed, replay may start 
at a later index.
    */
   public void seekToBeginning() {
+    seekToBeginning(true);
+  }
+
+  private void seekToBeginning(final boolean explicitSeek) {
     seekToResolvedPosition(
-        FIRST_CONSENSUS_SEARCH_INDEX, new 
RegionProgress(Collections.emptyMap()), "beginning");
+        FIRST_CONSENSUS_SEARCH_INDEX,
+        new RegionProgress(Collections.emptyMap()),
+        "beginning",
+        explicitSeek);
   }
 
   /**
    * Seeks to the current WAL write position. After this, only newly written 
data will be consumed.
    */
   public void seekToEnd() {
-    seekToResolvedPosition(nextSearchIndexAfterCurrent(), 
computeTailRegionProgress(), "end");
+    seekToEnd(true);
+  }
+
+  private void seekToEnd(final boolean explicitSeek) {
+    seekToResolvedPosition(
+        nextSearchIndexAfterCurrent(), computeTailRegionProgress(), "end", 
explicitSeek);
   }
 
   public void seekToRegionProgress(final RegionProgress regionProgress) {
@@ -4422,7 +4437,8 @@ public class ConsensusPrefetchingQueue {
         seekToResolvedPosition(
             replayTarget.getStartSearchIndex(),
             replayTarget.getRecoveryRegionProgress(),
-            "regionProgress");
+            "regionProgress",
+            true);
         return;
       case RETRY_LATER:
       case LOCATE_MISS:
@@ -4438,12 +4454,17 @@ public class ConsensusPrefetchingQueue {
   }
 
   public void seekAfterRegionProgress(final RegionProgress regionProgress) {
+    seekAfterRegionProgress(regionProgress, true);
+  }
+
+  private void seekAfterRegionProgress(
+      final RegionProgress regionProgress, final boolean explicitSeek) {
     if (!(consensusReqReader instanceof WALNode)) {
       LOGGER.warn(
           DataNodePipeMessages
               
.PIPE_LOG_CONSENSUSPREFETCHINGQUEUE_SEEKAFTERREGIONPROGRESS_NOT_SUPPORTED_55F36BE8,
           this);
-      seekToEnd();
+      seekToEnd(explicitSeek);
       return;
     }
     final WALNode walNode = (WALNode) consensusReqReader;
@@ -4464,7 +4485,8 @@ public class ConsensusPrefetchingQueue {
         seekToResolvedPosition(
             replayTarget.getStartSearchIndex(),
             replayTarget.getRecoveryRegionProgress(),
-            "regionProgressAfter");
+            "regionProgressAfter",
+            explicitSeek);
         return;
       case RETRY_LATER:
       case LOCATE_MISS:
@@ -4482,7 +4504,8 @@ public class ConsensusPrefetchingQueue {
   private synchronized void seekToResolvedPosition(
       final long targetSearchIndex,
       final RegionProgress committedRegionProgress,
-      final String seekReason) {
+      final String seekReason,
+      final boolean explicitSeek) {
     final PendingSeekRequest request;
 
     acquireWriteLock();
@@ -4500,6 +4523,7 @@ public class ConsensusPrefetchingQueue {
               targetSearchIndex,
               committedRegionProgress,
               seekReason,
+              explicitSeek,
               previousPrefetchInitialized,
               previousSeekGeneration,
               targetSeekGeneration);
@@ -4650,9 +4674,15 @@ public class ConsensusPrefetchingQueue {
     writerStates.clear();
     clearRecoveryWriterProgress();
     materializedProgressByWriter.clear();
-    if (Objects.nonNull(request.committedRegionProgress)
-        && !request.committedRegionProgress.getWriterPositions().isEmpty()) {
-      installRecoveryWriterProgress(request.committedRegionProgress);
+    final RegionProgress committedRegionProgress =
+        commitManager.resetState(
+            consumerGroupId,
+            topicName,
+            consensusGroupId,
+            request.committedRegionProgress,
+            request.explicitSeek);
+    if (!committedRegionProgress.getWriterPositions().isEmpty()) {
+      installRecoveryWriterProgress(committedRegionProgress);
     }
 
     // 4. Reset WAL read position. The target has been normalized by replay 
lookup: if the
@@ -4668,10 +4698,7 @@ public class ConsensusPrefetchingQueue {
     walGapWaitStartTimeMs = 0L;
     lastWalGapWaitLogTimeMs = 0L;
 
-    // 5. Reset commit state to the writer progress immediately before the 
first re-delivered
-    // entry so seek/rebind resumes from the intended frontier.
-    commitManager.resetState(
-        consumerGroupId, topicName, consensusGroupId, 
request.committedRegionProgress);
+    // 5. Refresh WAL retention from the installed committed frontier.
     refreshCommittedWalRetentionBoundAndNotify();
 
     LOGGER.info(
@@ -4680,9 +4707,7 @@ public class ConsensusPrefetchingQueue {
         this,
         request.seekReason,
         request.targetSearchIndex,
-        Objects.nonNull(request.committedRegionProgress)
-            ? request.committedRegionProgress.getWriterPositions().size()
-            : 0,
+        committedRegionProgress.getWriterPositions().size(),
         seekGeneration.get());
   }
 
@@ -5873,9 +5898,9 @@ public class ConsensusPrefetchingQueue {
       final RegionProgress authoritativeProgress = 
queryResult.getRegionProgress();
       if (prefetchInitialized) {
         if (authoritativeProgress.getWriterPositions().isEmpty()) {
-          seekToBeginning();
+          seekToBeginning(false);
         } else {
-          seekAfterRegionProgress(authoritativeProgress);
+          seekAfterRegionProgress(authoritativeProgress, false);
         }
       }
       pendingActivationRuntimeState = null;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java
index 9e926c0b490..29a4fad25d4 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusSubscriptionCommitManager.java
@@ -837,6 +837,15 @@ public class ConsensusSubscriptionCommitManager {
       final String topicName,
       final ConsensusGroupId regionId,
       final RegionProgress regionProgress) {
+    resetState(consumerGroupId, topicName, regionId, regionProgress, true);
+  }
+
+  RegionProgress resetState(
+      final String consumerGroupId,
+      final String topicName,
+      final ConsensusGroupId regionId,
+      final RegionProgress regionProgress,
+      final boolean explicitSeek) {
     final CommitStateKey stateKey =
         getCommitStateKey(consumerGroupId, topicName, regionId.toString());
     synchronized (getTopicPersistLock(stateKey.topicFileKey)) {
@@ -848,11 +857,23 @@ public class ConsensusSubscriptionCommitManager {
             consumerGroupId,
             topicName,
             regionId);
-        return;
+        return copyRegionProgress(regionProgress);
+      }
+      final RegionProgress resetProgress;
+      synchronized (state) {
+        // Activation applies its seek asynchronously, after the authoritative 
refresh. Preserve
+        // committed progress received since that refresh; only a user seek 
may rewind it.
+        resetProgress =
+            explicitSeek
+                ? copyRegionProgress(regionProgress)
+                : mergeRegionProgress(state.getCommittedRegionProgress(), 
regionProgress);
+        state.resetForSeek(resetProgress);
+        if (explicitSeek) {
+          explicitSeekResetKeys.add(stateKey.encodedStateKey);
+        }
       }
-      state.resetForSeek(regionProgress);
-      explicitSeekResetKeys.add(stateKey.encodedStateKey);
       persistRegionProgress(stateKey, state);
+      return resetProgress;
     }
   }
 
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java
index d113e172ca3..51f7bf1eae2 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueueSeekTest.java
@@ -40,6 +40,8 @@ import org.apache.iotdb.rpc.subscription.config.TopicConstant;
 import org.apache.iotdb.rpc.subscription.payload.poll.RegionProgress;
 import 
org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType;
 import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterId;
+import org.apache.iotdb.rpc.subscription.payload.poll.WriterProgress;
 
 import org.apache.tsfile.enums.ColumnCategory;
 import org.apache.tsfile.enums.TSDataType;
@@ -62,8 +64,10 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -133,6 +137,149 @@ public class ConsensusPrefetchingQueueSeekTest {
         Arrays.asList(createRequest(1L, 1L, 7), createRequest(-1L, 10L, 8)), 
2L, 0L, true);
   }
 
+  @Test(timeout = 20_000L)
+  public void testResolvedSeekDoesNotMaskNewerConfigNodeProgress() throws 
Exception {
+    assertActivationProgress(false, false);
+  }
+
+  @Test(timeout = 20_000L)
+  public void testActivationSeekPreservesConcurrentProgressBroadcast() throws 
Exception {
+    assertActivationProgress(false, true);
+  }
+
+  @Test(timeout = 20_000L)
+  public void testExplicitSeekSurvivesActivationQueueReset() throws Exception {
+    assertActivationProgress(true, false);
+  }
+
+  private void assertActivationProgress(
+      final boolean explicitSeek, final boolean broadcastDuringActivation) 
throws Exception {
+    final int originalPrefetchThreads =
+        config.getSubscriptionConsensusPrefetchExecutorMaxThreadNum();
+    final String originalSystemDir = 
IoTDBDescriptor.getInstance().getConfig().getSystemDir();
+    final File systemDir = temporaryFolder.newFolder();
+    ConsensusSubscriptionPrefetchExecutor executor = null;
+    ConsensusPrefetchingQueue queue = null;
+    try {
+      config.setSubscriptionConsensusPrefetchExecutorMaxThreadNum(1);
+      
IoTDBDescriptor.getInstance().getConfig().setSystemDir(systemDir.getAbsolutePath());
+      executor = new ConsensusSubscriptionPrefetchExecutor();
+      when(manager.getExecutor()).thenReturn(executor);
+      when(walNode.getCurrentSearchIndex()).thenReturn(10L);
+      when(walNode.getLogDirectory()).thenReturn(systemDir);
+      when(server.getWriterSafeFrontierTracker()).thenReturn(new 
WriterSafeFrontierTracker());
+      final DataRegionId regionId = new DataRegionId(1);
+      final WriterId writerId = new WriterId(regionId.toString(), 7);
+      final RegionProgress emptyProgress = new 
RegionProgress(Collections.emptyMap());
+      final RegionProgress initialProgress =
+          broadcastDuringActivation || explicitSeek
+              ? new RegionProgress(Collections.singletonMap(writerId, new 
WriterProgress(100L, 1L)))
+              : emptyProgress;
+      final AtomicReference<RegionProgress> configNodeProgress =
+          new AtomicReference<>(initialProgress);
+      final ConsensusSubscriptionCommitManager commitManager =
+          new ConsensusSubscriptionCommitManager(
+              (consumerGroupId, topicName, ignored) ->
+                  
ConsensusSubscriptionCommitManager.ConfigNodeProgressQueryResult.available(
+                      configNodeProgress.get()));
+      commitManager.getOrCreateState("consumerGroup", "topic", regionId);
+      final AtomicReference<Runnable> duringReplayLookup = new 
AtomicReference<>();
+      queue =
+          new ConsensusPrefetchingQueue(
+              "consumerGroup",
+              "topic",
+              TopicConstant.ORDER_MODE_LEADER_ONLY_VALUE,
+              regionId,
+              server,
+              new SubscriptionWalRetentionPolicy(
+                  "topic",
+                  SubscriptionWalRetentionPolicy.UNBOUNDED,
+                  SubscriptionWalRetentionPolicy.UNBOUNDED),
+              converter,
+              commitManager,
+              initialProgress,
+              1L,
+              1L,
+              true) {
+            @Override
+            protected ReplayLocateDecision locateReplayStartForRegionProgress(
+                final RegionProgress progress, final boolean seekAfter) {
+              final Runnable callback = duringReplayLookup.getAndSet(null);
+              if (callback != null) {
+                callback.run();
+              }
+              final long startSearchIndex =
+                  progress.getWriterPositions().values().stream()
+                          .mapToLong(WriterProgress::getLocalSeq)
+                          .max()
+                          .orElse(0L)
+                      + 1L;
+              return ReplayLocateDecision.found(startSearchIndex, progress, 
"activation replay");
+            }
+
+            @Override
+            protected ProgressWALIterator createSubscriptionWALIterator(
+                final long startSearchIndex) {
+              return new ProgressWALIterator(systemDir, startSearchIndex) {
+                @Override
+                boolean hasNext(final ScanBudget scanBudget) {
+                  return false;
+                }
+              };
+            }
+          };
+      queue.setSubscriptionMemoryManager(new SubscriptionMemoryManager(16L * 
1024 * 1024));
+      queue.poll("consumer");
+      queue.setActive(false);
+      // A seek while inactive initializes the cursor, so activation must 
apply a worker seek.
+      if (broadcastDuringActivation) {
+        queue.seekAfterRegionProgress(initialProgress);
+      } else {
+        queue.seekToBeginning();
+      }
+      final RegionProgress broadcastProgress =
+          new RegionProgress(Collections.singletonMap(writerId, new 
WriterProgress(200L, 2L)));
+      if (broadcastDuringActivation) {
+        // Advance after the manager's activation refresh, before the worker 
applies the seek.
+        duringReplayLookup.set(
+            () ->
+                commitManager.receiveProgressBroadcast(
+                    "consumerGroup",
+                    "topic",
+                    regionId.toString(),
+                    writerId,
+                    broadcastProgress.getWriterPositions().get(writerId)));
+      }
+      queue.applyRuntimeState(
+          new ConsensusRegionRuntimeState(2L, 7, true, 
Collections.singleton(7)));
+      assertNull(duringReplayLookup.get());
+      assertEquals(
+          explicitSeek
+              ? emptyProgress
+              : broadcastDuringActivation ? broadcastProgress : 
initialProgress,
+          commitManager.getCommittedRegionProgress("consumerGroup", "topic", 
regionId));
+
+      final RegionProgress newerConfigNodeProgress =
+          new RegionProgress(Collections.singletonMap(writerId, new 
WriterProgress(300L, 3L)));
+      configNodeProgress.set(newerConfigNodeProgress);
+      queue.setActive(false);
+      queue.applyRuntimeState(
+          new ConsensusRegionRuntimeState(3L, 7, true, 
Collections.singleton(7)));
+      assertEquals(
+          explicitSeek ? emptyProgress : newerConfigNodeProgress,
+          commitManager.getCommittedRegionProgress("consumerGroup", "topic", 
regionId));
+    } finally {
+      if (queue != null) {
+        queue.close();
+      }
+      if (executor != null) {
+        executor.shutdown();
+      }
+      
config.setSubscriptionConsensusPrefetchExecutorMaxThreadNum(originalPrefetchThreads);
+      
IoTDBDescriptor.getInstance().getConfig().setSystemDir(originalSystemDir);
+    }
+  }
+
   private void assertBeginningReplay(
       final List<IndexedConsensusRequest> requests,
       final long expectedNextIndex,

Reply via email to