This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 727b83af260 Fix Pipe leader-change startup ordering (#19318)
727b83af260 is described below

commit 727b83af26099403d973ac01a061438c6aeb16bd
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 19 15:18:00 2026 +0800

    Fix Pipe leader-change startup ordering (#19318)
    
    * Fix Pipe leader-change procedure ordering
    
    * Fix procedure recovery during leader startup
    
    * Fix leader readiness after procedure workers start
    
    * Fix internal procedure recovery readiness
---
 .../statemachine/ConfigRegionStateMachine.java     | 71 +++++++++++++++-------
 .../manager/consensus/ConsensusManager.java        | 10 +--
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
index 53bc04d38c4..d72d5de0f32 100644
--- 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
+++ 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
@@ -67,7 +67,6 @@ import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
 /** {@link IStateMachine} for ConfigRegion. */
@@ -95,10 +94,12 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
   private final ConfigPlanExecutor executor;
 
   /**
-   * Whether the leader services of the {@link #leaderServicesEpoch current 
epoch} have finished
-   * starting up. Read by {@link ConsensusManager#confirmLeader()} to gate 
external serving.
+   * Readiness of leader services in the {@link #leaderServicesEpoch current 
epoch}. Recovered
+   * procedures can run once the Pipe callback has completed, but external 
requests must also wait
+   * for worker startup to finish.
    */
-  private final AtomicBoolean leaderServicesReady;
+  private volatile LeaderServicesReadiness leaderServicesReadiness =
+      LeaderServicesReadiness.WARMING_UP;
 
   /**
    * Monotonically increasing leadership generation. Every become-leader / 
resign-leader transition
@@ -106,9 +107,15 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
    */
   private final AtomicLong leaderServicesEpoch;
 
-  /** Guards {@link #leaderServicesReady} and {@link #leaderServicesEpoch} as 
a unit. */
+  /** Guards {@link #leaderServicesReadiness} and {@link #leaderServicesEpoch} 
as a unit. */
   private final Object leaderServicesLock;
 
+  private enum LeaderServicesReadiness {
+    WARMING_UP,
+    READY_FOR_PROCEDURES,
+    READY
+  }
+
   private ConfigManager configManager;
 
   /** Variables for {@link ConfigNode} Simple Consensus. */
@@ -132,7 +139,6 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
 
   public ConfigRegionStateMachine(ConfigManager configManager, 
ConfigPlanExecutor executor) {
     this.executor = executor;
-    this.leaderServicesReady = new AtomicBoolean(false);
     this.leaderServicesEpoch = new AtomicLong(0);
     this.leaderServicesLock = new Object();
     this.configManager = configManager;
@@ -385,11 +391,23 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
     if (!isCurrentLeaderServicesEpoch(epoch)) {
       return;
     }
-    // The procedure executor may report readiness asynchronously once it has 
caught up.
+    final boolean procedureExecutorWasRunning =
+        configManager.getProcedureManager().getExecutor().isRunning();
     configManager
         .getProcedureManager()
-        .startExecutor(() -> markLeaderServicesReadyIfEpochCurrent(epoch));
-    markLeaderServicesReadyIfEpochCurrent(epoch);
+        .startExecutor(
+            () -> {
+              startPipeOnLeaderChangedIfEpochCurrent(epoch);
+              markLeaderServicesReadyIfEpochCurrent(
+                  epoch, LeaderServicesReadiness.READY_FOR_PROCEDURES);
+            });
+    if (!isCurrentLeaderServicesEpoch(epoch)) {
+      return;
+    }
+    if (procedureExecutorWasRunning) {
+      startPipeOnLeaderChangedIfEpochCurrent(epoch);
+    }
+    markLeaderServicesReadyIfEpochCurrent(epoch, 
LeaderServicesReadiness.READY);
 
     final boolean loadReady = waitForLoadReady(epoch);
     if (!isCurrentLeaderServicesEpoch(epoch)) {
@@ -427,13 +445,6 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
         new LeaderServiceStartup(
             "PipeHeartbeat",
             () -> 
configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeHeartbeat()),
-        new LeaderServiceStartup(
-            "PipeOnLeaderChanged",
-            () ->
-                configManager
-                    .getPipeManager()
-                    .getPipeRuntimeCoordinator()
-                    .onConfigRegionGroupLeaderChanged()),
         new LeaderServiceStartup(
             "SubscriptionMetaSync",
             () ->
@@ -510,10 +521,24 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
         leaderServicesStartupPool);
   }
 
-  private void markLeaderServicesReadyIfEpochCurrent(final long epoch) {
+  private void startPipeOnLeaderChangedIfEpochCurrent(final long epoch) {
+    startInParallelIfEpochCurrent(
+            epoch,
+            new LeaderServiceStartup(
+                "PipeOnLeaderChanged",
+                () ->
+                    configManager
+                        .getPipeManager()
+                        .getPipeRuntimeCoordinator()
+                        .onConfigRegionGroupLeaderChanged()))
+        .join();
+  }
+
+  private void markLeaderServicesReadyIfEpochCurrent(
+      final long epoch, final LeaderServicesReadiness readiness) {
     synchronized (leaderServicesLock) {
       if (isCurrentLeaderServicesEpoch(epoch)) {
-        leaderServicesReady.set(true);
+        leaderServicesReadiness = readiness;
       }
     }
   }
@@ -557,14 +582,16 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
     }
   }
 
-  public boolean areLeaderServicesReady() {
-    return leaderServicesReady.get();
+  public boolean areLeaderServicesReady(final boolean forInternalProcedure) {
+    final LeaderServicesReadiness readiness = leaderServicesReadiness;
+    return readiness == LeaderServicesReadiness.READY
+        || (forInternalProcedure && readiness == 
LeaderServicesReadiness.READY_FOR_PROCEDURES);
   }
 
   /** Open a new leadership generation, invalidating the previous one. */
   private long nextLeaderServicesEpoch() {
     synchronized (leaderServicesLock) {
-      leaderServicesReady.set(false);
+      leaderServicesReadiness = LeaderServicesReadiness.WARMING_UP;
       return leaderServicesEpoch.incrementAndGet();
     }
   }
@@ -572,7 +599,7 @@ public class ConfigRegionStateMachine implements 
IStateMachine, IStateMachine.Ev
   /** Invalidate the current leadership generation without opening a serving 
one. */
   private void invalidateLeaderServices() {
     synchronized (leaderServicesLock) {
-      leaderServicesReady.set(false);
+      leaderServicesReadiness = LeaderServicesReadiness.WARMING_UP;
       leaderServicesEpoch.incrementAndGet();
     }
   }
diff --git 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
index 9ee568a32b4..f3d5585875a 100644
--- 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
+++ 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
@@ -456,10 +456,10 @@ public class ConsensusManager {
    *     NEED_REDIRECTION otherwise
    */
   public TSStatus confirmLeader() {
-    return confirmLeader(true);
+    return confirmLeader(false);
   }
 
-  private TSStatus confirmLeader(final boolean checkLoadReady) {
+  private TSStatus confirmLeader(final boolean forInternalProcedure) {
     if (!isLeader()) {
       TSStatus result = new 
TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode());
       result.setMessage(
@@ -479,11 +479,11 @@ public class ConsensusManager {
           ManagerMessages
               
.MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_NOT_READY_YET_PLEASE_TRY_AGAIN_F0B10645);
     }
-    if (!stateMachine.areLeaderServicesReady()) {
+    if (!stateMachine.areLeaderServicesReady(forInternalProcedure)) {
       return getLeaderWarmingUpStatus(
           ManagerMessages.MESSAGE_CURRENT_CONFIGNODE_LEADER_SERVICE_NOT_READY);
     }
-    if (checkLoadReady && !configManager.getLoadManager().isLoadReady()) {
+    if (!forInternalProcedure && 
!configManager.getLoadManager().isLoadReady()) {
       return 
getLeaderWarmingUpStatus(configManager.getLoadManager().getLoadReadyReason());
     }
 
@@ -491,7 +491,7 @@ public class ConsensusManager {
   }
 
   public TSStatus confirmLeaderForInternalProcedure() {
-    return confirmLeader(false);
+    return confirmLeader(true);
   }
 
   private void waitForLeaderReady() {

Reply via email to