This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 727b83af260 Fix Pipe leader-change startup ordering (#19318)
727b83af260 is described below
commit 727b83af26099403d973ac01a061438c6aeb16bd
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 19 15:18:00 2026 +0800
Fix Pipe leader-change startup ordering (#19318)
* Fix Pipe leader-change procedure ordering
* Fix procedure recovery during leader startup
* Fix leader readiness after procedure workers start
* Fix internal procedure recovery readiness
---
.../statemachine/ConfigRegionStateMachine.java | 71 +++++++++++++++-------
.../manager/consensus/ConsensusManager.java | 10 +--
2 files changed, 54 insertions(+), 27 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
index 53bc04d38c4..d72d5de0f32 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
@@ -67,7 +67,6 @@ import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
/** {@link IStateMachine} for ConfigRegion. */
@@ -95,10 +94,12 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
private final ConfigPlanExecutor executor;
/**
- * Whether the leader services of the {@link #leaderServicesEpoch current
epoch} have finished
- * starting up. Read by {@link ConsensusManager#confirmLeader()} to gate
external serving.
+ * Readiness of leader services in the {@link #leaderServicesEpoch current
epoch}. Recovered
+ * procedures can run once the Pipe callback has completed, but external
requests must also wait
+ * for worker startup to finish.
*/
- private final AtomicBoolean leaderServicesReady;
+ private volatile LeaderServicesReadiness leaderServicesReadiness =
+ LeaderServicesReadiness.WARMING_UP;
/**
* Monotonically increasing leadership generation. Every become-leader /
resign-leader transition
@@ -106,9 +107,15 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
*/
private final AtomicLong leaderServicesEpoch;
- /** Guards {@link #leaderServicesReady} and {@link #leaderServicesEpoch} as
a unit. */
+ /** Guards {@link #leaderServicesReadiness} and {@link #leaderServicesEpoch}
as a unit. */
private final Object leaderServicesLock;
+ private enum LeaderServicesReadiness {
+ WARMING_UP,
+ READY_FOR_PROCEDURES,
+ READY
+ }
+
private ConfigManager configManager;
/** Variables for {@link ConfigNode} Simple Consensus. */
@@ -132,7 +139,6 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
public ConfigRegionStateMachine(ConfigManager configManager,
ConfigPlanExecutor executor) {
this.executor = executor;
- this.leaderServicesReady = new AtomicBoolean(false);
this.leaderServicesEpoch = new AtomicLong(0);
this.leaderServicesLock = new Object();
this.configManager = configManager;
@@ -385,11 +391,23 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
if (!isCurrentLeaderServicesEpoch(epoch)) {
return;
}
- // The procedure executor may report readiness asynchronously once it has
caught up.
+ final boolean procedureExecutorWasRunning =
+ configManager.getProcedureManager().getExecutor().isRunning();
configManager
.getProcedureManager()
- .startExecutor(() -> markLeaderServicesReadyIfEpochCurrent(epoch));
- markLeaderServicesReadyIfEpochCurrent(epoch);
+ .startExecutor(
+ () -> {
+ startPipeOnLeaderChangedIfEpochCurrent(epoch);
+ markLeaderServicesReadyIfEpochCurrent(
+ epoch, LeaderServicesReadiness.READY_FOR_PROCEDURES);
+ });
+ if (!isCurrentLeaderServicesEpoch(epoch)) {
+ return;
+ }
+ if (procedureExecutorWasRunning) {
+ startPipeOnLeaderChangedIfEpochCurrent(epoch);
+ }
+ markLeaderServicesReadyIfEpochCurrent(epoch,
LeaderServicesReadiness.READY);
final boolean loadReady = waitForLoadReady(epoch);
if (!isCurrentLeaderServicesEpoch(epoch)) {
@@ -427,13 +445,6 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
new LeaderServiceStartup(
"PipeHeartbeat",
() ->
configManager.getPipeManager().getPipeRuntimeCoordinator().startPipeHeartbeat()),
- new LeaderServiceStartup(
- "PipeOnLeaderChanged",
- () ->
- configManager
- .getPipeManager()
- .getPipeRuntimeCoordinator()
- .onConfigRegionGroupLeaderChanged()),
new LeaderServiceStartup(
"SubscriptionMetaSync",
() ->
@@ -510,10 +521,24 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
leaderServicesStartupPool);
}
- private void markLeaderServicesReadyIfEpochCurrent(final long epoch) {
+ private void startPipeOnLeaderChangedIfEpochCurrent(final long epoch) {
+ startInParallelIfEpochCurrent(
+ epoch,
+ new LeaderServiceStartup(
+ "PipeOnLeaderChanged",
+ () ->
+ configManager
+ .getPipeManager()
+ .getPipeRuntimeCoordinator()
+ .onConfigRegionGroupLeaderChanged()))
+ .join();
+ }
+
+ private void markLeaderServicesReadyIfEpochCurrent(
+ final long epoch, final LeaderServicesReadiness readiness) {
synchronized (leaderServicesLock) {
if (isCurrentLeaderServicesEpoch(epoch)) {
- leaderServicesReady.set(true);
+ leaderServicesReadiness = readiness;
}
}
}
@@ -557,14 +582,16 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
}
}
- public boolean areLeaderServicesReady() {
- return leaderServicesReady.get();
+ public boolean areLeaderServicesReady(final boolean forInternalProcedure) {
+ final LeaderServicesReadiness readiness = leaderServicesReadiness;
+ return readiness == LeaderServicesReadiness.READY
+ || (forInternalProcedure && readiness ==
LeaderServicesReadiness.READY_FOR_PROCEDURES);
}
/** Open a new leadership generation, invalidating the previous one. */
private long nextLeaderServicesEpoch() {
synchronized (leaderServicesLock) {
- leaderServicesReady.set(false);
+ leaderServicesReadiness = LeaderServicesReadiness.WARMING_UP;
return leaderServicesEpoch.incrementAndGet();
}
}
@@ -572,7 +599,7 @@ public class ConfigRegionStateMachine implements
IStateMachine, IStateMachine.Ev
/** Invalidate the current leadership generation without opening a serving
one. */
private void invalidateLeaderServices() {
synchronized (leaderServicesLock) {
- leaderServicesReady.set(false);
+ leaderServicesReadiness = LeaderServicesReadiness.WARMING_UP;
leaderServicesEpoch.incrementAndGet();
}
}
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
index 9ee568a32b4..f3d5585875a 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java
@@ -456,10 +456,10 @@ public class ConsensusManager {
* NEED_REDIRECTION otherwise
*/
public TSStatus confirmLeader() {
- return confirmLeader(true);
+ return confirmLeader(false);
}
- private TSStatus confirmLeader(final boolean checkLoadReady) {
+ private TSStatus confirmLeader(final boolean forInternalProcedure) {
if (!isLeader()) {
TSStatus result = new
TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode());
result.setMessage(
@@ -479,11 +479,11 @@ public class ConsensusManager {
ManagerMessages
.MESSAGE_CURRENT_CONFIGNODE_LEADER_BUT_NOT_READY_YET_PLEASE_TRY_AGAIN_F0B10645);
}
- if (!stateMachine.areLeaderServicesReady()) {
+ if (!stateMachine.areLeaderServicesReady(forInternalProcedure)) {
return getLeaderWarmingUpStatus(
ManagerMessages.MESSAGE_CURRENT_CONFIGNODE_LEADER_SERVICE_NOT_READY);
}
- if (checkLoadReady && !configManager.getLoadManager().isLoadReady()) {
+ if (!forInternalProcedure &&
!configManager.getLoadManager().isLoadReady()) {
return
getLeaderWarmingUpStatus(configManager.getLoadManager().getLoadReadyReason());
}
@@ -491,7 +491,7 @@ public class ConsensusManager {
}
public TSStatus confirmLeaderForInternalProcedure() {
- return confirmLeader(false);
+ return confirmLeader(true);
}
private void waitForLeaderReady() {