ahuang98 commented on code in PR #23168:
URL: https://github.com/apache/kafka/pull/23168#discussion_r4274605836
##########
raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java:
##########
@@ -195,6 +197,83 @@ void canElectNewLeaderAfterOldLeaderFailure() {
});
}
+ @ParameterizedTest
+ @ValueSource(ints = {4, 5, 6, 7, 8, 9})
+ void preferredSuccessorWinsElectionAfterGracefulLeaderShutdown(int
numVoters) {
+ Cluster cluster = new Cluster(numVoters, 0, 1234L);
+ /* The other simulations lower this to keep their runtime down, but
here the whole point is
+ * that the nominated successors stand for election one at a time, so
use the production
+ * default to keep the gap between them wide enough to be meaningful.
+ */
+ cluster.electionBackoffMaxMs =
QuorumConfig.DEFAULT_QUORUM_ELECTION_BACKOFF_MAX_MS;
+ MessageRouter router = new MessageRouter(cluster);
+ EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
+
+ cluster.startAll();
+ schedulePolling(scheduler, cluster, 3, 5);
+ scheduler.schedule(router::deliverAll, 0, 2, 1);
+ scheduler.runUntil(cluster::hasConsistentLeader);
+ // Every replica has fetched from the leader and is caught up with it
+ scheduler.runUntil(() -> cluster.allReachedHighWatermark(1));
+
+ int leaderId = cluster.latestLeader().orElseThrow(() ->
+ new AssertionError("Failed to find current leader")
+ );
+
+ // Read the ranking that the leader will put into its EndQuorumEpoch
requests
+ LeaderState<Integer> leaderState = cluster
+ .nodeIfRunning(leaderId)
+ .orElseThrow(() -> new AssertionError("Leader " + leaderId + " is
not running"))
+ .client
+ .quorum()
+ .leaderStateOrThrow();
+ List<Integer> preferredSuccessors = leaderState
+ .nonLeaderVotersByDescendingFetchOffset()
+ .stream()
+ .map(ReplicaKey::id)
+ .collect(Collectors.toList());
+
+ // A clean shutdown, so the leader resigns and nominates the
successors above
+ long resignedAtMs = cluster.time.milliseconds();
+ cluster.shutdown(leaderId);
+
+ scheduler.runUntil(() ->
+ cluster.latestLeader().isPresent() &&
cluster.latestLeader().getAsInt() != leaderId
+ );
+
+ int newLeaderId = cluster.latestLeader().getAsInt();
+ long electionDurationMs = cluster.time.milliseconds() - resignedAtMs;
+ int position = preferredSuccessors.indexOf(newLeaderId);
+ int backoffBaseMs = cluster.electionBackoffMaxMs >>
(preferredSuccessors.size() - 1);
+ long backoffOfWinnerMs = position <= 0 ? 0 : (long) backoffBaseMs <<
(position - 1);
+
+ System.out.printf(
+ "%s voters: leader %s resigned nominating %s. Node %s won at
position %s, whose " +
+ "election backoff is %s ms, and the election took %s ms%n",
+ numVoters,
+ leaderId,
+ preferredSuccessors,
+ newLeaderId,
+ position,
+ backoffOfWinnerMs,
+ electionDurationMs
+ );
Review Comment:
do we need this? seems to be covered by the assert below
##########
raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java:
##########
@@ -590,6 +669,7 @@ private static class Cluster {
final Map<Integer, Node> voters = new HashMap<>();
final Map<Integer, PersistentState> nodes = new HashMap<>();
final Map<Integer, RaftNode> running = new HashMap<>();
+ int electionBackoffMaxMs = ELECTION_JITTER_MS;
Review Comment:
do we need this change?
##########
raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java:
##########
@@ -195,6 +197,83 @@ void canElectNewLeaderAfterOldLeaderFailure() {
});
}
+ @ParameterizedTest
+ @ValueSource(ints = {4, 5, 6, 7, 8, 9})
Review Comment:
why not use `withVotersObservers`?
##########
raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java:
##########
@@ -195,6 +197,83 @@ void canElectNewLeaderAfterOldLeaderFailure() {
});
}
+ @ParameterizedTest
+ @ValueSource(ints = {4, 5, 6, 7, 8, 9})
+ void preferredSuccessorWinsElectionAfterGracefulLeaderShutdown(int
numVoters) {
+ Cluster cluster = new Cluster(numVoters, 0, 1234L);
+ /* The other simulations lower this to keep their runtime down, but
here the whole point is
+ * that the nominated successors stand for election one at a time, so
use the production
+ * default to keep the gap between them wide enough to be meaningful.
+ */
Review Comment:
nit: can we reword this slightly? I would prefer not using 'stand for
election one at a time' as it gives the impression of strict 'one after the
other'. maybe something like:
```
Keep the larger, default quorum.election.backoff.max.ms here to give
successors enough time to be
considered one after another. This config sets how long each successor waits
before it starts an election.
With a small value it is more likely two successors will have overlapping
elections.
```
##########
raft/src/test/java/org/apache/kafka/raft/RaftEventSimulationTest.java:
##########
@@ -195,6 +197,83 @@ void canElectNewLeaderAfterOldLeaderFailure() {
});
}
+ @ParameterizedTest
+ @ValueSource(ints = {4, 5, 6, 7, 8, 9})
+ void preferredSuccessorWinsElectionAfterGracefulLeaderShutdown(int
numVoters) {
+ Cluster cluster = new Cluster(numVoters, 0, 1234L);
+ /* The other simulations lower this to keep their runtime down, but
here the whole point is
+ * that the nominated successors stand for election one at a time, so
use the production
+ * default to keep the gap between them wide enough to be meaningful.
+ */
+ cluster.electionBackoffMaxMs =
QuorumConfig.DEFAULT_QUORUM_ELECTION_BACKOFF_MAX_MS;
+ MessageRouter router = new MessageRouter(cluster);
+ EventScheduler scheduler = schedulerWithDefaultInvariants(cluster);
+
+ cluster.startAll();
+ schedulePolling(scheduler, cluster, 3, 5);
+ scheduler.schedule(router::deliverAll, 0, 2, 1);
+ scheduler.runUntil(cluster::hasConsistentLeader);
+ // Every replica has fetched from the leader and is caught up with it
+ scheduler.runUntil(() -> cluster.allReachedHighWatermark(1));
+
+ int leaderId = cluster.latestLeader().orElseThrow(() ->
+ new AssertionError("Failed to find current leader")
+ );
+
+ // Read the ranking that the leader will put into its EndQuorumEpoch
requests
+ LeaderState<Integer> leaderState = cluster
+ .nodeIfRunning(leaderId)
+ .orElseThrow(() -> new AssertionError("Leader " + leaderId + " is
not running"))
+ .client
+ .quorum()
+ .leaderStateOrThrow();
+ List<Integer> preferredSuccessors = leaderState
+ .nonLeaderVotersByDescendingFetchOffset()
+ .stream()
+ .map(ReplicaKey::id)
+ .collect(Collectors.toList());
+
+ // A clean shutdown, so the leader resigns and nominates the
successors above
+ long resignedAtMs = cluster.time.milliseconds();
+ cluster.shutdown(leaderId);
+
+ scheduler.runUntil(() ->
+ cluster.latestLeader().isPresent() &&
cluster.latestLeader().getAsInt() != leaderId
+ );
+
+ int newLeaderId = cluster.latestLeader().getAsInt();
+ long electionDurationMs = cluster.time.milliseconds() - resignedAtMs;
+ int position = preferredSuccessors.indexOf(newLeaderId);
+ int backoffBaseMs = cluster.electionBackoffMaxMs >>
(preferredSuccessors.size() - 1);
+ long backoffOfWinnerMs = position <= 0 ? 0 : (long) backoffBaseMs <<
(position - 1);
Review Comment:
also if the calculation were to change in the production code then this
becomes out of date
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]