vvcephei commented on a change in pull request #8588:
URL: https://github.com/apache/kafka/pull/8588#discussion_r424820836
##########
File path:
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/TaskMovement.java
##########
@@ -53,75 +67,94 @@ private static boolean
taskIsCaughtUpOnClientOrNoCaughtUpClientsExist(final Task
/**
* @return whether any warmup replicas were assigned
*/
- static boolean assignTaskMovements(final Map<UUID, List<TaskId>>
statefulActiveTaskAssignment,
- final Map<TaskId, SortedSet<UUID>>
tasksToCaughtUpClients,
+ static boolean assignTaskMovements(final Map<TaskId, SortedSet<UUID>>
tasksToCaughtUpClients,
final Map<UUID, ClientState>
clientStates,
- final Map<TaskId, Integer>
tasksToRemainingStandbys,
final int maxWarmupReplicas) {
- boolean warmupReplicasAssigned = false;
+ final BiFunction<UUID, TaskId, Boolean> caughtUpPredicate =
+ (client, task) ->
taskIsCaughtUpOnClientOrNoCaughtUpClientsExist(task, client,
tasksToCaughtUpClients);
- final ValidClientsByTaskLoadQueue clientsByTaskLoad = new
ValidClientsByTaskLoadQueue(
- clientStates,
- (client, task) ->
taskIsCaughtUpOnClientOrNoCaughtUpClientsExist(task, client,
tasksToCaughtUpClients)
+ final ConstrainedPrioritySet clientsByTaskLoad = new
ConstrainedPrioritySet(
+ caughtUpPredicate,
+ client -> clientStates.get(client).taskLoad()
);
- final SortedSet<TaskMovement> taskMovements = new TreeSet<>(
- (movement, other) -> {
- final int numCaughtUpClients = movement.caughtUpClients.size();
- final int otherNumCaughtUpClients =
other.caughtUpClients.size();
- if (numCaughtUpClients != otherNumCaughtUpClients) {
- return Integer.compare(numCaughtUpClients,
otherNumCaughtUpClients);
- } else {
- return movement.task.compareTo(other.task);
- }
- }
+ final Queue<TaskMovement> taskMovements = new PriorityQueue<>(
+
Comparator.comparing(TaskMovement::numCaughtUpClients).thenComparing(TaskMovement::task)
);
- for (final Map.Entry<UUID, List<TaskId>> assignmentEntry :
statefulActiveTaskAssignment.entrySet()) {
- final UUID client = assignmentEntry.getKey();
- final ClientState state = clientStates.get(client);
- for (final TaskId task : assignmentEntry.getValue()) {
- if (taskIsCaughtUpOnClientOrNoCaughtUpClientsExist(task,
client, tasksToCaughtUpClients)) {
- state.assignActive(task);
- } else {
- final TaskMovement taskMovement = new TaskMovement(task,
client, tasksToCaughtUpClients.get(task));
- taskMovements.add(taskMovement);
+ for (final Map.Entry<UUID, ClientState> clientStateEntry :
clientStates.entrySet()) {
+ final UUID client = clientStateEntry.getKey();
+ final ClientState state = clientStateEntry.getValue();
+ for (final TaskId task : state.activeTasks()) {
+ // if the desired client is not caught up, and there is
another client that _is_ caught up, then
+ // we schedule a movement, so we can move the active task to
the caught-up client. We'll try to
+ // assign a warm-up to the desired client so that we can move
it later on.
+ if (!taskIsCaughtUpOnClientOrNoCaughtUpClientsExist(task,
client, tasksToCaughtUpClients)) {
+ taskMovements.add(new TaskMovement(task, client,
tasksToCaughtUpClients.get(task)));
}
}
clientsByTaskLoad.offer(client);
}
+ final boolean movementsNeeded = !taskMovements.isEmpty();
+
final AtomicInteger remainingWarmupReplicas = new
AtomicInteger(maxWarmupReplicas);
for (final TaskMovement movement : taskMovements) {
- final UUID sourceClient = clientsByTaskLoad.poll(movement.task);
- if (sourceClient == null) {
- throw new IllegalStateException("Tried to move task to
caught-up client but none exist");
- }
-
- final ClientState sourceClientState =
clientStates.get(sourceClient);
- sourceClientState.assignActive(movement.task);
- clientsByTaskLoad.offer(sourceClient);
+ final UUID standbySourceClient = clientsByTaskLoad.poll(
Review comment:
Agreed, it would just be good luck right now, but I figured we might as
well capitalize on the luck. I'm planning to follow up pretty soon with the
standby stickiness.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
