dajac commented on code in PR #22487: URL: https://github.com/apache/kafka/pull/22487#discussion_r4062285311
########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java: ########## @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.group.modern.Assignment; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TasksTuple; + +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Builds the records to persist a new target assignment for a group. + * + * Records are only created for members which have a new target assignment. If + * their assignment did not change, no new record is needed. + * + * When a member is deleted, it is assumed that its target assignment record + * is deleted as part of the member deletion process. In other words, this class + * does not yield a tombstone for removed members. + * + * @param <A> The member's target assignment type. + */ +public abstract class TargetAssignmentRecordsBuilder<A> { + + /** + * The logger. + */ + private final Logger log; + + /** + * The group id. + */ + private final String groupId; + + /** + * The target assignment metadata. + */ + private TargetAssignmentMetadata targetAssignmentMetadata; + + /** + * The static members in the group at the time the new target assignment was computed. + */ + private Map<String, String> previousStaticMembers = Map.of(); + + /** + * The current member ids in the group. + */ + private Set<String> currentMemberIds; + + /** + * The current static members in the group. + */ + private Map<String, String> currentStaticMembers = Map.of(); + + /** + * The current target assignment. + */ + private Map<String, A> currentTargetAssignment; + + /** + * The new target assignment. + */ + private Map<String, A> newTargetAssignment; + + /** + * Constructs the object. + * + * @param log The logger. + * @param groupId The group id. + */ + public TargetAssignmentRecordsBuilder( + Logger log, + String groupId + ) { + this.log = log; + this.groupId = Objects.requireNonNull(groupId); + } + + /** + * Sets the target assignment metadata. + * + * @param targetAssignmentMetadata The target assignment metadata. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withTargetAssignmentMetadata(TargetAssignmentMetadata targetAssignmentMetadata) { + this.targetAssignmentMetadata = Objects.requireNonNull(targetAssignmentMetadata); + return this; + } + + /** + * Sets the static members in the group at the time the new target assignment was computed. + * + * @param previousStaticMembers The static members in the group at the time the new target assignment was computed. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) { + this.previousStaticMembers = Objects.requireNonNull(previousStaticMembers); + return this; + } + + /** + * Sets the current member ids in the group. + * + * @param currentMemberIds The current member ids in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) { + this.currentMemberIds = Objects.requireNonNull(currentMemberIds); + return this; + } + + /** + * Sets the current static members in the group. + * + * @param currentStaticMembers The current static members in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) { + this.currentStaticMembers = Objects.requireNonNull(currentStaticMembers); + return this; + } + + /** + * Sets the current target assignment. + * + * @param currentTargetAssignment The current target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) { + this.currentTargetAssignment = Objects.requireNonNull(currentTargetAssignment); + return this; + } + + /** + * Sets the new target assignment. + * + * @param newTargetAssignment The new target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) { + this.newTargetAssignment = Objects.requireNonNull(newTargetAssignment); + return this; + } + + /** + * Builds the records for the new target assignment. + * + * @return The records for the new target assignment. + */ + public List<CoordinatorRecord> build() { + List<CoordinatorRecord> records = new ArrayList<>(); + build(records); + return records; + } + + /** + * Builds the records for the new target assignment. + * + * @param records The list to accumulate records. + */ + public void build(List<CoordinatorRecord> records) { + // The members in the group may have changed while the target assignment was computed. + // We want to act as if concurrent member operations such as leaves and static member + // replacements happened *after* the target assignment was computed. + // + // * When members leave the group, we tombstone their target assignment. + // * When static members are replaced, we move the assignment from the old member id to the + // new member id. + // * When members rejoin with the same member id but a different instance id, they keep + // their existing assignment. This operation will never happen when using the official + // Java client and may be forbidden in the future. + // + // Thus, when building the new target assignment records, + // * we should not emit records for members that have left the group + // * we should relabel records with the latest member id for static members + // * we should match up members using member ids first, then fall back to instance ids. + Review Comment: We must ensure that all the collections that we access in build have been set. In the current form, we may hit NPE. Alternatively, we could have default for all of them. I suppose that it is better to not have any defaults and force the caller to think about what he is doing. ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java: ########## @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.group.modern.Assignment; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TasksTuple; + +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Builds the records to persist a new target assignment for a group. + * + * Records are only created for members which have a new target assignment. If + * their assignment did not change, no new record is needed. + * + * When a member is deleted, it is assumed that its target assignment record + * is deleted as part of the member deletion process. In other words, this class + * does not yield a tombstone for removed members. + * + * @param <A> The member's target assignment type. + */ +public abstract class TargetAssignmentRecordsBuilder<A> { + + /** + * The logger. + */ + private final Logger log; + + /** + * The group id. + */ + private final String groupId; + + /** + * The target assignment metadata. + */ + private TargetAssignmentMetadata targetAssignmentMetadata; + + /** + * The static members in the group at the time the new target assignment was computed. + */ + private Map<String, String> previousStaticMembers = Map.of(); + + /** + * The current member ids in the group. + */ + private Set<String> currentMemberIds; + + /** + * The current static members in the group. + */ + private Map<String, String> currentStaticMembers = Map.of(); + + /** + * The current target assignment. + */ + private Map<String, A> currentTargetAssignment; + + /** + * The new target assignment. + */ + private Map<String, A> newTargetAssignment; + + /** + * Constructs the object. + * + * @param log The logger. + * @param groupId The group id. + */ + public TargetAssignmentRecordsBuilder( + Logger log, + String groupId + ) { + this.log = log; + this.groupId = Objects.requireNonNull(groupId); + } + + /** + * Sets the target assignment metadata. + * + * @param targetAssignmentMetadata The target assignment metadata. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withTargetAssignmentMetadata(TargetAssignmentMetadata targetAssignmentMetadata) { + this.targetAssignmentMetadata = Objects.requireNonNull(targetAssignmentMetadata); + return this; + } + + /** + * Sets the static members in the group at the time the new target assignment was computed. + * + * @param previousStaticMembers The static members in the group at the time the new target assignment was computed. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) { + this.previousStaticMembers = Objects.requireNonNull(previousStaticMembers); + return this; + } + + /** + * Sets the current member ids in the group. + * + * @param currentMemberIds The current member ids in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) { + this.currentMemberIds = Objects.requireNonNull(currentMemberIds); + return this; + } + + /** + * Sets the current static members in the group. + * + * @param currentStaticMembers The current static members in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) { + this.currentStaticMembers = Objects.requireNonNull(currentStaticMembers); + return this; + } + + /** + * Sets the current target assignment. + * + * @param currentTargetAssignment The current target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) { + this.currentTargetAssignment = Objects.requireNonNull(currentTargetAssignment); + return this; + } + + /** + * Sets the new target assignment. + * + * @param newTargetAssignment The new target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) { + this.newTargetAssignment = Objects.requireNonNull(newTargetAssignment); + return this; + } + + /** + * Builds the records for the new target assignment. + * + * @return The records for the new target assignment. + */ + public List<CoordinatorRecord> build() { + List<CoordinatorRecord> records = new ArrayList<>(); + build(records); + return records; + } + + /** + * Builds the records for the new target assignment. + * + * @param records The list to accumulate records. + */ + public void build(List<CoordinatorRecord> records) { + // The members in the group may have changed while the target assignment was computed. + // We want to act as if concurrent member operations such as leaves and static member + // replacements happened *after* the target assignment was computed. + // + // * When members leave the group, we tombstone their target assignment. + // * When static members are replaced, we move the assignment from the old member id to the + // new member id. + // * When members rejoin with the same member id but a different instance id, they keep + // their existing assignment. This operation will never happen when using the official + // Java client and may be forbidden in the future. + // + // Thus, when building the new target assignment records, + // * we should not emit records for members that have left the group + // * we should relabel records with the latest member id for static members + // * we should match up members using member ids first, then fall back to instance ids. + + // Build map of replacement member ids for static members that have churned. + Map<String, String> staticMemberIdRemapping = new HashMap<>(); + for (Map.Entry<String, String> entry : previousStaticMembers.entrySet()) { + String instanceId = entry.getKey(); + String oldMemberId = entry.getValue(); + String newMemberId = currentStaticMembers.get(instanceId); + + if (currentMemberIds.contains(oldMemberId)) { + // The old member id is still in the group. We must not create a remapping entry, + // otherwise we could give the same assignment to two different members. + continue; + } + + if (newTargetAssignment.containsKey(newMemberId)) { + // The new member id has been in the group the whole time. Avoid creating a + // remapping entry. Note that this is best effort, since newTargetAssignment may not + // contain entries for every member that used to be in the group. + continue; + } + + if (newMemberId != null) { + staticMemberIdRemapping.put(newMemberId, oldMemberId); + } + } + + if (log.isDebugEnabled()) { + logStaticMembers(); + logMembersLeftAndJoined(staticMemberIdRemapping); + } + + for (String memberId : currentMemberIds) { + String previousMemberId = staticMemberIdRemapping.getOrDefault(memberId, memberId); + + A oldMemberAssignment = currentTargetAssignment.get(memberId); + A newMemberAssignment = newTargetAssignment.get(previousMemberId); + if (newMemberAssignment == null) { + newMemberAssignment = emptyMemberAssignment(); + } + + if (oldMemberAssignment == null || + !newMemberAssignment.equals(oldMemberAssignment)) { Review Comment: nit: It seems to fit on the previous line. ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java: ########## @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.group.modern.Assignment; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TasksTuple; + +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Builds the records to persist a new target assignment for a group. + * + * Records are only created for members which have a new target assignment. If + * their assignment did not change, no new record is needed. + * + * When a member is deleted, it is assumed that its target assignment record + * is deleted as part of the member deletion process. In other words, this class + * does not yield a tombstone for removed members. + * + * @param <A> The member's target assignment type. + */ +public abstract class TargetAssignmentRecordsBuilder<A> { + + /** + * The logger. + */ + private final Logger log; + + /** + * The group id. + */ + private final String groupId; + + /** + * The target assignment metadata. + */ + private TargetAssignmentMetadata targetAssignmentMetadata; + + /** + * The static members in the group at the time the new target assignment was computed. + */ + private Map<String, String> previousStaticMembers = Map.of(); + + /** + * The current member ids in the group. + */ + private Set<String> currentMemberIds; + + /** + * The current static members in the group. + */ + private Map<String, String> currentStaticMembers = Map.of(); + + /** + * The current target assignment. + */ + private Map<String, A> currentTargetAssignment; + + /** + * The new target assignment. + */ + private Map<String, A> newTargetAssignment; + + /** + * Constructs the object. + * + * @param log The logger. + * @param groupId The group id. + */ + public TargetAssignmentRecordsBuilder( + Logger log, + String groupId + ) { + this.log = log; + this.groupId = Objects.requireNonNull(groupId); + } + + /** + * Sets the target assignment metadata. + * + * @param targetAssignmentMetadata The target assignment metadata. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withTargetAssignmentMetadata(TargetAssignmentMetadata targetAssignmentMetadata) { + this.targetAssignmentMetadata = Objects.requireNonNull(targetAssignmentMetadata); + return this; + } + + /** + * Sets the static members in the group at the time the new target assignment was computed. + * + * @param previousStaticMembers The static members in the group at the time the new target assignment was computed. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) { + this.previousStaticMembers = Objects.requireNonNull(previousStaticMembers); + return this; + } + + /** + * Sets the current member ids in the group. + * + * @param currentMemberIds The current member ids in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) { + this.currentMemberIds = Objects.requireNonNull(currentMemberIds); + return this; + } + + /** + * Sets the current static members in the group. + * + * @param currentStaticMembers The current static members in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) { + this.currentStaticMembers = Objects.requireNonNull(currentStaticMembers); + return this; + } + + /** + * Sets the current target assignment. + * + * @param currentTargetAssignment The current target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) { + this.currentTargetAssignment = Objects.requireNonNull(currentTargetAssignment); + return this; + } + + /** + * Sets the new target assignment. + * + * @param newTargetAssignment The new target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) { + this.newTargetAssignment = Objects.requireNonNull(newTargetAssignment); + return this; + } + + /** + * Builds the records for the new target assignment. + * + * @return The records for the new target assignment. + */ + public List<CoordinatorRecord> build() { + List<CoordinatorRecord> records = new ArrayList<>(); + build(records); + return records; + } + + /** + * Builds the records for the new target assignment. + * + * @param records The list to accumulate records. + */ + public void build(List<CoordinatorRecord> records) { + // The members in the group may have changed while the target assignment was computed. + // We want to act as if concurrent member operations such as leaves and static member + // replacements happened *after* the target assignment was computed. + // + // * When members leave the group, we tombstone their target assignment. + // * When static members are replaced, we move the assignment from the old member id to the + // new member id. + // * When members rejoin with the same member id but a different instance id, they keep + // their existing assignment. This operation will never happen when using the official + // Java client and may be forbidden in the future. + // + // Thus, when building the new target assignment records, + // * we should not emit records for members that have left the group + // * we should relabel records with the latest member id for static members + // * we should match up members using member ids first, then fall back to instance ids. + + // Build map of replacement member ids for static members that have churned. + Map<String, String> staticMemberIdRemapping = new HashMap<>(); + for (Map.Entry<String, String> entry : previousStaticMembers.entrySet()) { + String instanceId = entry.getKey(); + String oldMemberId = entry.getValue(); + String newMemberId = currentStaticMembers.get(instanceId); + + if (currentMemberIds.contains(oldMemberId)) { + // The old member id is still in the group. We must not create a remapping entry, + // otherwise we could give the same assignment to two different members. + continue; + } + + if (newTargetAssignment.containsKey(newMemberId)) { Review Comment: `newMemberId` may be `null` here. I think that `containsKey` could throw will `null` depending on the implementation used. It may be better to be defensive. ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java: ########## @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.group.modern.Assignment; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TasksTuple; + +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Builds the records to persist a new target assignment for a group. + * + * Records are only created for members which have a new target assignment. If + * their assignment did not change, no new record is needed. + * + * When a member is deleted, it is assumed that its target assignment record + * is deleted as part of the member deletion process. In other words, this class + * does not yield a tombstone for removed members. + * + * @param <A> The member's target assignment type. + */ +public abstract class TargetAssignmentRecordsBuilder<A> { + + /** + * The logger. + */ + private final Logger log; + + /** + * The group id. + */ + private final String groupId; + + /** + * The target assignment metadata. + */ + private TargetAssignmentMetadata targetAssignmentMetadata; + + /** + * The static members in the group at the time the new target assignment was computed. + */ + private Map<String, String> previousStaticMembers = Map.of(); + + /** + * The current member ids in the group. + */ + private Set<String> currentMemberIds; + + /** + * The current static members in the group. + */ + private Map<String, String> currentStaticMembers = Map.of(); + + /** + * The current target assignment. + */ + private Map<String, A> currentTargetAssignment; + + /** + * The new target assignment. + */ + private Map<String, A> newTargetAssignment; + + /** + * Constructs the object. + * + * @param log The logger. + * @param groupId The group id. + */ + public TargetAssignmentRecordsBuilder( + Logger log, + String groupId + ) { + this.log = log; + this.groupId = Objects.requireNonNull(groupId); + } + + /** + * Sets the target assignment metadata. + * + * @param targetAssignmentMetadata The target assignment metadata. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withTargetAssignmentMetadata(TargetAssignmentMetadata targetAssignmentMetadata) { + this.targetAssignmentMetadata = Objects.requireNonNull(targetAssignmentMetadata); + return this; + } + + /** + * Sets the static members in the group at the time the new target assignment was computed. + * + * @param previousStaticMembers The static members in the group at the time the new target assignment was computed. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) { + this.previousStaticMembers = Objects.requireNonNull(previousStaticMembers); + return this; + } + + /** + * Sets the current member ids in the group. + * + * @param currentMemberIds The current member ids in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) { + this.currentMemberIds = Objects.requireNonNull(currentMemberIds); + return this; + } + + /** + * Sets the current static members in the group. + * + * @param currentStaticMembers The current static members in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) { + this.currentStaticMembers = Objects.requireNonNull(currentStaticMembers); + return this; + } + + /** + * Sets the current target assignment. + * + * @param currentTargetAssignment The current target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) { + this.currentTargetAssignment = Objects.requireNonNull(currentTargetAssignment); + return this; + } + + /** + * Sets the new target assignment. + * + * @param newTargetAssignment The new target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) { + this.newTargetAssignment = Objects.requireNonNull(newTargetAssignment); + return this; + } + + /** + * Builds the records for the new target assignment. + * + * @return The records for the new target assignment. + */ + public List<CoordinatorRecord> build() { + List<CoordinatorRecord> records = new ArrayList<>(); + build(records); + return records; + } + + /** + * Builds the records for the new target assignment. + * + * @param records The list to accumulate records. + */ + public void build(List<CoordinatorRecord> records) { + // The members in the group may have changed while the target assignment was computed. + // We want to act as if concurrent member operations such as leaves and static member + // replacements happened *after* the target assignment was computed. + // + // * When members leave the group, we tombstone their target assignment. + // * When static members are replaced, we move the assignment from the old member id to the + // new member id. + // * When members rejoin with the same member id but a different instance id, they keep + // their existing assignment. This operation will never happen when using the official + // Java client and may be forbidden in the future. Review Comment: https://github.com/apache/kafka/pull/23542 ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java: ########## @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.common.utils.internals.LogContext; +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.group.modern.Assignment; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TasksTuple; + +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Builds the records to persist a new target assignment for a group. + * + * Records are only created for members which have a new target assignment. If + * their assignment did not change, no new record is needed. + * + * @param <A> The member's target assignment type. + */ +public abstract class TargetAssignmentRecordsBuilder<A> { + + /** + * The logger. + */ + private final Logger log; + + /** + * The group id. + */ + private final String groupId; + + /** + * The target assignment epoch. + */ + private int assignmentEpoch; + + /** + * The time at which the target assignment calculation finished. + */ + private long assignmentTimestampMs; + + /** + * The static members in the group at the time the new target assignment was computed. + */ + private Map<String, String> previousStaticMembers = Map.of(); + + /** + * The current member ids in the group. + */ + private Set<String> currentMemberIds; + + /** + * The current static members in the group. + */ + private Map<String, String> currentStaticMembers = Map.of(); + + /** + * The current target assignment. + */ + private Map<String, A> currentTargetAssignment = Map.of(); + + /** + * The new target assignment. + */ + private Map<String, A> newTargetAssignment = Map.of(); + + /** + * Constructs the object. + * + * @param logContext The log context. + * @param groupId The group id. + */ + public TargetAssignmentRecordsBuilder( + LogContext logContext, + String groupId + ) { + this.log = logContext.logger(this.getClass()); + this.groupId = Objects.requireNonNull(groupId); + } + + /** + * Sets the target assignment epoch. + * + * @param assignmentEpoch The target assignment epoch. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withAssignmentEpoch(int assignmentEpoch) { + this.assignmentEpoch = assignmentEpoch; + return this; + } + + /** + * Sets the time at which the target assignment calculation finished. + * + * @param assignmentTimestampMs The time at which the target assignment calculation finished. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withAssignmentTimestampMs(long assignmentTimestampMs) { + this.assignmentTimestampMs = assignmentTimestampMs; + return this; + } + + /** + * Sets the static members in the group at the time the new target assignment was computed. + * + * @param previousStaticMembers The static members in the group at the time the new target assignment was computed. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) { + this.previousStaticMembers = previousStaticMembers; + return this; + } + + /** + * Sets the current member ids in the group. + * + * @param currentMemberIds The current member ids in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) { + this.currentMemberIds = currentMemberIds; + return this; + } + + /** + * Sets the current static members in the group. + * + * @param currentStaticMembers The current static members in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) { + this.currentStaticMembers = currentStaticMembers; + return this; + } + + /** + * Sets the current target assignment. + * + * @param currentTargetAssignment The current target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) { + this.currentTargetAssignment = currentTargetAssignment; + return this; + } + + /** + * Sets the new target assignment. + * + * @param newTargetAssignment The new target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) { + this.newTargetAssignment = newTargetAssignment; + return this; + } + + /** + * Builds the records for the new target assignment. + * + * @return The records for the new target assignment. + */ + public List<CoordinatorRecord> build() { + List<CoordinatorRecord> records = new ArrayList<>(); + build(records); + return records; + } + + /** + * Builds the records for the new target assignment. + * + * @param records The list to accumulate records. + */ + @SuppressWarnings({"CyclomaticComplexity", "NPathComplexity"}) + public void build(List<CoordinatorRecord> records) { Review Comment: sg. Let's mention 2) somewhere in the comments. ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/TargetAssignmentRecordsBuilder.java: ########## @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group; + +import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; +import org.apache.kafka.coordinator.group.modern.Assignment; +import org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers; +import org.apache.kafka.coordinator.group.streams.TasksTuple; + +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Builds the records to persist a new target assignment for a group. + * + * Records are only created for members which have a new target assignment. If + * their assignment did not change, no new record is needed. + * + * When a member is deleted, it is assumed that its target assignment record + * is deleted as part of the member deletion process. In other words, this class + * does not yield a tombstone for removed members. + * + * @param <A> The member's target assignment type. + */ +public abstract class TargetAssignmentRecordsBuilder<A> { + + /** + * The logger. + */ + private final Logger log; + + /** + * The group id. + */ + private final String groupId; + + /** + * The target assignment metadata. + */ + private TargetAssignmentMetadata targetAssignmentMetadata; + + /** + * The static members in the group at the time the new target assignment was computed. + */ + private Map<String, String> previousStaticMembers = Map.of(); + + /** + * The current member ids in the group. + */ + private Set<String> currentMemberIds; + + /** + * The current static members in the group. + */ + private Map<String, String> currentStaticMembers = Map.of(); + + /** + * The current target assignment. + */ + private Map<String, A> currentTargetAssignment; + + /** + * The new target assignment. + */ + private Map<String, A> newTargetAssignment; + + /** + * Constructs the object. + * + * @param log The logger. + * @param groupId The group id. + */ + public TargetAssignmentRecordsBuilder( + Logger log, + String groupId + ) { + this.log = log; + this.groupId = Objects.requireNonNull(groupId); + } + + /** + * Sets the target assignment metadata. + * + * @param targetAssignmentMetadata The target assignment metadata. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withTargetAssignmentMetadata(TargetAssignmentMetadata targetAssignmentMetadata) { + this.targetAssignmentMetadata = Objects.requireNonNull(targetAssignmentMetadata); + return this; + } + + /** + * Sets the static members in the group at the time the new target assignment was computed. + * + * @param previousStaticMembers The static members in the group at the time the new target assignment was computed. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withPreviousStaticMembers(Map<String, String> previousStaticMembers) { + this.previousStaticMembers = Objects.requireNonNull(previousStaticMembers); + return this; + } + + /** + * Sets the current member ids in the group. + * + * @param currentMemberIds The current member ids in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentMemberIds(Set<String> currentMemberIds) { + this.currentMemberIds = Objects.requireNonNull(currentMemberIds); + return this; + } + + /** + * Sets the current static members in the group. + * + * @param currentStaticMembers The current static members in the group. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentStaticMembers(Map<String, String> currentStaticMembers) { + this.currentStaticMembers = Objects.requireNonNull(currentStaticMembers); + return this; + } + + /** + * Sets the current target assignment. + * + * @param currentTargetAssignment The current target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withCurrentTargetAssignment(Map<String, A> currentTargetAssignment) { + this.currentTargetAssignment = Objects.requireNonNull(currentTargetAssignment); + return this; + } + + /** + * Sets the new target assignment. + * + * @param newTargetAssignment The new target assignment. + * @return This object. + */ + public TargetAssignmentRecordsBuilder<A> withNewTargetAssignment(Map<String, A> newTargetAssignment) { + this.newTargetAssignment = Objects.requireNonNull(newTargetAssignment); + return this; + } + + /** + * Builds the records for the new target assignment. + * + * @return The records for the new target assignment. + */ + public List<CoordinatorRecord> build() { + List<CoordinatorRecord> records = new ArrayList<>(); + build(records); + return records; + } + + /** + * Builds the records for the new target assignment. + * + * @param records The list to accumulate records. + */ + public void build(List<CoordinatorRecord> records) { + // The members in the group may have changed while the target assignment was computed. + // We want to act as if concurrent member operations such as leaves and static member + // replacements happened *after* the target assignment was computed. + // + // * When members leave the group, we tombstone their target assignment. + // * When static members are replaced, we move the assignment from the old member id to the + // new member id. + // * When members rejoin with the same member id but a different instance id, they keep + // their existing assignment. This operation will never happen when using the official + // Java client and may be forbidden in the future. + // + // Thus, when building the new target assignment records, + // * we should not emit records for members that have left the group + // * we should relabel records with the latest member id for static members + // * we should match up members using member ids first, then fall back to instance ids. + + // Build map of replacement member ids for static members that have churned. + Map<String, String> staticMemberIdRemapping = new HashMap<>(); + for (Map.Entry<String, String> entry : previousStaticMembers.entrySet()) { + String instanceId = entry.getKey(); + String oldMemberId = entry.getValue(); + String newMemberId = currentStaticMembers.get(instanceId); + + if (currentMemberIds.contains(oldMemberId)) { + // The old member id is still in the group. We must not create a remapping entry, + // otherwise we could give the same assignment to two different members. + continue; + } + + if (newTargetAssignment.containsKey(newMemberId)) { + // The new member id has been in the group the whole time. Avoid creating a + // remapping entry. Note that this is best effort, since newTargetAssignment may not + // contain entries for every member that used to be in the group. + continue; + } + + if (newMemberId != null) { + staticMemberIdRemapping.put(newMemberId, oldMemberId); + } + } + + if (log.isDebugEnabled()) { + logStaticMembers(); + logMembersLeftAndJoined(staticMemberIdRemapping); + } Review Comment: I am not fan of this. Have you considered inlining the debug logging in the loop? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
