chia7712 commented on code in PR #15766: URL: https://github.com/apache/kafka/pull/15766#discussion_r1586367051
########## tools/src/test/java/org/apache/kafka/tools/consumer/group/ConsumerGroupExecutor.java: ########## @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.consumer.group; + +import org.apache.kafka.clients.consumer.GroupProtocol; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.RangeAssignor; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.Utils; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.Collections.singleton; +import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_REMOTE_ASSIGNOR_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.common.GroupType.CONSUMER; + +class ConsumerGroupExecutor { + + private ConsumerGroupExecutor() { + } + + static AutoCloseable buildConsumerGroup(String brokerAddress, + int numberOfConsumers, + String groupId, + String topic, + String groupProtocol, Review Comment: Maybe we should use a `Supplier<Comsumer>` instead of those arguments. This executor is used to run "consumer" background and we should keep it simpler. ########## tools/src/test/java/org/apache/kafka/tools/consumer/group/DeleteConsumerGroupsTest.java: ########## @@ -17,279 +17,375 @@ package org.apache.kafka.tools.consumer.group; import joptsimple.OptionException; +import kafka.test.ClusterConfig; +import kafka.test.ClusterGenerator; +import kafka.test.ClusterInstance; +import kafka.test.annotation.ClusterTemplate; +import kafka.test.junit.ClusterTestExtensions; +import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.consumer.GroupProtocol; import org.apache.kafka.clients.consumer.RangeAssignor; +import org.apache.kafka.common.ConsumerGroupState; import org.apache.kafka.common.errors.GroupIdNotFoundException; import org.apache.kafka.common.errors.GroupNotEmptyException; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.test.TestUtils; import org.apache.kafka.tools.ToolsTestUtils; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.extension.ExtendWith; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Properties; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static kafka.test.annotation.Type.CO_KRAFT; +import static kafka.test.annotation.Type.KRAFT; +import static kafka.test.annotation.Type.ZK; +import static org.apache.kafka.clients.consumer.GroupProtocol.CLASSIC; +import static org.apache.kafka.clients.consumer.GroupProtocol.CONSUMER; +import static org.apache.kafka.common.ConsumerGroupState.EMPTY; +import static org.apache.kafka.common.ConsumerGroupState.STABLE; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.NEW_GROUP_COORDINATOR_ENABLE_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public class DeleteConsumerGroupsTest extends ConsumerGroupCommandTest { - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteWithTopicOption(String quorum) { - createOffsetsTopic(listenerName(), new Properties()); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP, "--topic"}; - assertThrows(OptionException.class, () -> getConsumerGroupService(cgcArgs)); - } - - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteCmdNonExistingGroup(String quorum) { - createOffsetsTopic(listenerName(), new Properties()); - String missingGroup = "missing.group"; - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", missingGroup}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); +@ExtendWith(value = ClusterTestExtensions.class) +public class DeleteConsumerGroupsTest { + private final ClusterInstance cluster; + private final Iterable<GroupProtocol> groupProtocols; - String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups); - assertTrue(output.contains("Group '" + missingGroup + "' could not be deleted due to:") && output.contains(Errors.GROUP_ID_NOT_FOUND.message()), - "The expected error (" + Errors.GROUP_ID_NOT_FOUND + ") was not detected while deleting consumer group"); + public DeleteConsumerGroupsTest(ClusterInstance cluster) { + this.cluster = cluster; + this.groupProtocols = cluster.isKRaftTest() + ? Arrays.asList(CLASSIC, CONSUMER) + : singletonList(CLASSIC); } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteNonExistingGroup(String quorum) { - createOffsetsTopic(listenerName(), new Properties()); - String missingGroup = "missing.group"; - - // note the group to be deleted is a different (non-existing) group - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", missingGroup}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - Map<String, Throwable> result = service.deleteGroups(); - assertTrue(result.size() == 1 && result.containsKey(missingGroup) && result.get(missingGroup).getCause() instanceof GroupIdNotFoundException, - "The expected error (" + Errors.GROUP_ID_NOT_FOUND + ") was not detected while deleting consumer group"); + private static void generator(ClusterGenerator clusterGenerator) { + Map<String, String> serverProperties = new HashMap<>(); + serverProperties.put(OFFSETS_TOPIC_PARTITIONS_CONFIG, "1"); + serverProperties.put(OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, "1"); + serverProperties.put(NEW_GROUP_COORDINATOR_ENABLE_CONFIG, "false"); + + ClusterConfig zk = ClusterConfig.defaultBuilder() + .setType(ZK) + .setServerProperties(serverProperties) + .build(); + clusterGenerator.accept(zk); + + ClusterConfig raftWithLegacyCoordinator = ClusterConfig.defaultBuilder() + .setType(KRAFT) + .setServerProperties(serverProperties) + .build(); + clusterGenerator.accept(raftWithLegacyCoordinator); + + ClusterConfig combinedKRaftWithLegacyCoordinator = ClusterConfig.defaultBuilder() + .setType(CO_KRAFT) + .setServerProperties(serverProperties) + .build(); + clusterGenerator.accept(combinedKRaftWithLegacyCoordinator); + + // Following are test case config with new group coordinator + serverProperties.put(NEW_GROUP_COORDINATOR_ENABLE_CONFIG, "true"); + + ClusterConfig raftWithNewGroupCoordinator = ClusterConfig.defaultBuilder() + .setType(KRAFT) + .setName("newGroupCoordinator") + .setServerProperties(serverProperties).build(); + clusterGenerator.accept(raftWithNewGroupCoordinator); + + ClusterConfig combinedKRaftWithNewGroupCoordinator = ClusterConfig.defaultBuilder() + .setType(CO_KRAFT) + .setName("newGroupCoordinator") + .setServerProperties(serverProperties) + .build(); + clusterGenerator.accept(combinedKRaftWithNewGroupCoordinator); } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteCmdNonEmptyGroup(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - - // run one consumer in the group - addConsumerGroupExecutor(1); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - TestUtils.waitForCondition( - () -> service.collectGroupMembers(GROUP, false).getValue().get().size() == 1, - "The group did not initialize as expected." - ); - - String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups); - assertTrue(output.contains("Group '" + GROUP + "' could not be deleted due to:") && output.contains(Errors.NON_EMPTY_GROUP.message()), - "The expected error (" + Errors.NON_EMPTY_GROUP + ") was not detected while deleting consumer group. Output was: (" + output + ")"); + @ClusterTemplate("generator") + public void testDeleteWithTopicOption() { + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", getDummyGroupId(), "--topic"}; + assertThrows(OptionException.class, () -> getConsumerGroupService(cgcArgs)); } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteNonEmptyGroup(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - - // run one consumer in the group - addConsumerGroupExecutor(1); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - TestUtils.waitForCondition( - () -> service.collectGroupMembers(GROUP, false).getValue().get().size() == 1, - "The group did not initialize as expected." - ); - - Map<String, Throwable> result = service.deleteGroups(); - assertNotNull(result.get(GROUP), - "Group was deleted successfully, but it shouldn't have been. Result was:(" + result + ")"); - assertTrue(result.size() == 1 && result.containsKey(GROUP) && result.get(GROUP).getCause() instanceof GroupNotEmptyException, - "The expected error (" + Errors.NON_EMPTY_GROUP + ") was not detected while deleting consumer group. Result was:(" + result + ")"); + @ClusterTemplate("generator") + public void testDeleteCmdNonExistingGroup() { + String missingGroupId = composeMissingGroupId(CLASSIC); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", missingGroupId}; + try (ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs)) { + String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups); + assertTrue(output.contains("Group '" + missingGroupId + "' could not be deleted due to:") && output.contains(Errors.GROUP_ID_NOT_FOUND.message()), + "The expected error (" + Errors.GROUP_ID_NOT_FOUND + ") was not detected while deleting consumer group"); + } } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteCmdEmptyGroup(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - - // run one consumer in the group - ConsumerGroupExecutor executor = addConsumerGroupExecutor(1); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - TestUtils.waitForCondition( - () -> service.listConsumerGroups().contains(GROUP) && Objects.equals(service.collectGroupState(GROUP).state, "Stable"), - "The group did not initialize as expected." - ); - - executor.shutdown(); + @ClusterTemplate("generator") + public void testDeleteNonExistingGroup() { + String missingGroupId = composeMissingGroupId(CLASSIC); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", missingGroupId}; + try (ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs)) { + Map<String, Throwable> result = service.deleteGroups(); + assertTrue(result.size() == 1 && result.containsKey(missingGroupId) && result.get(missingGroupId).getCause() instanceof GroupIdNotFoundException, + "The expected error (" + Errors.GROUP_ID_NOT_FOUND + ") was not detected while deleting consumer group"); + } + } - TestUtils.waitForCondition( - () -> Objects.equals(service.collectGroupState(GROUP).state, "Empty"), - "The group did not become empty as expected." - ); + @ClusterTemplate("generator") + public void testDeleteNonEmptyGroup() throws Exception { + for (GroupProtocol groupProtocol : groupProtocols) { + String groupId = composeGroupId(groupProtocol); + String topicName = composeTopicName(groupProtocol); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", groupId}; + try ( + AutoCloseable consumerGroupCloseable = consumerGroupClosable(groupProtocol, groupId, topicName); + ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs) + ) { + TestUtils.waitForCondition( + () -> service.collectGroupMembers(groupId, false).getValue().get().size() == 1, + "The group did not initialize as expected." + ); + + String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups); + Map<String, Throwable> result = service.deleteGroups(); + + assertTrue(output.contains("Group '" + groupId + "' could not be deleted due to:") && output.contains(Errors.NON_EMPTY_GROUP.message()), + "The expected error (" + Errors.NON_EMPTY_GROUP + ") was not detected while deleting consumer group. Output was: (" + output + ")"); + + assertNotNull(result.get(groupId), + "Group was deleted successfully, but it shouldn't have been. Result was:(" + result + ")"); + + assertTrue(result.size() == 1 && result.containsKey(groupId) && result.get(groupId).getCause() instanceof GroupNotEmptyException, + "The expected error (" + Errors.NON_EMPTY_GROUP + ") was not detected while deleting consumer group. Result was:(" + result + ")"); + } + } + } - String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups); - assertTrue(output.contains("Deletion of requested consumer groups ('" + GROUP + "') was successful."), - "The consumer group could not be deleted as expected"); + @ClusterTemplate("generator") + void testDeleteEmptyGroup() throws Exception { + for (GroupProtocol groupProtocol : groupProtocols) { + String groupId = composeGroupId(groupProtocol); + String topicName = composeTopicName(groupProtocol); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", groupId}; + try ( + AutoCloseable consumerGroupCloseable = consumerGroupClosable(groupProtocol, groupId, topicName); + ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs) + ) { + TestUtils.waitForCondition( + () -> service.listConsumerGroups().contains(groupId) && checkGroupState(service, groupId, STABLE), + "The group did not initialize as expected." + ); + + consumerGroupCloseable.close(); + + TestUtils.waitForCondition( + () -> checkGroupState(service, groupId, EMPTY), + "The group did not become empty as expected." + ); + + Map<String, Throwable> result = new HashMap<>(); + String output = ToolsTestUtils.grabConsoleOutput(() -> result.putAll(service.deleteGroups())); + + assertTrue(output.contains("Deletion of requested consumer groups ('" + groupId + "') was successful."), + "The consumer group could not be deleted as expected"); + assertTrue(result.size() == 1 && result.containsKey(groupId) && result.get(groupId) == null, + "The consumer group could not be deleted as expected"); + } + } } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteCmdAllGroups(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - - // Create 3 groups with 1 consumer per each - Map<String, ConsumerGroupExecutor> groups = IntStream.rangeClosed(1, 3).mapToObj(i -> GROUP + i).collect(Collectors.toMap( - Function.identity(), - group -> addConsumerGroupExecutor(1, TOPIC, group, RangeAssignor.class.getName(), Optional.empty(), Optional.empty(), false, GroupProtocol.CLASSIC.name) - )); - - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--all-groups"}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - TestUtils.waitForCondition(() -> - new HashSet<>(service.listConsumerGroups()).equals(groups.keySet()) && - groups.keySet().stream().allMatch(groupId -> { - try { - return Objects.equals(service.collectGroupState(groupId).state, "Stable"); - } catch (Exception e) { - throw new RuntimeException(e); - } - }), - "The group did not initialize as expected."); - - // Shutdown consumers to empty out groups - groups.values().forEach(AbstractConsumerGroupExecutor::shutdown); - - TestUtils.waitForCondition(() -> - groups.keySet().stream().allMatch(groupId -> { - try { - return Objects.equals(service.collectGroupState(groupId).state, "Empty"); - } catch (Exception e) { - throw new RuntimeException(e); + @ClusterTemplate("generator") + public void testDeleteCmdAllGroups() throws Exception { + for (GroupProtocol groupProtocol : groupProtocols) { + String topicName = composeTopicName(groupProtocol); + // Create 3 groups with 1 consumer per each + Map<String, AutoCloseable> groupIdToExecutor = IntStream.rangeClosed(1, 3) + .mapToObj(i -> composeGroupId(groupProtocol) + i) + .collect(Collectors.toMap(Function.identity(), group -> consumerGroupClosable(groupProtocol, group, topicName))); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--all-groups"}; + + try (ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs)) { + TestUtils.waitForCondition(() -> + new HashSet<>(service.listConsumerGroups()).equals(groupIdToExecutor.keySet()) && + groupIdToExecutor.keySet().stream().allMatch(predicateGroupState(service, STABLE)), + "The group did not initialize as expected."); + + // Shutdown consumers to empty out groups + for (AutoCloseable consumerGroupExecutor : groupIdToExecutor.values()) { + consumerGroupExecutor.close(); } - }), - "The group did not become empty as expected."); - - String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups).trim(); - Set<String> expectedGroupsForDeletion = groups.keySet(); - Set<String> deletedGroupsGrepped = Arrays.stream(output.substring(output.indexOf('(') + 1, output.indexOf(')')).split(",")) - .map(str -> str.replaceAll("'", "").trim()).collect(Collectors.toSet()); - assertTrue(output.matches("Deletion of requested consumer groups (.*) was successful.") - && Objects.equals(deletedGroupsGrepped, expectedGroupsForDeletion), - "The consumer group(s) could not be deleted as expected"); + TestUtils.waitForCondition(() -> + groupIdToExecutor.keySet().stream().allMatch(predicateGroupState(service, EMPTY)), + "The group did not become empty as expected."); + + String output = ToolsTestUtils.grabConsoleOutput(service::deleteGroups).trim(); + Set<String> expectedGroupsForDeletion = groupIdToExecutor.keySet(); + Set<String> deletedGroupsGrepped = Arrays.stream(output.substring(output.indexOf('(') + 1, output.indexOf(')')).split(",")) + .map(str -> str.replaceAll("'", "").trim()) + .collect(Collectors.toSet()); + + assertTrue(output.matches("Deletion of requested consumer groups (.*) was successful.") + && Objects.equals(deletedGroupsGrepped, expectedGroupsForDeletion), + "The consumer group(s) could not be deleted as expected"); + } + } } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteEmptyGroup(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - - // run one consumer in the group - ConsumerGroupExecutor executor = addConsumerGroupExecutor(1); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - TestUtils.waitForCondition( - () -> service.listConsumerGroups().contains(GROUP) && Objects.equals(service.collectGroupState(GROUP).state, "Stable"), - "The group did not initialize as expected."); - - executor.shutdown(); - - TestUtils.waitForCondition( - () -> Objects.equals(service.collectGroupState(GROUP).state, "Empty"), - "The group did not become empty as expected."); - - Map<String, Throwable> result = service.deleteGroups(); - assertTrue(result.size() == 1 && result.containsKey(GROUP) && result.get(GROUP) == null, - "The consumer group could not be deleted as expected"); + @ClusterTemplate("generator") + public void testDeleteCmdWithMixOfSuccessAndError() throws Exception { + for (GroupProtocol groupProtocol : groupProtocols) { + String groupId = composeGroupId(groupProtocol); + String topicName = composeTopicName(groupProtocol); + String missingGroupId = composeMissingGroupId(groupProtocol); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", groupId}; + try ( + AutoCloseable consumerGroupClosable = consumerGroupClosable(groupProtocol, groupId, topicName); + ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs) + ) { + TestUtils.waitForCondition( + () -> service.listConsumerGroups().contains(groupId) && checkGroupState(service, groupId, STABLE), + "The group did not initialize as expected."); + + consumerGroupClosable.close(); + TestUtils.waitForCondition( + () -> checkGroupState(service, groupId, EMPTY), + "The group did not become empty as expected."); + + cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", groupId, "--group", missingGroupId}; + + ConsumerGroupCommand.ConsumerGroupService service2 = getConsumerGroupService(cgcArgs); + + String output = ToolsTestUtils.grabConsoleOutput(service2::deleteGroups); + assertTrue(output.contains("Group '" + missingGroupId + "' could not be deleted due to:") + && output.contains(Errors.GROUP_ID_NOT_FOUND.message()) + && output.contains("These consumer groups were deleted successfully: '" + groupId + "'"), + "The consumer group deletion did not work as expected"); + } + } } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteCmdWithMixOfSuccessAndError(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - String missingGroup = "missing.group"; - - // run one consumer in the group - ConsumerGroupExecutor executor = addConsumerGroupExecutor(1); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); - - TestUtils.waitForCondition( - () -> service.listConsumerGroups().contains(GROUP) && Objects.equals(service.collectGroupState(GROUP).state, "Stable"), - "The group did not initialize as expected."); - - executor.shutdown(); - - TestUtils.waitForCondition( - () -> Objects.equals(service.collectGroupState(GROUP).state, "Empty"), - "The group did not become empty as expected."); - - cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP, "--group", missingGroup}; - - ConsumerGroupCommand.ConsumerGroupService service2 = getConsumerGroupService(cgcArgs); + @ClusterTemplate("generator") + public void testDeleteWithMixOfSuccessAndError() throws Exception { + for (GroupProtocol groupProtocol : groupProtocols) { + String groupId = composeGroupId(groupProtocol); + String topicName = composeTopicName(groupProtocol); + String missingGroupId = composeMissingGroupId(groupProtocol); + String[] cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", groupId}; + try ( + AutoCloseable executor = consumerGroupClosable(groupProtocol, groupId, topicName); + ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs) + ) { + TestUtils.waitForCondition( + () -> service.listConsumerGroups().contains(groupId) && checkGroupState(service, groupId, STABLE), + "The group did not initialize as expected."); + + executor.close(); + + TestUtils.waitForCondition( + () -> checkGroupState(service, groupId, EMPTY), + "The group did not become empty as expected."); + + cgcArgs = new String[]{"--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", groupId, "--group", missingGroupId}; + + ConsumerGroupCommand.ConsumerGroupService service2 = getConsumerGroupService(cgcArgs); + Map<String, Throwable> result = service2.deleteGroups(); + assertTrue(result.size() == 2 && + result.containsKey(groupId) && result.get(groupId) == null && + result.containsKey(missingGroupId) && + result.get(missingGroupId).getMessage().contains(Errors.GROUP_ID_NOT_FOUND.message()), + "The consumer group deletion did not work as expected"); + } + } + } - String output = ToolsTestUtils.grabConsoleOutput(service2::deleteGroups); - assertTrue(output.contains("Group '" + missingGroup + "' could not be deleted due to:") - && output.contains(Errors.GROUP_ID_NOT_FOUND.message()) - && output.contains("These consumer groups were deleted successfully: '" + GROUP + "'"), - "The consumer group deletion did not work as expected"); + @ClusterTemplate("generator") + public void testDeleteWithUnrecognizedNewConsumerOption() { + String[] cgcArgs = new String[]{"--new-consumer", "--bootstrap-server", cluster.bootstrapServers(), "--delete", "--group", getDummyGroupId()}; + assertThrows(OptionException.class, () -> getConsumerGroupService(cgcArgs)); } - @ParameterizedTest - @ValueSource(strings = {"zk", "kraft"}) - public void testDeleteWithMixOfSuccessAndError(String quorum) throws Exception { - createOffsetsTopic(listenerName(), new Properties()); - String missingGroup = "missing.group"; + private String getDummyGroupId() { + return composeGroupId(null); + } - // run one consumer in the group - ConsumerGroupExecutor executor = addConsumerGroupExecutor(1); - String[] cgcArgs = new String[]{"--bootstrap-server", bootstrapServers(listenerName()), "--delete", "--group", GROUP}; - ConsumerGroupCommand.ConsumerGroupService service = getConsumerGroupService(cgcArgs); + private String composeGroupId(GroupProtocol protocol) { + String groupPrefix = "test."; + return protocol != null ? groupPrefix + protocol.name : groupPrefix + "dummy"; + } - TestUtils.waitForCondition( - () -> service.listConsumerGroups().contains(GROUP) && Objects.equals(service.collectGroupState(GROUP).state, "Stable"), - "The group did not initialize as expected."); + private String composeTopicName(GroupProtocol protocol) { + String topicPrefix = "foo."; + return protocol != null ? topicPrefix + protocol.name : topicPrefix + "dummy"; + } - executor.shutdown(); + private String composeMissingGroupId(GroupProtocol protocol) { + String missingGroupPrefix = "missing."; + return protocol != null ? missingGroupPrefix + protocol.name : missingGroupPrefix + "dummy"; + } - TestUtils.waitForCondition( - () -> Objects.equals(service.collectGroupState(GROUP).state, "Empty"), - "The group did not become empty as expected."); + private AutoCloseable consumerGroupClosable(GroupProtocol protocol, String groupId, String topicName) { + boolean isNewCoordinator = cluster.config() + .serverProperties() + .get(NEW_GROUP_COORDINATOR_ENABLE_CONFIG) + .equals("true"); + + if (isNewCoordinator) { + return ConsumerGroupExecutor.buildConsumerGroup( + cluster.bootstrapServers(), + 1, + groupId, + topicName, + protocol.name, + Optional.empty(), + emptyMap(), + false + ); + } + + return ConsumerGroupExecutor.buildClassicGroup( Review Comment: All cases we want to test are shown below. 1. zk + old consumer 2. kraft + old group coordinator + old consumer 3. kraft + new group coordinator + old consumer 4. kraft + new group coordinator + new consumer 5. combined kraft + old group coordinator + old consumer 6. combined kraft + new group coordinator + old consumer 7. combined kraft + new group coordinator + new consumer In this path, we will run case2 and case5 two times since `groupProtocols` is consist of `CLASSIC` and `CONSUMER`. Hence, we should create `groupProtocols` according to `isNewCoordinator`. For example: ```java this.groupProtocols = cluster.config().serverProperties() .get(NEW_GROUP_COORDINATOR_ENABLE_CONFIG).equals("true") ? Arrays.asList(CLASSIC, CONSUMER) : singletonList(CLASSIC); ``` ########## tools/src/test/java/org/apache/kafka/tools/consumer/group/ConsumerGroupExecutor.java: ########## @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.consumer.group; + +import org.apache.kafka.clients.consumer.GroupProtocol; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.RangeAssignor; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.Utils; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.Collections.singleton; +import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_REMOTE_ASSIGNOR_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.common.GroupType.CONSUMER; + +class ConsumerGroupExecutor { + + private ConsumerGroupExecutor() { + } + + static AutoCloseable buildConsumerGroup(String brokerAddress, + int numberOfConsumers, + String groupId, + String topic, + String groupProtocol, Review Comment: As this helper is used to create new consumer (AsyncConsumer), so it seems to me `groupProtocol` should always be `CONSUMER`. that means it is redundant. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
