frankvicky commented on code in PR #19582: URL: https://github.com/apache/kafka/pull/19582#discussion_r2065922081
########## clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java: ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.server.util.ShutdownableThread; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ConsumerAssignmentPoller extends ShutdownableThread { + private final Consumer<byte[], byte[]> consumer; + private final Set<TopicPartition> partitionsToAssign; + private final ConsumerRebalanceListener userRebalanceListener; + + private volatile Throwable thrownException = null; + private volatile int receivedMessages = 0; + + private final Set<TopicPartition> partitionAssignment = new HashSet<>(); + private volatile boolean subscriptionChanged = false; + private List<String> topicsSubscription; + private final ConsumerRebalanceListener rebalanceListener; + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe + ) { + this(consumer, topicsToSubscribe, Set.of(), null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + Set<TopicPartition> partitionsToAssign + ) { + this(consumer, List.of(), partitionsToAssign, null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe, + Set<TopicPartition> partitionsToAssign, + ConsumerRebalanceListener userRebalanceListener + ) { + super("daemon-consumer-assignment", false); + this.consumer = consumer; + this.topicsSubscription = topicsToSubscribe; + this.partitionsToAssign = partitionsToAssign; + this.userRebalanceListener = userRebalanceListener; + + this.rebalanceListener = new ConsumerRebalanceListener() { + @Override + public void onPartitionsAssigned(Collection<TopicPartition> partitions) { + partitionAssignment.addAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsAssigned(partitions); + } + + @Override + public void onPartitionsRevoked(Collection<TopicPartition> partitions) { + partitionAssignment.removeAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsRevoked(partitions); + } + }; + + if (partitionsToAssign.isEmpty()) { + consumer.subscribe(topicsToSubscribe, rebalanceListener); + } else { + consumer.assign(new ArrayList<>(partitionsToAssign)); + } + } + + public Set<TopicPartition> consumerAssignment() { + return Set.copyOf(partitionAssignment); + } + + /** + * Subscribe consumer to a new set of topics. + * Since this method most likely be called from a different thread, this function + * just "schedules" the subscription change, and actual call to consumer.subscribe is done + * in the doWork() method + * <p> + * This method does not allow to change subscription until doWork processes the previous call + * to this method. This is just to avoid race conditions and enough functionality for testing purposes Review Comment: ```suggestion * to this method. This is just to avoid race conditions and provide enough functionality for testing purposes ``` ########## clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java: ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.server.util.ShutdownableThread; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ConsumerAssignmentPoller extends ShutdownableThread { + private final Consumer<byte[], byte[]> consumer; + private final Set<TopicPartition> partitionsToAssign; + private final ConsumerRebalanceListener userRebalanceListener; + + private volatile Throwable thrownException = null; + private volatile int receivedMessages = 0; + + private final Set<TopicPartition> partitionAssignment = new HashSet<>(); + private volatile boolean subscriptionChanged = false; + private List<String> topicsSubscription; + private final ConsumerRebalanceListener rebalanceListener; + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe + ) { + this(consumer, topicsToSubscribe, Set.of(), null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + Set<TopicPartition> partitionsToAssign + ) { + this(consumer, List.of(), partitionsToAssign, null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe, + Set<TopicPartition> partitionsToAssign, + ConsumerRebalanceListener userRebalanceListener + ) { + super("daemon-consumer-assignment", false); + this.consumer = consumer; + this.topicsSubscription = topicsToSubscribe; + this.partitionsToAssign = partitionsToAssign; + this.userRebalanceListener = userRebalanceListener; + + this.rebalanceListener = new ConsumerRebalanceListener() { + @Override + public void onPartitionsAssigned(Collection<TopicPartition> partitions) { + partitionAssignment.addAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsAssigned(partitions); + } + + @Override + public void onPartitionsRevoked(Collection<TopicPartition> partitions) { + partitionAssignment.removeAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsRevoked(partitions); + } + }; + + if (partitionsToAssign.isEmpty()) { + consumer.subscribe(topicsToSubscribe, rebalanceListener); + } else { + consumer.assign(new ArrayList<>(partitionsToAssign)); + } + } + + public Set<TopicPartition> consumerAssignment() { + return Set.copyOf(partitionAssignment); + } + + /** + * Subscribe consumer to a new set of topics. + * Since this method most likely be called from a different thread, this function Review Comment: ```suggestion * Since this method is most likely to be called from a different thread, this function ``` ########## clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java: ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.server.util.ShutdownableThread; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ConsumerAssignmentPoller extends ShutdownableThread { + private final Consumer<byte[], byte[]> consumer; + private final Set<TopicPartition> partitionsToAssign; + private final ConsumerRebalanceListener userRebalanceListener; + + private volatile Throwable thrownException = null; + private volatile int receivedMessages = 0; + + private final Set<TopicPartition> partitionAssignment = new HashSet<>(); + private volatile boolean subscriptionChanged = false; + private List<String> topicsSubscription; + private final ConsumerRebalanceListener rebalanceListener; + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe + ) { + this(consumer, topicsToSubscribe, Set.of(), null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + Set<TopicPartition> partitionsToAssign + ) { + this(consumer, List.of(), partitionsToAssign, null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe, + Set<TopicPartition> partitionsToAssign, + ConsumerRebalanceListener userRebalanceListener + ) { + super("daemon-consumer-assignment", false); + this.consumer = consumer; + this.topicsSubscription = topicsToSubscribe; + this.partitionsToAssign = partitionsToAssign; + this.userRebalanceListener = userRebalanceListener; + + this.rebalanceListener = new ConsumerRebalanceListener() { + @Override + public void onPartitionsAssigned(Collection<TopicPartition> partitions) { + partitionAssignment.addAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsAssigned(partitions); + } + + @Override + public void onPartitionsRevoked(Collection<TopicPartition> partitions) { + partitionAssignment.removeAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsRevoked(partitions); + } + }; + + if (partitionsToAssign.isEmpty()) { + consumer.subscribe(topicsToSubscribe, rebalanceListener); + } else { + consumer.assign(new ArrayList<>(partitionsToAssign)); Review Comment: ```suggestion consumer.assign(List.copyOf(partitionsToAssign)); ``` ########## clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java: ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.server.util.ShutdownableThread; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ConsumerAssignmentPoller extends ShutdownableThread { + private final Consumer<byte[], byte[]> consumer; + private final Set<TopicPartition> partitionsToAssign; + private final ConsumerRebalanceListener userRebalanceListener; + + private volatile Throwable thrownException = null; + private volatile int receivedMessages = 0; + + private final Set<TopicPartition> partitionAssignment = new HashSet<>(); + private volatile boolean subscriptionChanged = false; + private List<String> topicsSubscription; + private final ConsumerRebalanceListener rebalanceListener; + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe + ) { + this(consumer, topicsToSubscribe, Set.of(), null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + Set<TopicPartition> partitionsToAssign + ) { + this(consumer, List.of(), partitionsToAssign, null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe, + Set<TopicPartition> partitionsToAssign, + ConsumerRebalanceListener userRebalanceListener + ) { + super("daemon-consumer-assignment", false); + this.consumer = consumer; + this.topicsSubscription = topicsToSubscribe; + this.partitionsToAssign = partitionsToAssign; + this.userRebalanceListener = userRebalanceListener; + + this.rebalanceListener = new ConsumerRebalanceListener() { + @Override + public void onPartitionsAssigned(Collection<TopicPartition> partitions) { + partitionAssignment.addAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsAssigned(partitions); + } + + @Override + public void onPartitionsRevoked(Collection<TopicPartition> partitions) { + partitionAssignment.removeAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsRevoked(partitions); + } + }; + + if (partitionsToAssign.isEmpty()) { + consumer.subscribe(topicsToSubscribe, rebalanceListener); + } else { + consumer.assign(new ArrayList<>(partitionsToAssign)); + } + } + + public Set<TopicPartition> consumerAssignment() { + return Set.copyOf(partitionAssignment); + } + + /** + * Subscribe consumer to a new set of topics. + * Since this method most likely be called from a different thread, this function + * just "schedules" the subscription change, and actual call to consumer.subscribe is done + * in the doWork() method + * <p> + * This method does not allow to change subscription until doWork processes the previous call Review Comment: ```suggestion * This method does not allow changing the subscription until doWork processes the previous call ``` ########## clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/PlaintextConsumerPollTest.java: ########## @@ -0,0 +1,686 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.test.ClusterInstance; +import org.apache.kafka.common.test.TestUtils; +import org.apache.kafka.common.test.api.ClusterConfigProperty; +import org.apache.kafka.common.test.api.ClusterTest; +import org.apache.kafka.common.test.api.ClusterTestDefaults; +import org.apache.kafka.common.test.api.Type; +import org.apache.kafka.common.utils.Utils; + +import org.junit.jupiter.api.BeforeEach; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.IntStream; + +import static org.apache.kafka.clients.ClientsTestUtils.consumeAndVerifyRecords; +import static org.apache.kafka.clients.ClientsTestUtils.sendRecords; +import static org.apache.kafka.clients.CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.SESSION_TIMEOUT_MS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.CLIENT_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_PROTOCOL_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.MAX_POLL_RECORDS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.CONSUMER_GROUP_HEARTBEAT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.CONSUMER_GROUP_MIN_HEARTBEAT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_PARTITIONS_CONFIG; +import static org.apache.kafka.coordinator.group.GroupCoordinatorConfig.OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ClusterTestDefaults( + types = {Type.KRAFT}, + brokers = PlaintextConsumerPollTest.BROKER_COUNT, + serverProperties = { + @ClusterConfigProperty(key = OFFSETS_TOPIC_PARTITIONS_CONFIG, value = "1"), + @ClusterConfigProperty(key = OFFSETS_TOPIC_REPLICATION_FACTOR_CONFIG, value = "3"), + @ClusterConfigProperty(key = GROUP_MIN_SESSION_TIMEOUT_MS_CONFIG, value = "100"), + @ClusterConfigProperty(key = CONSUMER_GROUP_HEARTBEAT_INTERVAL_MS_CONFIG, value = "500"), + @ClusterConfigProperty(key = CONSUMER_GROUP_MIN_HEARTBEAT_INTERVAL_MS_CONFIG, value = "500"), + @ClusterConfigProperty(key = GROUP_INITIAL_REBALANCE_DELAY_MS_CONFIG, value = "10"), + } +) +public class PlaintextConsumerPollTest { + + public static final int BROKER_COUNT = 3; Review Comment: Use `short` to eliminate the casting. ########## clients/clients-integration-tests/src/test/java/org/apache/kafka/clients/consumer/ConsumerAssignmentPoller.java: ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.server.util.ShutdownableThread; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ConsumerAssignmentPoller extends ShutdownableThread { + private final Consumer<byte[], byte[]> consumer; + private final Set<TopicPartition> partitionsToAssign; + private final ConsumerRebalanceListener userRebalanceListener; + + private volatile Throwable thrownException = null; + private volatile int receivedMessages = 0; + + private final Set<TopicPartition> partitionAssignment = new HashSet<>(); + private volatile boolean subscriptionChanged = false; + private List<String> topicsSubscription; + private final ConsumerRebalanceListener rebalanceListener; + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe + ) { + this(consumer, topicsToSubscribe, Set.of(), null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + Set<TopicPartition> partitionsToAssign + ) { + this(consumer, List.of(), partitionsToAssign, null); + } + + public ConsumerAssignmentPoller( + Consumer<byte[], byte[]> consumer, + List<String> topicsToSubscribe, + Set<TopicPartition> partitionsToAssign, + ConsumerRebalanceListener userRebalanceListener + ) { + super("daemon-consumer-assignment", false); + this.consumer = consumer; + this.topicsSubscription = topicsToSubscribe; + this.partitionsToAssign = partitionsToAssign; + this.userRebalanceListener = userRebalanceListener; + + this.rebalanceListener = new ConsumerRebalanceListener() { + @Override + public void onPartitionsAssigned(Collection<TopicPartition> partitions) { + partitionAssignment.addAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsAssigned(partitions); + } + + @Override + public void onPartitionsRevoked(Collection<TopicPartition> partitions) { + partitionAssignment.removeAll(partitions); + if (userRebalanceListener != null) + userRebalanceListener.onPartitionsRevoked(partitions); + } + }; + + if (partitionsToAssign.isEmpty()) { + consumer.subscribe(topicsToSubscribe, rebalanceListener); + } else { + consumer.assign(new ArrayList<>(partitionsToAssign)); + } + } + + public Set<TopicPartition> consumerAssignment() { + return Set.copyOf(partitionAssignment); + } + + /** + * Subscribe consumer to a new set of topics. + * Since this method most likely be called from a different thread, this function + * just "schedules" the subscription change, and actual call to consumer.subscribe is done Review Comment: ```suggestion * just "schedules" the subscription change, and actual call to {@link org.apache.kafka.clients.consumer.Consumer#subscribe(Collection)} is done ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
