hachikuji commented on a change in pull request #11688:
URL: https://github.com/apache/kafka/pull/11688#discussion_r803159171
##########
File path:
clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
##########
@@ -699,11 +702,15 @@ public void handle(JoinGroupResponse joinResponse,
RequestFuture<ByteBuffer> fut
return sendSyncGroupRequest(requestBuilder);
}
- private RequestFuture<ByteBuffer> onJoinLeader(JoinGroupResponse
joinResponse) {
+ private RequestFuture<ByteBuffer> onLeaderElected(JoinGroupResponse
joinResponse) {
try {
// perform the leader synchronization and send back the assignment
for the group
- Map<String, ByteBuffer> groupAssignment =
performAssignment(joinResponse.data().leader(),
joinResponse.data().protocolName(),
- joinResponse.data().members());
+ Map<String, ByteBuffer> groupAssignment = onLeaderElected(
+ joinResponse.data().leader(),
+ joinResponse.data().protocolName(),
+ joinResponse.data().members(),
+ joinResponse.data().skipAssignment()
+ );
Review comment:
Do you think it is worthwhile validating that `groupAssignment` is empty
when `skipAssignment` is set?
##########
File path:
clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
##########
@@ -1691,6 +1717,64 @@ public void testMetadataChangeTriggersRebalance() {
assertTrue(coordinator.rejoinNeededOrPending());
}
+ @Test
+ public void testStaticLeaderRejoinsGroupAndCanTriggersRebalance() {
+ // ensure metadata is up-to-date for leader
+ subscriptions.subscribe(singleton(topic1), rebalanceListener);
+ client.updateMetadata(metadataResponse);
+
+ client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+ coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
+
+ // the leader is responsible for picking up metadata changes and
forcing a group rebalance.
+ // note that `MockPartitionAssignor.prepare` is not called therefore
calling `MockPartitionAssignor.assign`
+ // will throw a IllegalStateException. this indirectly verifies that
`assign` is correctly skipped.
+ Map<String, List<String>> memberSubscriptions =
singletonMap(consumerId, singletonList(topic1));
+ client.prepareResponse(joinGroupLeaderResponse(1, consumerId,
memberSubscriptions, true, Errors.NONE));
+ client.prepareResponse(syncGroupResponse(singletonList(t1p),
Errors.NONE));
+
+ coordinator.poll(time.timer(Long.MAX_VALUE));
+
+ assertFalse(coordinator.rejoinNeededOrPending());
+
+ // a new partition is added to the topic
+
metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWith(1,
singletonMap(topic1, 2)), false, time.milliseconds());
+ coordinator.maybeUpdateSubscriptionMetadata();
+
+ // we should detect the change and ask for reassignment
+ assertTrue(coordinator.rejoinNeededOrPending());
+ }
+
+ @Test
+ public void
testStaticLeaderRejoinsGroupAndCanDetectMetadataChangesForOtherMembers() {
+ // ensure metadata is up-to-date for leader
+ subscriptions.subscribe(singleton(topic1), rebalanceListener);
+ client.updateMetadata(metadataResponse);
+
+ client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
+ coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
+
+ // the leader is responsible for picking up metadata changes and
forcing a group rebalance.
+ // note that `MockPartitionAssignor.prepare` is not called therefore
calling `MockPartitionAssignor.assign`
+ // will throw a IllegalStateException. this indirectly verifies that
`assign` is correctly skipped.
+ Map<String, List<String>> memberSubscriptions = new HashMap<>();
+ memberSubscriptions.put(consumerId, singletonList(topic1));
+ memberSubscriptions.put(consumerId2, singletonList(topic2));
+ client.prepareResponse(joinGroupLeaderResponse(1, consumerId,
memberSubscriptions, true, Errors.NONE));
+ client.prepareResponse(syncGroupResponse(singletonList(t1p),
Errors.NONE));
+
+ coordinator.poll(time.timer(Long.MAX_VALUE));
+
+ assertFalse(coordinator.rejoinNeededOrPending());
+
Review comment:
Could we add an assertion for `SubscriptionState.metadataTopics`?
##########
File path: clients/src/main/resources/common/message/JoinGroupResponse.json
##########
@@ -49,6 +51,8 @@
"about": "The group protocol selected by the coordinator." },
{ "name": "Leader", "type": "string", "versions": "0+",
"about": "The leader of the group." },
+ { "name": "SkipAssignment", "type": "bool", "versions": "9+", "default":
"false",
+ "about": "True is the leader must skip running the assignment." },
Review comment:
nit: is -> if
##########
File path: core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
##########
@@ -1791,4 +1793,34 @@ class PlaintextConsumerTest extends BaseConsumerTest {
assertTrue(records2.count() == 1 &&
records2.records(tp).asScala.head.offset == 1,
"Expected consumer2 to consume one message from offset 1, which is the
committed offset of consumer1")
}
+
+ @Test
+ def testStaticConsumerDetectsNewPartitionCreatedAfterRestart(): Unit = {
+ val foo = "foo"
+ val foo0 = new TopicPartition(foo, 0)
+ val foo1 = new TopicPartition(foo, 1)
+
+ val admin = createAdminClient()
+ admin.createTopics(Seq(new NewTopic(foo, 1,
1.asInstanceOf[Short])).asJava).all.get
Review comment:
nit: I think 1.toShort works?
##########
File path:
clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
##########
@@ -645,6 +651,15 @@ private void maybeUpdateGroupSubscription(String
assignorName,
isLeader = true;
+ if (skipAssignment) {
+ log.info("Skipped assignment for returning static leader at
generation {}. The static leader " +
+ "will continue with its existing assignment.",
generation().generationId);
+ assignmentSnapshot = metadataSnapshot;
+ return Collections.emptyMap();
+ }
+
+ Map<String, ByteBuffer> groupAssignment = new HashMap<>();
Review comment:
nit: seemed reasonable at its original location
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
