This is an automated email from the ASF dual-hosted git repository.
lucasbru pushed a change to branch kip1071
in repository https://gitbox.apache.org/repos/asf/kafka.git
from ab0df20489a KAFKA-17592; Support for SubscribedTopicsRegex in
ConsumerGroupHeartbeat RPC (#17257)
new 9271d829d05 Very initial (compiling) streams RPCs.
new 4096e719157 StreamsInitialize RPC and some errors
new e13a5d18d72 Define RPCs to perform client-side assignment
new 1be15da2751 Specify AsyncKafkaConsumer interface with Streams for the
Rebalance PoC
new e79b30048b9 Implement StreamsInitialize request manager
new 7a5293a52ef Create and inject StreamsHeartbeatRequestManager
new dd9387faa20 Minor: Test fixes
new 31135af1cbc Define initial record types for the consumer offset topic
new fd4600b9b62 Implement InitStreamsApp RPC in the group coordinator
new 2fa4de23780 Add Assignment class in group coordinator for Streams
new aff4d176073 Minor: Rename TaskId to TaskIds
new 025a0a50c93 Basic heartbeat RPC handler
new 168a1c7f72c Add streams group member
new 1257cb3a95b Define the assignor interface for streams and implement a
simple assignor
new b165df7a7bd Create assignment-related classes for streams groups
new f57440bcb86 Replay all streams-related records
new 76472c7dcfa Get SmokeTestDriverIntegrationTest working
new 61f68434e7e Rebased on AK trunk 2024-07-16
new 33a535d9e74 Update RPCs
new 6ac06566243 Implement DescribeStreamsGroup RPC handling
new 4dc40157ada Rebase on AK trunk 2024-08-15
new 656916309bf Minor: Revert test changes
new ddf464d3b1c StickyTaskAssignor
new ff570e1677a Rename streamsHeartbeatX and streamsInitializeX to
streamsGroupX
new 7c032dc7adb Improve the Streams group initialization handler
new 7e9d6028612 Streams reconciliation logic
new 5138d3df823 Add broker configs for streams group
new d4fb972f45c Rebase on AK trunk 2024-09-25
The 28 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
checkstyle/suppressions.xml | 10 +-
.../consumer/internals/AsyncKafkaConsumer.java | 24 +-
.../internals/ConsumerDelegateCreator.java | 6 +-
.../consumer/internals/RequestManagers.java | 44 +-
.../internals/StreamsAssignmentInterface.java | 296 ++++
.../StreamsGroupHeartbeatRequestManager.java | 628 +++++++
.../StreamsGroupInitializeRequestManager.java | 150 ++
.../events/ApplicationEventProcessor.java | 12 +-
.../errors/StreamsGroupUninitializedException.java | 23 +
.../StreamsInconsistentTopologyException.java | 23 +
.../errors/StreamsInvalidAssignmentException.java | 23 +
.../errors/StreamsInvalidTopologyException.java | 23 +
.../StreamsMissingSourceTopicsException.java | 23 +
.../StreamsShutdownApplicationException.java | 23 +
.../org/apache/kafka/common/protocol/ApiKeys.java | 6 +-
.../org/apache/kafka/common/protocol/Errors.java | 12 +-
.../kafka/common/requests/AbstractRequest.java | 6 +
.../kafka/common/requests/AbstractResponse.java | 6 +
.../requests/StreamsGroupDescribeRequest.java | 98 ++
.../requests/StreamsGroupDescribeResponse.java | 76 +
.../requests/StreamsGroupHeartbeatRequest.java | 88 +
.../requests/StreamsGroupHeartbeatResponse.java | 78 +
.../requests/StreamsGroupInitializeRequest.java | 77 +
.../requests/StreamsGroupInitializeResponse.java | 71 +
.../common/message/OffsetCommitRequest.json | 2 +-
.../common/message/OffsetFetchRequest.json | 4 +-
.../message/StreamsGroupDescribeRequest.json | 29 +
.../message/StreamsGroupDescribeResponse.json | 151 ++
.../message/StreamsGroupHeartbeatRequest.json | 89 +
.../message/StreamsGroupHeartbeatResponse.json | 111 ++
.../message/StreamsGroupInitializeRequest.json | 64 +
.../message/StreamsGroupInitializeResponse.json | 42 +
.../consumer/internals/AsyncKafkaConsumerTest.java | 6 +-
.../consumer/internals/RequestManagersTest.java | 3 +-
.../StreamsGroupHeartbeatRequestManagerTest.java | 382 +++++
.../StreamsGroupInitializeRequestManagerTest.java | 131 ++
.../events/ApplicationEventProcessorTest.java | 6 +-
.../kafka/common/requests/RequestResponseTest.java | 51 +
.../common/runtime/CoordinatorRuntime.java | 5 +
.../common/runtime/CoordinatorTimer.java | 9 +
.../common/runtime/MockCoordinatorTimer.java | 3 +-
.../group/GroupCoordinatorAdapter.scala | 29 +-
core/src/main/scala/kafka/server/KafkaApis.scala | 116 ++
core/src/main/scala/kafka/server/KafkaConfig.scala | 7 +
.../kafka/api/IntegrationTestHarness.scala | 4 +
.../kafka/server/QuorumTestHarness.scala | 6 +-
.../src/test/scala/kafka/utils/TestInfoUtils.scala | 4 +
.../group/GroupCoordinatorAdapterTest.scala | 34 +-
.../scala/unit/kafka/server/KafkaApisTest.scala | 268 +++
.../scala/unit/kafka/server/KafkaConfigTest.scala | 6 +
gradle/spotbugs-exclude.xml | 5 +
.../org/apache/kafka/coordinator/group/Group.java | 1 +
.../kafka/coordinator/group/GroupCoordinator.java | 46 +
.../coordinator/group/GroupCoordinatorConfig.java | 51 +
.../group/GroupCoordinatorRecordHelpers.java | 58 +-
.../coordinator/group/GroupCoordinatorService.java | 118 ++
.../coordinator/group/GroupCoordinatorShard.java | 147 +-
.../coordinator/group/GroupMetadataManager.java | 1784 +++++++++++++++++++-
.../org/apache/kafka/coordinator/group/Utils.java | 28 +
.../group/metrics/GroupCoordinatorMetrics.java | 115 +-
.../metrics/GroupCoordinatorMetricsShard.java | 140 ++
.../coordinator/group/streams/Assignment.java | 129 ++
.../streams/CoordinatorStreamsRecordHelpers.java | 432 +++++
.../group/streams/CurrentAssignmentBuilder.java | 585 +++++++
.../coordinator/group/streams/MemberState.java | 74 +
.../coordinator/group/streams/StreamsGroup.java | 1232 ++++++++++++++
.../group/streams/StreamsGroupMember.java | 751 ++++++++
.../coordinator/group/streams/StreamsTopology.java | 154 ++
.../group/streams/TargetAssignmentBuilder.java | 368 ++++
.../coordinator/group/streams/TopicMetadata.java | 165 ++
.../group/streams/TopologyMetadata.java | 111 ++
.../group/taskassignor/AssignmentMemberSpec.java | 182 ++
.../group/taskassignor/GroupAssignment.java | 67 +
.../coordinator/group/taskassignor/GroupSpec.java | 39 +
.../group/taskassignor/GroupSpecImpl.java | 96 ++
.../group/taskassignor/MemberAssignment.java | 91 +
.../group/taskassignor/MockAssignor.java | 115 ++
.../group/taskassignor/ProcessState.java | 144 ++
.../group/taskassignor/StickyTaskAssignor.java | 437 +++++
.../group/taskassignor/TaskAssignor.java | 45 +
.../group/taskassignor/TaskAssignorException.java | 33 +
.../coordinator/group/taskassignor/TaskId.java | 70 +
.../group/taskassignor/TopologyDescriber.java | 39 +
.../StreamsGroupCurrentMemberAssignmentKey.json | 28 +
.../StreamsGroupCurrentMemberAssignmentValue.json | 50 +
.../message/StreamsGroupMemberMetadataKey.json | 28 +
.../message/StreamsGroupMemberMetadataValue.json | 60 +
.../common/message/StreamsGroupMetadataKey.json | 26 +
.../common/message/StreamsGroupMetadataValue.json | 26 +
.../message/StreamsGroupPartitionMetadataKey.json | 26 +
.../StreamsGroupPartitionMetadataValue.json | 40 +
.../StreamsGroupTargetAssignmentMemberKey.json | 28 +
.../StreamsGroupTargetAssignmentMemberValue.json | 38 +
.../StreamsGroupTargetAssignmentMetadataKey.json | 26 +
.../StreamsGroupTargetAssignmentMetadataValue.json | 26 +
.../common/message/StreamsGroupTopologyKey.json | 26 +
.../common/message/StreamsGroupTopologyValue.json | 61 +
.../group/GroupCoordinatorConfigTest.java | 30 +-
.../group/GroupCoordinatorServiceTest.java | 376 +++++
.../group/GroupCoordinatorShardTest.java | 79 +-
.../group/GroupMetadataManagerTest.java | 644 +++++++
.../group/GroupMetadataManagerTestContext.java | 176 ++
.../kafka/coordinator/group/MockTaskAssignor.java | 46 +
.../group/OffsetMetadataManagerTest.java | 2 +
.../coordinator/group/streams/AssignmentTest.java | 134 ++
.../CoordinatorStreamsRecordHelpersTest.java | 110 ++
.../streams/CurrentAssignmentBuilderTest.java | 728 ++++++++
.../group/streams/StreamsGroupBuilder.java | 99 ++
.../group/streams/StreamsGroupMemberTest.java | 394 +++++
.../group/streams/StreamsGroupTest.java | 763 +++++++++
.../group/streams/StreamsTopologyTest.java | 182 ++
.../group/streams/TargetAssignmentBuilderTest.java | 815 +++++++++
.../group/streams/TaskAssignmentTestUtil.java | 117 ++
.../group/taskassignor/GroupSpecImplTest.java | 72 +
.../group/taskassignor/MockAssignorTest.java | 211 +++
.../group/taskassignor/StickyTaskAssignorTest.java | 1201 +++++++++++++
.../kafka/server/config/AbstractKafkaConfig.java | 1 +
.../org/apache/kafka/streams/GroupProtocol.java | 43 +
.../apache/kafka/streams/KafkaClientSupplier.java | 11 +
.../org/apache/kafka/streams/StreamsConfig.java | 17 +
.../internals/DefaultKafkaClientSupplier.java | 12 +
.../streams/processor/internals/StreamThread.java | 195 ++-
.../internals/StreamsRebalanceListener.java | 3 +
.../streams/processor/internals/TaskManager.java | 6 +-
.../SmokeTestDriverIntegrationTest.java | 89 +-
.../processor/internals/StreamThreadTest.java | 50 +-
.../org/apache/kafka/test/MockClientSupplier.java | 5 +
streams/src/test/resources/log4j.properties | 1 +
.../apache/kafka/streams/TopologyTestDriver.java | 6 +-
129 files changed, 18165 insertions(+), 182 deletions(-)
create mode 100644
clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsAssignmentInterface.java
create mode 100644
clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManager.java
create mode 100644
clients/src/main/java/org/apache/kafka/clients/consumer/internals/StreamsGroupInitializeRequestManager.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/errors/StreamsGroupUninitializedException.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/errors/StreamsInconsistentTopologyException.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/errors/StreamsInvalidAssignmentException.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/errors/StreamsInvalidTopologyException.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/errors/StreamsMissingSourceTopicsException.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/errors/StreamsShutdownApplicationException.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/requests/StreamsGroupDescribeRequest.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/requests/StreamsGroupDescribeResponse.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/requests/StreamsGroupHeartbeatRequest.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/requests/StreamsGroupHeartbeatResponse.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/requests/StreamsGroupInitializeRequest.java
create mode 100644
clients/src/main/java/org/apache/kafka/common/requests/StreamsGroupInitializeResponse.java
create mode 100644
clients/src/main/resources/common/message/StreamsGroupDescribeRequest.json
create mode 100644
clients/src/main/resources/common/message/StreamsGroupDescribeResponse.json
create mode 100644
clients/src/main/resources/common/message/StreamsGroupHeartbeatRequest.json
create mode 100644
clients/src/main/resources/common/message/StreamsGroupHeartbeatResponse.json
create mode 100644
clients/src/main/resources/common/message/StreamsGroupInitializeRequest.json
create mode 100644
clients/src/main/resources/common/message/StreamsGroupInitializeResponse.json
create mode 100644
clients/src/test/java/org/apache/kafka/clients/consumer/internals/StreamsGroupHeartbeatRequestManagerTest.java
create mode 100644
clients/src/test/java/org/apache/kafka/clients/consumer/internals/StreamsGroupInitializeRequestManagerTest.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/Assignment.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/CoordinatorStreamsRecordHelpers.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/CurrentAssignmentBuilder.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberState.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroup.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMember.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TopicMetadata.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TopologyMetadata.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/AssignmentMemberSpec.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/GroupAssignment.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/GroupSpec.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/GroupSpecImpl.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/MemberAssignment.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/MockAssignor.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/ProcessState.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/StickyTaskAssignor.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/TaskAssignor.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/TaskAssignorException.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/TaskId.java
create mode 100644
group-coordinator/src/main/java/org/apache/kafka/coordinator/group/taskassignor/TopologyDescriber.java
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupCurrentMemberAssignmentKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupCurrentMemberAssignmentValue.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupMemberMetadataKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupMemberMetadataValue.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupMetadataKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupMetadataValue.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupPartitionMetadataKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupPartitionMetadataValue.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupTargetAssignmentMemberKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupTargetAssignmentMemberValue.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupTargetAssignmentMetadataKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupTargetAssignmentMetadataValue.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupTopologyKey.json
create mode 100644
group-coordinator/src/main/resources/common/message/StreamsGroupTopologyValue.json
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/MockTaskAssignor.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/AssignmentTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/CoordinatorStreamsRecordHelpersTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/CurrentAssignmentBuilderTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupBuilder.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupMemberTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsTopologyTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TaskAssignmentTestUtil.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/taskassignor/GroupSpecImplTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/taskassignor/MockAssignorTest.java
create mode 100644
group-coordinator/src/test/java/org/apache/kafka/coordinator/group/taskassignor/StickyTaskAssignorTest.java
create mode 100644
streams/src/main/java/org/apache/kafka/streams/GroupProtocol.java
