lucasbru commented on code in PR #18446: URL: https://github.com/apache/kafka/pull/18446#discussion_r1908358367
########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Contains all information related to a topology of a Streams group. + * <p> + * This class is immutable and is fully backed by records stored in the __consumer_offsets topic. + * + * @param topologyEpoch The epoch of the topology. + * @param subtopologies The subtopologies of the topology containing information about source topics, + * repartition topics, changelog topics, co-partition groups etc. + */ + public record StreamsTopology (int topologyEpoch, Review Comment: extra space ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Contains all information related to a topology of a Streams group. Review Comment: We should specify that this is the requested topology that has not been configured according to the topics available on the broker. So differentiate to ConfiguredTopology. It doesn't really contain "all information related to a topology". ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Contains all information related to a topology of a Streams group. + * <p> + * This class is immutable and is fully backed by records stored in the __consumer_offsets topic. + * + * @param topologyEpoch The epoch of the topology. + * @param subtopologies The subtopologies of the topology containing information about source topics, + * repartition topics, changelog topics, co-partition groups etc. + */ + public record StreamsTopology (int topologyEpoch, + Map<String, Subtopology> subtopologies) { + + /** + * Returns the set of topics required by the topology. Review Comment: These javadocs seem somewhat void of content. They say nothing that isn't implied by the function name. If we are going to put these javadocs, please specify what it means to be "required". ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Contains all information related to a topology of a Streams group. + * <p> + * This class is immutable and is fully backed by records stored in the __consumer_offsets topic. Review Comment: I know I wrote that, so I'm talking to myself here, but the `subtopologies` objects aren't actually immutable. We don't want to mutate it, but saying it's immutable is probably misleady. ########## group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsTopologyTest.java: ########## @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.kafka.common.utils.Utils.mkEntry; +import static org.apache.kafka.common.utils.Utils.mkMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class StreamsTopologyTest { + + private static final String SUBTOPOLOGY_ID_1 = "subtopology-1"; + private static final String SUBTOPOLOGY_ID_2 = "subtopology-2"; + private static final String SOURCE_TOPIC_1 = "source-topic-1"; + private static final String SOURCE_TOPIC_2 = "source-topic-2"; + private static final String SOURCE_TOPIC_3 = "source-topic-3"; + private static final String REPARTITION_TOPIC_1 = "repartition-topic-1"; + private static final String REPARTITION_TOPIC_2 = "repartition-topic-2"; + private static final String REPARTITION_TOPIC_3 = "repartition-topic-3"; + private static final String CHANGELOG_TOPIC_1 = "changelog-1"; + private static final String CHANGELOG_TOPIC_2 = "changelog-2"; + private static final String CHANGELOG_TOPIC_3 = "changelog-3"; + + @Test + public void requiredTopicsShouldBeCorrect() { + Map<String, Subtopology> subtopologies = mkMap( + mkEntry(SUBTOPOLOGY_ID_1, mkSubtopology1()), + mkEntry(SUBTOPOLOGY_ID_2, mkSubtopology2()) + ); + StreamsTopology topology = new StreamsTopology(1, subtopologies); + Set<String> expectedTopics = Set.of( + SOURCE_TOPIC_1, SOURCE_TOPIC_2, SOURCE_TOPIC_3, + REPARTITION_TOPIC_1, REPARTITION_TOPIC_2, REPARTITION_TOPIC_3, + CHANGELOG_TOPIC_1, CHANGELOG_TOPIC_2, CHANGELOG_TOPIC_3 + ); + + assertEquals(expectedTopics, topology.requiredTopics()); + } + + @Test + public void fromRecordShouldCreateCorrectTopology() { + StreamsGroupTopologyValue record = new StreamsGroupTopologyValue() + .setEpoch(1) + .setSubtopologies(Arrays.asList(mkSubtopology1(), mkSubtopology2())); + StreamsTopology topology = StreamsTopology.fromRecord(record); + assertEquals(1, topology.topologyEpoch()); + assertEquals(2, topology.subtopologies().size()); + assertTrue(topology.subtopologies().containsKey(SUBTOPOLOGY_ID_1)); + assertEquals(mkSubtopology1(), topology.subtopologies().get(SUBTOPOLOGY_ID_1)); + assertTrue(topology.subtopologies().containsKey(SUBTOPOLOGY_ID_2)); + assertEquals(mkSubtopology2(), topology.subtopologies().get(SUBTOPOLOGY_ID_2)); + } + + private Subtopology mkSubtopology1() { + return new Subtopology() + .setSubtopologyId(SUBTOPOLOGY_ID_1) + .setSourceTopics(List.of( + SOURCE_TOPIC_1, + SOURCE_TOPIC_2, + REPARTITION_TOPIC_1, + REPARTITION_TOPIC_2 + )) + .setRepartitionSourceTopics(List.of( + new TopicInfo().setName(REPARTITION_TOPIC_1), + new TopicInfo().setName(REPARTITION_TOPIC_2) + )) + .setRepartitionSinkTopics(List.of( + REPARTITION_TOPIC_3 + )) + .setStateChangelogTopics(List.of( + new TopicInfo().setName(CHANGELOG_TOPIC_1), + new TopicInfo().setName(CHANGELOG_TOPIC_2) + )) + .setCopartitionGroups(List.of( + new StreamsGroupTopologyValue.CopartitionGroup() + .setRepartitionSourceTopics(List.of((short) 0)) + .setSourceTopics(List.of((short) 0)), + new StreamsGroupTopologyValue.CopartitionGroup() + .setRepartitionSourceTopics(List.of((short) 1)) + .setSourceTopics(List.of((short) 1)) + )); + } + + private Subtopology mkSubtopology2() { Review Comment: I personally preferred the unit tests when they were self-contained, which is easier to understand. There is no benefit to making this a separate method, and `mkSubtopology2` doesn't mean anything. ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Contains all information related to a topology of a Streams group. + * <p> + * This class is immutable and is fully backed by records stored in the __consumer_offsets topic. + * Review Comment: I think it's also useful to specify, that the main purpose of this class compared to `StreamGroupTopologyValue` is that we have a map for looking up a subtopology by name in constant time. ########## group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/StreamsTopology.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams; + +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.Subtopology; +import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue.TopicInfo; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Contains all information related to a topology of a Streams group. + * <p> + * This class is immutable and is fully backed by records stored in the __consumer_offsets topic. + * + * @param topologyEpoch The epoch of the topology. + * @param subtopologies The subtopologies of the topology containing information about source topics, + * repartition topics, changelog topics, co-partition groups etc. + */ + public record StreamsTopology (int topologyEpoch, + Map<String, Subtopology> subtopologies) { + + /** + * Returns the set of topics required by the topology. + * + * @return set of topics required by the topology + */ + public Set<String> requiredTopics() { + return subtopologies.values().stream() + .flatMap(x -> + Stream.concat( + Stream.concat( + x.sourceTopics().stream(), + x.repartitionSourceTopics().stream().map(TopicInfo::name) + ), + x.stateChangelogTopics().stream().map(TopicInfo::name) + ) + ).collect(Collectors.toSet()); + } + + /** + * Creates a instance of StreamsTopology from a StreamsGroupTopologyValue record. + * + * @param record StreamsGroupTopologyValue record Review Comment: I think most other classes use full sentences, with initial character capitalized and trailing period in `@param` and `@return`. Applies to several javadoc comments here. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
