chia7712 commented on code in PR #22576: URL: https://github.com/apache/kafka/pull/22576#discussion_r3412734383
########## transaction-coordinator/src/main/java/org/apache/kafka/coordinator/transaction/TxnMarkerQueue.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.transaction; + +import org.apache.kafka.common.Node; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.BiConsumer; + +public class TxnMarkerQueue { + + private static final Logger LOG = LoggerFactory.getLogger(TxnMarkerQueue.class); + + // keep track of the requests per txn topic partition so we can easily clear the queue + // during partition emigration + private final ConcurrentHashMap<Integer, BlockingQueue<PendingCompleteTxnAndMarkerEntry>> markersPerTxnTopicPartition = + new ConcurrentHashMap<>(); + + private volatile Node destination; + + public TxnMarkerQueue(Node destination) { + this.destination = destination; + } + + public Node destination() { + return destination; + } + + public void setDestination(Node destination) { + this.destination = destination; + } + + public Optional<BlockingQueue<PendingCompleteTxnAndMarkerEntry>> removeMarkersForTxnTopicPartition(int partition) { + return Optional.ofNullable(markersPerTxnTopicPartition.remove(partition)); + } + + public void addMarkers(int txnTopicPartition, PendingCompleteTxnAndMarkerEntry pendingCompleteTxnAndMarker) { + BlockingQueue<PendingCompleteTxnAndMarkerEntry> queue = markersPerTxnTopicPartition.computeIfAbsent(txnTopicPartition, partition -> { + // Note that this may get called more than once if threads have a close race while adding new queue. + LOG.info("Creating new marker queue for txn partition {} to destination broker {}", txnTopicPartition, destination.id()); + return new LinkedBlockingQueue<>(); + }); + queue.add(pendingCompleteTxnAndMarker); + + if (markersPerTxnTopicPartition.get(txnTopicPartition) != queue) { + // This could happen if the queue got removed concurrently. + // Note that it could create an unexpected state when the queue is removed from + // removeMarkersForTxnTopicPartition, we could have: + // + // 1. [addMarkers] Retrieve queue. + // 2. [removeMarkersForTxnTopicPartition] Remove queue. + // 3. [removeMarkersForTxnTopicPartition] Iterate over queue, but not removeMarkersForTxn because queue is empty. + // 4. [addMarkers] Add markers to the queue. + // + // Now we've effectively removed the markers while transactionsWithPendingMarkers has an entry. + // + // While this could lead to an orphan entry in transactionsWithPendingMarkers, sending new markers + // will fix the state, so it shouldn't impact the state machine operation. + LOG.warn("Added {} to dead queue for txn partition {} to destination broker {}", + pendingCompleteTxnAndMarker, txnTopicPartition, destination.id()); + } + } + + public void forEachTxnTopicPartition(BiConsumer<Integer, BlockingQueue<PendingCompleteTxnAndMarkerEntry>> f) { + markersPerTxnTopicPartition.forEach((partition, queue) -> { + if (!queue.isEmpty()) { Review Comment: I'm not sure why we need this check. It seems we can safely remove it without letting the smoke out ########## transaction-coordinator/src/main/java/org/apache/kafka/coordinator/transaction/PendingCompleteTxn.java: ########## @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.transaction; + +public record PendingCompleteTxn( + String transactionalId, + int coordinatorEpoch, + TransactionMetadata txnMetadata, + TxnTransitMetadata newMetadata +) { + + @Override Review Comment: What do you think about relying on the auto-generated code? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
