poorbarcode commented on code in PR #16931: URL: https://github.com/apache/pulsar/pull/16931#discussion_r999249164
########## pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java: ########## @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.systopic; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.TopicName; + +@Slf4j +public class TransactionBufferSnapshotBaseSystemTopicClient<T> extends SystemTopicClientBase<T> { + + protected final SystemTopicTxnBufferSnapshotService<T> systemTopicTxnBufferSnapshotService; + protected final Class<T> schemaType; + + public TransactionBufferSnapshotBaseSystemTopicClient(PulsarClient client, + TopicName topicName, + SystemTopicTxnBufferSnapshotService<T> + systemTopicTxnBufferSnapshotService, + Class<T> schemaType) { + super(client, topicName); + this.systemTopicTxnBufferSnapshotService = systemTopicTxnBufferSnapshotService; + this.schemaType = schemaType; + } + + protected void removeWriter(Writer<T> writer) { + writers.remove(writer); + this.systemTopicTxnBufferSnapshotService.removeClient(topicName, this); + } + + protected void removeReader(Reader<T> reader) { + readers.remove(reader); + this.systemTopicTxnBufferSnapshotService.removeClient(topicName, this); + } + + protected static class TransactionBufferSnapshotWriter<T> implements Writer<T> { + + protected final Producer<T> producer; + protected final TransactionBufferSnapshotBaseSystemTopicClient<T> + transactionBufferSnapshotBaseSystemTopicClient; + + protected TransactionBufferSnapshotWriter(Producer<T> producer, + TransactionBufferSnapshotBaseSystemTopicClient<T> + transactionBufferSnapshotBaseSystemTopicClient) { + this.producer = producer; + this.transactionBufferSnapshotBaseSystemTopicClient = transactionBufferSnapshotBaseSystemTopicClient; + } + + @Override + public MessageId write(T t, String key) + throws PulsarClientException { + return producer.newMessage().key(key) + .value(t).send(); + } + + @Override + public CompletableFuture<MessageId> writeAsync(T t, String key) { + return producer.newMessage() + .key(key) + .value(t).sendAsync(); + } + + @Override + public MessageId delete(T t, String key) + throws PulsarClientException { + return producer.newMessage() + .key(key) + .value(null) + .send(); + } + + @Override + public CompletableFuture<MessageId> deleteAsync(T t, String key) { + return producer.newMessage() + .key(key) + .value(null) + .sendAsync(); + } + + @Override + public void close() throws IOException { + this.producer.close(); + transactionBufferSnapshotBaseSystemTopicClient.removeWriter(this); Review Comment: This code `transactionBufferSnapshotBaseSystemTopicClient.removeWriter(this); ` has already been executed once in `closeAsync`. should we remove this line? Same for `TransactionBufferSnapshotReader` ########## pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java: ########## @@ -23,64 +23,63 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; -import org.apache.pulsar.broker.systopic.SystemTopicClient.Reader; -import org.apache.pulsar.broker.systopic.SystemTopicClient.Writer; -import org.apache.pulsar.broker.systopic.TransactionBufferSystemTopicClient; -import org.apache.pulsar.broker.transaction.buffer.matadata.TransactionBufferSnapshot; +import org.apache.pulsar.broker.systopic.SystemTopicClientBase; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException.InvalidTopicNameException; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; -public class SystemTopicBaseTxnBufferSnapshotService implements TransactionBufferSnapshotService { +public class SystemTopicTxnBufferSnapshotService<T> { - private final Map<TopicName, SystemTopicClient<TransactionBufferSnapshot>> clients; + protected final Map<TopicName, SystemTopicClient<T>> clients; + protected final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory; - private final NamespaceEventsSystemTopicFactory namespaceEventsSystemTopicFactory; + protected final Class<T> schemaType; + protected final EventType systemTopicType; - public SystemTopicBaseTxnBufferSnapshotService(PulsarClient client) { + public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType systemTopicType, + Class<T> schemaType) { this.namespaceEventsSystemTopicFactory = new NamespaceEventsSystemTopicFactory(client); + this.systemTopicType = systemTopicType; + this.schemaType = schemaType; this.clients = new ConcurrentHashMap<>(); } - @Override - public CompletableFuture<Writer<TransactionBufferSnapshot>> createWriter(TopicName topicName) { + public CompletableFuture<SystemTopicClient.Writer<T>> createWriter(TopicName topicName) { return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newWriterAsync); } - private CompletableFuture<SystemTopicClient<TransactionBufferSnapshot>> getTransactionBufferSystemTopicClient( - TopicName topicName) { - TopicName systemTopicName = NamespaceEventsSystemTopicFactory - .getSystemTopicName(topicName.getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT); - if (systemTopicName == null) { - return FutureUtil.failedFuture( - new InvalidTopicNameException("Can't create SystemTopicBaseTxnBufferSnapshotService, " - + "because the topicName is null!")); - } - return CompletableFuture.completedFuture(clients.computeIfAbsent(systemTopicName, - (v) -> namespaceEventsSystemTopicFactory - .createTransactionBufferSystemTopicClient(topicName.getNamespaceObject(), this))); - } - - @Override - public CompletableFuture<Reader<TransactionBufferSnapshot>> createReader(TopicName topicName) { + public CompletableFuture<SystemTopicClient.Reader<T>> createReader(TopicName topicName) { return getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newReaderAsync); } - @Override - public void removeClient(TopicName topicName, - TransactionBufferSystemTopicClient transactionBufferSystemTopicClient) { + public void removeClient(TopicName topicName, SystemTopicClientBase<T> transactionBufferSystemTopicClient) { if (transactionBufferSystemTopicClient.getReaders().size() == 0 && transactionBufferSystemTopicClient.getWriters().size() == 0) { clients.remove(topicName); Review Comment: There are many System topics under each namespace. I think these resources should be released( `gc(SystemTopicClient)` ) on the namespace deleted event. Now such an implementation will have a race condition: - close the last `writer` - registry a new `writer` | close the last `writer` | registry a new `writer` | |---|---| | check `transactionBufferSystemTopicClient.getWriters().size() == 0` : `true` | | | | registry new one | | remove(topic) | | -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
