poorbarcode commented on code in PR #19641:
URL: https://github.com/apache/pulsar/pull/19641#discussion_r1121218809
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java:
##########
@@ -442,11 +443,13 @@ public class PersistentWorker {
private final PersistentTopic topic;
//Persistent snapshot segment and index at the single thread.
- private final
CompletableFuture<SystemTopicClient.Writer<TransactionBufferSnapshotSegment>>
+ private final ReferenceCountedWriter<TransactionBufferSnapshotSegment>
snapshotSegmentsWriterFuture;
Review Comment:
Should we rename the variable `snapshotSegmentsWriterFuture` to
`snapshotSegmentsWriter` ?
Same for the variable `snapshotIndexWriterFuture`.
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java:
##########
@@ -21,63 +21,150 @@
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
import org.apache.pulsar.broker.systopic.SystemTopicClient;
import org.apache.pulsar.broker.systopic.SystemTopicClientBase;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.common.events.EventType;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.util.FutureUtil;
+@Slf4j
public class SystemTopicTxnBufferSnapshotService<T> {
- protected final Map<TopicName, SystemTopicClient<T>> clients;
+ protected final ConcurrentHashMap<NamespaceName, SystemTopicClient<T>>
clients;
protected final NamespaceEventsSystemTopicFactory
namespaceEventsSystemTopicFactory;
protected final Class<T> schemaType;
protected final EventType systemTopicType;
+ private final ConcurrentHashMap<NamespaceName, ReferenceCountedWriter<T>>
refCountedWriterMap;
+
+ // The class ReferenceCountedWriter will maintain the reference count,
+ // when the reference count decrement to 0, it will be removed from
writerFutureMap, the writer will be closed.
+ public static class ReferenceCountedWriter<T> {
+
+ private final AtomicLong referenceCount;
+ private final NamespaceName namespaceName;
+ private final CompletableFuture<SystemTopicClient.Writer<T>> future;
+ private final SystemTopicTxnBufferSnapshotService<T> snapshotService;
+
+ public ReferenceCountedWriter(NamespaceName namespaceName,
+
CompletableFuture<SystemTopicClient.Writer<T>> future,
+ SystemTopicTxnBufferSnapshotService<T>
snapshotService) {
+ this.referenceCount = new AtomicLong(1);
+ this.namespaceName = namespaceName;
+ this.snapshotService = snapshotService;
+ this.future = future;
+ this.future.exceptionally(t -> {
+ log.error("[{}] Failed to create transaction buffer
snapshot writer.", namespaceName, t);
+ snapshotService.refCountedWriterMap.remove(namespaceName,
this);
+ return null;
+ });
+ }
+
+ public CompletableFuture<SystemTopicClient.Writer<T>> getFuture() {
+ return future;
+ }
+
+ private void retain() {
+ operationValidate(true);
+ this.referenceCount.incrementAndGet();
+ }
+
+ private long release() {
+ operationValidate(false);
+ return this.referenceCount.decrementAndGet();
+ }
+
+ private void operationValidate(boolean isRetain) {
+ if (this.referenceCount.get() == 0) {
+ throw new RuntimeException(
+ "[" + namespaceName + "] The reference counted
transaction buffer snapshot writer couldn't "
+ + "be " + (isRetain ? "retained" : "released")
+ ", refCnt is 0.");
+ }
+ }
+
+ public void close() {
+ if (release() == 0) {
+ snapshotService.refCountedWriterMap.remove(namespaceName,
this);
+ future.thenAccept(writer -> {
+ final String topicName =
writer.getSystemTopicClient().getTopicName().toString();
+ writer.closeAsync().exceptionally(t -> {
+ if (t != null) {
+ log.error("[{}] Failed to close writer.",
topicName, t);
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Success to close writer.",
topicName);
+ }
+ }
+ return null;
+ });
+ });
+ }
+ }
+
+ }
+
public SystemTopicTxnBufferSnapshotService(PulsarClient client, EventType
systemTopicType,
Class<T> schemaType) {
this.namespaceEventsSystemTopicFactory = new
NamespaceEventsSystemTopicFactory(client);
this.systemTopicType = systemTopicType;
this.schemaType = schemaType;
this.clients = new ConcurrentHashMap<>();
+ this.refCountedWriterMap = new ConcurrentHashMap<>();
}
public CompletableFuture<SystemTopicClient.Writer<T>>
createWriter(TopicName topicName) {
- return
getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newWriterAsync);
+ return
getTransactionBufferSystemTopicClient(topicName).newWriterAsync();
}
public CompletableFuture<SystemTopicClient.Reader<T>>
createReader(TopicName topicName) {
- return
getTransactionBufferSystemTopicClient(topicName).thenCompose(SystemTopicClient::newReaderAsync);
+ return
getTransactionBufferSystemTopicClient(topicName).newReaderAsync();
}
public void removeClient(TopicName topicName, SystemTopicClientBase<T>
transactionBufferSystemTopicClient) {
if (transactionBufferSystemTopicClient.getReaders().size() == 0
&& transactionBufferSystemTopicClient.getWriters().size() ==
0) {
- clients.remove(topicName);
+ clients.remove(topicName.getNamespaceObject());
}
}
- protected CompletableFuture<SystemTopicClient<T>>
getTransactionBufferSystemTopicClient(TopicName topicName) {
+ public ReferenceCountedWriter<T> getReferenceWriter(TopicName topicName) {
+ return refCountedWriterMap.compute(topicName.getNamespaceObject(), (k,
v) -> {
+ if (v == null) {
+ return new
ReferenceCountedWriter<>(topicName.getNamespaceObject(),
+
getTransactionBufferSystemTopicClient(topicName).newWriterAsync(), this);
+ } else {
+ v.retain();
+ }
+ return v;
+ });
+ }
+
+ private SystemTopicClient<T>
getTransactionBufferSystemTopicClient(TopicName topicName) {
Review Comment:
Why not change the param `TopicName` to `Namespace`?
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java:
##########
@@ -175,8 +178,12 @@ public long getLastSnapshotTimestamps() {
}
@Override
- public CompletableFuture<Void> closeAsync() {
- return
takeSnapshotWriter.thenCompose(SystemTopicClient.Writer::closeAsync);
+ public synchronized CompletableFuture<Void> closeAsync() {
+ if (!isClosed) {
+ isClosed = true;
+ takeSnapshotWriter.close();
Review Comment:
We should call `release` instead of `close` here, right?
##########
pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java:
##########
@@ -760,11 +763,13 @@ private CompletableFuture<Void>
clearAllSnapshotSegments() {
});
}
-
- CompletableFuture<Void> closeAsync() {
- return CompletableFuture.allOf(
-
this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync),
-
this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync));
+ synchronized CompletableFuture<Void> closeAsync() {
+ if (!closed) {
+ closed = true;
+ this.snapshotIndexWriterFuture.close();
+ this.snapshotSegmentsWriterFuture.close();
Review Comment:
We should call `release` instead of `close` here, right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
