yandrey321 commented on code in PR #11874:
URL: https://github.com/apache/ozone/pull/11874#discussion_r4626079744


##########
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java:
##########
@@ -815,63 +835,353 @@ protected boolean shouldBlockAndWaitAsyncReply(
     return !HddsUtils.isReadOnly(request);
   }
 
+  /**
+   * Sends the request to the given datanode for a caller that waits for the
+   * reply with {@link CompletableFuture#get()}: the response is delivered on
+   * the waiting thread, see {@link CallerRunsReplyFuture}.
+   */
   @VisibleForTesting
   public XceiverClientReply sendCommandAsync(
       ContainerCommandRequestProto request, DatanodeDetails dn)
       throws IOException, InterruptedException {
+    return sendCommandAsync(request, dn, new CallerRunsReplyFuture());
+  }
+
+  /**
+   * @param replyFuture completed with the reply; if it is an {@link Executor}
+   *        the call's callbacks run on it, otherwise on gRPC's default 
executor
+   */
+  private XceiverClientReply sendCommandAsync(
+      ContainerCommandRequestProto request, DatanodeDetails dn,
+      CompletableFuture<ContainerCommandResponseProto> replyFuture)
+      throws IOException, InterruptedException {
     checkOpen(dn);
     DatanodeID dnId = dn.getID();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Send command {} to datanode {}",
           request.getCmdType(), dn);
     }
-    final CompletableFuture<ContainerCommandResponseProto> replyFuture =
-        new CompletableFuture<>();
     semaphore.acquire();
     long requestTime = Time.monotonicNow();
     metrics.incrPendingContainerOpsMetrics(request.getCmdType());
 
+    final ChannelInfo channelInfo = dnChannelInfoMap.get(dnId);
+    final ReplyObserver reply = new ReplyObserver(request, dn, replyFuture, 
requestTime);
+    if (pooledStreamsEnabled && replyFuture instanceof CallerRunsReplyFuture
+        && !Span.current().getSpanContext().isValid()) {
+      // No trace context to send, so a pooled stream carries the same headers 
as a new one would.
+      channelInfo.leaseStream().send(request, reply, (CallerRunsReplyFuture) 
replyFuture,
+          System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout));
+      return new XceiverClientReply(replyFuture);
+    }
+
+    XceiverClientProtocolServiceStub stub = 
channelInfo.getStub().withDeadlineAfter(timeout, TimeUnit.MILLISECONDS);
+    if (replyFuture instanceof Executor) {
+      stub = stub.withExecutor((Executor) replyFuture);
+    }
     // create a new grpc message stream pair for each call.
-    final StreamObserver<ContainerCommandRequestProto> requestObserver =
-        dnChannelInfoMap.get(dnId).getStub().withDeadlineAfter(timeout, 
TimeUnit.MILLISECONDS)
-            .send(new StreamObserver<ContainerCommandResponseProto>() {
-              @Override
-              public void onNext(ContainerCommandResponseProto value) {
-                replyFuture.complete(value);
-                decreasePendingMetricsAndReleaseSemaphore();
-              }
-
-              @Override
-              public void onError(Throwable t) {
-                replyFuture.completeExceptionally(t);
-                decreasePendingMetricsAndReleaseSemaphore();
-              }
-
-              @Override
-              public void onCompleted() {
-                if (!replyFuture.isDone()) {
-                  replyFuture.completeExceptionally(new IOException(
-                      "Stream completed but no reply for request " +
-                          processForDebug(request)));
-                }
-              }
-
-              private void decreasePendingMetricsAndReleaseSemaphore() {
-                metrics.decrPendingContainerOpsMetrics(request.getCmdType());
-                long cost = Time.monotonicNow() - requestTime;
-                metrics.addContainerOpsLatency(request.getCmdType(), cost);
-                if (LOG.isDebugEnabled()) {
-                  LOG.debug("Executed command {} on datanode {}, cost = {}, 
cmdType = {}",
-                      processForDebug(request), dn, cost, 
request.getCmdType());
-                }
-                semaphore.release();
-              }
-            });
+    final StreamObserver<ContainerCommandRequestProto> requestObserver = 
stub.send(reply);
     requestObserver.onNext(request);
     requestObserver.onCompleted();
     return new XceiverClientReply(replyFuture);
   }
 
+  /**
+   * Receives the reply to one request: completes its future, then releases
+   * its semaphore permit and updates the metrics.
+   */
+  @VisibleForTesting
+  final class ReplyObserver implements 
StreamObserver<ContainerCommandResponseProto> {
+    private final ContainerCommandRequestProto request;
+    private final DatanodeDetails dn;
+    private final CompletableFuture<ContainerCommandResponseProto> replyFuture;
+    private final long requestTime;
+
+    ReplyObserver(ContainerCommandRequestProto request, DatanodeDetails dn,
+        CompletableFuture<ContainerCommandResponseProto> replyFuture, long 
requestTime) {
+      this.request = request;
+      this.dn = dn;
+      this.replyFuture = replyFuture;
+      this.requestTime = requestTime;
+    }
+
+    @Override
+    public void onNext(ContainerCommandResponseProto value) {
+      replyFuture.complete(value);
+      decreasePendingMetricsAndReleaseSemaphore();
+    }
+
+    @Override
+    public void onError(Throwable t) {
+      replyFuture.completeExceptionally(t);
+      decreasePendingMetricsAndReleaseSemaphore();
+    }
+
+    @Override
+    public void onCompleted() {
+      if (!replyFuture.isDone()) {
+        replyFuture.completeExceptionally(new IOException(
+            "Stream completed but no reply for request " +
+                processForDebug(request)));
+      }
+    }
+
+    private void decreasePendingMetricsAndReleaseSemaphore() {
+      metrics.decrPendingContainerOpsMetrics(request.getCmdType());
+      long cost = Time.monotonicNow() - requestTime;
+      metrics.addContainerOpsLatency(request.getCmdType(), cost);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Executed command {} on datanode {}, cost = {}, cmdType = 
{}",
+            processForDebug(request), dn, cost, request.getCmdType());
+      }
+      semaphore.release();
+    }
+  }
+
+  /**
+   * A send stream kept open across requests, so a request does not pay for
+   * opening and closing an HTTP/2 stream. It carries one request at a time:
+   * the datanode serves the requests of a stream in order and replies to each
+   * one, so the next reply always belongs to the current request. The gRPC
+   * callbacks run on the current request's {@link CallerRunsReplyFuture}, as
+   * for a stream opened per request.
+   */
+  @VisibleForTesting
+  static final class PooledStream implements Executor,
+      ClientResponseObserver<ContainerCommandRequestProto, 
ContainerCommandResponseProto> {
+    private final ChannelInfo owner;
+    private ClientCallStreamObserver<ContainerCommandRequestProto> 
requestStream;
+    private final AtomicReference<ReplyObserver> current = new 
AtomicReference<>();
+    private volatile CallerRunsReplyFuture currentFuture;
+    private volatile boolean closed;
+    private volatile long idleSinceNanos;
+
+    PooledStream(ChannelInfo owner) {
+      this.owner = owner;
+    }
+
+    @Override
+    public void 
beforeStart(ClientCallStreamObserver<ContainerCommandRequestProto> stream) {
+      requestStream = stream;
+    }
+
+    void send(ContainerCommandRequestProto request, ReplyObserver reply, 
CallerRunsReplyFuture future,
+        long deadlineNanos) {
+      future.setDeadline(deadlineNanos, () -> cancel(reply));
+      currentFuture = future;
+      current.set(reply);
+      if (closed && current.compareAndSet(reply, null)) {
+        // the stream ended after it was leased, before the request was 
attached, so its callbacks did not fail it
+        currentFuture = null;
+        reply.onError(Status.UNAVAILABLE.withDescription("pooled stream closed 
before the request was sent")
+            .asRuntimeException());
+        return;
+      }
+      requestStream.onNext(request);
+    }
+
+    @Override
+    public void execute(Runnable task) {
+      final CallerRunsReplyFuture future = current.get() != null ? 
currentFuture : null;
+      if (future != null) {
+        future.execute(task);
+      } else {
+        task.run();
+      }
+    }
+
+    @Override
+    public void onNext(ContainerCommandResponseProto value) {
+      final ReplyObserver reply = current.getAndSet(null);
+      currentFuture = null;
+      if (reply == null) {

Review Comment:
   [Minor] a stray reply after the stream was re-leased goes to the next request
   
   
   The null guard handles a reply that arrives while no request is attached. 
But onNext re-pools the stream (owner.releaseStream) before the current 
request's reply is processed. So if a second reply for request R1 arrives after 
another thread has leased the stream and attached R2, `current.getAndSet(null)` 
returns R2's observer.
   R2 then completes with R1's response, and R2's real reply arrives later and 
drops the stream.
   
   This only happens if a datanode answers a request twice, which 
GrpcXceiverService doesn't do today. 
`duplicateReplyDropsTheStreamWithoutLeakingPermits` covers only the case where 
the duplicate arrives while the stream is idle. A cheap guard would catch the 
re-leased case: compare `value.getCmdType()` with the attached request's
   cmdType, and on a mismatch fail that request with a retriable status and 
drop the stream. It doesn't prove the reply belongs to R2, but it catches the 
realistic mix-ups, such as a ReadChunk reply delivered to a GetBlock. Response 
validators and checksum verification would catch most of the rest, but not 
every command has them.
   



##########
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java:
##########
@@ -815,63 +835,353 @@ protected boolean shouldBlockAndWaitAsyncReply(
     return !HddsUtils.isReadOnly(request);
   }
 
+  /**
+   * Sends the request to the given datanode for a caller that waits for the
+   * reply with {@link CompletableFuture#get()}: the response is delivered on
+   * the waiting thread, see {@link CallerRunsReplyFuture}.
+   */
   @VisibleForTesting
   public XceiverClientReply sendCommandAsync(
       ContainerCommandRequestProto request, DatanodeDetails dn)
       throws IOException, InterruptedException {
+    return sendCommandAsync(request, dn, new CallerRunsReplyFuture());
+  }
+
+  /**
+   * @param replyFuture completed with the reply; if it is an {@link Executor}
+   *        the call's callbacks run on it, otherwise on gRPC's default 
executor
+   */
+  private XceiverClientReply sendCommandAsync(
+      ContainerCommandRequestProto request, DatanodeDetails dn,
+      CompletableFuture<ContainerCommandResponseProto> replyFuture)
+      throws IOException, InterruptedException {
     checkOpen(dn);
     DatanodeID dnId = dn.getID();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Send command {} to datanode {}",
           request.getCmdType(), dn);
     }
-    final CompletableFuture<ContainerCommandResponseProto> replyFuture =
-        new CompletableFuture<>();
     semaphore.acquire();
     long requestTime = Time.monotonicNow();
     metrics.incrPendingContainerOpsMetrics(request.getCmdType());
 
+    final ChannelInfo channelInfo = dnChannelInfoMap.get(dnId);
+    final ReplyObserver reply = new ReplyObserver(request, dn, replyFuture, 
requestTime);
+    if (pooledStreamsEnabled && replyFuture instanceof CallerRunsReplyFuture
+        && !Span.current().getSpanContext().isValid()) {
+      // No trace context to send, so a pooled stream carries the same headers 
as a new one would.
+      channelInfo.leaseStream().send(request, reply, (CallerRunsReplyFuture) 
replyFuture,
+          System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout));
+      return new XceiverClientReply(replyFuture);
+    }
+
+    XceiverClientProtocolServiceStub stub = 
channelInfo.getStub().withDeadlineAfter(timeout, TimeUnit.MILLISECONDS);
+    if (replyFuture instanceof Executor) {
+      stub = stub.withExecutor((Executor) replyFuture);
+    }
     // create a new grpc message stream pair for each call.
-    final StreamObserver<ContainerCommandRequestProto> requestObserver =
-        dnChannelInfoMap.get(dnId).getStub().withDeadlineAfter(timeout, 
TimeUnit.MILLISECONDS)
-            .send(new StreamObserver<ContainerCommandResponseProto>() {
-              @Override
-              public void onNext(ContainerCommandResponseProto value) {
-                replyFuture.complete(value);
-                decreasePendingMetricsAndReleaseSemaphore();
-              }
-
-              @Override
-              public void onError(Throwable t) {
-                replyFuture.completeExceptionally(t);
-                decreasePendingMetricsAndReleaseSemaphore();
-              }
-
-              @Override
-              public void onCompleted() {
-                if (!replyFuture.isDone()) {
-                  replyFuture.completeExceptionally(new IOException(
-                      "Stream completed but no reply for request " +
-                          processForDebug(request)));
-                }
-              }
-
-              private void decreasePendingMetricsAndReleaseSemaphore() {
-                metrics.decrPendingContainerOpsMetrics(request.getCmdType());
-                long cost = Time.monotonicNow() - requestTime;
-                metrics.addContainerOpsLatency(request.getCmdType(), cost);
-                if (LOG.isDebugEnabled()) {
-                  LOG.debug("Executed command {} on datanode {}, cost = {}, 
cmdType = {}",
-                      processForDebug(request), dn, cost, 
request.getCmdType());
-                }
-                semaphore.release();
-              }
-            });
+    final StreamObserver<ContainerCommandRequestProto> requestObserver = 
stub.send(reply);
     requestObserver.onNext(request);
     requestObserver.onCompleted();
     return new XceiverClientReply(replyFuture);
   }
 
+  /**
+   * Receives the reply to one request: completes its future, then releases
+   * its semaphore permit and updates the metrics.
+   */
+  @VisibleForTesting
+  final class ReplyObserver implements 
StreamObserver<ContainerCommandResponseProto> {
+    private final ContainerCommandRequestProto request;
+    private final DatanodeDetails dn;
+    private final CompletableFuture<ContainerCommandResponseProto> replyFuture;
+    private final long requestTime;
+
+    ReplyObserver(ContainerCommandRequestProto request, DatanodeDetails dn,
+        CompletableFuture<ContainerCommandResponseProto> replyFuture, long 
requestTime) {
+      this.request = request;
+      this.dn = dn;
+      this.replyFuture = replyFuture;
+      this.requestTime = requestTime;
+    }
+
+    @Override
+    public void onNext(ContainerCommandResponseProto value) {
+      replyFuture.complete(value);
+      decreasePendingMetricsAndReleaseSemaphore();
+    }
+
+    @Override
+    public void onError(Throwable t) {
+      replyFuture.completeExceptionally(t);
+      decreasePendingMetricsAndReleaseSemaphore();
+    }
+
+    @Override
+    public void onCompleted() {
+      if (!replyFuture.isDone()) {
+        replyFuture.completeExceptionally(new IOException(
+            "Stream completed but no reply for request " +
+                processForDebug(request)));
+      }
+    }
+
+    private void decreasePendingMetricsAndReleaseSemaphore() {
+      metrics.decrPendingContainerOpsMetrics(request.getCmdType());
+      long cost = Time.monotonicNow() - requestTime;
+      metrics.addContainerOpsLatency(request.getCmdType(), cost);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Executed command {} on datanode {}, cost = {}, cmdType = 
{}",
+            processForDebug(request), dn, cost, request.getCmdType());
+      }
+      semaphore.release();
+    }
+  }
+
+  /**
+   * A send stream kept open across requests, so a request does not pay for
+   * opening and closing an HTTP/2 stream. It carries one request at a time:
+   * the datanode serves the requests of a stream in order and replies to each
+   * one, so the next reply always belongs to the current request. The gRPC
+   * callbacks run on the current request's {@link CallerRunsReplyFuture}, as
+   * for a stream opened per request.
+   */
+  @VisibleForTesting
+  static final class PooledStream implements Executor,
+      ClientResponseObserver<ContainerCommandRequestProto, 
ContainerCommandResponseProto> {
+    private final ChannelInfo owner;
+    private ClientCallStreamObserver<ContainerCommandRequestProto> 
requestStream;
+    private final AtomicReference<ReplyObserver> current = new 
AtomicReference<>();
+    private volatile CallerRunsReplyFuture currentFuture;
+    private volatile boolean closed;
+    private volatile long idleSinceNanos;
+
+    PooledStream(ChannelInfo owner) {
+      this.owner = owner;
+    }
+
+    @Override
+    public void 
beforeStart(ClientCallStreamObserver<ContainerCommandRequestProto> stream) {
+      requestStream = stream;
+    }
+
+    void send(ContainerCommandRequestProto request, ReplyObserver reply, 
CallerRunsReplyFuture future,
+        long deadlineNanos) {
+      future.setDeadline(deadlineNanos, () -> cancel(reply));
+      currentFuture = future;
+      current.set(reply);
+      if (closed && current.compareAndSet(reply, null)) {

Review Comment:
   [Question] a stream that ended between lease and send fails the request
   
   
   `requestOnAStreamThatEndedAfterTheLeaseFailsAtOnce` shows this is 
deliberate: if the datanode ended an idle stream after leaseStream() saw it 
open, send() fails the request with UNAVAILABLE. Nothing has been sent at that 
point, so a single retry on a freshly leased stream would be safe and invisible 
to the caller.
   
   Today the caller sees an error. sendCommandWithRetry then fails over to the 
next datanode, or, for a single-node pipeline (an EC replica index), the read 
fails and the EC reader falls back to reconstruction. The window is small, and 
it mostly opens when the datanode really went away (restart, GOAWAY), when 
failing over is right anyway. But
   after a rolling restart, the channel reconnects to a healthy datanode while 
some idle streams are still in the deque, and then the failover is spurious. 
Worth re-leasing once here?
   



##########
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java:
##########
@@ -815,63 +835,353 @@ protected boolean shouldBlockAndWaitAsyncReply(
     return !HddsUtils.isReadOnly(request);
   }
 
+  /**
+   * Sends the request to the given datanode for a caller that waits for the
+   * reply with {@link CompletableFuture#get()}: the response is delivered on
+   * the waiting thread, see {@link CallerRunsReplyFuture}.
+   */
   @VisibleForTesting
   public XceiverClientReply sendCommandAsync(
       ContainerCommandRequestProto request, DatanodeDetails dn)
       throws IOException, InterruptedException {
+    return sendCommandAsync(request, dn, new CallerRunsReplyFuture());
+  }
+
+  /**
+   * @param replyFuture completed with the reply; if it is an {@link Executor}
+   *        the call's callbacks run on it, otherwise on gRPC's default 
executor
+   */
+  private XceiverClientReply sendCommandAsync(
+      ContainerCommandRequestProto request, DatanodeDetails dn,
+      CompletableFuture<ContainerCommandResponseProto> replyFuture)
+      throws IOException, InterruptedException {
     checkOpen(dn);
     DatanodeID dnId = dn.getID();
     if (LOG.isDebugEnabled()) {
       LOG.debug("Send command {} to datanode {}",
           request.getCmdType(), dn);
     }
-    final CompletableFuture<ContainerCommandResponseProto> replyFuture =
-        new CompletableFuture<>();
     semaphore.acquire();
     long requestTime = Time.monotonicNow();
     metrics.incrPendingContainerOpsMetrics(request.getCmdType());
 
+    final ChannelInfo channelInfo = dnChannelInfoMap.get(dnId);
+    final ReplyObserver reply = new ReplyObserver(request, dn, replyFuture, 
requestTime);
+    if (pooledStreamsEnabled && replyFuture instanceof CallerRunsReplyFuture
+        && !Span.current().getSpanContext().isValid()) {
+      // No trace context to send, so a pooled stream carries the same headers 
as a new one would.
+      channelInfo.leaseStream().send(request, reply, (CallerRunsReplyFuture) 
replyFuture,
+          System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeout));
+      return new XceiverClientReply(replyFuture);
+    }
+
+    XceiverClientProtocolServiceStub stub = 
channelInfo.getStub().withDeadlineAfter(timeout, TimeUnit.MILLISECONDS);
+    if (replyFuture instanceof Executor) {
+      stub = stub.withExecutor((Executor) replyFuture);
+    }
     // create a new grpc message stream pair for each call.
-    final StreamObserver<ContainerCommandRequestProto> requestObserver =
-        dnChannelInfoMap.get(dnId).getStub().withDeadlineAfter(timeout, 
TimeUnit.MILLISECONDS)
-            .send(new StreamObserver<ContainerCommandResponseProto>() {
-              @Override
-              public void onNext(ContainerCommandResponseProto value) {
-                replyFuture.complete(value);
-                decreasePendingMetricsAndReleaseSemaphore();
-              }
-
-              @Override
-              public void onError(Throwable t) {
-                replyFuture.completeExceptionally(t);
-                decreasePendingMetricsAndReleaseSemaphore();
-              }
-
-              @Override
-              public void onCompleted() {
-                if (!replyFuture.isDone()) {
-                  replyFuture.completeExceptionally(new IOException(
-                      "Stream completed but no reply for request " +
-                          processForDebug(request)));
-                }
-              }
-
-              private void decreasePendingMetricsAndReleaseSemaphore() {
-                metrics.decrPendingContainerOpsMetrics(request.getCmdType());
-                long cost = Time.monotonicNow() - requestTime;
-                metrics.addContainerOpsLatency(request.getCmdType(), cost);
-                if (LOG.isDebugEnabled()) {
-                  LOG.debug("Executed command {} on datanode {}, cost = {}, 
cmdType = {}",
-                      processForDebug(request), dn, cost, 
request.getCmdType());
-                }
-                semaphore.release();
-              }
-            });
+    final StreamObserver<ContainerCommandRequestProto> requestObserver = 
stub.send(reply);
     requestObserver.onNext(request);
     requestObserver.onCompleted();
     return new XceiverClientReply(replyFuture);
   }
 
+  /**
+   * Receives the reply to one request: completes its future, then releases
+   * its semaphore permit and updates the metrics.
+   */
+  @VisibleForTesting
+  final class ReplyObserver implements 
StreamObserver<ContainerCommandResponseProto> {
+    private final ContainerCommandRequestProto request;
+    private final DatanodeDetails dn;
+    private final CompletableFuture<ContainerCommandResponseProto> replyFuture;
+    private final long requestTime;
+
+    ReplyObserver(ContainerCommandRequestProto request, DatanodeDetails dn,
+        CompletableFuture<ContainerCommandResponseProto> replyFuture, long 
requestTime) {
+      this.request = request;
+      this.dn = dn;
+      this.replyFuture = replyFuture;
+      this.requestTime = requestTime;
+    }
+
+    @Override
+    public void onNext(ContainerCommandResponseProto value) {
+      replyFuture.complete(value);
+      decreasePendingMetricsAndReleaseSemaphore();
+    }
+
+    @Override
+    public void onError(Throwable t) {
+      replyFuture.completeExceptionally(t);
+      decreasePendingMetricsAndReleaseSemaphore();
+    }
+
+    @Override
+    public void onCompleted() {
+      if (!replyFuture.isDone()) {
+        replyFuture.completeExceptionally(new IOException(
+            "Stream completed but no reply for request " +
+                processForDebug(request)));
+      }
+    }
+
+    private void decreasePendingMetricsAndReleaseSemaphore() {
+      metrics.decrPendingContainerOpsMetrics(request.getCmdType());
+      long cost = Time.monotonicNow() - requestTime;
+      metrics.addContainerOpsLatency(request.getCmdType(), cost);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Executed command {} on datanode {}, cost = {}, cmdType = 
{}",
+            processForDebug(request), dn, cost, request.getCmdType());
+      }
+      semaphore.release();
+    }
+  }
+
+  /**
+   * A send stream kept open across requests, so a request does not pay for
+   * opening and closing an HTTP/2 stream. It carries one request at a time:
+   * the datanode serves the requests of a stream in order and replies to each
+   * one, so the next reply always belongs to the current request. The gRPC
+   * callbacks run on the current request's {@link CallerRunsReplyFuture}, as
+   * for a stream opened per request.
+   */
+  @VisibleForTesting
+  static final class PooledStream implements Executor,
+      ClientResponseObserver<ContainerCommandRequestProto, 
ContainerCommandResponseProto> {
+    private final ChannelInfo owner;
+    private ClientCallStreamObserver<ContainerCommandRequestProto> 
requestStream;
+    private final AtomicReference<ReplyObserver> current = new 
AtomicReference<>();
+    private volatile CallerRunsReplyFuture currentFuture;
+    private volatile boolean closed;
+    private volatile long idleSinceNanos;
+
+    PooledStream(ChannelInfo owner) {
+      this.owner = owner;
+    }
+
+    @Override
+    public void 
beforeStart(ClientCallStreamObserver<ContainerCommandRequestProto> stream) {
+      requestStream = stream;
+    }
+
+    void send(ContainerCommandRequestProto request, ReplyObserver reply, 
CallerRunsReplyFuture future,
+        long deadlineNanos) {
+      future.setDeadline(deadlineNanos, () -> cancel(reply));
+      currentFuture = future;
+      current.set(reply);
+      if (closed && current.compareAndSet(reply, null)) {
+        // the stream ended after it was leased, before the request was 
attached, so its callbacks did not fail it
+        currentFuture = null;
+        reply.onError(Status.UNAVAILABLE.withDescription("pooled stream closed 
before the request was sent")
+            .asRuntimeException());
+        return;
+      }
+      requestStream.onNext(request);
+    }
+
+    @Override
+    public void execute(Runnable task) {
+      final CallerRunsReplyFuture future = current.get() != null ? 
currentFuture : null;
+      if (future != null) {
+        future.execute(task);
+      } else {
+        task.run();
+      }
+    }
+
+    @Override
+    public void onNext(ContainerCommandResponseProto value) {
+      final ReplyObserver reply = current.getAndSet(null);
+      currentFuture = null;
+      if (reply == null) {
+        // no request is attached, e.g. it was abandoned just before this 
reply: never reuse the stream
+        closed = true;
+        requestStream.cancel("reply without a request", null);
+        return;
+      }
+      // back in the pool before the permit is released, so the thread woken 
by the permit finds the stream
+      owner.releaseStream(this);
+      reply.onNext(value);
+    }
+
+    @Override
+    public void onError(Throwable t) {
+      closed = true;
+      final ReplyObserver reply = current.getAndSet(null);
+      currentFuture = null;
+      if (reply != null) {
+        reply.onError(t);
+      }
+    }
+
+    @Override
+    public void onCompleted() {
+      closed = true;
+      final ReplyObserver reply = current.getAndSet(null);
+      currentFuture = null;
+      if (reply != null) {
+        reply.onCompleted();
+      }
+    }
+
+    boolean isClosed() {
+      return closed;
+    }
+
+    void setIdleSince(long nanos) {
+      idleSinceNanos = nanos;
+    }
+
+    long getIdleSince() {
+      return idleSinceNanos;
+    }
+
+    /** Ends an idle stream, the datanode then completes it. */
+    void halfClose() {
+      closed = true;
+      requestStream.onCompleted();
+    }
+
+    /**
+     * Called when the request timed out or its waiter gave up: fails the 
request and drops the stream, unless
+     * its reply already returned the stream to the pool, where the stream may 
carry another request by now.
+     */
+    private void cancel(ReplyObserver reply) {
+      if (current.compareAndSet(reply, null)) {
+        currentFuture = null;
+        closed = true;
+        requestStream.cancel("request abandoned by the client", null);
+        reply.onError(Status.CANCELLED.withDescription("request abandoned by 
the client").asRuntimeException());
+      }
+    }
+  }
+
+  /**
+   * Reply future that is also the gRPC call executor. The callbacks of the
+   * call are queued and run by the thread waiting in {@link #get()}, so the
+   * response is parsed and the future completed on the caller, without a
+   * hand-off through gRPC's shared executor (as gRPC blocking stubs do).
+   * Once the waiter returns, queued and later callbacks run directly.
+   * Only {@code get} runs the callbacks and enforces the deadline: without a
+   * caller in {@code get}, {@code join()} or a dependent stage never 
completes.
+   */
+  @VisibleForTesting
+  static final class CallerRunsReplyFuture
+      extends CompletableFuture<ContainerCommandResponseProto> implements 
Executor {
+    private static final Runnable WAKE_UP = () -> { };
+    private final LinkedBlockingQueue<Runnable> tasks = new 
LinkedBlockingQueue<>();
+    private boolean detached;
+    // set for a request on a pooled stream, which has no gRPC deadline
+    private long deadlineNanos;
+    private Runnable onAbandon;
+
+    /**
+     * The waiter fails the request with DEADLINE_EXCEEDED once {@code 
deadline}
+     * (in {@link System#nanoTime()}) passes, and runs {@code abandon} when it
+     * times out or gives up before the reply.
+     */
+    void setDeadline(long deadline, Runnable abandon) {
+      this.deadlineNanos = deadline;
+      this.onAbandon = abandon;
+    }
+
+    // A pooled stream's callbacks may run on another request's waiter, so 
wake this one up.
+    @Override
+    public boolean complete(ContainerCommandResponseProto value) {
+      final boolean completed = super.complete(value);
+      execute(WAKE_UP);
+      return completed;
+    }
+
+    @Override
+    public boolean completeExceptionally(Throwable ex) {
+      final boolean completed = super.completeExceptionally(ex);
+      execute(WAKE_UP);
+      return completed;
+    }
+
+    @Override
+    public void execute(Runnable task) {
+      synchronized (tasks) {
+        if (!detached) {
+          tasks.add(task);
+          return;
+        }
+      }
+      task.run();
+    }
+
+    @Override
+    public ContainerCommandResponseProto get() throws InterruptedException, 
ExecutionException {
+      try {
+        if (onAbandon == null) {
+          while (!isDone()) {
+            tasks.take().run();
+          }
+        } else {
+          runTasksUntil(deadlineNanos);
+        }
+      } finally {
+        detach();
+      }
+      return super.get();
+    }
+
+    @Override
+    public ContainerCommandResponseProto get(long time, TimeUnit unit)

Review Comment:
   [Nit] `get(time, unit)` reports a timeout as CANCELLED
   
   When the caller's timeout is earlier than the request deadline, 
runTasksUntil returns without completing, and detach() runs onAbandon → 
cancel(reply). That completes the future with CANCELLED via reply.onError, so 
`super.get(0, NANOSECONDS)` throws ExecutionException(CANCELLED) instead of the 
TimeoutException the CompletableFuture contract promises. No in-tree caller 
uses get(timeout) (all use get()), so this is only about the contract. Either 
throw TimeoutException when the caller's deadline passed first, or document it 
in the class javadoc next to the join() note.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to