chungen0126 commented on code in PR #12009:
URL: https://github.com/apache/ozone/pull/12009#discussion_r4637954750
##########
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java:
##########
@@ -665,42 +674,78 @@ public void initStreamRead(BlockID blockID,
StreamingReaderSpi streamObserver) t
*/
public void initStreamRead(BlockID blockID, StreamingReaderSpi
streamObserver,
Set<DatanodeID> excludedDatanodes) throws IOException {
- final List<DatanodeDetails> datanodeList = sortDatanodes(null,
ContainerProtos.Type.ReadBlock);
- IOException lastException = null;
- for (DatanodeDetails dn : datanodeList) {
+ // Select the candidates before waiting for a permit: an open that no
datanode can serve must not queue for one.
+ final List<DatanodeDetails> datanodeList = new ArrayList<>();
+ for (DatanodeDetails dn : sortDatanodes(null,
ContainerProtos.Type.ReadBlock)) {
if (excludedDatanodes.contains(dn.getID())) {
LOG.debug("Skipping excluded datanode {} (uuid={}) for initStreamRead
{}",
dn, dn.getUuidString(), blockID.getContainerBlockID());
- continue;
+ } else {
+ datanodeList.add(dn);
}
- try {
- checkOpen(dn);
- semaphore.acquire();
- XceiverClientProtocolServiceStub stub =
dnChannelInfoMap.get(dn.getID()).getStub();
- if (stub == null) {
- throw new IOException("Failed to get gRPC stub for DataNode: " + dn);
+ }
+ if (datanodeList.isEmpty()) {
+ throw new IOException("Failed to start streaming read to any available
DataNodes");
+ }
+ acquireStreamPermit(blockID);
+ boolean started = false;
+ try {
+ IOException lastException = null;
+ for (DatanodeDetails dn : datanodeList) {
+ try {
+ checkOpen(dn);
+ XceiverClientProtocolServiceStub stub =
dnChannelInfoMap.get(dn.getID()).getStub();
+ if (stub == null) {
+ throw new IOException("Failed to get gRPC stub for DataNode: " +
dn);
+ }
+ LOG.debug("initStreamRead {} on datanode {}",
blockID.getContainerBlockID(), dn);
+ // No deadline: it would bound the entire long-lived streaming call.
Per-request timeliness is
+ // enforced by streamReadTimeout in streamRead() and
StreamingReader.poll().
+ stub.send(new ReadyAwareResponseObserver(dn, streamObserver));
+ started = true;
+ return;
+ } catch (IOException e) {
+ LOG.error("Failed to start streaming read to DataNode {}", dn, e);
+ lastException = e;
}
- LOG.debug("initStreamRead {} on datanode {}",
blockID.getContainerBlockID(), dn);
- // No deadline: it would bound the entire long-lived streaming call.
Per-request timeliness is
- // enforced by streamReadTimeout in streamRead() and
StreamingReader.poll().
- stub.send(new ReadyAwareResponseObserver(dn, streamObserver));
- return;
- } catch (IOException e) {
- LOG.error("Failed to start streaming read to DataNode {}", dn, e);
- semaphore.release();
- lastException = e;
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IOException("Interrupted initStreamRead to " + dn + " for "
+ blockID, e);
}
- }
- if (lastException != null) {
throw lastException;
- } else {
- throw new IOException("Failed to start streaming read to any available
DataNodes");
+ } finally {
+ if (!started) {
+ streamSemaphore.release();
+ }
}
}
+ @VisibleForTesting
+ void acquireStreamPermit(BlockID blockID) throws IOException {
+ try {
+ // Bounded: a reader that already holds a stream on this client
(sequential read plus pread, or seeks across
+ // blocks) would otherwise wait forever for a permit only it can release.
+ if (!streamSemaphore.tryAcquire(streamReadTimeoutNanos,
TimeUnit.NANOSECONDS)) {
+ // Retriable for the reader: a stream of another reader may be closed
within its read retries.
+ throw new TimeoutIOException("Failed to start streaming read for " +
blockID.getContainerBlockID()
+ + " on pipeline " + pipeline.getId() + ": " + maxConcurrentStreams
+ + " block streams are already open on this pipeline client."
+ + " Close unused input streams or increase
ozone.client.stream.read.max-concurrent-streams.");
+ }
Review Comment:
From my understanding, throwing a timeout here will subsequently be treated
as a DataNode side timeout, causing the corresponding DataNode to be excluded
and the block to be refreshed. This error handling is incorrect, though it
could probably be addressed in a follow-up patch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]