vanzin commented on a change in pull request #25299: [SPARK-27651][Core] Avoid
the network when shuffle blocks are fetched from the same host
URL: https://github.com/apache/spark/pull/25299#discussion_r346947325
##########
File path:
common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
##########
@@ -182,14 +183,54 @@ public void onSuccess(ByteBuffer response) {
@Override
public void onFailure(Throwable e) {
logger.warn("Error trying to remove RDD blocks " +
Arrays.toString(blockIds) +
- " via external shuffle service from executor: " + execId, e);
+ " via external shuffle service from executor: " + execId, e);
numRemovedBlocksFuture.complete(0);
client.close();
}
});
return numRemovedBlocksFuture;
}
+ public void getHostLocalDirs(
+ String host,
+ int port,
+ String[] execIds,
+ CompletableFuture<Map<String, String[]>> hostLocalDirsCompletable) {
+ checkInit();
+ GetLocalDirsForExecutors getLocalDirsMessage = new
GetLocalDirsForExecutors(appId, execIds);
+ try {
+ TransportClient client = clientFactory.createClient(host, port);
+ client.sendRpc(getLocalDirsMessage.toByteBuffer(), new
RpcResponseCallback() {
+ @Override
+ public void onSuccess(ByteBuffer response) {
+ try {
+ BlockTransferMessage msgObj =
BlockTransferMessage.Decoder.fromByteBuffer(response);
+ hostLocalDirsCompletable.complete(
+ ((LocalDirsForExecutors) msgObj).getLocalDirsByExec());
+ } catch (Throwable t) {
+ logger.warn("Error trying to get the host local dirs for " +
+ Arrays.toString(getLocalDirsMessage.execIds) + " via external
shuffle service",
Review comment:
Indentation.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
