vanzin commented on a change in pull request #24554: [SPARK-27622][Core]
Avoiding the network when block manager fetches disk persisted RDD blocks from
the same host
URL: https://github.com/apache/spark/pull/24554#discussion_r288344962
##########
File path: core/src/main/scala/org/apache/spark/storage/BlockManager.scala
##########
@@ -861,22 +913,12 @@ private[spark] class BlockManager(
}
/**
- * Get block from remote block managers as a ManagedBuffer.
+ * Fetch the block from remote block managers as a ManagedBuffer.
*/
- private def getRemoteManagedBuffer(blockId: BlockId): Option[ManagedBuffer]
= {
- logDebug(s"Getting remote block $blockId")
- require(blockId != null, "BlockId is null")
- var runningFailureCount = 0
- var totalFailureCount = 0
-
- // Because all the remote blocks are registered in driver, it is not
necessary to ask
- // all the slave executors to get block status.
- val locationsAndStatus = master.getLocationsAndStatus(blockId)
- val blockSize = locationsAndStatus.map { b =>
- b.status.diskSize.max(b.status.memSize)
- }.getOrElse(0L)
- val blockLocations =
locationsAndStatus.map(_.locations).getOrElse(Seq.empty)
-
+ def fetchRemoteManagedBuffer(
Review comment:
private?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
