Reynold Xin created SPARK-2940:
----------------------------------
Summary: Support fetching multiple blocks in a single request in
Netty network module
Key: SPARK-2940
URL: https://issues.apache.org/jira/browse/SPARK-2940
Project: Spark
Issue Type: Sub-task
Reporter: Reynold Xin
ShuffleCopier.getBlock gets one block at a time (in one request).
{code}
def getBlock(host: String, port: Int, blockId: BlockId,
resultCollectCallback: (BlockId, Long, ByteBuf) => Unit) {
val handler = new ShuffleCopier.ShuffleClientHandler(resultCollectCallback)
val connectTimeout = conf.getInt("spark.shuffle.netty.connect.timeout",
60000)
val fc = new FileClient(handler, connectTimeout)
try {
fc.init()
fc.connect(host, port)
fc.sendRequest(blockId.name)
fc.waitForClose()
fc.close()
} catch {
// Handle any socket-related exceptions in FileClient
case e: Exception => {
logError("Shuffle copy of block " + blockId + " from " + host + ":" +
port + " failed", e)
handler.handleError(blockId)
}
}
}
{code}
--
This message was sent by Atlassian JIRA
(v6.2#6252)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
