Repository: spark Updated Branches: refs/heads/master e3486e1b9 -> 2c8274568
[SPARK-24307][CORE] Add conf to revert to old code. In case there are any issues in converting FileSegmentManagedBuffer to ChunkedByteBuffer, add a conf to go back to old code path. Followup to 7e847646d1f377f46dc3154dea37148d4e557a03 Author: Imran Rashid <iras...@cloudera.com> Closes #21867 from squito/SPARK-24307-p2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2c827456 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2c827456 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2c827456 Branch: refs/heads/master Commit: 2c82745686f4456c4d5c84040a431dcb5b6cb60b Parents: e3486e1 Author: Imran Rashid <iras...@cloudera.com> Authored: Thu Jul 26 12:13:27 2018 -0700 Committer: Xiao Li <gatorsm...@gmail.com> Committed: Thu Jul 26 12:13:27 2018 -0700 ---------------------------------------------------------------------- .../scala/org/apache/spark/storage/BlockManager.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/2c827456/core/src/main/scala/org/apache/spark/storage/BlockManager.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala index 1db0327..5cd21e3 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala @@ -132,6 +132,8 @@ private[spark] class BlockManager( conf.getBoolean("spark.shuffle.service.enabled", false) private val chunkSize = conf.getSizeAsBytes("spark.storage.memoryMapLimitForTests", Int.MaxValue.toString).toInt + private val remoteReadNioBufferConversion = + conf.getBoolean("spark.network.remoteReadNioBufferConversion", false) val diskBlockManager = { // Only perform cleanup if an external service is not serving our shuffle files. @@ -731,7 +733,14 @@ private[spark] class BlockManager( } if (data != null) { - return Some(ChunkedByteBuffer.fromManagedBuffer(data, chunkSize)) + // SPARK-24307 undocumented "escape-hatch" in case there are any issues in converting to + // ChunkedByteBuffer, to go back to old code-path. Can be removed post Spark 2.4 if + // new path is stable. + if (remoteReadNioBufferConversion) { + return Some(new ChunkedByteBuffer(data.nioByteBuffer())) + } else { + return Some(ChunkedByteBuffer.fromManagedBuffer(data, chunkSize)) + } } logDebug(s"The value of block $blockId is null") } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
