Minor cleanup in ShuffleBlockManager
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/1ba11b1c Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/1ba11b1c Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/1ba11b1c Branch: refs/heads/master Commit: 1ba11b1c6aeda084cb158262ec0aa37a7b70fe32 Parents: 6201e5e Author: Aaron Davidson <[email protected]> Authored: Mon Nov 4 17:16:20 2013 -0800 Committer: Aaron Davidson <[email protected]> Committed: Mon Nov 4 17:16:41 2013 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/storage/ShuffleBlockManager.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/1ba11b1c/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala index 6346db3..2f1b049 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala @@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicInteger import scala.collection.JavaConversions._ -import org.apache.spark.Logging import org.apache.spark.serializer.Serializer import org.apache.spark.util.{MetadataCleanerType, MetadataCleaner, TimeStampedHashMap} import org.apache.spark.util.collection.{PrimitiveKeyOpenHashMap, PrimitiveVector} @@ -59,7 +58,7 @@ private[spark] trait ShuffleWriterGroup { * files within a ShuffleFileGroups associated with the block's reducer. */ private[spark] -class ShuffleBlockManager(blockManager: BlockManager) extends Logging { +class ShuffleBlockManager(blockManager: BlockManager) { // Turning off shuffle file consolidation causes all shuffle Blocks to get their own file. // TODO: Remove this once the shuffle file consolidation feature is stable. val consolidateShuffleFiles = @@ -83,7 +82,7 @@ class ShuffleBlockManager(blockManager: BlockManager) extends Logging { private val metadataCleaner = new MetadataCleaner(MetadataCleanerType.SHUFFLE_BLOCK_MANAGER, this.cleanup) - def forMapTask(shuffleId: Int, mapId: Int, numBuckets: Int, serializer: Serializer) = + def forMapTask(shuffleId: Int, mapId: Int, numBuckets: Int, serializer: Serializer) = { new ShuffleWriterGroup { shuffleStates.putIfAbsent(shuffleId, new ShuffleState()) private val shuffleState = shuffleStates(shuffleId) @@ -133,6 +132,7 @@ class ShuffleBlockManager(blockManager: BlockManager) extends Logging { shuffleState.unusedFileGroups.add(group) } } + } /** * Returns the physical file segment in which the given BlockId is located. @@ -177,7 +177,7 @@ object ShuffleBlockManager { * Note: mapIdToIndex(mapId) returns the index of the mapper into the vector for every * reducer. */ - private val blockOffsetsByReducer = Array.tabulate[PrimitiveVector[Long]](files.length) { _ => + private val blockOffsetsByReducer = Array.fill[PrimitiveVector[Long]](files.length) { new PrimitiveVector[Long]() }
