Address minor comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/7453f311 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/7453f311 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/7453f311 Branch: refs/heads/master Commit: 7453f31181a173f1cacb2c957455bf05e52f43c2 Parents: 84991a1 Author: Aaron Davidson <[email protected]> Authored: Fri Nov 1 21:01:04 2013 -0700 Committer: Aaron Davidson <[email protected]> Committed: Sun Nov 3 21:34:43 2013 -0800 ---------------------------------------------------------------------- .../apache/spark/storage/DiskBlockManager.scala | 3 +- .../spark/storage/ShuffleBlockManager.scala | 7 +-- .../org/apache/spark/util/PrimitiveVector.scala | 48 -------------------- .../spark/util/collection/PrimitiveVector.scala | 47 +++++++++++++++++++ 4 files changed, 53 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/7453f311/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index 4f9537d..bde3d1f 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -34,7 +34,8 @@ import org.apache.spark.util.Utils * * @param rootDirs The directories to use for storing block files. Data will be hashed among these. */ -private[spark] class DiskBlockManager(shuffleManager: ShuffleBlockManager, rootDirs: String) extends PathResolver with Logging { +private[spark] class DiskBlockManager(shuffleManager: ShuffleBlockManager, rootDirs: String) + extends PathResolver with Logging { private val MAX_DIR_CREATION_ATTEMPTS: Int = 10 private val subDirsPerLocalDir = System.getProperty("spark.diskStore.subDirectories", "64").toInt http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/7453f311/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala index c61febf..d718c87 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockManager.scala @@ -27,7 +27,8 @@ import scala.collection.mutable import org.apache.spark.Logging import org.apache.spark.serializer.Serializer -import org.apache.spark.util.{MetadataCleanerType, MetadataCleaner, AGodDamnPrimitiveVector, TimeStampedHashMap} +import org.apache.spark.util.{MetadataCleanerType, MetadataCleaner, TimeStampedHashMap} +import org.apache.spark.util.collection.PrimitiveVector private[spark] class ShuffleWriterGroup( @@ -203,7 +204,7 @@ class ShuffleBlockManager(blockManager: BlockManager) extends Logging { */ private[spark] class ShuffleFileGroup(val shuffleId: Int, val fileId: Int, val files: Array[ShuffleFile]) { - private val mapIds = new AGodDamnPrimitiveVector[Int]() + private val mapIds = new PrimitiveVector[Int]() files.foreach(_.setShuffleFileGroup(this)) @@ -238,7 +239,7 @@ class ShuffleFile(val file: File) { * Consecutive offsets of blocks into the file, ordered by position in the file. * This ordering allows us to compute block lengths by examining the following block offset. */ - val blockOffsets = new AGodDamnPrimitiveVector[Long]() + val blockOffsets = new PrimitiveVector[Long]() /** Back pointer to whichever ShuffleFileGroup this file is a part of. */ private var shuffleFileGroup : ShuffleFileGroup = _ http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/7453f311/core/src/main/scala/org/apache/spark/util/PrimitiveVector.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/PrimitiveVector.scala b/core/src/main/scala/org/apache/spark/util/PrimitiveVector.scala deleted file mode 100644 index d316601..0000000 --- a/core/src/main/scala/org/apache/spark/util/PrimitiveVector.scala +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.util - -/** Provides a simple, non-threadsafe, array-backed vector that can store primitives. */ -class AGodDamnPrimitiveVector[@specialized(Long, Int, Double) V: ClassManifest] - (initialSize: Int = 64) -{ - private var numElements = 0 - private var array = new Array[V](initialSize) - - def apply(index: Int): V = { - require(index < numElements) - array(index) - } - - def +=(value: V) { - if (numElements == array.length) { resize(array.length * 2) } - array(numElements) = value - numElements += 1 - } - - def length = numElements - - def getUnderlyingArray = array - - /** Resizes the array, dropping elements if the total length decreases. */ - def resize(newLength: Int) { - val newArray = new Array[V](newLength) - array.copyToArray(newArray) - array = newArray - } -} http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/7453f311/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala new file mode 100644 index 0000000..721f12b --- /dev/null +++ b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util.collection + +/** Provides a simple, non-threadsafe, array-backed vector that can store primitives. */ +private[spark] +class PrimitiveVector[@specialized(Long, Int, Double) V: ClassManifest](initialSize: Int = 64) { + private var numElements = 0 + private var array = new Array[V](initialSize) + + def apply(index: Int): V = { + require(index < numElements) + array(index) + } + + def +=(value: V) { + if (numElements == array.length) { resize(array.length * 2) } + array(numElements) = value + numElements += 1 + } + + def length = numElements + + def getUnderlyingArray = array + + /** Resizes the array, dropping elements if the total length decreases. */ + def resize(newLength: Int) { + val newArray = new Array[V](newLength) + array.copyToArray(newArray) + array = newArray + } +}
