Merge branch 'master' of github.com:apache/incubator-spark into scala-2.10-temp
Conflicts:
core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/95d8dbce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/95d8dbce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/95d8dbce
Branch: refs/heads/master
Commit: 95d8dbce91f49467050250d5cf3671aaaa648d76
Parents: 199e9cf 2fead51
Author: Prashant Sharma <[email protected]>
Authored: Thu Nov 21 12:34:46 2013 +0530
Committer: Prashant Sharma <[email protected]>
Committed: Thu Nov 21 12:34:46 2013 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/SparkContext.scala | 1 -
.../spark/deploy/FaultToleranceTest.scala | 28 ++---
.../apache/spark/deploy/LocalSparkCluster.scala | 6 +-
.../executor/CoarseGrainedExecutorBackend.scala | 2 +-
.../apache/spark/executor/ExecutorSource.scala | 2 -
.../spark/network/netty/ShuffleCopier.scala | 2 +-
.../org/apache/spark/rdd/CartesianRDD.scala | 2 +-
.../apache/spark/rdd/PartitionPruningRDD.scala | 8 +-
.../scheduler/cluster/ClusterScheduler.scala | 13 +--
.../cluster/ClusterTaskSetManager.scala | 9 ++
.../cluster/CoarseGrainedSchedulerBackend.scala | 1 +
.../cluster/SimrSchedulerBackend.scala | 13 +--
.../org/apache/spark/ui/exec/ExecutorsUI.scala | 23 ++--
.../org/apache/spark/ui/jobs/StagePage.scala | 2 +-
.../spark/util/collection/PrimitiveVector.scala | 46 +++++---
.../org/apache/spark/LocalSparkContext.scala | 2 +-
.../apache/spark/PartitionPruningRDDSuite.scala | 45 -------
.../org/apache/spark/PartitioningSuite.scala | 10 +-
.../spark/rdd/PartitionPruningRDDSuite.scala | 86 ++++++++++++++
.../util/collection/PrimitiveVectorSuite.scala | 117 +++++++++++++++++++
docs/running-on-yarn.md | 2 +
.../apache/spark/examples/BroadcastTest.scala | 10 +-
.../org/apache/spark/examples/LocalALS.scala | 2 +-
.../spark/examples/MultiBroadcastTest.scala | 15 ++-
.../org/apache/spark/examples/SparkTC.scala | 2 +-
.../streaming/examples/ActorWordCount.scala | 2 +-
.../streaming/examples/MQTTWordCount.scala | 4 +-
.../org/apache/spark/streaming/Checkpoint.scala | 6 +-
.../api/java/JavaStreamingContext.scala | 7 +-
.../streaming/dstream/FlumeInputDStream.scala | 4 +-
.../spark/streaming/InputStreamsSuite.scala | 4 +-
.../apache/spark/streaming/TestSuiteBase.scala | 2 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 53 +++++----
.../org/apache/spark/deploy/yarn/Client.scala | 63 +++++-----
.../yarn/ClientDistributedCacheManager.scala | 6 +-
.../spark/deploy/yarn/WorkerRunnable.scala | 13 +--
.../deploy/yarn/YarnAllocationHandler.scala | 16 ++-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 5 +-
.../ClientDistributedCacheManagerSuite.scala | 2 +-
39 files changed, 415 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --cc core/src/main/scala/org/apache/spark/SparkContext.scala
index a12f886,42b2985..b9fe7f6
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@@ -24,11 -24,8 +24,10 @@@ import java.util.concurrent.atomic.Atom
import scala.collection.Map
import scala.collection.generic.Growable
- import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.collection.mutable.HashMap
+import scala.reflect.{ ClassTag, classTag}
+import scala.util.DynamicVariable
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
----------------------------------------------------------------------
diff --cc
core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index 16d8f81,8332631..a98ec06
---
a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++
b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@@ -19,10 -19,10 +19,10 @@@ package org.apache.spark.executo
import java.nio.ByteBuffer
-import akka.actor.{ActorRef, Actor, Props, Terminated}
-import akka.remote.{RemoteClientLifeCycleEvent, RemoteClientShutdown,
RemoteClientDisconnected}
+import akka.actor._
+import akka.remote._
- import org.apache.spark.{Logging, SparkEnv}
+ import org.apache.spark.Logging
import org.apache.spark.TaskState.TaskState
import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
import org.apache.spark.util.{Utils, AkkaUtils}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/scheduler/cluster/ClusterScheduler.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
----------------------------------------------------------------------
diff --cc
core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
index 465c221,20554f0..b84eb65
--- a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
@@@ -17,21 -17,21 +17,23 @@@
package org.apache.spark.util.collection
+import scala.reflect.ClassTag
+
- /** Provides a simple, non-threadsafe, array-backed vector that can store
primitives. */
+ /**
+ * An append-only, non-threadsafe, array-backed vector that is optimized for
primitive types.
+ */
private[spark]
-class PrimitiveVector[@specialized(Long, Int, Double) V:
ClassManifest](initialSize: Int = 64) {
+class PrimitiveVector[@specialized(Long, Int, Double) V:
ClassTag](initialSize: Int = 64) {
- private var numElements = 0
- private var array: Array[V] = _
+ private var _numElements = 0
+ private var _array: Array[V] = _
// NB: This must be separate from the declaration, otherwise the
specialized parent class
- // will get its own array with the same initial size. TODO: Figure out
why...
- array = new Array[V](initialSize)
+ // will get its own array with the same initial size.
+ _array = new Array[V](initialSize)
def apply(index: Int): V = {
- require(index < numElements)
- array(index)
+ require(index < _numElements)
+ _array(index)
}
def +=(value: V) {
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
----------------------------------------------------------------------
diff --cc
streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
index ca0c905,7f9dab0..80dcf87
---
a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
+++
b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
@@@ -309,13 -304,13 +308,13 @@@ class JavaStreamingContext(val ssc: Str
* @tparam F Input format for reading HDFS file
*/
def fileStream[K, V, F <: NewInputFormat[K, V]](directory: String):
JavaPairDStream[K, V] = {
- implicit val cmk: ClassManifest[K] =
- implicitly[ClassManifest[AnyRef]].asInstanceOf[ClassManifest[K]]
- implicit val cmv: ClassManifest[V] =
- implicitly[ClassManifest[AnyRef]].asInstanceOf[ClassManifest[V]]
- implicit val cmf: ClassManifest[F] =
- implicitly[ClassManifest[AnyRef]].asInstanceOf[ClassManifest[F]]
+ implicit val cmk: ClassTag[K] =
+ implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[K]]
+ implicit val cmv: ClassTag[V] =
+ implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[V]]
+ implicit val cmf: ClassTag[F] =
+ implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[F]]
- ssc.fileStream[K, V, F](directory);
+ ssc.fileStream[K, V, F](directory)
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/streaming/src/main/scala/org/apache/spark/streaming/dstream/FlumeInputDStream.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/streaming/src/test/scala/org/apache/spark/streaming/TestSuiteBase.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/95d8dbce/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
----------------------------------------------------------------------
