Merge branch 'master' of https://github.com/apache/incubator-spark
Updating local branch Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/6c6b146f Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/6c6b146f Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/6c6b146f Branch: refs/heads/master Commit: 6c6b146fc20f2ba85f1cf1bb9e35283ee195f59a Parents: d2c86e7 e33b183 Author: KarthikTunga <karthik.tu...@gmail.com> Authored: Tue Oct 15 00:46:35 2013 -0700 Committer: KarthikTunga <karthik.tu...@gmail.com> Committed: Tue Oct 15 00:46:35 2013 -0700 ---------------------------------------------------------------------- bin/stop-slaves.sh | 2 - core/pom.xml | 4 + .../spark/network/netty/FileClientHandler.java | 3 +- .../spark/network/netty/FileServerHandler.java | 6 +- .../scala/org/apache/spark/Aggregator.scala | 49 ++- .../apache/spark/BlockStoreShuffleFetcher.scala | 23 +- .../scala/org/apache/spark/CacheManager.scala | 27 +- .../scala/org/apache/spark/FutureAction.scala | 250 +++++++++++ .../apache/spark/InterruptibleIterator.scala | 30 ++ .../org/apache/spark/MapOutputTracker.scala | 4 +- .../scala/org/apache/spark/ShuffleFetcher.scala | 5 +- .../scala/org/apache/spark/SparkContext.scala | 102 +++-- .../scala/org/apache/spark/TaskContext.scala | 23 +- .../scala/org/apache/spark/TaskEndReason.scala | 2 + .../spark/broadcast/BitTorrentBroadcast.scala | 4 +- .../apache/spark/broadcast/HttpBroadcast.scala | 13 +- .../apache/spark/broadcast/TreeBroadcast.scala | 8 +- .../org/apache/spark/deploy/DeployMessage.scala | 29 +- .../spark/deploy/ExecutorDescription.scala | 34 ++ .../spark/deploy/FaultToleranceTest.scala | 420 +++++++++++++++++++ .../org/apache/spark/deploy/JsonProtocol.scala | 3 +- .../apache/spark/deploy/LocalSparkCluster.scala | 7 +- .../org/apache/spark/deploy/client/Client.scala | 84 +++- .../spark/deploy/client/ClientListener.scala | 4 + .../apache/spark/deploy/client/TestClient.scala | 7 +- .../spark/deploy/master/ApplicationInfo.scala | 53 ++- .../spark/deploy/master/ApplicationSource.scala | 2 +- .../spark/deploy/master/ApplicationState.scala | 4 +- .../spark/deploy/master/ExecutorInfo.scala | 7 +- .../master/FileSystemPersistenceEngine.scala | 90 ++++ .../deploy/master/LeaderElectionAgent.scala | 45 ++ .../org/apache/spark/deploy/master/Master.scala | 228 ++++++++-- .../spark/deploy/master/MasterMessages.scala | 46 ++ .../spark/deploy/master/MasterSource.scala | 6 +- .../spark/deploy/master/PersistenceEngine.scala | 53 +++ .../spark/deploy/master/RecoveryState.scala | 26 ++ .../deploy/master/SparkZooKeeperSession.scala | 203 +++++++++ .../apache/spark/deploy/master/WorkerInfo.scala | 42 +- .../spark/deploy/master/WorkerState.scala | 6 +- .../master/ZooKeeperLeaderElectionAgent.scala | 136 ++++++ .../master/ZooKeeperPersistenceEngine.scala | 85 ++++ .../spark/deploy/worker/ExecutorRunner.scala | 13 +- .../org/apache/spark/deploy/worker/Worker.scala | 175 +++++--- .../spark/deploy/worker/WorkerArguments.scala | 8 +- .../spark/deploy/worker/WorkerSource.scala | 10 +- .../spark/deploy/worker/ui/WorkerWebUI.scala | 2 +- .../org/apache/spark/executor/Executor.scala | 122 ++++-- .../apache/spark/executor/ExecutorSource.scala | 18 +- .../spark/executor/MesosExecutorBackend.scala | 18 +- .../executor/StandaloneExecutorBackend.scala | 10 +- .../apache/spark/network/netty/FileHeader.scala | 22 +- .../spark/network/netty/ShuffleCopier.scala | 27 +- .../spark/network/netty/ShuffleSender.scala | 8 +- .../org/apache/spark/rdd/AsyncRDDActions.scala | 122 ++++++ .../scala/org/apache/spark/rdd/BlockRDD.scala | 6 +- .../org/apache/spark/rdd/CheckpointRDD.scala | 2 +- .../org/apache/spark/rdd/CoGroupedRDD.scala | 26 +- .../scala/org/apache/spark/rdd/HadoopRDD.scala | 116 +++-- .../spark/rdd/MapPartitionsWithContextRDD.scala | 41 ++ .../spark/rdd/MapPartitionsWithIndexRDD.scala | 41 -- .../org/apache/spark/rdd/NewHadoopRDD.scala | 79 ++-- .../org/apache/spark/rdd/PairRDDFunctions.scala | 16 +- .../spark/rdd/ParallelCollectionRDD.scala | 5 +- .../main/scala/org/apache/spark/rdd/RDD.scala | 95 +++-- .../org/apache/spark/rdd/ShuffledRDD.scala | 2 +- .../org/apache/spark/rdd/SubtractedRDD.scala | 2 +- .../apache/spark/scheduler/DAGScheduler.scala | 128 +++--- .../spark/scheduler/DAGSchedulerEvent.scala | 24 +- .../spark/scheduler/DAGSchedulerSource.scala | 12 +- .../org/apache/spark/scheduler/JobWaiter.scala | 62 +-- .../scala/org/apache/spark/scheduler/Pool.scala | 5 +- .../org/apache/spark/scheduler/ResultTask.scala | 48 ++- .../spark/scheduler/SchedulableBuilder.scala | 3 + .../apache/spark/scheduler/ShuffleMapTask.scala | 50 ++- .../apache/spark/scheduler/SparkListener.scala | 2 +- .../spark/scheduler/SparkListenerBus.scala | 18 + .../scala/org/apache/spark/scheduler/Task.scala | 63 ++- .../org/apache/spark/scheduler/TaskResult.scala | 3 +- .../apache/spark/scheduler/TaskScheduler.scala | 3 + .../org/apache/spark/scheduler/TaskSet.scala | 4 + .../scheduler/cluster/ClusterScheduler.scala | 37 +- .../cluster/ClusterTaskSetManager.scala | 85 ++-- .../scheduler/cluster/SchedulerBackend.scala | 6 +- .../cluster/SparkDeploySchedulerBackend.scala | 14 +- .../cluster/StandaloneClusterMessage.scala | 2 + .../cluster/StandaloneSchedulerBackend.scala | 7 + .../spark/scheduler/local/LocalScheduler.scala | 190 +++------ .../scheduler/local/LocalTaskSetManager.scala | 13 +- .../spark/serializer/KryoSerializer.scala | 30 +- .../apache/spark/storage/BlockException.scala | 2 +- .../spark/storage/BlockFetcherIterator.scala | 24 +- .../org/apache/spark/storage/BlockId.scala | 96 +++++ .../org/apache/spark/storage/BlockManager.scala | 130 +++--- .../spark/storage/BlockManagerMaster.scala | 8 +- .../spark/storage/BlockManagerMasterActor.scala | 21 +- .../spark/storage/BlockManagerMessages.scala | 16 +- .../spark/storage/BlockManagerSource.scala | 8 +- .../spark/storage/BlockManagerWorker.scala | 4 +- .../org/apache/spark/storage/BlockMessage.scala | 38 +- .../spark/storage/BlockMessageArray.scala | 7 +- .../spark/storage/BlockObjectWriter.scala | 2 +- .../org/apache/spark/storage/BlockStore.scala | 14 +- .../org/apache/spark/storage/DiskStore.scala | 37 +- .../org/apache/spark/storage/MemoryStore.scala | 34 +- .../spark/storage/ShuffleBlockManager.scala | 16 +- .../org/apache/spark/storage/StorageUtils.scala | 47 +-- .../apache/spark/storage/ThreadingTest.scala | 6 +- .../apache/spark/ui/UIWorkloadGenerator.scala | 2 +- .../org/apache/spark/ui/storage/RDDPage.scala | 23 +- .../org/apache/spark/util/AppendOnlyMap.scala | 230 ++++++++++ .../org/apache/spark/util/MetadataCleaner.scala | 36 +- .../org/apache/spark/CacheManagerSuite.scala | 21 +- .../org/apache/spark/CheckpointSuite.scala | 10 +- .../org/apache/spark/DistributedSuite.scala | 16 +- .../scala/org/apache/spark/JavaAPISuite.java | 2 +- .../org/apache/spark/JobCancellationSuite.scala | 177 ++++++++ .../apache/spark/deploy/JsonProtocolSuite.scala | 7 +- .../apache/spark/rdd/AsyncRDDActionsSuite.scala | 176 ++++++++ .../spark/rdd/PairRDDFunctionsSuite.scala | 2 +- .../spark/scheduler/DAGSchedulerSuite.scala | 27 +- .../spark/scheduler/SparkListenerSuite.scala | 36 +- .../spark/scheduler/cluster/FakeTask.scala | 5 +- .../cluster/TaskResultGetterSuite.scala | 3 +- .../scheduler/local/LocalSchedulerSuite.scala | 28 +- .../spark/serializer/KryoSerializerSuite.scala | 21 + .../org/apache/spark/storage/BlockIdSuite.scala | 114 +++++ .../spark/storage/BlockManagerSuite.scala | 102 ++--- .../scala/org/apache/spark/ui/UISuite.scala | 7 +- .../apache/spark/util/AppendOnlyMapSuite.scala | 154 +++++++ docker/README.md | 5 + docker/build | 22 + docker/spark-test/README.md | 10 + docker/spark-test/base/Dockerfile | 38 ++ docker/spark-test/build | 22 + docker/spark-test/master/Dockerfile | 21 + docker/spark-test/master/default_cmd | 22 + docker/spark-test/worker/Dockerfile | 22 + docker/spark-test/worker/default_cmd | 22 + docs/mllib-guide.md | 24 +- docs/python-programming-guide.md | 2 +- docs/running-on-yarn.md | 9 +- docs/spark-standalone.md | 75 ++++ docs/streaming-programming-guide.md | 5 +- docs/tuning.md | 2 +- ec2/spark_ec2.py | 2 +- .../org/apache/spark/examples/SparkKMeans.scala | 2 - .../apache/spark/mllib/recommendation/ALS.scala | 199 +++++++-- .../mllib/recommendation/JavaALSSuite.java | 85 +++- .../spark/mllib/recommendation/ALSSuite.scala | 75 +++- pom.xml | 11 + project/SparkBuild.scala | 14 +- python/pyspark/rdd.py | 60 ++- .../spark/streaming/NetworkInputTracker.scala | 11 +- .../streaming/dstream/NetworkInputDStream.scala | 14 +- .../streaming/dstream/RawInputDStream.scala | 4 +- .../streaming/receivers/ActorReceiver.scala | 4 +- .../spark/deploy/yarn/ApplicationMaster.scala | 55 ++- .../org/apache/spark/deploy/yarn/Client.scala | 160 ++++++- .../spark/deploy/yarn/ClientArguments.scala | 25 +- .../spark/deploy/yarn/WorkerRunnable.scala | 58 ++- 160 files changed, 5357 insertions(+), 1402 deletions(-) ----------------------------------------------------------------------
