Merge remote-tracking branch 'upstream/master' into sparsesvd
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/61405785 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/61405785 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/61405785 Branch: refs/heads/master Commit: 61405785bc561b55681100fc3ef7e15ae8c4b113 Parents: 2612164 3713f81 Author: Reza Zadeh <riz...@gmail.com> Authored: Thu Jan 2 01:50:30 2014 -0800 Committer: Reza Zadeh <riz...@gmail.com> Committed: Thu Jan 2 01:50:30 2014 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/Accumulators.scala | 8 +- .../org/apache/spark/MapOutputTracker.scala | 11 +- .../scala/org/apache/spark/Partitioner.scala | 4 +- .../main/scala/org/apache/spark/SparkConf.scala | 189 +++++++++++++++ .../scala/org/apache/spark/SparkContext.scala | 241 +++++++++++-------- .../main/scala/org/apache/spark/SparkEnv.scala | 54 +++-- .../spark/api/java/JavaSparkContext.scala | 21 +- .../org/apache/spark/api/python/PythonRDD.scala | 4 +- .../org/apache/spark/broadcast/Broadcast.scala | 8 +- .../spark/broadcast/BroadcastFactory.scala | 4 +- .../apache/spark/broadcast/HttpBroadcast.scala | 43 ++-- .../spark/broadcast/TorrentBroadcast.scala | 45 ++-- .../spark/deploy/FaultToleranceTest.scala | 4 +- .../apache/spark/deploy/LocalSparkCluster.scala | 7 +- .../apache/spark/deploy/SparkHadoopUtil.scala | 14 +- .../org/apache/spark/deploy/client/Client.scala | 13 +- .../apache/spark/deploy/client/TestClient.scala | 10 +- .../org/apache/spark/deploy/master/Master.scala | 41 ++-- .../spark/deploy/master/MasterArguments.scala | 11 +- .../deploy/master/SparkZooKeeperSession.scala | 7 +- .../master/ZooKeeperLeaderElectionAgent.scala | 9 +- .../master/ZooKeeperPersistenceEngine.scala | 8 +- .../spark/deploy/master/ui/MasterWebUI.scala | 2 +- .../org/apache/spark/deploy/worker/Worker.scala | 34 ++- .../spark/deploy/worker/ui/WorkerWebUI.scala | 6 +- .../executor/CoarseGrainedExecutorBackend.scala | 6 +- .../org/apache/spark/executor/Executor.scala | 21 +- .../org/apache/spark/io/CompressionCodec.scala | 19 +- .../apache/spark/metrics/MetricsSystem.scala | 10 +- .../spark/network/ConnectionManager.scala | 22 +- .../org/apache/spark/network/ReceiverTest.scala | 12 +- .../org/apache/spark/network/SenderTest.scala | 16 +- .../spark/network/netty/ShuffleCopier.scala | 10 +- .../org/apache/spark/rdd/CheckpointRDD.scala | 5 +- .../org/apache/spark/rdd/CoGroupedRDD.scala | 2 +- .../main/scala/org/apache/spark/rdd/RDD.scala | 1 + .../org/apache/spark/rdd/ShuffledRDD.scala | 2 +- .../org/apache/spark/rdd/SubtractedRDD.scala | 2 +- .../apache/spark/scheduler/DAGScheduler.scala | 3 +- .../spark/scheduler/InputFormatInfo.scala | 14 +- .../org/apache/spark/scheduler/ResultTask.scala | 4 +- .../spark/scheduler/SchedulableBuilder.scala | 6 +- .../spark/scheduler/SchedulerBackend.scala | 3 - .../apache/spark/scheduler/ShuffleMapTask.scala | 6 +- .../spark/scheduler/TaskResultGetter.scala | 3 +- .../spark/scheduler/TaskSchedulerImpl.scala | 20 +- .../apache/spark/scheduler/TaskSetManager.scala | 18 +- .../cluster/CoarseGrainedSchedulerBackend.scala | 20 +- .../cluster/SimrSchedulerBackend.scala | 4 +- .../cluster/SparkDeploySchedulerBackend.scala | 8 +- .../mesos/CoarseMesosSchedulerBackend.scala | 14 +- .../cluster/mesos/MesosSchedulerBackend.scala | 8 +- .../spark/scheduler/local/LocalBackend.scala | 3 +- .../spark/serializer/JavaSerializer.scala | 3 +- .../spark/serializer/KryoSerializer.scala | 14 +- .../spark/serializer/SerializerManager.scala | 12 +- .../spark/storage/BlockFetcherIterator.scala | 4 +- .../org/apache/spark/storage/BlockManager.scala | 58 ++--- .../spark/storage/BlockManagerMaster.scala | 11 +- .../spark/storage/BlockManagerMasterActor.scala | 14 +- .../spark/storage/BlockObjectWriter.scala | 5 +- .../apache/spark/storage/DiskBlockManager.scala | 2 +- .../spark/storage/ShuffleBlockManager.scala | 10 +- .../spark/storage/StoragePerfTester.scala | 2 +- .../apache/spark/storage/ThreadingTest.scala | 8 +- .../scala/org/apache/spark/ui/SparkUI.scala | 4 +- .../apache/spark/ui/UIWorkloadGenerator.scala | 17 +- .../org/apache/spark/ui/env/EnvironmentUI.scala | 15 +- .../spark/ui/jobs/JobProgressListener.scala | 4 +- .../scala/org/apache/spark/util/AkkaUtils.scala | 25 +- .../org/apache/spark/util/MetadataCleaner.scala | 35 ++- .../org/apache/spark/util/SizeEstimator.scala | 14 +- .../scala/org/apache/spark/util/Utils.scala | 25 +- core/src/test/resources/spark.conf | 8 + .../apache/spark/MapOutputTrackerSuite.scala | 16 +- .../org/apache/spark/SharedSparkContext.scala | 4 +- .../scala/org/apache/spark/SparkConfSuite.scala | 110 +++++++++ .../apache/spark/io/CompressionCodecSuite.scala | 8 +- .../spark/metrics/MetricsSystemSuite.scala | 8 +- .../spark/scheduler/ClusterSchedulerSuite.scala | 2 +- .../spark/scheduler/DAGSchedulerSuite.scala | 23 +- .../apache/spark/scheduler/JobLoggerSuite.scala | 2 +- .../spark/scheduler/TaskResultGetterSuite.scala | 6 +- .../spark/scheduler/TaskSetManagerSuite.scala | 4 +- .../spark/serializer/KryoSerializerSuite.scala | 29 +-- .../spark/storage/BlockManagerSuite.scala | 97 ++++---- .../spark/storage/DiskBlockManagerSuite.scala | 18 +- .../apache/spark/util/SizeEstimatorSuite.scala | 2 +- docs/_config.yml | 2 +- docs/configuration.md | 71 ++++-- docs/css/bootstrap.min.css | 2 +- docs/job-scheduling.md | 21 +- docs/monitoring.md | 3 +- docs/python-programming-guide.md | 15 +- docs/quick-start.md | 52 ++-- docs/running-on-mesos.md | 19 +- docs/scala-programming-guide.md | 4 +- docs/spark-standalone.md | 15 +- docs/streaming-programming-guide.md | 4 +- docs/tuning.md | 21 +- .../examples/bagel/WikipediaPageRank.scala | 10 +- .../bagel/WikipediaPageRankStandalone.scala | 8 +- .../streaming/examples/ActorWordCount.scala | 3 +- .../apache/spark/mllib/recommendation/ALS.scala | 13 +- .../spark/deploy/yarn/ApplicationMaster.scala | 52 ++-- .../org/apache/spark/deploy/yarn/Client.scala | 40 +-- .../spark/deploy/yarn/ClientArguments.scala | 2 +- .../spark/deploy/yarn/WorkerLauncher.scala | 4 +- .../deploy/yarn/YarnAllocationHandler.scala | 4 +- .../cluster/YarnClientSchedulerBackend.scala | 4 +- project/SparkBuild.scala | 1 + python/epydoc.conf | 2 +- python/pyspark/__init__.py | 32 ++- python/pyspark/broadcast.py | 11 + python/pyspark/conf.py | 171 +++++++++++++ python/pyspark/context.py | 59 +++-- python/pyspark/java_gateway.py | 1 + python/run-tests | 3 +- .../org/apache/spark/repl/SparkILoop.scala | 17 +- .../org/apache/spark/repl/SparkIMain.scala | 7 +- .../org/apache/spark/streaming/Checkpoint.scala | 18 +- .../org/apache/spark/streaming/DStream.scala | 2 +- .../spark/streaming/StreamingContext.scala | 55 +++-- .../api/java/JavaStreamingContext.scala | 9 + .../streaming/dstream/NetworkInputDStream.scala | 6 +- .../streaming/scheduler/JobGenerator.scala | 8 +- .../streaming/scheduler/JobScheduler.scala | 4 +- .../spark/streaming/util/RawTextSender.scala | 4 +- .../apache/spark/streaming/JavaAPISuite.java | 10 +- .../spark/streaming/BasicOperationsSuite.scala | 8 +- .../spark/streaming/CheckpointSuite.scala | 15 +- .../spark/streaming/InputStreamsSuite.scala | 18 +- .../apache/spark/streaming/TestSuiteBase.scala | 34 +-- .../spark/streaming/WindowOperationsSuite.scala | 1 - .../spark/deploy/yarn/ApplicationMaster.scala | 64 ++--- .../org/apache/spark/deploy/yarn/Client.scala | 52 ++-- .../spark/deploy/yarn/ClientArguments.scala | 2 +- .../spark/deploy/yarn/WorkerLauncher.scala | 4 +- .../deploy/yarn/YarnAllocationHandler.scala | 2 +- .../cluster/YarnClientSchedulerBackend.scala | 4 +- 140 files changed, 1731 insertions(+), 941 deletions(-) ----------------------------------------------------------------------
