Merge remote-tracking branch 'upstream/master' into sparsesvd
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/caf97a25 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/caf97a25 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/caf97a25 Branch: refs/heads/master Commit: caf97a25a2bd70ef5164c3ce0e8b59a8e39eb288 Parents: 4e96757 d749d47 Author: Reza Zadeh <riz...@gmail.com> Authored: Fri Jan 17 14:34:03 2014 -0800 Committer: Reza Zadeh <riz...@gmail.com> Committed: Fri Jan 17 14:34:03 2014 -0800 ---------------------------------------------------------------------- assembly/pom.xml | 116 +++++- assembly/src/deb/RELEASE | 2 + assembly/src/deb/control/control | 8 + bin/spark-class2.cmd | 2 +- bin/spark-shell.cmd | 4 +- .../scala/org/apache/spark/Accumulators.scala | 40 +- .../scala/org/apache/spark/Aggregator.scala | 25 +- .../main/scala/org/apache/spark/SparkConf.scala | 20 +- .../scala/org/apache/spark/SparkContext.scala | 4 +- .../spark/api/java/JavaSparkContext.scala | 127 +++++- .../org/apache/spark/api/java/package.scala | 23 ++ .../org/apache/spark/broadcast/Broadcast.scala | 33 +- .../org/apache/spark/broadcast/package.scala | 25 ++ .../spark/deploy/worker/CommandUtils.scala | 17 + .../spark/deploy/worker/DriverWrapper.scala | 17 + .../org/apache/spark/deploy/worker/Worker.scala | 5 +- .../spark/deploy/worker/WorkerWatcher.scala | 17 + .../org/apache/spark/rdd/CoGroupedRDD.scala | 3 +- .../apache/spark/scheduler/TaskSetManager.scala | 2 +- .../org/apache/spark/storage/BlockManager.scala | 2 +- .../org/apache/spark/DistributedSuite.scala | 17 + .../scala/org/apache/spark/SparkConfSuite.scala | 17 + .../spark/deploy/worker/DriverRunnerTest.scala | 17 + .../deploy/worker/WorkerWatcherSuite.scala | 17 + .../collection/ExternalAppendOnlyMapSuite.scala | 17 + docs/building-with-maven.md | 6 +- docs/configuration.md | 2 +- docs/graphx-programming-guide.md | 315 +++++++++----- docs/mllib-guide.md | 3 +- docs/python-programming-guide.md | 4 +- docs/streaming-programming-guide.md | 32 +- docs/tuning.md | 3 +- examples/pom.xml | 6 + .../streaming/examples/NetworkWordCount.scala | 3 +- .../streaming/examples/StreamingExamples.scala | 17 + .../spark/streaming/flume/FlumeUtils.scala | 1 - .../flume/src/test/resources/log4j.properties | 2 +- .../streaming/flume/FlumeStreamSuite.scala | 2 +- .../spark/streaming/kafka/KafkaUtils.scala | 4 +- .../kafka/src/test/resources/log4j.properties | 2 +- .../streaming/kafka/KafkaStreamSuite.scala | 1 + .../apache/spark/streaming/mqtt/MQTTUtils.scala | 4 +- .../mqtt/src/test/resources/log4j.properties | 2 +- .../spark/streaming/mqtt/MQTTStreamSuite.scala | 1 + .../spark/streaming/twitter/TwitterUtils.scala | 4 +- .../twitter/src/test/resources/log4j.properties | 2 +- .../streaming/twitter/TwitterStreamSuite.scala | 1 + .../zeromq/src/test/resources/log4j.properties | 2 +- .../streaming/zeromq/ZeroMQStreamSuite.scala | 1 + graphx/pom.xml | 1 - .../scala/org/apache/spark/graphx/Edge.scala | 25 +- .../org/apache/spark/graphx/EdgeDirection.scala | 17 + .../scala/org/apache/spark/graphx/EdgeRDD.scala | 21 +- .../org/apache/spark/graphx/EdgeTriplet.scala | 21 +- .../scala/org/apache/spark/graphx/Graph.scala | 35 +- .../spark/graphx/GraphKryoRegistrator.scala | 19 +- .../org/apache/spark/graphx/GraphLoader.scala | 17 + .../org/apache/spark/graphx/GraphOps.scala | 49 ++- .../apache/spark/graphx/PartitionStrategy.scala | 31 +- .../scala/org/apache/spark/graphx/Pregel.scala | 25 +- .../org/apache/spark/graphx/VertexRDD.scala | 42 +- .../spark/graphx/impl/EdgePartition.scala | 33 +- .../graphx/impl/EdgePartitionBuilder.scala | 27 +- .../spark/graphx/impl/EdgeTripletIterator.scala | 19 +- .../apache/spark/graphx/impl/GraphImpl.scala | 49 ++- .../spark/graphx/impl/MessageToPartition.scala | 29 +- .../graphx/impl/ReplicatedVertexView.scala | 47 ++- .../apache/spark/graphx/impl/RoutingTable.scala | 33 +- .../apache/spark/graphx/impl/Serializers.scala | 27 +- .../spark/graphx/impl/VertexPartition.scala | 61 ++- .../org/apache/spark/graphx/impl/package.scala | 19 +- .../org/apache/spark/graphx/lib/Analytics.scala | 17 + .../spark/graphx/lib/ConnectedComponents.scala | 21 +- .../org/apache/spark/graphx/lib/PageRank.scala | 21 +- .../apache/spark/graphx/lib/SVDPlusPlus.scala | 29 +- .../lib/StronglyConnectedComponents.scala | 23 +- .../apache/spark/graphx/lib/TriangleCount.scala | 19 +- .../scala/org/apache/spark/graphx/package.scala | 21 +- .../spark/graphx/util/BytecodeUtils.scala | 17 + .../spark/graphx/util/GraphGenerators.scala | 29 +- graphx/src/test/resources/als-test.data | 16 + .../org/apache/spark/graphx/GraphOpsSuite.scala | 27 +- .../org/apache/spark/graphx/GraphSuite.scala | 45 +- .../apache/spark/graphx/LocalSparkContext.scala | 17 + .../org/apache/spark/graphx/PregelSuite.scala | 25 +- .../apache/spark/graphx/SerializerSuite.scala | 35 +- .../apache/spark/graphx/VertexRDDSuite.scala | 17 + .../spark/graphx/impl/EdgePartitionSuite.scala | 19 +- .../graphx/impl/VertexPartitionSuite.scala | 17 + .../graphx/lib/ConnectedComponentsSuite.scala | 19 +- .../apache/spark/graphx/lib/PageRankSuite.scala | 17 + .../spark/graphx/lib/SVDPlusPlusSuite.scala | 27 +- .../lib/StronglyConnectedComponentsSuite.scala | 17 + .../spark/graphx/lib/TriangleCountSuite.scala | 17 + .../spark/graphx/util/BytecodeUtilsSuite.scala | 17 + .../spark/mllib/api/python/PythonMLLibAPI.scala | 4 +- .../apache/spark/mllib/classification/SVM.scala | 2 - .../spark/mllib/clustering/KMeansModel.scala | 4 +- .../mllib/regression/LinearRegression.scala | 2 +- .../mllib/regression/RidgeRegression.scala | 6 +- .../spark/mllib/util/LinearDataGenerator.scala | 4 +- .../spark/mllib/util/MFDataGenerator.scala | 17 +- .../org/apache/spark/mllib/util/MLUtils.scala | 2 +- .../spark/mllib/util/SVMDataGenerator.scala | 2 +- .../classification/JavaNaiveBayesSuite.java | 17 + .../LogisticRegressionSuite.scala | 6 +- .../spark/mllib/classification/SVMSuite.scala | 9 +- .../spark/mllib/clustering/KMeansSuite.scala | 3 - .../spark/mllib/recommendation/ALSSuite.scala | 1 - .../spark/mllib/regression/LassoSuite.scala | 6 +- .../regression/LinearRegressionSuite.scala | 5 +- .../mllib/regression/RidgeRegressionSuite.scala | 3 - pom.xml | 10 - repl-bin/pom.xml | 184 --------- repl-bin/src/deb/bin/run | 57 --- repl-bin/src/deb/bin/spark-executor | 22 - repl-bin/src/deb/bin/spark-shell | 21 - repl-bin/src/deb/control/control | 8 - .../scala/org/apache/spark/repl/ReplSuite.scala | 17 + .../org/apache/spark/streaming/Checkpoint.scala | 10 +- .../apache/spark/streaming/ContextWaiter.scala | 17 + .../apache/spark/streaming/DStreamGraph.scala | 2 + .../spark/streaming/StreamingContext.scala | 40 +- .../api/java/JavaStreamingContext.scala | 13 +- .../spark/streaming/dstream/DStream.scala | 20 +- .../dstream/DStreamCheckpointData.scala | 26 +- .../spark/streaming/dstream/InputDStream.scala | 24 +- .../streaming/dstream/NetworkInputDStream.scala | 9 +- .../dstream/PairDStreamFunctions.scala | 5 + .../org/apache/spark/streaming/package.scala | 38 ++ .../streaming/util/MasterFailureTest.scala | 2 +- .../util/RateLimitedOutputStream.scala | 1 + .../spark/streaming/util/RawTextHelper.scala | 1 + .../spark/streaming/util/RawTextSender.scala | 1 + .../apache/spark/streaming/JavaAPISuite.java | 6 +- .../apache/spark/streaming/JavaTestUtils.scala | 3 +- .../spark/streaming/BasicOperationsSuite.scala | 1 - .../spark/streaming/CheckpointSuite.scala | 30 +- .../spark/streaming/InputStreamsSuite.scala | 8 +- .../spark/streaming/StreamingContextSuite.scala | 1 - .../apache/spark/streaming/TestSuiteBase.scala | 7 +- .../org/apache/spark/deploy/yarn/Client.scala | 371 +---------------- .../spark/deploy/yarn/WorkerRunnable.scala | 126 +----- .../apache/spark/deploy/yarn/ClientBase.scala | 410 +++++++++++++++++++ .../spark/deploy/yarn/WorkerRunnableUtil.scala | 176 ++++++++ .../org/apache/spark/deploy/yarn/Client.scala | 365 +---------------- .../spark/deploy/yarn/WorkerRunnable.scala | 130 +----- 147 files changed, 2621 insertions(+), 1807 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/caf97a25/docs/mllib-guide.md ----------------------------------------------------------------------
