Merge remote-tracking branch 'apache/master' into filestream-fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0e15bd78 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0e15bd78 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0e15bd78 Branch: refs/heads/master Commit: 0e15bd7827d5acb5c1ccb071e358338817f95a79 Parents: 1f4718c 74b46ac Author: Tathagata Das <tathagata.das1...@gmail.com> Authored: Tue Jan 14 22:21:20 2014 -0800 Committer: Tathagata Das <tathagata.das1...@gmail.com> Committed: Tue Jan 14 22:21:20 2014 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/Accumulators.scala | 40 ++++++++++--- .../scala/org/apache/spark/Aggregator.scala | 22 +++++-- .../main/scala/org/apache/spark/SparkConf.scala | 17 ++++++ .../scala/org/apache/spark/SparkContext.scala | 4 +- .../spark/api/java/JavaSparkContext.scala | 11 ++-- .../org/apache/spark/api/java/package.scala | 23 +++++++ .../org/apache/spark/broadcast/Broadcast.scala | 33 +++++++++- .../org/apache/spark/broadcast/package.scala | 25 ++++++++ .../spark/deploy/worker/CommandUtils.scala | 17 ++++++ .../spark/deploy/worker/DriverWrapper.scala | 17 ++++++ .../spark/deploy/worker/WorkerWatcher.scala | 17 ++++++ .../org/apache/spark/storage/BlockManager.scala | 2 +- .../scala/org/apache/spark/SparkConfSuite.scala | 17 ++++++ .../spark/deploy/worker/DriverRunnerTest.scala | 17 ++++++ .../deploy/worker/WorkerWatcherSuite.scala | 17 ++++++ .../collection/ExternalAppendOnlyMapSuite.scala | 17 ++++++ docs/configuration.md | 2 +- docs/graphx-programming-guide.md | 63 ++++++++++++-------- examples/pom.xml | 6 ++ .../streaming/examples/StreamingExamples.scala | 17 ++++++ .../scala/org/apache/spark/graphx/Edge.scala | 17 ++++++ .../org/apache/spark/graphx/EdgeDirection.scala | 17 ++++++ .../scala/org/apache/spark/graphx/EdgeRDD.scala | 17 ++++++ .../org/apache/spark/graphx/EdgeTriplet.scala | 17 ++++++ .../scala/org/apache/spark/graphx/Graph.scala | 17 ++++++ .../spark/graphx/GraphKryoRegistrator.scala | 17 ++++++ .../org/apache/spark/graphx/GraphLoader.scala | 17 ++++++ .../org/apache/spark/graphx/GraphOps.scala | 17 ++++++ .../apache/spark/graphx/PartitionStrategy.scala | 17 ++++++ .../scala/org/apache/spark/graphx/Pregel.scala | 17 ++++++ .../spark/graphx/impl/EdgePartition.scala | 17 ++++++ .../graphx/impl/EdgePartitionBuilder.scala | 17 ++++++ .../spark/graphx/impl/EdgeTripletIterator.scala | 17 ++++++ .../apache/spark/graphx/impl/GraphImpl.scala | 17 ++++++ .../spark/graphx/impl/MessageToPartition.scala | 17 ++++++ .../graphx/impl/ReplicatedVertexView.scala | 17 ++++++ .../apache/spark/graphx/impl/RoutingTable.scala | 17 ++++++ .../apache/spark/graphx/impl/Serializers.scala | 17 ++++++ .../spark/graphx/impl/VertexPartition.scala | 17 ++++++ .../org/apache/spark/graphx/impl/package.scala | 17 ++++++ .../org/apache/spark/graphx/lib/Analytics.scala | 17 ++++++ .../spark/graphx/lib/ConnectedComponents.scala | 17 ++++++ .../org/apache/spark/graphx/lib/PageRank.scala | 17 ++++++ .../apache/spark/graphx/lib/SVDPlusPlus.scala | 17 ++++++ .../lib/StronglyConnectedComponents.scala | 17 ++++++ .../apache/spark/graphx/lib/TriangleCount.scala | 17 ++++++ .../scala/org/apache/spark/graphx/package.scala | 17 ++++++ .../spark/graphx/util/BytecodeUtils.scala | 17 ++++++ .../spark/graphx/util/GraphGenerators.scala | 17 ++++++ .../org/apache/spark/graphx/GraphOpsSuite.scala | 17 ++++++ .../org/apache/spark/graphx/GraphSuite.scala | 17 ++++++ .../apache/spark/graphx/LocalSparkContext.scala | 17 ++++++ .../org/apache/spark/graphx/PregelSuite.scala | 17 ++++++ .../apache/spark/graphx/SerializerSuite.scala | 17 ++++++ .../apache/spark/graphx/VertexRDDSuite.scala | 17 ++++++ .../spark/graphx/impl/EdgePartitionSuite.scala | 17 ++++++ .../graphx/impl/VertexPartitionSuite.scala | 17 ++++++ .../graphx/lib/ConnectedComponentsSuite.scala | 17 ++++++ .../apache/spark/graphx/lib/PageRankSuite.scala | 17 ++++++ .../spark/graphx/lib/SVDPlusPlusSuite.scala | 17 ++++++ .../lib/StronglyConnectedComponentsSuite.scala | 17 ++++++ .../spark/graphx/lib/TriangleCountSuite.scala | 17 ++++++ .../spark/graphx/util/BytecodeUtilsSuite.scala | 17 ++++++ .../classification/JavaNaiveBayesSuite.java | 17 ++++++ .../scala/org/apache/spark/repl/ReplSuite.scala | 17 ++++++ .../apache/spark/streaming/ContextWaiter.scala | 17 ++++++ 66 files changed, 1114 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0e15bd78/core/src/main/scala/org/apache/spark/Aggregator.scala ---------------------------------------------------------------------- diff --cc core/src/main/scala/org/apache/spark/Aggregator.scala index bcf8ae5,edbea6e..c4579cf --- a/core/src/main/scala/org/apache/spark/Aggregator.scala +++ b/core/src/main/scala/org/apache/spark/Aggregator.scala @@@ -31,10 -33,15 +33,14 @@@ case class Aggregator[K, V, C] mergeValue: (C, V) => C, mergeCombiners: (C, C) => C) { - private val sparkConf = SparkEnv.get.conf - private val externalSorting = sparkConf.getBoolean("spark.shuffle.spill", true) + private val externalSorting = SparkEnv.get.conf.getBoolean("spark.shuffle.spill", true) + @deprecated("use combineValuesByKey with TaskContext argument", "0.9.0") + def combineValuesByKey(iter: Iterator[_ <: Product2[K, V]]): Iterator[(K, C)] = + combineValuesByKey(iter, null) + def combineValuesByKey(iter: Iterator[_ <: Product2[K, V]], - context: TaskContext) : Iterator[(K, C)] = { + context: TaskContext): Iterator[(K, C)] = { if (!externalSorting) { val combiners = new AppendOnlyMap[K,C] var kv: Product2[K, V] = null http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0e15bd78/core/src/main/scala/org/apache/spark/SparkConf.scala ----------------------------------------------------------------------
