Repository: spark Updated Branches: refs/heads/master 06a979379 -> 9b4da7b79
[SPARK-21491][GRAPHX] Enhance GraphX performance: breakOut instead of .toMap ## What changes were proposed in this pull request? `Traversable.toMap` changed to 'collections.breakOut', that eliminates intermediate tuple collection creation, see [Stack Overflow article](https://stackoverflow.com/questions/1715681/scala-2-8-breakout). ## How was this patch tested? Unit tests run. No performance tests performed yet. Please review http://spark.apache.org/contributing.html before opening a pull request. Author: iurii.ant <[email protected]> Closes #18693 from SereneAnt/performance_toMap-breakOut. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9b4da7b7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9b4da7b7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9b4da7b7 Branch: refs/heads/master Commit: 9b4da7b7906131e42fcac109d40e4d663f57c291 Parents: 06a9793 Author: iurii.ant <[email protected]> Authored: Tue Jul 25 21:43:39 2017 +0100 Committer: Sean Owen <[email protected]> Committed: Tue Jul 25 21:43:39 2017 +0100 ---------------------------------------------------------------------- .../scala/org/apache/spark/graphx/lib/LabelPropagation.scala | 2 +- .../main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/9b4da7b7/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala ---------------------------------------------------------------------- diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala index fc7547a..cb3025f 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala @@ -55,7 +55,7 @@ object LabelPropagation { val count1Val = count1.getOrElse(i, 0L) val count2Val = count2.getOrElse(i, 0L) i -> (count1Val + count2Val) - }.toMap + }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]] } def vertexProgram(vid: VertexId, attr: Long, message: Map[VertexId, Long]): VertexId = { if (message.isEmpty) attr else message.maxBy(_._2)._1 http://git-wip-us.apache.org/repos/asf/spark/blob/9b4da7b7/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala ---------------------------------------------------------------------- diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala index f0c6bcb..4cac633 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala @@ -33,10 +33,11 @@ object ShortestPaths { private def incrementMap(spmap: SPMap): SPMap = spmap.map { case (v, d) => v -> (d + 1) } - private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap = + private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap = { (spmap1.keySet ++ spmap2.keySet).map { k => k -> math.min(spmap1.getOrElse(k, Int.MaxValue), spmap2.getOrElse(k, Int.MaxValue)) - }.toMap + }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]] + } /** * Computes shortest paths to the given set of landmark vertices. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
