Repository: spark Updated Branches: refs/heads/master ef3c73535 -> bf7bb4977
[SPARK-19679][ML] Destroy broadcasted object without blocking ## What changes were proposed in this pull request? Destroy broadcasted object without blocking use `find mllib -name '*.scala' | xargs -i bash -c 'egrep "destroy" -n {} && echo {}'` ## How was this patch tested? existing tests Author: Zheng RuiFeng <ruife...@foxmail.com> Closes #17016 from zhengruifeng/destroy_without_block. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf7bb497 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf7bb497 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf7bb497 Branch: refs/heads/master Commit: bf7bb497784dd97de75516f300e3add478ab472d Parents: ef3c735 Author: Zheng RuiFeng <ruife...@foxmail.com> Authored: Wed Feb 22 16:36:03 2017 +0200 Committer: Nick Pentreath <ni...@za.ibm.com> Committed: Wed Feb 22 16:36:03 2017 +0200 ---------------------------------------------------------------------- .../scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala | 2 +- .../src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala | 2 +- .../org/apache/spark/mllib/tree/model/treeEnsembleModels.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/bf7bb497/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala ---------------------------------------------------------------------- diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala index f3bace8..4c525c0 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala @@ -226,7 +226,7 @@ private[spark] object GradientBoostedTrees extends Logging { (a, b) => treesIndices.map(idx => a(idx) + b(idx))) .map(_ / dataCount) - broadcastTrees.destroy() + broadcastTrees.destroy(blocking = false) evaluation.toArray } http://git-wip-us.apache.org/repos/asf/spark/blob/bf7bb497/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala ---------------------------------------------------------------------- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala index 7a714db..efedebe 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala @@ -261,7 +261,7 @@ object LBFGS extends Logging { val (gradientSum, lossSum) = data.treeAggregate((zeroSparseVector, 0.0))(seqOp, combOp) // broadcasted model is not needed anymore - bcW.destroy() + bcW.destroy(blocking = false) /** * regVal is sum of weight squares if it's L2 updater; http://git-wip-us.apache.org/repos/asf/spark/blob/bf7bb497/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala ---------------------------------------------------------------------- diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala index fc1d412..b1e8265 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala @@ -167,7 +167,7 @@ class GradientBoostedTreesModel @Since("1.2.0") ( (a, b) => treesIndices.map(idx => a(idx) + b(idx))) .map(_ / dataCount) - broadcastTrees.destroy() + broadcastTrees.destroy(blocking = false) evaluation.toArray } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org