Repository: spark Updated Branches: refs/heads/master 31c74fec2 -> 8a5eb5068
[SPARK-21941] Stop storing unused attemptId in SQLTaskMetrics ## What changes were proposed in this pull request? In a driver heap dump containing 390,105 instances of SQLTaskMetrics this would have saved me approximately 3.2MB of memory. Since we're not getting any benefit from storing this unused value, let's eliminate it until a future PR makes use of it. ## How was this patch tested? Existing unit tests Author: Andrew Ash <and...@andrewash.com> Closes #19153 from ash211/aash/trim-sql-listener. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8a5eb506 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8a5eb506 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8a5eb506 Branch: refs/heads/master Commit: 8a5eb5068104f527426fb2d0908f45c8eff0749f Parents: 31c74fe Author: Andrew Ash <and...@andrewash.com> Authored: Fri Sep 8 23:33:15 2017 -0700 Committer: gatorsmile <gatorsm...@gmail.com> Committed: Fri Sep 8 23:33:15 2017 -0700 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/execution/ui/SQLListener.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/8a5eb506/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala index b4a9123..8c27af3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLListener.scala @@ -255,10 +255,8 @@ class SQLListener(conf: SparkConf) extends SparkListener with Logging { // heartbeat reports } case None => - // TODO Now just set attemptId to 0. Should fix here when we can get the attempt - // id from SparkListenerExecutorMetricsUpdate stageMetrics.taskIdToMetricUpdates(taskId) = new SQLTaskMetrics( - attemptId = 0, finished = finishTask, accumulatorUpdates) + finished = finishTask, accumulatorUpdates) } } case None => @@ -478,10 +476,11 @@ private[ui] class SQLStageMetrics( val stageAttemptId: Long, val taskIdToMetricUpdates: mutable.HashMap[Long, SQLTaskMetrics] = mutable.HashMap.empty) + +// TODO Should add attemptId here when we can get it from SparkListenerExecutorMetricsUpdate /** * Store all accumulatorUpdates for a Spark task. */ private[ui] class SQLTaskMetrics( - val attemptId: Long, // TODO not used yet var finished: Boolean, var accumulatorUpdates: Seq[(Long, Any)]) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
