Repository: spark Updated Branches: refs/heads/master 3a9851936 -> c07a50b86
[SPARK-10930] History "Stages" page "duration" can be confusing Author: Derek Dagit <[email protected]> Closes #9051 from d2r/spark-10930-ui-max-task-dur. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c07a50b8 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c07a50b8 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c07a50b8 Branch: refs/heads/master Commit: c07a50b86254578625be777b1890ff95e832ac6e Parents: 3a98519 Author: Derek Dagit <[email protected]> Authored: Wed Nov 18 15:56:54 2015 -0800 Committer: Andrew Or <[email protected]> Committed: Wed Nov 18 15:56:54 2015 -0800 ---------------------------------------------------------------------- .../org/apache/spark/ui/jobs/StageTable.scala | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/c07a50b8/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala index ea806d0..2a1c3c1 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala @@ -145,9 +145,22 @@ private[ui] class StageTableBase( case None => "Unknown" } val finishTime = s.completionTime.getOrElse(System.currentTimeMillis) - val duration = s.submissionTime.map { t => - if (finishTime > t) finishTime - t else System.currentTimeMillis - t - } + + // The submission time for a stage is misleading because it counts the time + // the stage waits to be launched. (SPARK-10930) + val taskLaunchTimes = + stageData.taskData.values.map(_.taskInfo.launchTime).filter(_ > 0) + val duration: Option[Long] = + if (taskLaunchTimes.nonEmpty) { + val startTime = taskLaunchTimes.min + if (finishTime > startTime) { + Some(finishTime - startTime) + } else { + Some(System.currentTimeMillis() - startTime) + } + } else { + None + } val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown") val inputRead = stageData.inputBytes --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
