Repository: spark Updated Branches: refs/heads/master be7a2fc07 -> 715a19d56
[SPARK-12637][CORE] Print stage info of finished stages properly Improve printing of StageInfo in onStageCompleted See also https://github.com/apache/spark/pull/10585 Author: Sean Owen <[email protected]> Closes #10922 from srowen/SPARK-12637. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/715a19d5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/715a19d5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/715a19d5 Branch: refs/heads/master Commit: 715a19d56fc934d4aec5025739ff650daf4580b7 Parents: be7a2fc Author: Sean Owen <[email protected]> Authored: Mon Feb 1 16:23:17 2016 -0800 Committer: Andrew Or <[email protected]> Committed: Mon Feb 1 16:23:17 2016 -0800 ---------------------------------------------------------------------- .../org/apache/spark/scheduler/SparkListener.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/715a19d5/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala index ed3adbd..7b09c2e 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala @@ -270,7 +270,7 @@ class StatsReportListener extends SparkListener with Logging { override def onStageCompleted(stageCompleted: SparkListenerStageCompleted) { implicit val sc = stageCompleted - this.logInfo("Finished stage: " + stageCompleted.stageInfo) + this.logInfo(s"Finished stage: ${getStatusDetail(stageCompleted.stageInfo)}") showMillisDistribution("task runtime:", (info, _) => Some(info.duration), taskInfoMetrics) // Shuffle write @@ -297,6 +297,17 @@ class StatsReportListener extends SparkListener with Logging { taskInfoMetrics.clear() } + private def getStatusDetail(info: StageInfo): String = { + val failureReason = info.failureReason.map("(" + _ + ")").getOrElse("") + val timeTaken = info.submissionTime.map( + x => info.completionTime.getOrElse(System.currentTimeMillis()) - x + ).getOrElse("-") + + s"Stage(${info.stageId}, ${info.attemptId}); Name: '${info.name}'; " + + s"Status: ${info.getStatusString}$failureReason; numTasks: ${info.numTasks}; " + + s"Took: $timeTaken msec" + } + } private[spark] object StatsReportListener extends Logging { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
