Repository: spark Updated Branches: refs/heads/master b7df31eb3 -> 43e4a29da
SPARK-1399: show stage failure reason in UI https://issues.apache.org/jira/browse/SPARK-1399 refactor StageTable a bit to support additional column for failed stage Author: CodingCat <[email protected]> Author: Nan Zhu <[email protected]> Closes #421 from CodingCat/SPARK-1399 and squashes the following commits: 2caba36 [CodingCat] remove dummy tag 77cf305 [CodingCat] create dummy element to wrap columns 3989ce2 [CodingCat] address Aaron's comments 18fc09f [Nan Zhu] fix compile error 00ea30a [Nan Zhu] address Kay's comments 16ac83d [CodingCat] set a default value of failureReason 35df3df [CodingCat] address andrew's comments 06d21a4 [CodingCat] address andrew's comments 25a6db6 [CodingCat] style fix dc8856d [CodingCat] show stage failure reason in UI Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/43e4a29d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/43e4a29d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/43e4a29d Branch: refs/heads/master Commit: 43e4a29dacf61bea870ed5010d6df77bc8009aa3 Parents: b7df31e Author: CodingCat <[email protected]> Authored: Mon Apr 21 14:10:23 2014 -0700 Committer: Patrick Wendell <[email protected]> Committed: Mon Apr 21 14:10:23 2014 -0700 ---------------------------------------------------------------------- .../apache/spark/ui/jobs/JobProgressPage.scala | 8 +- .../org/apache/spark/ui/jobs/PoolPage.scala | 3 +- .../org/apache/spark/ui/jobs/StageTable.scala | 89 ++++++++++++-------- 3 files changed, 61 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/43e4a29d/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala index 34ff2ac..0da6289 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala @@ -41,10 +41,12 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("") val now = System.currentTimeMillis val activeStagesTable = - new StageTable(activeStages.sortBy(_.submissionTime).reverse, parent, parent.killEnabled) + new StageTableBase(activeStages.sortBy(_.submissionTime).reverse, + parent, parent.killEnabled) val completedStagesTable = - new StageTable(completedStages.sortBy(_.submissionTime).reverse, parent) - val failedStagesTable = new StageTable(failedStages.sortBy(_.submissionTime).reverse, parent) + new StageTableBase(completedStages.sortBy(_.submissionTime).reverse, parent) + val failedStagesTable = + new FailedStageTable(failedStages.sortBy(_.submissionTime).reverse, parent) // For now, pool information is only accessible in live UIs val pools = if (live) sc.getAllPools else Seq[Schedulable]() http://git-wip-us.apache.org/repos/asf/spark/blob/43e4a29d/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala index fd83d37..0a2bf31 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala @@ -40,7 +40,8 @@ private[ui] class PoolPage(parent: JobProgressTab) extends WebUIPage("pool") { case Some(s) => s.values.toSeq case None => Seq[StageInfo]() } - val activeStagesTable = new StageTable(activeStages.sortBy(_.submissionTime).reverse, parent) + val activeStagesTable = + new StageTableBase(activeStages.sortBy(_.submissionTime).reverse, parent) // For now, pool information is only accessible in live UIs val pools = if (live) Seq(sc.getPoolForName(poolName).get) else Seq[Schedulable]() http://git-wip-us.apache.org/repos/asf/spark/blob/43e4a29d/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala index 8c5b1f5..2eb8c7a 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala @@ -27,34 +27,37 @@ import org.apache.spark.ui.UIUtils import org.apache.spark.util.Utils /** Page showing list of all ongoing and recently finished stages */ -private[ui] class StageTable( +private[ui] class StageTableBase( stages: Seq[StageInfo], parent: JobProgressTab, killEnabled: Boolean = false) { private val basePath = parent.basePath private val listener = parent.listener - private lazy val isFairScheduler = parent.isFairScheduler + protected def isFairScheduler = parent.isFairScheduler + + protected def columns: Seq[Node] = { + // create dummy element to wrap the columns + <th>Stage Id</th> ++ + {if (isFairScheduler) {<th>Pool Name</th>} else Seq.empty} ++ + <th>Description</th> + <th>Submitted</th> + <th>Duration</th> + <th>Tasks: Succeeded/Total</th> + <th>Shuffle Read</th> + <th>Shuffle Write</th> + } def toNodeSeq: Seq[Node] = { listener.synchronized { - stageTable(stageRow, stages) + stageTable(renderStageRow, stages) } } /** Special table that merges two header cells. */ - private def stageTable[T](makeRow: T => Seq[Node], rows: Seq[T]): Seq[Node] = { + protected def stageTable[T](makeRow: T => Seq[Node], rows: Seq[T]): Seq[Node] = { <table class="table table-bordered table-striped table-condensed sortable"> - <thead> - <th>Stage Id</th> - {if (isFairScheduler) {<th>Pool Name</th>} else {}} - <th>Description</th> - <th>Submitted</th> - <th>Duration</th> - <th>Tasks: Succeeded/Total</th> - <th>Shuffle Read</th> - <th>Shuffle Write</th> - </thead> + <thead>{columns}</thead> <tbody> {rows.map(r => makeRow(r))} </tbody> @@ -94,8 +97,7 @@ private[ui] class StageTable( .getOrElse(<div> {killLink}{nameLink}</div>) } - /** Render an HTML row that represents a stage */ - private def stageRow(s: StageInfo): Seq[Node] = { + protected def stageRow(s: StageInfo): Seq[Node] = { val poolName = listener.stageIdToPool.get(s.stageId) val submissionTime = s.submissionTime match { case Some(t) => UIUtils.formatDate(new Date(t)) @@ -124,25 +126,42 @@ private[ui] class StageTable( case 0 => "" case b => Utils.bytesToString(b) } - - <tr> - <td>{s.stageId}</td> - {if (isFairScheduler) { - <td> - <a href={"%s/stages/pool?poolname=%s" - .format(UIUtils.prependBaseUri(basePath), poolName.get)}> - {poolName.get} - </a> - </td> - }} - <td>{makeDescription(s)}</td> - <td valign="middle">{submissionTime}</td> - <td sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td> - <td class="progress-cell"> - {makeProgressBar(startedTasks, completedTasks, failedTasks, totalTasks)} + <td>{s.stageId}</td> ++ + {if (isFairScheduler) { + <td> + <a href={"%s/stages/pool?poolname=%s" + .format(UIUtils.prependBaseUri(basePath), poolName.get)}> + {poolName.get} + </a> </td> - <td sorttable_customekey={shuffleReadSortable.toString}>{shuffleRead}</td> - <td sorttable_customekey={shuffleWriteSortable.toString}>{shuffleWrite}</td> - </tr> + } else { + Seq.empty + }} ++ + <td>{makeDescription(s)}</td> + <td valign="middle">{submissionTime}</td> + <td sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td> + <td class="progress-cell"> + {makeProgressBar(startedTasks, completedTasks, failedTasks, totalTasks)} + </td> + <td sorttable_customekey={shuffleReadSortable.toString}>{shuffleRead}</td> + <td sorttable_customekey={shuffleWriteSortable.toString}>{shuffleWrite}</td> + } + + /** Render an HTML row that represents a stage */ + private def renderStageRow(s: StageInfo): Seq[Node] = <tr>{stageRow(s)}</tr> +} + +private[ui] class FailedStageTable( + stages: Seq[StageInfo], + parent: JobProgressTab, + killEnabled: Boolean = false) + extends StageTableBase(stages, parent, killEnabled) { + + override protected def columns: Seq[Node] = super.columns ++ <th>Failure Reason</th> + + override protected def stageRow(s: StageInfo): Seq[Node] = { + val basicColumns = super.stageRow(s) + val failureReason = <td valign="middle">{s.failureReason.getOrElse("")}</td> + basicColumns ++ failureReason } }
