Repository: spark Updated Branches: refs/heads/branch-2.3 285b841ff -> 578607b30
[SPARK-23475][UI][BACKPORT-2.3] Show also skipped stages ## What changes were proposed in this pull request? SPARK-20648 introduced the status `SKIPPED` for the stages. On the UI, previously, skipped stages were shown as `PENDING`; after this change, they are not shown on the UI. The PR introduce a new section in order to show also `SKIPPED` stages in a proper table. Manual backport from to branch-2.3. ## How was this patch tested? added UT Author: Marco Gaido <marcogaid...@gmail.com> Closes #20662 from mgaido91/SPARK-23475_2.3. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/578607b3 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/578607b3 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/578607b3 Branch: refs/heads/branch-2.3 Commit: 578607b30a65d101380acbe9c9740dc267a5d55c Parents: 285b841 Author: Marco Gaido <marcogaid...@gmail.com> Authored: Fri Feb 23 18:27:33 2018 -0800 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Fri Feb 23 18:27:33 2018 -0800 ---------------------------------------------------------------------- .../apache/spark/ui/jobs/AllStagesPage.scala | 27 ++++++++++++++++++++ .../org/apache/spark/ui/UISeleniumSuite.scala | 17 ++++++++++++ 2 files changed, 44 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/578607b3/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala index b1e3434..9325b90 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala @@ -36,6 +36,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { val activeStages = allStages.filter(_.status == StageStatus.ACTIVE) val pendingStages = allStages.filter(_.status == StageStatus.PENDING) + val skippedStages = allStages.filter(_.status == StageStatus.SKIPPED) val completedStages = allStages.filter(_.status == StageStatus.COMPLETE) val failedStages = allStages.filter(_.status == StageStatus.FAILED).reverse @@ -51,6 +52,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { val completedStagesTable = new StageTableBase(parent.store, request, completedStages, "completed", "completedStage", parent.basePath, subPath, parent.isFairScheduler, false, false) + val skippedStagesTable = + new StageTableBase(parent.store, request, skippedStages, "skipped", "skippedStage", + parent.basePath, subPath, parent.isFairScheduler, false, false) val failedStagesTable = new StageTableBase(parent.store, request, failedStages, "failed", "failedStage", parent.basePath, subPath, parent.isFairScheduler, false, true) @@ -66,6 +70,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { val shouldShowActiveStages = activeStages.nonEmpty val shouldShowPendingStages = pendingStages.nonEmpty val shouldShowCompletedStages = completedStages.nonEmpty + val shouldShowSkippedStages = skippedStages.nonEmpty val shouldShowFailedStages = failedStages.nonEmpty val appSummary = parent.store.appSummary() @@ -103,6 +108,14 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { } } { + if (shouldShowSkippedStages) { + <li> + <a href="#skipped"><strong>Skipped Stages:</strong></a> + {skippedStages.size} + </li> + } + } + { if (shouldShowFailedStages) { <li> <a href="#failed"><strong>Failed Stages:</strong></a> @@ -133,6 +146,20 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { content ++= <h4 id="completed">Completed Stages ({completedStageNumStr})</h4> ++ completedStagesTable.toNodeSeq } + if (shouldShowSkippedStages) { + content ++= + <span id="skipped" class="collapse-aggregated-allSkippedStages collapse-table" + onClick="collapseTable('collapse-aggregated-allSkippedStages', + 'aggregated-allSkippedStages')"> + <h4> + <span class="collapse-table-arrow arrow-open"></span> + <a>Skipped Stages ({skippedStages.size})</a> + </h4> + </span> ++ + <div class="aggregated-allSkippedStages collapsible-table"> + {skippedStagesTable.toNodeSeq} + </div> + } if (shouldShowFailedStages) { content ++= <h4 id ="failed">Failed Stages ({numFailedStages})</h4> ++ failedStagesTable.toNodeSeq http://git-wip-us.apache.org/repos/asf/spark/blob/578607b3/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 3265467..0f20eea 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -706,6 +706,23 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B } } + test("stages page should show skipped stages") { + withSpark(newSparkContext()) { sc => + val rdd = sc.parallelize(0 to 100, 100).repartition(10).cache() + rdd.count() + rdd.count() + + eventually(timeout(5 seconds), interval(50 milliseconds)) { + goToUi(sc, "/stages") + find(id("skipped")).get.text should be("Skipped Stages (1)") + } + val stagesJson = getJson(sc.ui.get, "stages") + stagesJson.children.size should be (4) + val stagesStatus = stagesJson.children.map(_ \ "status") + stagesStatus.count(_ == JString(StageStatus.SKIPPED.name())) should be (1) + } + } + def goToUi(sc: SparkContext, path: String): Unit = { goToUi(sc.ui.get, path) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org