Repository: spark Updated Branches: refs/heads/master c24aeb6a3 -> 7fe0f3f2b
http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/scheduler/StageInfo.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/StageInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/StageInfo.scala index c6dc336..cf3db0b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/StageInfo.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/StageInfo.scala @@ -47,6 +47,18 @@ class StageInfo( failureReason = Some(reason) completionTime = Some(System.currentTimeMillis) } + + private[spark] def getStatusString: String = { + if (completionTime.isDefined) { + if (failureReason.isDefined) { + "failed" + } else { + "succeeded" + } + } else { + "running" + } + } } private[spark] object StageInfo { http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/UIUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index f078641..395af2e 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -159,12 +159,17 @@ private[spark] object UIUtils extends Logging { type="text/css" /> <link rel="stylesheet" href={prependBaseUri("/static/webui.css")} type="text/css" /> + <link rel="stylesheet" href={prependBaseUri("/static/vis.min.css")} + typ="text/css" /> + <link rel="stylesheet" href={prependBaseUri("/static/timeline-view.css")}></link> <script src={prependBaseUri("/static/sorttable.js")} ></script> <script src={prependBaseUri("/static/jquery-1.11.1.min.js")}></script> + <script src={prependBaseUri("/static/vis.min.js")}></script> <script src={prependBaseUri("/static/bootstrap-tooltip.js")}></script> <script src={prependBaseUri("/static/initialize-tooltips.js")}></script> <script src={prependBaseUri("/static/table.js")}></script> <script src={prependBaseUri("/static/additional-metrics.js")}></script> + <script src={prependBaseUri("/static/timeline-view.js")}></script> } /** Returns a spark page with correctly formatted headers */ http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index 69053fe..0a08b00 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -24,6 +24,7 @@ import org.apache.spark.annotation.DeveloperApi import org.apache.spark.scheduler._ import org.apache.spark.storage.{StorageStatus, StorageStatusListener} import org.apache.spark.ui.{SparkUI, SparkUITab} +import org.apache.spark.ui.jobs.UIData.ExecutorUIData private[ui] class ExecutorsTab(parent: SparkUI) extends SparkUITab(parent, "executors") { val listener = parent.executorsListener @@ -54,12 +55,22 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp val executorToShuffleRead = HashMap[String, Long]() val executorToShuffleWrite = HashMap[String, Long]() val executorToLogUrls = HashMap[String, Map[String, String]]() + val executorIdToData = HashMap[String, ExecutorUIData]() def storageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = synchronized { val eid = executorAdded.executorId executorToLogUrls(eid) = executorAdded.executorInfo.logUrlMap + executorIdToData(eid) = ExecutorUIData(executorAdded.time) + } + + override def onExecutorRemoved( + executorRemoved: SparkListenerExecutorRemoved): Unit = synchronized { + val eid = executorRemoved.executorId + val uiData = executorIdToData(eid) + uiData.finishTime = Some(executorRemoved.time) + uiData.finishReason = Some(executorRemoved.reason) } override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = synchronized { http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index bd923d7..a7ea12b 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -17,17 +17,183 @@ package org.apache.spark.ui.jobs -import scala.xml.{Node, NodeSeq} +import scala.collection.mutable.{HashMap, ListBuffer} +import scala.xml.{Node, NodeSeq, Unparsed} +import java.util.Date import javax.servlet.http.HttpServletRequest -import org.apache.spark.ui.{WebUIPage, UIUtils} -import org.apache.spark.ui.jobs.UIData.JobUIData +import org.apache.spark.ui.{UIUtils, WebUIPage} +import org.apache.spark.ui.jobs.UIData.{ExecutorUIData, JobUIData} +import org.apache.spark.JobExecutionStatus /** Page showing list of all ongoing and recently finished jobs */ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { - private val startTime: Option[Long] = parent.sc.map(_.startTime) - private val listener = parent.listener + private val JOBS_LEGEND = + <div class="legend-area"><svg width="150px" height="85px"> + <rect class="succeeded-job-legend" + x="5px" y="5px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="17px">Succeeded</text> + <rect class="failed-job-legend" + x="5px" y="30px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="42px">Failed</text> + <rect class="running-job-legend" + x="5px" y="55px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="67px">Running</text> + </svg></div>.toString.filter(_ != '\n') + + private val EXECUTORS_LEGEND = + <div class="legend-area"><svg width="150px" height="55px"> + <rect class="executor-added-legend" + x="5px" y="5px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="17px">Added</text> + <rect class="executor-removed-legend" + x="5px" y="30px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="42px">Removed</text> + </svg></div>.toString.filter(_ != '\n') + + private def getLastStageNameAndDescription(job: JobUIData): (String, String) = { + val lastStageInfo = Option(job.stageIds) + .filter(_.nonEmpty) + .flatMap { ids => parent.jobProgresslistener.stageIdToInfo.get(ids.max)} + val lastStageData = lastStageInfo.flatMap { s => + parent.jobProgresslistener.stageIdToData.get((s.stageId, s.attemptId)) + } + val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)") + val description = lastStageData.flatMap(_.description).getOrElse("") + (name, description) + } + + private def makeJobEvent(jobUIDatas: Seq[JobUIData]): Seq[String] = { + jobUIDatas.filter { jobUIData => + jobUIData.status != JobExecutionStatus.UNKNOWN && jobUIData.submissionTime.isDefined + }.map { jobUIData => + val jobId = jobUIData.jobId + val status = jobUIData.status + val (jobName, jobDescription) = getLastStageNameAndDescription(jobUIData) + val displayJobDescription = if (jobDescription.isEmpty) jobName else jobDescription + val submissionTime = jobUIData.submissionTime.get + val completionTimeOpt = jobUIData.completionTime + val completionTime = completionTimeOpt.getOrElse(System.currentTimeMillis()) + val classNameByStatus = status match { + case JobExecutionStatus.SUCCEEDED => "succeeded" + case JobExecutionStatus.FAILED => "failed" + case JobExecutionStatus.RUNNING => "running" + } + + val jobEventJsonAsStr = + s""" + |{ + | 'className': 'job application-timeline-object ${classNameByStatus}', + | 'group': 'jobs', + | 'start': new Date(${submissionTime}), + | 'end': new Date(${completionTime}), + | 'content': '<div class="application-timeline-content"' + + | 'data-html="true" data-placement="top" data-toggle="tooltip"' + + | 'data-title="${displayJobDescription} (Job ${jobId})<br>Status: ${status}<br>' + + | 'Submission Time: ${UIUtils.formatDate(new Date(submissionTime))}' + + | '${ + if (status != JobExecutionStatus.RUNNING) { + s"""<br>Completion Time: ${UIUtils.formatDate(new Date(completionTime))}""" + } else { + "" + } + }">' + + | '${displayJobDescription} (Job ${jobId})</div>' + |} + """.stripMargin + jobEventJsonAsStr + } + } + + private def makeExecutorEvent(executorUIDatas: HashMap[String, ExecutorUIData]): Seq[String] = { + val events = ListBuffer[String]() + executorUIDatas.foreach { + case (executorId, event) => + val addedEvent = + s""" + |{ + | 'className': 'executor added', + | 'group': 'executors', + | 'start': new Date(${event.startTime}), + | 'content': '<div class="executor-event-content"' + + | 'data-toggle="tooltip" data-placement="bottom"' + + | 'data-title="Executor ${executorId}<br>' + + | 'Added at ${UIUtils.formatDate(new Date(event.startTime))}"' + + | 'data-html="true">Executor ${executorId} added</div>' + |} + """.stripMargin + events += addedEvent + + if (event.finishTime.isDefined) { + val removedEvent = + s""" + |{ + | 'className': 'executor removed', + | 'group': 'executors', + | 'start': new Date(${event.finishTime.get}), + | 'content': '<div class="executor-event-content"' + + | 'data-toggle="tooltip" data-placement="bottom"' + + | 'data-title="Executor ${executorId}<br>' + + | 'Removed at ${UIUtils.formatDate(new Date(event.finishTime.get))}' + + | '${ + if (event.finishReason.isDefined) { + s"""<br>Reason: ${event.finishReason.get}""" + } else { + "" + } + }"' + + | 'data-html="true">Executor ${executorId} removed</div>' + |} + """.stripMargin + events += removedEvent + } + } + events.toSeq + } + + private def makeTimeline( + jobs: Seq[JobUIData], + executors: HashMap[String, ExecutorUIData], + startTime: Long): Seq[Node] = { + + val jobEventJsonAsStrSeq = makeJobEvent(jobs) + val executorEventJsonAsStrSeq = makeExecutorEvent(executors) + + val groupJsonArrayAsStr = + s""" + |[ + | { + | 'id': 'executors', + | 'content': '<div>Executors</div>${EXECUTORS_LEGEND}', + | }, + | { + | 'id': 'jobs', + | 'content': '<div>Jobs</div>${JOBS_LEGEND}', + | } + |] + """.stripMargin + + val eventArrayAsStr = + (jobEventJsonAsStrSeq ++ executorEventJsonAsStrSeq).mkString("[", ",", "]") + + <span class="expand-application-timeline"> + <span class="expand-application-timeline-arrow arrow-closed"></span> + <strong>Event Timeline</strong> + </span> ++ + <div id="application-timeline" class="collapsed"> + <div class="control-panel"> + <div id="application-timeline-zoom-lock"> + <input type="checkbox" checked="checked"></input> + <span>Zoom Lock</span> + </div> + </div> + </div> ++ + <script type="text/javascript"> + {Unparsed(s"drawApplicationTimeline(${groupJsonArrayAsStr}," + + s"${eventArrayAsStr}, ${startTime});")} + </script> + } private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined) @@ -42,15 +208,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { } def makeRow(job: JobUIData): Seq[Node] = { - val lastStageInfo = Option(job.stageIds) - .filter(_.nonEmpty) - .flatMap { ids => listener.stageIdToInfo.get(ids.max) } - val lastStageData = lastStageInfo.flatMap { s => - listener.stageIdToData.get((s.stageId, s.attemptId)) - } - - val lastStageName = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)") - val lastStageDescription = lastStageData.flatMap(_.description).getOrElse("") + val (lastStageName, lastStageDescription) = getLastStageNameAndDescription(job) val duration: Option[Long] = { job.submissionTime.map { start => val end = job.completionTime.getOrElse(System.currentTimeMillis()) @@ -61,7 +219,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { val formattedSubmissionTime = job.submissionTime.map(UIUtils.formatDate).getOrElse("Unknown") val detailUrl = "%s/jobs/job?id=%s".format(UIUtils.prependBaseUri(parent.basePath), job.jobId) - <tr> + <tr id={"job-" + job.jobId}> <td sorttable_customkey={job.jobId.toString}> {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")} </td> @@ -95,11 +253,12 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { } def render(request: HttpServletRequest): Seq[Node] = { + val listener = parent.jobProgresslistener listener.synchronized { + val startTime = listener.startTime val activeJobs = listener.activeJobs.values.toSeq val completedJobs = listener.completedJobs.reverse.toSeq val failedJobs = listener.failedJobs.reverse.toSeq - val now = System.currentTimeMillis val activeJobsTable = jobsTable(activeJobs.sortBy(_.submissionTime.getOrElse(-1L)).reverse) @@ -115,11 +274,11 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { val summary: NodeSeq = <div> <ul class="unstyled"> - {if (startTime.isDefined) { + {if (parent.sc.isDefined) { // Total duration is not meaningful unless the UI is live <li> <strong>Total Duration: </strong> - {UIUtils.formatDuration(now - startTime.get)} + {UIUtils.formatDuration(System.currentTimeMillis() - startTime)} </li> }} <li> @@ -154,6 +313,10 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { </div> var content = summary + val executorListener = parent.executorListener + content ++= makeTimeline(activeJobs ++ completedJobs ++ failedJobs, + executorListener.executorIdToData, startTime) + if (shouldShowActiveJobs) { content ++= <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++ activeJobsTable @@ -166,6 +329,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { content ++= <h4 id ="failed">Failed Jobs ({failedJobs.size})</h4> ++ failedJobsTable } + val helpText = """A job is triggered by an action, like "count()" or "saveAsTextFile()".""" + " Click on a job's title to see information about the stages of tasks associated with" + " the job." http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index 7541d3e..dd968e1 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -17,20 +17,167 @@ package org.apache.spark.ui.jobs -import scala.collection.mutable -import scala.xml.{NodeSeq, Node} +import java.util.Date + +import scala.collection.mutable.{Buffer, HashMap, ListBuffer} +import scala.xml.{NodeSeq, Node, Unparsed} import javax.servlet.http.HttpServletRequest import org.apache.spark.JobExecutionStatus import org.apache.spark.scheduler.StageInfo import org.apache.spark.ui.{UIUtils, WebUIPage} +import org.apache.spark.ui.jobs.UIData.ExecutorUIData /** Page showing statistics and stage list for a given job */ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") { - private val listener = parent.listener + private val STAGES_LEGEND = + <div class="legend-area"><svg width="150px" height="85px"> + <rect class="completed-stage-legend" + x="5px" y="5px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="17px">Completed</text> + <rect class="failed-stage-legend" + x="5px" y="30px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="42px">Failed</text> + <rect class="active-stage-legend" + x="5px" y="55px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="67px">Active</text> + </svg></div>.toString.filter(_ != '\n') + + private val EXECUTORS_LEGEND = + <div class="legend-area"><svg width="150px" height="55px"> + <rect class="executor-added-legend" + x="5px" y="5px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="17px">Added</text> + <rect class="executor-removed-legend" + x="5px" y="30px" width="20px" height="15px" rx="2px" ry="2px"></rect> + <text x="35px" y="42px">Removed</text> + </svg></div>.toString.filter(_ != '\n') + + private def makeStageEvent(stageInfos: Seq[StageInfo]): Seq[String] = { + stageInfos.map { stage => + val stageId = stage.stageId + val attemptId = stage.attemptId + val name = stage.name + val status = stage.getStatusString + val submissionTime = stage.submissionTime.get + val completionTime = stage.completionTime.getOrElse(System.currentTimeMillis()) + + s""" + |{ + | 'className': 'stage job-timeline-object ${status}', + | 'group': 'stages', + | 'start': new Date(${submissionTime}), + | 'end': new Date(${completionTime}), + | 'content': '<div class="job-timeline-content" data-toggle="tooltip"' + + | 'data-placement="top" data-html="true"' + + | 'data-title="${name} (Stage ${stageId}.${attemptId})<br>' + + | 'Status: ${status.toUpperCase}<br>' + + | 'Submission Time: ${UIUtils.formatDate(new Date(submissionTime))}' + + | '${ + if (status != "running") { + s"""<br>Completion Time: ${UIUtils.formatDate(new Date(completionTime))}""" + } else { + "" + } + }">' + + | '${name} (Stage ${stageId}.${attemptId})</div>', + |} + """.stripMargin + } + } + + def makeExecutorEvent(executorUIDatas: HashMap[String, ExecutorUIData]): Seq[String] = { + val events = ListBuffer[String]() + executorUIDatas.foreach { + case (executorId, event) => + val addedEvent = + s""" + |{ + | 'className': 'executor added', + | 'group': 'executors', + | 'start': new Date(${event.startTime}), + | 'content': '<div class="executor-event-content"' + + | 'data-toggle="tooltip" data-placement="bottom"' + + | 'data-title="Executor ${executorId}<br>' + + | 'Added at ${UIUtils.formatDate(new Date(event.startTime))}"' + + | 'data-html="true">Executor ${executorId} added</div>' + |} + """.stripMargin + events += addedEvent + + if (event.finishTime.isDefined) { + val removedEvent = + s""" + |{ + | 'className': 'executor removed', + | 'group': 'executors', + | 'start': new Date(${event.finishTime.get}), + | 'content': '<div class="executor-event-content"' + + | 'data-toggle="tooltip" data-placement="bottom"' + + | 'data-title="Executor ${executorId}<br>' + + | 'Removed at ${UIUtils.formatDate(new Date(event.finishTime.get))}' + + | '${ + if (event.finishReason.isDefined) { + s"""<br>Reason: ${event.finishReason.get}""" + } else { + "" + } + }"' + + | 'data-html="true">Executor ${executorId} removed</div>' + |} + """.stripMargin + events += removedEvent + } + } + events.toSeq + } + + private def makeTimeline( + stages: Seq[StageInfo], + executors: HashMap[String, ExecutorUIData], + appStartTime: Long): Seq[Node] = { + + val stageEventJsonAsStrSeq = makeStageEvent(stages) + val executorsJsonAsStrSeq = makeExecutorEvent(executors) + + val groupJsonArrayAsStr = + s""" + |[ + | { + | 'id': 'executors', + | 'content': '<div>Executors</div>${EXECUTORS_LEGEND}', + | }, + | { + | 'id': 'stages', + | 'content': '<div>Stages</div>${STAGES_LEGEND}', + | } + |] + """.stripMargin + + val eventArrayAsStr = + (stageEventJsonAsStrSeq ++ executorsJsonAsStrSeq).mkString("[", ",", "]") + + <span class="expand-job-timeline"> + <span class="expand-job-timeline-arrow arrow-closed"></span> + <strong>Event Timeline</strong> + </span> ++ + <div id="job-timeline" class="collapsed"> + <div class="control-panel"> + <div id="job-timeline-zoom-lock"> + <input type="checkbox" checked="checked"></input> + <span>Zoom Lock</span> + </div> + </div> + </div> ++ + <script type="text/javascript"> + {Unparsed(s"drawJobTimeline(${groupJsonArrayAsStr}, ${eventArrayAsStr}, ${appStartTime});")} + </script> + } def render(request: HttpServletRequest): Seq[Node] = { + val listener = parent.jobProgresslistener + listener.synchronized { val parameterId = request.getParameter("id") require(parameterId != null && parameterId.nonEmpty, "Missing id parameter") @@ -54,11 +201,11 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") { new StageInfo(stageId, 0, "Unknown", 0, Seq.empty, "Unknown")) } - val activeStages = mutable.Buffer[StageInfo]() - val completedStages = mutable.Buffer[StageInfo]() + val activeStages = Buffer[StageInfo]() + val completedStages = Buffer[StageInfo]() // If the job is completed, then any pending stages are displayed as "skipped": - val pendingOrSkippedStages = mutable.Buffer[StageInfo]() - val failedStages = mutable.Buffer[StageInfo]() + val pendingOrSkippedStages = Buffer[StageInfo]() + val failedStages = Buffer[StageInfo]() for (stage <- stages) { if (stage.submissionTime.isEmpty) { pendingOrSkippedStages += stage @@ -75,18 +222,18 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") { val activeStagesTable = new StageTableBase(activeStages.sortBy(_.submissionTime).reverse, - parent.basePath, parent.listener, isFairScheduler = parent.isFairScheduler, + parent.basePath, parent.jobProgresslistener, isFairScheduler = parent.isFairScheduler, killEnabled = parent.killEnabled) val pendingOrSkippedStagesTable = new StageTableBase(pendingOrSkippedStages.sortBy(_.stageId).reverse, - parent.basePath, parent.listener, isFairScheduler = parent.isFairScheduler, + parent.basePath, parent.jobProgresslistener, isFairScheduler = parent.isFairScheduler, killEnabled = false) val completedStagesTable = new StageTableBase(completedStages.sortBy(_.submissionTime).reverse, parent.basePath, - parent.listener, isFairScheduler = parent.isFairScheduler, killEnabled = false) + parent.jobProgresslistener, isFairScheduler = parent.isFairScheduler, killEnabled = false) val failedStagesTable = new FailedStageTable(failedStages.sortBy(_.submissionTime).reverse, parent.basePath, - parent.listener, isFairScheduler = parent.isFairScheduler) + parent.jobProgresslistener, isFairScheduler = parent.isFairScheduler) val shouldShowActiveStages = activeStages.nonEmpty val shouldShowPendingStages = !isComplete && pendingOrSkippedStages.nonEmpty @@ -154,6 +301,11 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") { </div> var content = summary + val appStartTime = listener.startTime + val executorListener = parent.executorListener + content ++= makeTimeline(activeStages ++ completedStages ++ failedStages, + executorListener.executorIdToData, appStartTime) + if (shouldShowActiveStages) { content ++= <h4 id="active">Active Stages ({activeStages.size})</h4> ++ activeStagesTable.toNodeSeq http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 6255968..d6d716d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -50,6 +50,9 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { type PoolName = String type ExecutorId = String + // Applicatin: + @volatile var startTime = -1L + // Jobs: val activeJobs = new HashMap[JobId, JobUIData] val completedJobs = ListBuffer[JobUIData]() @@ -75,6 +78,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { // Misc: val executorIdToBlockManagerId = HashMap[ExecutorId, BlockManagerId]() + def blockManagerIds: Seq[BlockManagerId] = executorIdToBlockManagerId.values.toSeq var schedulingMode: Option[SchedulingMode] = None @@ -516,6 +520,9 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { } } + override def onApplicationStart(appStarted: SparkListenerApplicationStart) { + startTime = appStarted.time + } } private object JobProgressListener { http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala index 7ffcf29..342787f 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala @@ -24,8 +24,10 @@ import org.apache.spark.ui.{SparkUI, SparkUITab} private[ui] class JobsTab(parent: SparkUI) extends SparkUITab(parent, "jobs") { val sc = parent.sc val killEnabled = parent.killEnabled - def isFairScheduler: Boolean = listener.schedulingMode.exists(_ == SchedulingMode.FAIR) - val listener = parent.jobProgressListener + def isFairScheduler: Boolean = + jobProgresslistener.schedulingMode.exists(_ == SchedulingMode.FAIR) + val jobProgresslistener = parent.jobProgressListener + val executorListener = parent.executorsListener attachPage(new AllJobsPage(this)) attachPage(new JobPage(this)) http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala index cb72890..6d8c7e1 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala @@ -174,7 +174,8 @@ private[ui] class StageTableBase( } /** Render an HTML row that represents a stage */ - private def renderStageRow(s: StageInfo): Seq[Node] = <tr>{stageRow(s)}</tr> + private def renderStageRow(s: StageInfo): Seq[Node] = + <tr id={"stage-" + s.stageId + "-" + s.attemptId}>{stageRow(s)}</tr> } private[ui] class FailedStageTable( http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala index 935c8a4..3d96113 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala @@ -108,4 +108,9 @@ private[spark] object UIData { var taskInfo: TaskInfo, var taskMetrics: Option[TaskMetrics] = None, var errorMessage: Option[String] = None) + + case class ExecutorUIData( + val startTime: Long, + var finishTime: Option[Long] = None, + var finishReason: Option[String] = None) } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
