Github user squito commented on a diff in the pull request:
https://github.com/apache/spark/pull/19698#discussion_r150622074
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala ---
@@ -182,173 +185,183 @@ private[ui] class JobPage(parent: JobsTab) extends
WebUIPage("job") {
}
def render(request: HttpServletRequest): Seq[Node] = {
- val listener = parent.jobProgresslistener
+ // stripXSS is called first to remove suspicious characters used in
XSS attacks
+ val parameterId = UIUtils.stripXSS(request.getParameter("id"))
+ require(parameterId != null && parameterId.nonEmpty, "Missing id
parameter")
- listener.synchronized {
- // stripXSS is called first to remove suspicious characters used in
XSS attacks
- val parameterId = UIUtils.stripXSS(request.getParameter("id"))
- require(parameterId != null && parameterId.nonEmpty, "Missing id
parameter")
-
- val jobId = parameterId.toInt
- val jobDataOption = listener.jobIdToData.get(jobId)
- if (jobDataOption.isEmpty) {
- val content =
- <div id="no-info">
- <p>No information to display for job {jobId}</p>
- </div>
- return UIUtils.headerSparkPage(
- s"Details for Job $jobId", content, parent)
- }
- val jobData = jobDataOption.get
- val isComplete = jobData.status != JobExecutionStatus.RUNNING
- val stages = jobData.stageIds.map { stageId =>
- // This could be empty if the JobProgressListener hasn't received
information about the
- // stage or if the stage information has been garbage collected
- listener.stageIdToInfo.getOrElse(stageId,
- new StageInfo(stageId, 0, "Unknown", 0, Seq.empty, Seq.empty,
"Unknown"))
+ val jobId = parameterId.toInt
+ val jobDataOption = Try(store.job(jobId)).toOption
+ if (jobDataOption.isEmpty) {
+ val content =
+ <div id="no-info">
+ <p>No information to display for job {jobId}</p>
+ </div>
+ return UIUtils.headerSparkPage(
+ s"Details for Job $jobId", content, parent)
+ }
+ val jobData = jobDataOption.get
+ val isComplete = jobData.status != JobExecutionStatus.RUNNING
+ val stages = jobData.stageIds.map { stageId =>
+ // This could be empty if the listener hasn't received information
about the
+ // stage or if the stage information has been garbage collected
+ store.stageData(stageId).lastOption.getOrElse {
+ new v1.StageData(
+ v1.StageStatus.PENDING,
+ stageId,
+ 0, 0, 0, 0, 0, 0, 0,
+ 0L, 0L, None, None, None, None,
+ 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
+ "Unknown",
+ None,
+ "Unknown",
+ null,
+ Nil,
+ Nil,
+ None,
+ None,
+ Map())
}
+ }
- val activeStages = Buffer[StageInfo]()
- val completedStages = Buffer[StageInfo]()
- // If the job is completed, then any pending stages are displayed as
"skipped":
- val pendingOrSkippedStages = Buffer[StageInfo]()
- val failedStages = Buffer[StageInfo]()
- for (stage <- stages) {
- if (stage.submissionTime.isEmpty) {
- pendingOrSkippedStages += stage
- } else if (stage.completionTime.isDefined) {
- if (stage.failureReason.isDefined) {
- failedStages += stage
- } else {
- completedStages += stage
- }
+ val activeStages = Buffer[v1.StageData]()
+ val completedStages = Buffer[v1.StageData]()
+ // If the job is completed, then any pending stages are displayed as
"skipped":
+ val pendingOrSkippedStages = Buffer[v1.StageData]()
+ val failedStages = Buffer[v1.StageData]()
+ for (stage <- stages) {
+ if (stage.submissionTime.isEmpty) {
+ pendingOrSkippedStages += stage
+ } else if (stage.completionTime.isDefined) {
+ if (stage.status == v1.StageStatus.FAILED) {
+ failedStages += stage
} else {
- activeStages += stage
+ completedStages += stage
}
+ } else {
+ activeStages += stage
}
+ }
- val basePath = "jobs/job"
+ val basePath = "jobs/job"
- val pendingOrSkippedTableId =
- if (isComplete) {
- "pending"
- } else {
- "skipped"
- }
+ val pendingOrSkippedTableId =
+ if (isComplete) {
+ "pending"
+ } else {
+ "skipped"
+ }
- val activeStagesTable =
- new StageTableBase(request, activeStages, "active", "activeStage",
parent.basePath,
- basePath, parent.jobProgresslistener, parent.isFairScheduler,
- killEnabled = parent.killEnabled, isFailedStage = false)
- val pendingOrSkippedStagesTable =
- new StageTableBase(request, pendingOrSkippedStages,
pendingOrSkippedTableId, "pendingStage",
- parent.basePath, basePath, parent.jobProgresslistener,
parent.isFairScheduler,
- killEnabled = false, isFailedStage = false)
- val completedStagesTable =
- new StageTableBase(request, completedStages, "completed",
"completedStage", parent.basePath,
- basePath, parent.jobProgresslistener, parent.isFairScheduler,
- killEnabled = false, isFailedStage = false)
- val failedStagesTable =
- new StageTableBase(request, failedStages, "failed", "failedStage",
parent.basePath,
- basePath, parent.jobProgresslistener, parent.isFairScheduler,
- killEnabled = false, isFailedStage = true)
+ val activeStagesTable =
+ new StageTableBase(store, request, activeStages, "active",
"activeStage", parent.basePath,
+ basePath, parent.isFairScheduler,
+ killEnabled = parent.killEnabled, isFailedStage = false)
+ val pendingOrSkippedStagesTable =
+ new StageTableBase(store, request, pendingOrSkippedStages,
pendingOrSkippedTableId,
+ "pendingStage", parent.basePath, basePath, parent.isFairScheduler,
+ killEnabled = false, isFailedStage = false)
+ val completedStagesTable =
+ new StageTableBase(store, request, completedStages, "completed",
"completedStage",
+ parent.basePath, basePath, parent.isFairScheduler,
+ killEnabled = false, isFailedStage = false)
+ val failedStagesTable =
+ new StageTableBase(store, request, failedStages, "failed",
"failedStage", parent.basePath,
+ basePath, parent.isFairScheduler,
+ killEnabled = false, isFailedStage = true)
- val shouldShowActiveStages = activeStages.nonEmpty
- val shouldShowPendingStages = !isComplete &&
pendingOrSkippedStages.nonEmpty
- val shouldShowCompletedStages = completedStages.nonEmpty
- val shouldShowSkippedStages = isComplete &&
pendingOrSkippedStages.nonEmpty
- val shouldShowFailedStages = failedStages.nonEmpty
+ val shouldShowActiveStages = activeStages.nonEmpty
+ val shouldShowPendingStages = !isComplete &&
pendingOrSkippedStages.nonEmpty
+ val shouldShowCompletedStages = completedStages.nonEmpty
+ val shouldShowSkippedStages = isComplete &&
pendingOrSkippedStages.nonEmpty
+ val shouldShowFailedStages = failedStages.nonEmpty
- val summary: NodeSeq =
- <div>
- <ul class="unstyled">
- <li>
- <Strong>Status:</Strong>
- {jobData.status}
- </li>
- {
- if (jobData.jobGroup.isDefined) {
- <li>
- <strong>Job Group:</strong>
- {jobData.jobGroup.get}
- </li>
- }
- }
- {
- if (shouldShowActiveStages) {
- <li>
- <a href="#active"><strong>Active Stages:</strong></a>
- {activeStages.size}
- </li>
- }
- }
- {
- if (shouldShowPendingStages) {
- <li>
- <a href="#pending">
- <strong>Pending Stages:</strong>
- </a>{pendingOrSkippedStages.size}
- </li>
- }
+ val summary: NodeSeq =
+ <div>
+ <ul class="unstyled">
+ <li>
+ <Strong>Status:</Strong>
+ {jobData.status}
+ </li>
+ {
+ if (jobData.jobGroup.isDefined) {
+ <li>
+ <strong>Job Group:</strong>
+ {jobData.jobGroup.get}
+ </li>
}
- {
- if (shouldShowCompletedStages) {
- <li>
- <a href="#completed"><strong>Completed
Stages:</strong></a>
- {completedStages.size}
- </li>
- }
+ }
+ {
+ if (shouldShowActiveStages) {
+ <li>
+ <a href="#active"><strong>Active Stages:</strong></a>
+ {activeStages.size}
+ </li>
}
- {
- if (shouldShowSkippedStages) {
+ }
+ {
+ if (shouldShowPendingStages) {
<li>
- <a href="#skipped"><strong>Skipped Stages:</strong></a>
- {pendingOrSkippedStages.size}
+ <a href="#pending">
+ <strong>Pending Stages:</strong>
+ </a>{pendingOrSkippedStages.size}
</li>
}
+ }
+ {
+ if (shouldShowCompletedStages) {
+ <li>
+ <a href="#completed"><strong>Completed Stages:</strong></a>
+ {completedStages.size}
+ </li>
}
- {
- if (shouldShowFailedStages) {
- <li>
- <a href="#failed"><strong>Failed Stages:</strong></a>
- {failedStages.size}
- </li>
- }
+ }
+ {
+ if (shouldShowSkippedStages) {
+ <li>
+ <a href="#skipped"><strong>Skipped Stages:</strong></a>
+ {pendingOrSkippedStages.size}
+ </li>
+ }
+ }
+ {
+ if (shouldShowFailedStages) {
+ <li>
+ <a href="#failed"><strong>Failed Stages:</strong></a>
+ {failedStages.size}
+ </li>
}
- </ul>
- </div>
+ }
+ </ul>
+ </div>
- var content = summary
- val appStartTime = listener.startTime
- val operationGraphListener = parent.operationGraphListener
+ var content = summary
+ val appStartTime =
store.applicationInfo().attempts.head.startTime.getTime()
- content ++= makeTimeline(activeStages ++ completedStages ++
failedStages,
- parent.parent.store.executorList(false), appStartTime)
+ content ++= makeTimeline(activeStages ++ completedStages ++
failedStages,
+ store.executorList(false), appStartTime)
--- End diff --
nit: indentation
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
