Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/22645#discussion_r223193077
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
---
@@ -229,73 +406,88 @@ private[ui] abstract class ExecutionTable(
}
val desc = if (execution.description != null &&
execution.description.nonEmpty) {
- <a href={executionURL(request,
execution.executionId)}>{execution.description}</a>
+ <a
href={executionURL(execution.executionId)}>{execution.description}</a>
} else {
- <a href={executionURL(request,
execution.executionId)}>{execution.executionId}</a>
+ <a
href={executionURL(execution.executionId)}>{execution.executionId}</a>
}
- <div>{desc} {details}</div>
- }
-
- def toNodeSeq(request: HttpServletRequest): Seq[Node] = {
- UIUtils.listingTable[SQLExecutionUIData](
- header, row(request, currentTime, _), executionUIDatas, id =
Some(tableId))
+ <div>{desc}{details}</div>
}
private def jobURL(request: HttpServletRequest, jobId: Long): String =
"%s/jobs/job/?id=%s".format(UIUtils.prependBaseUri(request,
parent.basePath), jobId)
- private def executionURL(request: HttpServletRequest, executionID:
Long): String =
+ private def executionURL(executionID: Long): String =
s"${UIUtils.prependBaseUri(
request,
parent.basePath)}/${parent.prefix}/execution/?id=$executionID"
}
-private[ui] class RunningExecutionTable(
- parent: SQLTab,
- currentTime: Long,
- executionUIDatas: Seq[SQLExecutionUIData])
- extends ExecutionTable(
- parent,
- "running-execution-table",
- currentTime,
- executionUIDatas,
- showRunningJobs = true,
- showSucceededJobs = true,
- showFailedJobs = true) {
- override protected def header: Seq[String] =
- baseHeader ++ Seq("Running Job IDs", "Succeeded Job IDs", "Failed Job
IDs")
-}
+private[ui] class ExecutionTableRowData(
+ val submissionTime: Long,
+ val duration: Long,
+ val executionUIData: SQLExecutionUIData)
+
-private[ui] class CompletedExecutionTable(
+private[ui] class ExecutionDataSource(
+ request: HttpServletRequest,
parent: SQLTab,
+ executionData: Seq[SQLExecutionUIData],
+ basePath: String,
currentTime: Long,
- executionUIDatas: Seq[SQLExecutionUIData])
- extends ExecutionTable(
- parent,
- "completed-execution-table",
- currentTime,
- executionUIDatas,
- showRunningJobs = false,
- showSucceededJobs = true,
- showFailedJobs = false) {
+ pageSize: Int,
+ sortColumn: String,
+ desc: Boolean) extends
PagedDataSource[ExecutionTableRowData](pageSize) {
- override protected def header: Seq[String] = baseHeader ++ Seq("Job IDs")
-}
+ // Convert ExecutionData to ExecutionTableRowData which contains the
final contents to show
+ // in the table so that we can avoid creating duplicate contents during
sorting the data
+ private val data =
executionData.map(executionRow).sorted(ordering(sortColumn, desc))
-private[ui] class FailedExecutionTable(
- parent: SQLTab,
- currentTime: Long,
- executionUIDatas: Seq[SQLExecutionUIData])
- extends ExecutionTable(
- parent,
- "failed-execution-table",
- currentTime,
- executionUIDatas,
- showRunningJobs = false,
- showSucceededJobs = true,
- showFailedJobs = true) {
+ private var _slicedJobIds: Set[Int] = _
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[ExecutionTableRowData] =
{
+ val r = data.slice(from, to)
+ _slicedJobIds = r.map(_.executionUIData.executionId.toInt).toSet
+ r
+ }
- override protected def header: Seq[String] =
- baseHeader ++ Seq("Succeeded Job IDs", "Failed Job IDs")
+ private def executionRow(executionUIData: SQLExecutionUIData):
ExecutionTableRowData = {
+ val submissionTime = executionUIData.submissionTime
+ val duration = executionUIData.completionTime.map(_.getTime())
+ .getOrElse(currentTime) - submissionTime
+
+ new ExecutionTableRowData(
+ submissionTime,
+ duration,
+ executionUIData)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[ExecutionTableRowData] = {
+ val ordering: Ordering[ExecutionTableRowData] = sortColumn match {
+ case "ID" => Ordering.by(_.executionUIData.executionId)
+ case "Description" => Ordering.by(_.executionUIData.description)
+ case "Submitted" => Ordering.by(_.executionUIData.submissionTime)
+ case "Duration" => Ordering.by(_.duration)
+ case "Job IDs" | "Succeeded Job IDs" => Ordering by
(_.executionUIData.jobs.filter {
--- End diff --
The expression here is still a little complex for my taste, but can't be
simplified much. Maybe omit jobId in the case statement, and the second
toString below it? the Seq[String] should already have the same ordering.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
