Github user shahidki31 commented on a diff in the pull request:
https://github.com/apache/spark/pull/22645#discussion_r223195007
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
---
@@ -229,73 +406,88 @@ private[ui] abstract class ExecutionTable(
}
val desc = if (execution.description != null &&
execution.description.nonEmpty) {
- <a href={executionURL(request,
execution.executionId)}>{execution.description}</a>
+ <a
href={executionURL(execution.executionId)}>{execution.description}</a>
} else {
- <a href={executionURL(request,
execution.executionId)}>{execution.executionId}</a>
+ <a
href={executionURL(execution.executionId)}>{execution.executionId}</a>
}
- <div>{desc} {details}</div>
- }
-
- def toNodeSeq(request: HttpServletRequest): Seq[Node] = {
- UIUtils.listingTable[SQLExecutionUIData](
- header, row(request, currentTime, _), executionUIDatas, id =
Some(tableId))
+ <div>{desc}{details}</div>
}
private def jobURL(request: HttpServletRequest, jobId: Long): String =
"%s/jobs/job/?id=%s".format(UIUtils.prependBaseUri(request,
parent.basePath), jobId)
- private def executionURL(request: HttpServletRequest, executionID:
Long): String =
+ private def executionURL(executionID: Long): String =
s"${UIUtils.prependBaseUri(
request,
parent.basePath)}/${parent.prefix}/execution/?id=$executionID"
}
-private[ui] class RunningExecutionTable(
- parent: SQLTab,
- currentTime: Long,
- executionUIDatas: Seq[SQLExecutionUIData])
- extends ExecutionTable(
- parent,
- "running-execution-table",
- currentTime,
- executionUIDatas,
- showRunningJobs = true,
- showSucceededJobs = true,
- showFailedJobs = true) {
- override protected def header: Seq[String] =
- baseHeader ++ Seq("Running Job IDs", "Succeeded Job IDs", "Failed Job
IDs")
-}
+private[ui] class ExecutionTableRowData(
+ val submissionTime: Long,
+ val duration: Long,
+ val executionUIData: SQLExecutionUIData)
+
-private[ui] class CompletedExecutionTable(
+private[ui] class ExecutionDataSource(
+ request: HttpServletRequest,
parent: SQLTab,
+ executionData: Seq[SQLExecutionUIData],
+ basePath: String,
currentTime: Long,
- executionUIDatas: Seq[SQLExecutionUIData])
- extends ExecutionTable(
- parent,
- "completed-execution-table",
- currentTime,
- executionUIDatas,
- showRunningJobs = false,
- showSucceededJobs = true,
- showFailedJobs = false) {
+ pageSize: Int,
+ sortColumn: String,
+ desc: Boolean) extends
PagedDataSource[ExecutionTableRowData](pageSize) {
- override protected def header: Seq[String] = baseHeader ++ Seq("Job IDs")
-}
+ // Convert ExecutionData to ExecutionTableRowData which contains the
final contents to show
+ // in the table so that we can avoid creating duplicate contents during
sorting the data
+ private val data =
executionData.map(executionRow).sorted(ordering(sortColumn, desc))
-private[ui] class FailedExecutionTable(
- parent: SQLTab,
- currentTime: Long,
- executionUIDatas: Seq[SQLExecutionUIData])
- extends ExecutionTable(
- parent,
- "failed-execution-table",
- currentTime,
- executionUIDatas,
- showRunningJobs = false,
- showSucceededJobs = true,
- showFailedJobs = true) {
+ private var _slicedJobIds: Set[Int] = _
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[ExecutionTableRowData] =
{
+ val r = data.slice(from, to)
+ _slicedJobIds = r.map(_.executionUIData.executionId.toInt).toSet
+ r
+ }
- override protected def header: Seq[String] =
- baseHeader ++ Seq("Succeeded Job IDs", "Failed Job IDs")
+ private def executionRow(executionUIData: SQLExecutionUIData):
ExecutionTableRowData = {
+ val submissionTime = executionUIData.submissionTime
+ val duration = executionUIData.completionTime.map(_.getTime())
+ .getOrElse(currentTime) - submissionTime
+
+ new ExecutionTableRowData(
+ submissionTime,
+ duration,
+ executionUIData)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[ExecutionTableRowData] = {
+ val ordering: Ordering[ExecutionTableRowData] = sortColumn match {
+ case "ID" => Ordering.by(_.executionUIData.executionId)
+ case "Description" => Ordering.by(_.executionUIData.description)
+ case "Submitted" => Ordering.by(_.executionUIData.submissionTime)
+ case "Duration" => Ordering.by(_.duration)
+ case "Job IDs" | "Succeeded Job IDs" => Ordering by
(_.executionUIData.jobs.filter {
--- End diff --
I have modified the code by pre computing the job data based on status and
put it in the 'ExecutionTableRowData'.
Now from there, both 'ordering' and 'jobLinks' uses the job data.
Thanks you.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
