This is an automated email from the ASF dual-hosted git repository.
csy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 49456bb81 [KYUUBI #6108] Display the CPU time consumed by the
statement in the Spark Engine tab
49456bb81 is described below
commit 49456bb81f5bc1fdd6851cc9a93db8912f8a0db0
Author: bkhan <[email protected]>
AuthorDate: Tue Mar 5 16:33:01 2024 +0800
[KYUUBI #6108] Display the CPU time consumed by the statement in the Spark
Engine tab
# :mag: Description
## Issue References ๐
This pull request fixes #6108
## Describe Your Solution ๐ง
Follow #6112
Display the run time and CPU time of statements or sessions in the Spark UI.
<img width="1429" alt="screenshot 2024-03-05 10 37 26"
src="https://github.com/apache/kyuubi/assets/23011702/337772e0-a681-4989-b6f9-ee3633bb6287";>
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
#### Behavior With This Pull Request :tada:
#### Related Unit Tests
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6113 from XorSum/features/spark-engine-cpu-time-UI.
Closes #6108
aad2bbb6c [bkhan] session immediately
8e957c7ec [bkhan] display run time
9018a07ed [bkhan] Apply suggestions from code review
85233640f [bkhan] Display the CPU time consumed by the statement in the
Engine tab
Authored-by: bkhan <[email protected]>
Signed-off-by: Shaoyun Chen <[email protected]>
---
.../kyuubi/engine/spark/operation/SparkOperation.scala | 5 +++--
.../kyuubi/engine/spark/session/SparkSessionImpl.scala | 9 +++------
.../src/main/scala/org/apache/spark/ui/EnginePage.scala | 12 ++++++++++++
3 files changed, 18 insertions(+), 8 deletions(-)
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
index da65d9f64..ec9db84e4 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala
@@ -135,8 +135,9 @@ abstract class SparkOperation(session: Session)
info(s"statementId=${statementId}, " +
s"operationRunTime=${formatDuration(l.getOperationRunTime)}, " +
s"operationCpuTime=${formatDuration(l.getOperationCpuTime /
1000000)}")
-
session.asInstanceOf[SparkSessionImpl].increaseRunTime(l.getOperationRunTime)
-
session.asInstanceOf[SparkSessionImpl].increaseCpuTime(l.getOperationCpuTime)
+ session.asInstanceOf[SparkSessionImpl].increaseRunAndCpuTime(
+ l.getOperationRunTime,
+ l.getOperationCpuTime)
})
}
}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
index e899c28c3..b0e2d91c5 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
@@ -129,11 +129,8 @@ class SparkSessionImpl(
handle)
}
- def increaseRunTime(time: Long): Unit = {
- sessionRunTime.getAndAdd(time)
- }
-
- def increaseCpuTime(time: Long): Unit = {
- sessionCpuTime.getAndAdd(time)
+ def increaseRunAndCpuTime(runTime: Long, cpuTime: Long): Unit = {
+ sessionEvent.sessionRunTime = sessionRunTime.addAndGet(runTime)
+ sessionEvent.sessionCpuTime = sessionCpuTime.addAndGet(cpuTime)
}
}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
index cae0c03bf..d59b64dd9 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala
@@ -402,6 +402,8 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
("Start Time", true, None),
("Finish Time", true, None),
("Duration", true, None),
+ ("Run Time", true, None),
+ ("CPU Time", true, None),
("Total Statements", true, None))
headerStatRow(
@@ -428,6 +430,8 @@ case class EnginePage(parent: EngineTab) extends
WebUIPage("") {
<td> {formatDate(session.startTime)} </td>
<td> {if (session.endTime > 0) formatDate(session.endTime)} </td>
<td> {formatDuration(session.duration)} </td>
+ <td> {formatDuration(session.sessionRunTime)} </td>
+ <td> {formatDuration(session.sessionCpuTime / 1000000)} </td>
<td> {session.totalOperations} </td>
</tr>
}
@@ -484,6 +488,8 @@ private class StatementStatsPagedTable(
("Create Time", true, None),
("Finish Time", true, None),
("Duration", true, None),
+ ("Run Time", true, None),
+ ("CPU Time", true, None),
("Statement", true, None),
("State", true, None),
("Query Details", true, None),
@@ -523,6 +529,8 @@ private class StatementStatsPagedTable(
<td >
{formatDuration(event.duration)}
</td>
+ <td> {formatDuration(event.operationRunTime.getOrElse(0L))} </td>
+ <td> {formatDuration(event.operationCpuTime.getOrElse(0L) / 1000000)}
</td>
<td>
<span class="description-input">
{event.statement}
@@ -592,6 +600,8 @@ private class SessionStatsTableDataSource(
case "Start Time" => Ordering.by(_.startTime)
case "Finish Time" => Ordering.by(_.endTime)
case "Duration" => Ordering.by(_.duration)
+ case "Run Time" => Ordering.by(_.sessionRunTime)
+ case "CPU Time" => Ordering.by(_.sessionCpuTime)
case "Total Statements" => Ordering.by(_.totalOperations)
case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
}
@@ -627,6 +637,8 @@ private class StatementStatsTableDataSource(
case "Create Time" => Ordering.by(_.createTime)
case "Finish Time" => Ordering.by(_.completeTime)
case "Duration" => Ordering.by(_.duration)
+ case "Run Time" => Ordering.by(_.operationRunTime.getOrElse(0L))
+ case "CPU Time" => Ordering.by(_.operationCpuTime.getOrElse(0L))
case "Statement" => Ordering.by(_.statement)
case "State" => Ordering.by(_.state)
case "Query Details" => Ordering.by(_.executionId)
