This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 05972c5dde5a [SPARK-55876][SQL] Add query statistics summary bar to
SQL tab
05972c5dde5a is described below
commit 05972c5dde5aabd190e46fbc805bb4bb079a597b
Author: Adithya Ajith <[email protected]>
AuthorDate: Tue Jun 2 14:00:54 2026 +0900
[SPARK-55876][SQL] Add query statistics summary bar to SQL tab
### What changes were proposed in this pull request?
This PR adds a SQL workload summary table at the top of the SQL / DataFrame
page ([SPARK-55876](https://issues.apache.org/jira/browse/SPARK-55876)).
The summary is returned as part of the existing SQL table REST response and
rendered above the SQL executions table. The summary is a fixed one-row table
styled consistently with the existing SQL executions table. It is not
initialized as a DataTable because paging, searching, ordering, and info
controls are not needed for a fixed 1x5 summary.
The summary is computed from the same execution set represented by the
current SQL table mode:
- When sub-execution grouping is enabled, the summary counts root SQL
executions only, matching the rows shown in the SQL executions table.
- When sub-execution grouping is disabled, the summary counts all SQL
executions, matching the flat table view.
The summary contains the following fields:
- Total Queries: total number of SQL query rows represented by the current
table mode.
- Average Duration: average duration across those query rows. Completed and
failed executions use their final duration. Running executions use their
current elapsed duration, computed with a single timestamp per response and
consistent with how the SQL executions table displays duration.
- Running Queries: number of query rows whose current status is RUNNING.
- Failed Queries: number of query rows whose current status is FAILED.
- Failure Rate: failed queries divided by total queries, displayed as a
percentage. When there are no queries, this is displayed as 0.0%.
The SQL executions table remains unchanged functionally.
### Why are the changes needed?
The SQL tab currently requires users to inspect the full executions table
to understand the overall SQL workload state.
Adding a summary table provides an at-a-glance view of SQL workload health,
including total query volume, average duration, currently running queries, and
failure rate.
### Does this PR introduce _any_ user-facing change?
Yes.
The SQL / DataFrame page now shows a Summary table above the SQL Executions
table.
### How was this patch tested?
Ran:
- `build/sbt 'sql/testOnly
org.apache.spark.status.api.v1.sql.SqlResourceWithActualMetricsSuite'`
- `build/sbt 'sql/testOnly
org.apache.spark.sql.execution.ui.AllExecutionsPageWithInMemoryStoreSuite'`
- `git diff --check`
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex GPT-5.5
### Before
<img width="1512" height="468" alt="before"
src="https://github.com/user-attachments/assets/9bd28a2a-2af5-49a4-b6f0-9480cc6a8f01";
/>
### After
<img width="1512" height="671" alt="Screenshot 2026-05-28 at 11 05 42 PM"
src="https://github.com/user-attachments/assets/e7cb8666-29c0-4d9b-824e-3f7b1b18986f";
/>
Closes #56090 from XdithyX/SPARK-55876.
Authored-by: Adithya Ajith <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
(cherry picked from commit f7c103196e0583c9a0206189c08b1e93c0e4ccc7)
Signed-off-by: Kousuke Saruta <[email protected]>
---
.../sql/execution/ui/static/allexecutionspage.js | 31 ++++++++++++++++++++--
.../spark/sql/execution/ui/AllExecutionsPage.scala | 3 +++
.../spark/status/api/v1/sql/SqlResource.scala | 26 ++++++++++++++++++
.../sql/execution/ui/AllExecutionsPageSuite.scala | 1 +
.../v1/sql/SqlResourceWithActualMetricsSuite.scala | 13 +++++++++
5 files changed, 72 insertions(+), 2 deletions(-)
diff --git
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/allexecutionspage.js
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/allexecutionspage.js
index ed4561c73224..5973ab896de1 100644
---
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/allexecutionspage.js
+++
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/allexecutionspage.js
@@ -32,16 +32,40 @@ $(document).ready(function () {
var container = document.getElementById("sql-executions-table");
container.innerHTML =
+ '<h4 class="title-table">SQL Executions</h4>' +
+ '<div>' +
'<select id="status-filter" class="form-select form-select-sm ' +
'd-inline-block w-auto mb-2">' +
'<option value="">All Statuses</option>' +
'<option value="RUNNING">Running</option>' +
'<option value="COMPLETED">Completed</option>' +
'<option value="FAILED">Failed</option>' +
- '</select>' +
+ '</select></div>' +
'<table id="sql-table" class="table table-striped compact cell-border" '
+
'style="width:100%"></table>';
+ function renderSummary(summary) {
+ var summaryBar = document.getElementById("sql-summary-bar");
+ if (!summaryBar || !summary) return;
+ var failureRate = (summary.failureRate || 0) * 100;
+ summaryBar.innerHTML =
+ '<h4 class="title-table">Summary</h4>' +
+ '<table id="sql-summary-table" class="table table-striped compact ' +
+ 'cell-border dataTable" style="width:100%"><thead><tr>' +
+ '<th>Total Queries</th>' +
+ '<th>Average Duration</th>' +
+ '<th>Running Queries</th>' +
+ '<th>Failed Queries</th>' +
+ '<th>Failure Rate</th>' +
+ '</tr></thead><tbody><tr>' +
+ '<td>' + (summary.totalQueries || 0) + '</td>' +
+ '<td>' + formatDurationSql(summary.averageDuration || 0) + '</td>' +
+ '<td>' + (summary.runningQueries || 0) + '</td>' +
+ '<td>' + (summary.failedQueries || 0) + '</td>' +
+ '<td>' + failureRate.toFixed(1) + '%</td>' +
+ '</tr></tbody></table>';
+ }
+
var columns = getSqlTableColumns({ detail: false });
if (groupSubExecEnabled) {
// Trailing "Sub Executions" column matching the SPARK-41752 / 4.1
layout:
@@ -80,7 +104,10 @@ $(document).ready(function () {
}
d.groupSubExecution = groupSubExecEnabled ? "true" : "false";
},
- dataSrc: function (json) { return json.aaData; },
+ dataSrc: function (json) {
+ renderSummary(json.summary);
+ return json.aaData;
+ },
error: function () {
$("#sql-table_processing").css("display", "none");
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
index 5801aba8c769..2f120731ca20 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
@@ -39,6 +39,9 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends
WebUIPage("") {
<span>
<div id="group-sub-exec-config" style="display:none"
data-value={groupSubExec.toString}></div>
+ <div id="sql-summary-bar" class="mb-3">
+ {spinner}
+ </div>
<div id="sql-executions-table">
{spinner}
</div>
diff --git
a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
index 59a1264993e1..b2c734ec27ac 100644
---
a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
+++
b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
@@ -134,6 +134,7 @@ private[v1] class SqlResource extends BaseAppResource {
} else {
(filteredExecs, Map.empty[Long, Seq[SQLExecutionUIData]])
}
+ val summary = SqlResource.executionSummary(rootRows)
// Sort
val sortCol = Option(uriParams.getFirst("order[0][column]"))
@@ -182,6 +183,7 @@ private[v1] class SqlResource extends BaseAppResource {
ret.put("aaData", aaData)
ret.put("recordsTotal", java.lang.Long.valueOf(recordsTotal))
ret.put("recordsFiltered", java.lang.Long.valueOf(recordsFiltered))
+ ret.put("summary", summary)
ret
}
}
@@ -315,6 +317,30 @@ private[v1] class SqlResource extends BaseAppResource {
private[v1] object SqlResource {
+ def executionSummary(execs: Seq[SQLExecutionUIData]):
java.util.LinkedHashMap[String, Object] = {
+ val totalQueries = execs.size
+ val runningQueries = execs.count(_.executionStatus == "RUNNING")
+ val failedQueries = execs.count(_.executionStatus == "FAILED")
+ val now = System.currentTimeMillis()
+ val totalDuration = execs.foldLeft(0L) { case (sum, exec) =>
+ sum + (exec.completionTime.map(_.getTime).getOrElse(now) -
exec.submissionTime)
+ }
+ val averageDuration = if (totalQueries > 0) totalDuration / totalQueries
else 0L
+ val failureRate = if (totalQueries > 0) {
+ failedQueries.toDouble / totalQueries.toDouble
+ } else {
+ 0.0
+ }
+
+ val summary = new java.util.LinkedHashMap[String, Object]()
+ summary.put("totalQueries", java.lang.Long.valueOf(totalQueries))
+ summary.put("averageDuration", java.lang.Long.valueOf(averageDuration))
+ summary.put("runningQueries", java.lang.Long.valueOf(runningQueries))
+ summary.put("failedQueries", java.lang.Long.valueOf(failedQueries))
+ summary.put("failureRate", java.lang.Double.valueOf(failureRate))
+ summary
+ }
+
/**
* Split a set of executions into root rows and a sub-execution map. A root
row is
* either an execution whose id equals its rootExecutionId, or an orphan sub
whose
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
index 635968838406..6c3419f7b10a 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
@@ -65,6 +65,7 @@ abstract class AllExecutionsPageSuite extends
SharedSparkSession with BeforeAndA
val page = new AllExecutionsPage(tab)
val html = page.render(request).toString().toLowerCase(Locale.ROOT)
+ assert(html.contains("sql-summary-bar"))
assert(html.contains("sql-executions-table"))
assert(html.contains("sql-table-utils.js"))
assert(html.contains("allexecutionspage.js"))
diff --git
a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
index f03aff39b532..e7072a1c0d28 100644
---
a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceWithActualMetricsSuite.scala
@@ -180,10 +180,16 @@ class SqlResourceWithActualMetricsSuite
val recordsTotal = (json \ "recordsTotal").extract[Long]
val recordsFiltered = (json \ "recordsFiltered").extract[Long]
val aaData = (json \ "aaData").children
+ val summary = json \ "summary"
assert(draw === 1, "draw should be echoed back")
assert(recordsTotal > 0, "should have some executions")
assert(recordsFiltered === recordsTotal, "no filter applied")
assert(aaData.size <= 5, "should respect page length")
+ assert((summary \ "totalQueries").extract[Long] === recordsTotal)
+ assert((summary \ "averageDuration").extract[Long] >= 0)
+ assert((summary \ "runningQueries").extract[Long] >= 0)
+ assert((summary \ "failedQueries").extract[Long] >= 0)
+ assert((summary \ "failureRate").extract[Double] >= 0.0)
// Verify row data fields
val firstRow = aaData.head
@@ -237,11 +243,14 @@ class SqlResourceWithActualMetricsSuite
val groupedJson = JsonMethods.parse(groupedOpt.get)
val groupedRecordsTotal = (groupedJson \ "recordsTotal").extract[Long]
val groupedRecordsFiltered = (groupedJson \
"recordsFiltered").extract[Long]
+ val groupedSummary = groupedJson \ "summary"
val groupedRows = (groupedJson \ "aaData").children
assert(groupedRecordsTotal === groupedRows.size,
"with no filter, recordsTotal should match returned root count")
assert(groupedRecordsFiltered === groupedRows.size,
"with no filter, recordsFiltered should match returned root count")
+ assert((groupedSummary \ "totalQueries").extract[Long] ===
groupedRecordsFiltered,
+ "grouped summary should count root rows only")
// Every row in grouped mode is either a true root (id ==
rootExecutionId)
// or an orphan sub whose real parent is absent from the result set.
val visibleIds = groupedRows.map(r => (r \ "id").extract[Long]).toSet
@@ -272,9 +281,13 @@ class SqlResourceWithActualMetricsSuite
val (flatCode, flatOpt, _) = getContentAndCode(flatUrl)
assert(flatCode === HttpServletResponse.SC_OK)
val flatJson = JsonMethods.parse(flatOpt.get)
+ val flatRecordsFiltered = (flatJson \ "recordsFiltered").extract[Long]
+ val flatSummary = flatJson \ "summary"
val flatRows = (flatJson \ "aaData").children
assert(flatRows.size > groupedRows.size,
"flat listing should contain at least one extra sub-execution row")
+ assert((flatSummary \ "totalQueries").extract[Long] ===
flatRecordsFiltered,
+ "flat summary should count all execution rows")
val embeddedSubs = groupedRows.map(r => (r \
"subExecutions").children.size).sum
assert(flatRows.size === groupedRows.size + embeddedSubs,
"flat size should equal grouped roots plus embedded sub rows")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
