This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b42b1f2bc993 [SPARK-55785][UI] Compact SQL plan visualization with
detail side panel
b42b1f2bc993 is described below
commit b42b1f2bc993ffaad6c9ccdd8bb87c7fdf95c9c5
Author: Kent Yao <[email protected]>
AuthorDate: Thu Mar 5 23:44:32 2026 +0800
[SPARK-55785][UI] Compact SQL plan visualization with detail side panel
### What changes were proposed in this pull request?
Improve the SQL plan visualization on the execution detail page by making
node labels compact and adding a clickable detail side panel for metrics.
**Key changes:**
- **Compact node labels**: Show only operator name as plain text (removed
HTML `labelType` and `<b>` tags)
- **Compact cluster labels**: Shortened from `WholeStageCodegen (1) /
duration: total (min, med, max)...` to `(1) / 9.3s`
- **Detail side panel** (col-4 right sidebar): Click any node to see its
metrics in a structured table
- **Cluster click**: Shows cluster-level metrics + all child node metrics
grouped by operator
- **Structured metric tables**: Uses `metricType` from server JSON to
render Total/Min/Med/Max sub-tables (no regex parsing)
- **Panel collapses** with Plan Visualization toggle
- **Scrollable panel body** (`max-height: 80vh`) for large metric sets
- **Layout tuning**: Reduced `ranksep`/`nodesep`, cluster padding via dagre
pre-layout
### Why are the changes needed?
The existing SQL plan visualization bakes all metrics into node labels,
making them hard to read for large plans with many operators. The compact
layout with on-demand detail panel provides a cleaner overview while keeping
full metric details accessible.
### Does this PR introduce _any_ user-facing change?
Yes. The SQL execution detail page now shows:
- Compact plan graph with operator names only
- A right-side panel showing full metrics on node click
- Cluster labels with just the codegen stage number and total duration
### How was this patch tested?
- Updated `SparkPlanGraphSuite` for new label format
- Added test for `makeNodeDetailsJson` verifying cluster prefix and
children array
- Manual verification with simple and complex queries (multi-way joins,
unions, window functions)
- Scalastyle passes
### Was this patch authored or co-authored using generative AI tooling?
Yes, co-authored with GitHub Copilot.
Closes #54565 from yaooqinn/SPARK-55785.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../sql/execution/ui/static/spark-sql-viz.css | 36 +++
.../spark/sql/execution/ui/static/spark-sql-viz.js | 337 +++++++++++++++++++--
.../spark/sql/execution/ui/ExecutionPage.scala | 29 +-
.../spark/sql/execution/ui/SparkPlanGraph.scala | 104 ++++---
.../sql/execution/ui/SparkPlanGraphSuite.scala | 27 +-
5 files changed, 474 insertions(+), 59 deletions(-)
diff --git
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index 032957940681..1b03c36ae0f2 100644
---
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -28,6 +28,11 @@ svg g.cluster rect {
stroke-width: 1px;
}
+svg g.cluster > g.label {
+ font-size: 0.75em;
+ font-style: italic;
+}
+
svg g.node rect {
fill: #C3EBFF;
stroke: #3EC0FF;
@@ -37,6 +42,7 @@ svg g.node rect {
/* Highlight the SparkPlan node name */
svg text :first-child:not(.stageId-and-taskId-metrics) {
font-weight: bold;
+ font-size: 0.85em;
}
svg text {
@@ -75,3 +81,33 @@ svg path.linked {
stroke: #317EACFF;
stroke-width: 2px;
}
+
+/* Detail side panel */
+#plan-viz-details-panel .card {
+ font-size: 0.75rem;
+}
+
+#plan-viz-details-panel .card-body {
+ max-height: 80vh;
+ overflow-y: auto;
+}
+
+#plan-viz-details-panel .card-header {
+ background-color: #C3EBFF;
+ border-bottom-color: #3EC0FF;
+}
+
+#plan-viz-details-panel .table th,
+#plan-viz-details-panel .table td:first-child {
+ white-space: nowrap;
+}
+
+#plan-viz-details-panel .table td:last-child {
+ word-break: break-word;
+}
+
+/* Edge labels showing row counts */
+.edgeLabel text {
+ font-size: 10px;
+ fill: #6c757d;
+}
diff --git
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index 2b13c93ad4f3..007f2963ce15 100644
---
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -22,6 +22,10 @@ var PlanVizConstants = {
svgMarginY: 16
};
+// Track selected node for re-rendering the detail panel on checkbox toggle
+var selectedNodeId = null;
+var cachedNodeDetails = null;
+
function shouldRenderPlanViz() {
return planVizContainer().selectAll("svg").empty();
}
@@ -51,6 +55,7 @@ function renderPlanViz() {
setupTooltipForSparkPlanNode(g);
resizeSvg(svg);
postprocessForAdditionalMetrics();
+ setupDetailedLabelsToggle();
}
/* -------------------- *
@@ -87,19 +92,21 @@ function setupLayoutForSparkPlanCluster(g, svg) {
const bbox = labelGroup.node().getBBox();
const rect = cluster.select("rect");
const oldWidth = parseFloat(rect.attr("width"));
- const newWidth = Math.max(oldWidth, bbox.width) + 10;
- const oldHeight = parseFloat(rect.attr("height"));
- const newHeight = oldHeight + bbox.height;
- rect
- .attr("width", (_ignored_i) => newWidth)
- .attr("height", (_ignored_i) => newHeight)
- .attr("x", (_ignored_i) => parseFloat(rect.attr("x")) - (newWidth -
oldWidth) / 2)
- .attr("y", (_ignored_i) => parseFloat(rect.attr("y")) - (newHeight -
oldHeight) / 2);
+ const newWidth = Math.max(oldWidth, bbox.width + 20);
+ const height = parseFloat(rect.attr("height"));
+
+ // Expand width if label is wider than cluster
+ if (newWidth > oldWidth) {
+ rect
+ .attr("width", newWidth)
+ .attr("x", parseFloat(rect.attr("x")) - (newWidth - oldWidth) / 2);
+ }
+ // Move label to top-right corner
labelGroup
.select("g")
.attr("text-anchor", "end")
- .attr("transform", "translate(" + (newWidth / 2 - 5) + "," + (-newHeight
/ 2 + 5) + ")");
+ .attr("transform", "translate(" + (newWidth / 2 - 5) + "," + (-height /
2 + 5) + ")");
})
}
@@ -113,10 +120,18 @@ var stageAndTaskMetricsPattern =
"^(.*)(\\(stage.*task[^)]*\\))(.*)$";
* and sizes of graph elements, e.g. padding, font style, shape.
*/
function preprocessGraphLayout(g) {
- g.graph().ranksep = "70";
+ g.graph().ranksep = "35";
+ g.graph().nodesep = "15";
g.nodes().forEach(function (v) {
const node = g.node(v);
- node.padding = "5";
+ node.padding = "3";
+ if (node.isCluster) {
+ node.clusterLabelPos = "top";
+ node.paddingTop = "35";
+ node.paddingBottom = "10";
+ node.paddingLeft = "10";
+ node.paddingRight = "10";
+ }
var firstSeparator;
var secondSeparator;
@@ -139,10 +154,14 @@ function preprocessGraphLayout(g) {
}
});
});
- // Curve the edges
+ // Curve the edges and preserve labels from DOT
g.edges().forEach(function (edge) {
+ var edgeObj = g.edge(edge);
g.setEdge(edge.v, edge.w, {
- curve: d3.curveBasis
+ curve: d3.curveBasis,
+ label: edgeObj.label || "",
+ style: "fill: none",
+ labelStyle: "font-size: 10px; fill: #6c757d;"
})
})
}
@@ -237,7 +256,7 @@ function postprocessForAdditionalMetrics() {
var checkboxNode = $("#stageId-and-taskId-checkbox");
checkboxNode.click(function() {
- onClickAdditionalMetricsCheckbox($(this));
+ onClickAdditionalMetricsCheckbox($(this), true);
});
var isChecked = window.localStorage.getItem("stageId-and-taskId-checked")
=== "true";
checkboxNode.prop("checked", isChecked);
@@ -247,7 +266,7 @@ function postprocessForAdditionalMetrics() {
/*
* Helper function which defines the action on click the checkbox.
*/
-function onClickAdditionalMetricsCheckbox(checkboxNode) {
+function onClickAdditionalMetricsCheckbox(checkboxNode, fromUserClick) {
var additionalMetrics = $(".stageId-and-taskId-metrics");
var isChecked = checkboxNode.prop("checked");
if (isChecked) {
@@ -256,6 +275,17 @@ function onClickAdditionalMetricsCheckbox(checkboxNode) {
additionalMetrics.hide();
}
window.localStorage.setItem("stageId-and-taskId-checked", isChecked);
+ // Re-render the detail panel to update stage/task column visibility
+ if (selectedNodeId && cachedNodeDetails) {
+ updateDetailsPanel(selectedNodeId, cachedNodeDetails);
+ }
+ // Re-render graph nodes if in detailed mode (only on user click, not init)
+ if (fromUserClick) {
+ var detailedCheckbox = document.getElementById("detailed-labels-checkbox");
+ if (detailedCheckbox && detailedCheckbox.checked) {
+ rerenderWithDetailedLabels();
+ }
+ }
}
function togglePlanViz() { // eslint-disable-line no-unused-vars
@@ -264,18 +294,193 @@ function togglePlanViz() { // eslint-disable-line
no-unused-vars
$(this).toggleClass("arrow-open").toggleClass("arrow-closed")
});
if (arrow.classed("arrow-open")) {
- planVizContainer().style("display", "block");
+ d3.select("#plan-viz-content").style("display", "flex");
+ } else {
+ d3.select("#plan-viz-content").style("display", "none");
+ }
+}
+
+/*
+ * Parse the node details JSON from the hidden metadata div.
+ */
+function getNodeDetails() {
+ var detailsText = d3.select("#plan-viz-metadata
.node-details").text().trim();
+ try {
+ return JSON.parse(detailsText);
+ } catch (e) {
+ return {};
+ }
+}
+
+/*
+ * Format a metric value for display in the detail panel.
+ * Detects "total (min, med, max ...)\nVALUE (v1, v2, v3 (...))" patterns
+ * and renders them as a structured sub-table.
+ */
+/*
+ * Format a metric value for display in the detail panel.
+ * Uses metricType to determine layout instead of regex parsing.
+ * Types: "size", "timing", "nsTiming" have total + (min, med, max).
+ * "average" has (min, med, max) only.
+ * "sum" is a plain value.
+ */
+function formatMetricValue(val, metricType, showStageTask) {
+ var lines = val.split("\n");
+ if (lines.length < 2) return val;
+
+ // Multi-line: header line + data line
+ var dataLine = lines[1].trim();
+ var hasTotal = (metricType === "size" || metricType === "timing"
+ || metricType === "nsTiming");
+
+ // Parse data: "TOTAL (MIN, MED, MAX (stage X: task Y))" or "(MIN, MED, MAX
(...))"
+ var total = null, min, med, maxVal, stageId = "", taskId = "";
+ if (hasTotal) {
+ // Split "7.5 KiB (240.0 B, 240.0 B, 240.0 B (stage 3.0: task 36))"
+ var idx = dataLine.indexOf("(");
+ if (idx < 0) return val;
+ total = dataLine.substring(0, idx).trim();
+ dataLine = dataLine.substring(idx);
+ }
+ // Now dataLine is "(MIN, MED, MAX ...)" — strip outer parens
+ var inner = dataLine.replace(/^\(/, "").replace(/\)$/, "");
+ // Split by comma, but respect parenthesized stage/task
+ var parts = [];
+ var depth = 0, start = 0;
+ for (var i = 0; i < inner.length; i++) {
+ if (inner[i] === "(") depth++;
+ else if (inner[i] === ")") depth--;
+ else if (inner[i] === "," && depth === 0) {
+ parts.push(inner.substring(start, i).trim());
+ start = i + 1;
+ }
+ }
+ parts.push(inner.substring(start).trim());
+
+ min = parts[0] || "";
+ med = parts[1] || "";
+ // Max may contain "(stage X: task Y)"
+ var maxPart = parts[2] || "";
+ var stageMatch = maxPart.match(/^(.+?)\s*\(stage\s+(.+?):\s*task\s+(.+)\)$/);
+ if (stageMatch) {
+ maxVal = stageMatch[1];
+ stageId = stageMatch[2];
+ taskId = stageMatch[3];
} else {
- planVizContainer().style("display", "none");
+ maxVal = maxPart;
+ }
+
+ return buildStatTable(total, min, med, maxVal,
+ stageId, taskId, showStageTask);
+}
+
+function buildStatTable(total, min, med, maxVal,
+ stageId, taskId, showStageTask) {
+ var h = '<table class="table table-sm table-bordered mb-0"><thead><tr>';
+ if (total !== null) h += "<th>Total</th>";
+ h += "<th>Min</th><th>Med</th><th>Max</th>";
+ if (showStageTask) h += "<th>Stage</th><th>Task</th>";
+ h += "</tr></thead><tbody><tr>";
+ if (total !== null) h += "<td>" + total + "</td>";
+ h += "<td>" + min + "</td><td>" + med +
+ "</td><td>" + maxVal + "</td>";
+ if (showStageTask) {
+ h += "<td>" + stageId + "</td><td>" + taskId + "</td>";
+ }
+ h += "</tr></tbody></table>";
+ return h;
+}
+
+/*
+ * Update the detail side panel with the selected node's information.
+ */
+function updateDetailsPanel(nodeId, nodeDetails) {
+ var titleEl = document.getElementById("plan-viz-details-title");
+ var bodyEl = document.getElementById("plan-viz-details-body");
+ if (!titleEl || !bodyEl) return;
+
+ selectedNodeId = nodeId;
+ cachedNodeDetails = nodeDetails;
+
+ var details = nodeDetails[nodeId];
+ if (!details) {
+ titleEl.textContent = "Details";
+ bodyEl.innerHTML = '<p class="text-muted mb-0">No details available</p>';
+ return;
+ }
+
+ titleEl.textContent = details.name;
+
+ // Show node description as a tooltip on the title
+ var existingDesc = document.getElementById("plan-viz-details-desc");
+ if (existingDesc) existingDesc.remove();
+ if (details.desc) {
+ titleEl.setAttribute("data-bs-toggle", "tooltip");
+ titleEl.setAttribute("title", details.desc);
+ var existing = bootstrap.Tooltip.getInstance(titleEl);
+ if (existing) existing.dispose();
+ new bootstrap.Tooltip(titleEl, {container: "body"});
+ } else {
+ titleEl.removeAttribute("data-bs-toggle");
+ titleEl.removeAttribute("title");
+ }
+
+ var showStageTask = document.getElementById("stageId-and-taskId-checkbox")
+ && document.getElementById("stageId-and-taskId-checkbox").checked;
+
+ var html = "";
+ if (details.metrics && details.metrics.length > 0) {
+ html += buildMetricsTable(details.metrics, showStageTask);
+ } else if (!details.children) {
+ html += '<p class="text-muted mb-0">No metrics</p>';
+ }
+
+ // For clusters, show child node metrics
+ if (details.children && details.children.length > 0) {
+ details.children.forEach(function (childId) {
+ var child = nodeDetails[childId];
+ if (child) {
+ html += '<h6 class="mt-2 mb-1 fw-bold">' + htmlEscape(child.name) +
'</h6>';
+ if (child.metrics && child.metrics.length > 0) {
+ html += buildMetricsTable(child.metrics, showStageTask);
+ } else {
+ html += '<p class="text-muted mb-0">No metrics</p>';
+ }
+ }
+ });
}
+ bodyEl.innerHTML = html;
+}
+
+function htmlEscape(str) {
+ return str.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
+ .replace(/"/g, """).replace(/'/g, "'");
+}
+
+/*
+ * Build an HTML metrics table from a metrics array.
+ */
+function buildMetricsTable(metrics, showStageTask) {
+ var html = '<table class="table table-sm table-bordered mb-0">';
+ html += "<thead><tr><th>Metric</th><th>Value</th></tr></thead><tbody>";
+ metrics.forEach(function (m) {
+ var name = htmlEscape(m.name);
+ var val = htmlEscape(m.value);
+ html += "<tr><td>" + name + "</td><td>" +
+ formatMetricValue(val, m.type, showStageTask) + "</td></tr>";
+ });
+ html += "</tbody></table>";
+ return html;
}
/*
* Light up the selected node and its linked nodes and edges.
+ * Also updates the detail side panel with the selected node's metrics.
*/
function setupSelectionForSparkPlanNode(g) {
const linkedNodes = new Map();
const linkedEdges = new Map();
+ const nodeDetails = getNodeDetails();
g.edges().forEach(function (e) {
const edge = g.edge(e);
@@ -288,7 +493,8 @@ function setupSelectionForSparkPlanNode(g) {
});
linkedNodes.forEach((linkedNodes, selectNode) => {
- d3.select("#" + selectNode).on("click", () => {
+ d3.select("#" + selectNode).on("click", (event) => {
+ event.stopPropagation();
planVizContainer().selectAll(".selected").classed("selected", false);
planVizContainer().selectAll(".linked").classed("linked", false);
d3.select("#" + selectNode + " rect").classed("selected", true);
@@ -301,8 +507,25 @@ function setupSelectionForSparkPlanNode(g) {
const arrowShaft =
$(arrowHead.node()).parents("g.edgePath").children("path");
arrowShaft.addClass("linked");
});
+ updateDetailsPanel(selectNode, nodeDetails);
});
});
+
+ // Add click handlers for clusters (WholeStageCodegen groups)
+ planVizContainer().selectAll("g.cluster").each(function() {
+ const clusterId = d3.select(this).attr("id");
+ if (clusterId && nodeDetails[clusterId]) {
+ d3.select(this).on("click", (event) => {
+ // Skip if click was on a child node
+ if (event.target.closest("g.node")) return;
+ event.stopPropagation();
+ planVizContainer().selectAll(".selected").classed("selected", false);
+ planVizContainer().selectAll(".linked").classed("linked", false);
+ d3.select(this).select(":scope > rect").classed("selected", true);
+ updateDetailsPanel(clusterId, nodeDetails);
+ });
+ }
+ });
}
function collectLinks(map, key, value) {
@@ -352,6 +575,84 @@ function clickPhysicalPlanDetails() {
$('#physical-plan-details-arrow').toggleClass('arrow-open').toggleClass('arrow-closed');
}
+/*
+ * Set up the toggle for detailed node labels (show metrics inside graph
nodes).
+ * Stores original compact labels so they can be restored on toggle-off.
+ */
+/*
+ * Set up the toggle for detailed node labels.
+ * Re-renders the graph from the detailed DOT file instead of patching SVG
in-place.
+ */
+/*
+ * Toggle between compact and detailed node labels.
+ * Re-renders the graph with metrics injected into node labels from the JSON
metadata.
+ */
+function setupDetailedLabelsToggle() {
+ var checkbox = document.getElementById("detailed-labels-checkbox");
+ if (!checkbox) return;
+
+ var isChecked = window.localStorage.getItem("detailed-labels-checked") ===
"true";
+ checkbox.checked = isChecked;
+ if (isChecked) {
+ rerenderWithDetailedLabels();
+ }
+
+ checkbox.addEventListener("click", function () {
+ window.localStorage.setItem("detailed-labels-checked",
String(checkbox.checked));
+ rerenderWithDetailedLabels();
+ });
+}
+
+function rerenderWithDetailedLabels() {
+ var metadata = d3.select("#plan-viz-metadata");
+ var dot = metadata.select(".dot-file").text().trim();
+ if (!dot) return;
+
+ var container = planVizContainer();
+ container.selectAll("svg").remove();
+
+ var svg = container
+ .append("svg")
+ .attr("width", window.innerWidth || 1920)
+ .attr("height", 1000);
+ var graph = svg.append("g");
+
+ var g = graphlibDot.read(dot);
+
+ // If detailed mode, inject HTML labels with metrics tables
+ var detailed = document.getElementById("detailed-labels-checkbox");
+ if (detailed && detailed.checked) {
+ var nodeDetails = getNodeDetails();
+ var showStageTask = document.getElementById("stageId-and-taskId-checkbox")
+ && document.getElementById("stageId-and-taskId-checkbox").checked;
+ g.nodes().forEach(function (v) {
+ var node = g.node(v);
+ if (!node.isCluster && nodeDetails[node.id]) {
+ var details = nodeDetails[node.id];
+ if (details.metrics && details.metrics.length > 0) {
+ var html = '<div
style="padding:4px;text-align:left;font-size:10px;">';
+ html += '<strong>' + htmlEscape(details.name) + '</strong>';
+ html += buildMetricsTable(details.metrics, showStageTask);
+ html += '</div>';
+ node.labelType = "html";
+ node.label = html;
+ }
+ }
+ });
+ }
+
+ preprocessGraphLayout(g);
+ var renderer = new dagreD3.render();
+ renderer(graph, g);
+
+ svg.selectAll("rect").attr("rx", "5").attr("ry", "5");
+
+ setupLayoutForSparkPlanCluster(g, svg);
+ setupSelectionForSparkPlanNode(g);
+ setupTooltipForSparkPlanNode(g);
+ resizeSvg(svg);
+ postprocessForAdditionalMetrics();
+}
document.addEventListener("DOMContentLoaded", function () {
if (shouldRenderPlanViz()) {
renderPlanViz();
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
index 4860620d19d2..4bd897fa43f1 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
@@ -168,16 +168,37 @@ class ExecutionPage(parent: SQLTab) extends
WebUIPage("execution") with Logging
</span>
</div>
- <div id="plan-viz-graph">
- <div>
- <input type="checkbox" id="stageId-and-taskId-checkbox"></input>
- <span>Show the Stage ID and Task ID that corresponds to the max
metric</span>
+ <div id="plan-viz-content" class="row">
+ <div class="col-8">
+ <div id="plan-viz-graph">
+ <div>
+ <input type="checkbox" id="stageId-and-taskId-checkbox"></input>
+ <span>Show the Stage ID and Task ID that corresponds to the max
metric</span>
+ </div>
+ <div>
+ <input type="checkbox" id="detailed-labels-checkbox"></input>
+ <span>Show metrics in graph nodes (detailed mode)</span>
+ </div>
+ </div>
+ </div>
+ <div class="col-4">
+ <div id="plan-viz-details-panel" class="sticky-top" style="top:
4rem; z-index: 1;">
+ <div class="card">
+ <div class="card-header fw-bold"
id="plan-viz-details-title">Details</div>
+ <div class="card-body" id="plan-viz-details-body">
+ <p class="text-muted mb-0">Click a node to view details</p>
+ </div>
+ </div>
+ </div>
</div>
</div>
<div id="plan-viz-metadata" style="display:none">
<div class="dot-file">
{graph.makeDotFile(metrics)}
</div>
+ <div class="node-details">
+ {graph.makeNodeDetailsJson(metrics)}
+ </div>
</div>
{planVisualizationResources(request)}
</div>
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index 3ddf0b69e762..cd5c5bd5ad0f 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -40,11 +40,41 @@ case class SparkPlanGraph(
val dotFile = new StringBuilder
dotFile.append("digraph G {\n")
nodes.foreach(node => dotFile.append(node.makeDotNode(metrics) + "\n"))
- edges.foreach(edge => dotFile.append(edge.makeDotEdge + "\n"))
+ val nodeMap = allNodes.map(n => n.id -> n).toMap
+ edges.foreach(edge => dotFile.append(edge.makeDotEdge(nodeMap, metrics) +
"\n"))
dotFile.append("}")
dotFile.toString()
}
+ /**
+ * Generate a JSON string containing node details (name, metrics, and
optional children)
+ * for the detail side panel. This keeps the DOT node labels compact (name
only) while
+ * providing full metrics on demand via JavaScript.
+ */
+ def makeNodeDetailsJson(metrics: Map[Long, String]): String = {
+ val entries = allNodes.map { node =>
+ val metricsJson = node.metrics.flatMap { m =>
+ metrics.get(m.accumulatorId).map { v =>
+ val n = StringEscapeUtils.escapeJson(m.name)
+ val mv = StringEscapeUtils.escapeJson(v)
+ val mt = StringEscapeUtils.escapeJson(m.metricType)
+ s"""{"name":"$n","value":"$mv","type":"$mt"}"""
+ }
+ }.mkString("[", ",", "]")
+ val (prefix, extra) = node match {
+ case cluster: SparkPlanGraphCluster =>
+ val childIds = cluster.nodes
+ .map(n => s""""node${n.id}"""").mkString("[", ",", "]")
+ ("cluster", s""","children":$childIds""")
+ case _ => ("node", "")
+ }
+ s"""
"$prefix${node.id}":{"name":"${StringEscapeUtils.escapeJson(node.name)}",""" +
+ s""""desc":"${StringEscapeUtils.escapeJson(node.desc)}",""" +
+ s""""metrics":$metricsJson$extra}"""
+ }
+ entries.mkString("{\n", ",\n", "\n}")
+ }
+
/**
* All the SparkPlanGraphNodes, including those inside of WholeStageCodegen.
*/
@@ -167,35 +197,10 @@ class SparkPlanGraphNode(
val metrics: collection.Seq[SQLPlanMetric]) {
def makeDotNode(metricsValue: Map[Long, String]): String = {
- val builder = new mutable.StringBuilder("<b>" + name + "</b>")
-
- val values = for {
- metric <- metrics
- value <- metricsValue.get(metric.accumulatorId)
- } yield {
- // The value may contain ":" to extend the name, like `total (min, med,
max): ...`
- if (value.contains(":")) {
- metric.name + " " + value
- } else {
- metric.name + ": " + value
- }
- }
val nodeId = s"node$id"
val tooltip = StringEscapeUtils.escapeJava(desc)
- val labelStr = if (values.nonEmpty) {
- // If there are metrics, display each entry in a separate line.
- // Note: whitespace between two "\n"s is to create an empty line between
the name of
- // SparkPlan and metrics. If removing it, it won't display the empty
line in UI.
- builder ++= "<br><br>"
- builder ++= values.mkString("<br>")
- StringEscapeUtils.escapeJava(builder.toString().replaceAll("\n", "<br>"))
- } else {
- // SPARK-30684: when there is no metrics, add empty lines to increase
the height of the node,
- // so that there won't be gaps between an edge and a small node.
- s"<br><b>${StringEscapeUtils.escapeJava(name)}</b><br><br>"
- }
- s""" $id [id="$nodeId" labelType="html" label="$labelStr"
tooltip="$tooltip"];"""
-
+ val labelStr = StringEscapeUtils.escapeJava(name)
+ s""" $id [id="$nodeId" label="$labelStr" tooltip="$tooltip"];"""
}
}
@@ -211,17 +216,31 @@ class SparkPlanGraphCluster(
extends SparkPlanGraphNode(id, name, desc, metrics) {
override def makeDotNode(metricsValue: Map[Long, String]): String = {
- val duration =
metrics.filter(_.name.startsWith(WholeStageCodegenExec.PIPELINE_DURATION_METRIC))
+ val duration = metrics.filter(m =>
+ m.name != null &&
m.name.startsWith(WholeStageCodegenExec.PIPELINE_DURATION_METRIC))
+ // Extract short label: "(1)" from "WholeStageCodegen (1)"
+ val shortName = if (name != null) {
+ "\\(\\d+\\)".r.findFirstIn(name).getOrElse(name)
+ } else {
+ ""
+ }
val labelStr = if (duration.nonEmpty) {
require(duration.length == 1)
- val id = duration(0).accumulatorId
- if (metricsValue.contains(id)) {
- name + "\n \n" + duration(0).name + ": " + metricsValue(id)
+ val durationMetricId = duration(0).accumulatorId
+ if (metricsValue.contains(durationMetricId)) {
+ // For multi-line values like "total (min, med, max)\n10.0 s (...)",
+ // extract just the total number from the data line
+ val raw = metricsValue(durationMetricId)
+ val lines = raw.split("\n")
+ val dataLine = if (lines.length > 1) lines(1).trim else lines(0).trim
+ // Extract just the total value (first token before any parenthesized
breakdown)
+ val total = dataLine.split("\\s*\\(")(0).trim
+ shortName + " / " + total
} else {
- name
+ shortName
}
} else {
- name
+ shortName
}
val clusterId = s"cluster$id"
s"""
@@ -243,5 +262,20 @@ class SparkPlanGraphCluster(
*/
case class SparkPlanGraphEdge(fromId: Long, toId: Long) {
- def makeDotEdge: String = s""" $fromId->$toId;\n"""
+ def makeDotEdge(
+ nodeMap: Map[Long, SparkPlanGraphNode],
+ metricsValue: Map[Long, String]): String = {
+ val label = nodeMap.get(fromId).flatMap { node =>
+ node.metrics
+ .find(_.name == "number of output rows")
+ .flatMap(m => metricsValue.get(m.accumulatorId))
+ }
+ label match {
+ case Some(rows) =>
+ val escapedRows = StringEscapeUtils.escapeJava(rows)
+ s""" $fromId->$toId [label="$escapedRows" labeltooltip="$escapedRows
rows"];\n"""
+ case None =>
+ s""" $fromId->$toId;\n"""
+ }
+ }
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanGraphSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanGraphSuite.scala
index 975dbc1a1d8d..4188f2b0896e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanGraphSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanGraphSuite.scala
@@ -36,11 +36,34 @@ class SparkPlanGraphSuite extends SparkFunSuite {
accumulatorId = 35,
metricType = "nsTiming")))
val dotNode = planGraphNode.makeDotNode(Map.empty[Long, String])
- val expectedDotNode = " 24 [id=\"node24\" labelType=\"html\" label=\"" +
- "<br><b>Scan JDBCRelation(\\\"test-schema\\\".tickets)
[numPartitions=1]</b><br><br>\" " +
+ val expectedDotNode = " 24 [id=\"node24\" label=\"" +
+ "Scan JDBCRelation(\\\"test-schema\\\".tickets) [numPartitions=1]\" " +
"tooltip=\"Scan JDBCRelation(\\\"test-schema\\\".tickets)
[numPartitions=1] [ticket_no#0] " +
"PushedFilters: [], ReadSchema: struct<ticket_no:string>\"];"
assertResult(expectedDotNode)(dotNode)
}
+
+ test("SPARK-55786: makeNodeDetailsJson uses cluster prefix and includes
children") {
+ import scala.collection.mutable
+ val childNode = new SparkPlanGraphNode(
+ id = 1, name = "HashAggregate", desc = "HashAggregate(keys=[])",
+ metrics = Seq(SQLPlanMetric("peak memory", 10, "size")))
+ val cluster = new SparkPlanGraphCluster(
+ id = 2, name = "WholeStageCodegen (1)", desc = "WholeStageCodegen (1)",
+ nodes = mutable.ArrayBuffer(childNode),
+ metrics = Seq(SQLPlanMetric("duration", 20, "timing")))
+ val graph = SparkPlanGraph(Seq(cluster), Seq.empty)
+ val json = graph.makeNodeDetailsJson(Map(10L -> "256.0 KiB", 20L -> "5
ms"))
+ assert(json.contains("\"cluster2\""), "cluster should use cluster prefix")
+ assert(json.contains("\"node1\""), "child should use node prefix")
+ assert(json.contains("\"children\":[\"node1\"]"),
+ "cluster should list children")
+ assert(!json.contains("\"node2\""),
+ "cluster should not use node prefix")
+ assert(json.contains("\"desc\":\"HashAggregate(keys=[])\""),
+ "node should include desc field")
+ assert(json.contains("\"desc\":\"WholeStageCodegen (1)\""),
+ "cluster should include desc field")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
