This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 999c9ed1 [SPARK-31081][UI][SQL] Make display of
stageId/stageAttemptId/taskId of sql metrics toggleable
999c9ed1 is described below
commit 999c9ed10c2362d89afd3bbb48e35f3c7ac3cf89
Author: Kousuke Saruta <saru...@oss.nttdata.com>
AuthorDate: Tue Mar 24 13:37:13 2020 -0700
[SPARK-31081][UI][SQL] Make display of stageId/stageAttemptId/taskId of sql
metrics toggleable
### What changes were proposed in this pull request?
This is another solution for `SPARK-31081` and #27849 .
I added a checkbox which can toggle display of stageId/taskid in the SQL's
DAG page.
Mainly, I implemented the toggleable texts in boxes with HTML label feature
provided by `dagre-d3`.
The additional metrics are enclosed by `<span>` and control the appearance
of the text.
But the exception is additional metrics in clusters.
We can use HTML label for cluster but layout will be broken so I choosed
normal text label for clusters.
Due to that, this solution contains a little bit tricky code
in`spark-sql-viz.js` to manipulate the metric texts and generate DOMs.
### Why are the changes needed?
It makes metrics harder to read after #26843 and user may not interest in
extra info(stageId/StageAttemptId/taskId ) when they do not need debug.
#27849 control the appearance by a new configuration property but providing
a checkbox is more flexible.
### Does this PR introduce any user-facing change?
Yes.
[Additional metrics shown]
[Additional metrics hidden]
### How was this patch tested?
Tested manually with a simple DataFrame operation.
* The appearance of additional metrics in the boxes are controlled by the
newly added checkbox.
* No error found with JS-debugger.
* Checked/not-checked state is preserved after reloading.
Closes #27927 from sarutak/SPARK-31081.
Authored-by: Kousuke Saruta <saru...@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.w...@databricks.com>
---
.../sql/execution/ui/static/spark-sql-viz.css | 2 +-
.../spark/sql/execution/ui/static/spark-sql-viz.js | 74 +++++++++++++++++++++-
.../spark/sql/execution/ui/ExecutionPage.scala | 4 ++
.../spark/sql/execution/ui/SparkPlanGraph.scala | 10 +--
4 files changed, 83 insertions(+), 7 deletions(-)
diff --git
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index 94a6bd8..6ba79dc 100644
---
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -35,7 +35,7 @@
}
/* Highlight the SparkPlan node name */
-#plan-viz-graph svg text :first-child {
+#plan-viz-graph svg text :first-child:not(.stageId-and-taskId-metrics) {
font-weight: bold;
}
diff --git
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index 6244b2c..0fb7dab 100644
---
a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++
b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -47,6 +47,7 @@ function renderPlanViz() {
}
resizeSvg(svg);
+ postprocessForAdditionalMetrics();
}
/* -------------------- *
@@ -70,6 +71,10 @@ function setupTooltipForSparkPlanNode(nodeId) {
})
}
+// labelSeparator should be a non-graphical character in order not to affect
the width of boxes.
+var labelSeparator = "\x01";
+var stageAndTaskMetricsPattern = "^(.*)(\\(stage.*attempt.*task[^)]*\\))(.*)$";
+
/*
* Helper function to pre-process the graph layout.
* This step is necessary for certain styles that affect the positioning
@@ -79,8 +84,29 @@ function preprocessGraphLayout(g) {
g.graph().ranksep = "70";
var nodes = g.nodes();
for (var i = 0; i < nodes.length; i++) {
- var node = g.node(nodes[i]);
- node.padding = "5";
+ var node = g.node(nodes[i]);
+ node.padding = "5";
+
+ var firstSearator;
+ var secondSeparator;
+ var splitter;
+ if (node.isCluster) {
+ firstSearator = secondSeparator = labelSeparator;
+ splitter = "\\n";
+ } else {
+ firstSearator = "<span class='stageId-and-taskId-metrics'>";
+ secondSeparator = "</span>";
+ splitter = "<br>";
+ }
+
+ node.label.split(splitter).forEach(function(text, i) {
+ var newTexts = text.match(stageAndTaskMetricsPattern);
+ if (newTexts) {
+ node.label = node.label.replace(
+ newTexts[0],
+ newTexts[1] + firstSearator + newTexts[2] + secondSeparator +
newTexts[3]);
+ }
+ });
}
// Curve the edges
var edges = g.edges();
@@ -158,3 +184,47 @@ function getAbsolutePosition(d3selection) {
}
return { x: _x, y: _y };
}
+
+/*
+ * Helper function for postprocess for additional metrics.
+ */
+function postprocessForAdditionalMetrics() {
+ // With dagre-d3, we can choose normal text (default) or HTML as a label
type.
+ // HTML label for node works well but not for cluster so we need to choose
the default label type
+ // and manipulate DOM.
+ $("g.cluster text tspan")
+ .each(function() {
+ var originalText = $(this).text();
+ if (originalText.indexOf(labelSeparator) > 0) {
+ var newTexts = originalText.split(labelSeparator);
+ var thisD3Node = d3.selectAll($(this));
+ thisD3Node.text(newTexts[0]);
+ thisD3Node.append("tspan").attr("class",
"stageId-and-taskId-metrics").text(newTexts[1]);
+ $(this).append(newTexts[2]);
+ } else {
+ return originalText;
+ }
+ });
+
+ var checkboxNode = $("#stageId-and-taskId-checkbox");
+ checkboxNode.click(function() {
+ onClickAdditionalMetricsCheckbox($(this));
+ });
+ var isChecked = window.localStorage.getItem("stageId-and-taskId-checked") ==
"true";
+ $("#stageId-and-taskId-checkbox").prop("checked", isChecked);
+ onClickAdditionalMetricsCheckbox(checkboxNode);
+}
+
+/*
+ * Helper function which defines the action on click the checkbox.
+ */
+function onClickAdditionalMetricsCheckbox(checkboxNode) {
+ var additionalMetrics = $(".stageId-and-taskId-metrics");
+ var isChecked = checkboxNode.prop("checked");
+ if (isChecked) {
+ additionalMetrics.show();
+ } else {
+ additionalMetrics.hide();
+ }
+ window.localStorage.setItem("stageId-and-taskId-checked", isChecked);
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
index 6084aaf..d304369 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
@@ -71,6 +71,10 @@ class ExecutionPage(parent: SQLTab) extends
WebUIPage("execution") with Logging
{jobLinks(JobExecutionStatus.FAILED, "Failed Jobs:")}
</ul>
</div>
+ <div>
+ <input type="checkbox" id="stageId-and-taskId-checkbox"></input>
+ <span>Show the Stage (Stage Attempt): Task ID that corresponds to
the max metric</span>
+ </div>
val metrics = sqlStore.executionMetrics(executionId)
val graph = sqlStore.planGraph(executionId)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index d31d778..6762802 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -160,7 +160,7 @@ private[ui] class SparkPlanGraphNode(
val metrics: Seq[SQLPlanMetric]) {
def makeDotNode(metricsValue: Map[Long, String]): String = {
- val builder = new mutable.StringBuilder(name)
+ val builder = new mutable.StringBuilder("<b>" + name + "</b>")
val values = for {
metric <- metrics
@@ -173,9 +173,10 @@ private[ui] class SparkPlanGraphNode(
// If there are metrics, display each entry in a separate line.
// Note: whitespace between two "\n"s is to create an empty line between
the name of
// SparkPlan and metrics. If removing it, it won't display the empty
line in UI.
- builder ++= "\n \n"
- builder ++= values.mkString("\n")
- s""" $id
[label="${StringEscapeUtils.escapeJava(builder.toString())}"];"""
+ builder ++= "<br><br>"
+ builder ++= values.mkString("<br>")
+ val labelStr =
StringEscapeUtils.escapeJava(builder.toString().replaceAll("\n", "<br>"))
+ s""" $id [labelType="html" label="${labelStr}"];"""
} else {
// SPARK-30684: when there is no metrics, add empty lines to increase
the height of the node,
// so that there won't be gaps between an edge and a small node.
@@ -210,6 +211,7 @@ private[ui] class SparkPlanGraphCluster(
}
s"""
| subgraph cluster${id} {
+ | isCluster="true";
| label="${StringEscapeUtils.escapeJava(labelStr)}";
| ${nodes.map(_.makeDotNode(metricsValue)).mkString(" \n")}
| }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
