This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new a28f6c6 [SPARK-31882][WEBUI] DAG-viz is not rendered correctly with
pagination
a28f6c6 is described below
commit a28f6c6db6d071a49601b9185f24f1872c625251
Author: Kousuke Saruta <saru...@oss.nttdata.com>
AuthorDate: Tue Jun 2 16:45:16 2020 -0700
[SPARK-31882][WEBUI] DAG-viz is not rendered correctly with pagination
### What changes were proposed in this pull request?
This PR fix an issue related to DAG-viz.
Because DAG-viz for a job fetches link urls for each stage from the stage
table, rendering can fail with pagination.
You can reproduce this issue with the following operation.
```
sc.parallelize(1 to 10).map(value => (value
,value)).repartition(1).repartition(1).repartition(1).reduceByKey(_ + _).collect
```
And then, visit the corresponding job page.
There are 5 stages so show <5 stages in the paged table.
<img width="1440" alt="dag-rendering-issue1"
src="https://user-images.githubusercontent.com/4736016/83376286-c29f3d00-a40c-11ea-891b-eb8f42afbb27.png";>
<img width="1439" alt="dag-rendering-issue2"
src="https://user-images.githubusercontent.com/4736016/83376288-c3d06a00-a40c-11ea-8bb2-38542e5010c1.png";>
### Why are the changes needed?
This is a bug.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Newly added test case with following command.
`build/sbt -Dtest.default.exclude.tags=
-Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly
org.apache.spark.ui.ChromeUISeleniumSuite -- -z SPARK-31882"`
Closes #28690 from sarutak/fix-dag-rendering-issue.
Authored-by: Kousuke Saruta <saru...@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.w...@databricks.com>
(cherry picked from commit 271eb26c026490f7307ae888200bfc81a645592f)
Signed-off-by: Gengliang Wang <gengliang.w...@databricks.com>
---
.../org/apache/spark/ui/static/spark-dag-viz.js | 7 ++-----
.../resources/org/apache/spark/ui/static/webui.js | 7 ++++++-
.../main/scala/org/apache/spark/ui/UIUtils.scala | 1 +
.../spark/ui/RealBrowserUISeleniumSuite.scala | 22 ++++++++++++++++++++++
4 files changed, 31 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index ae02def..fd4a48d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -216,7 +216,7 @@ function renderDagVizForJob(svgContainer) {
var dot = metadata.select(".dot-file").text();
var stageId = metadata.attr("stage-id");
var containerId = VizConstants.graphPrefix + stageId;
- var isSkipped = metadata.attr("skipped") == "true";
+ var isSkipped = metadata.attr("skipped") === "true";
var container;
if (isSkipped) {
container = svgContainer
@@ -225,11 +225,8 @@ function renderDagVizForJob(svgContainer) {
.attr("skipped", "true");
} else {
// Link each graph to the corresponding stage page (TODO: handle stage
attempts)
- // Use the link from the stage table so it also works for the history
server
var attemptId = 0;
- var stageLink = d3.select("#stage-" + stageId + "-" + attemptId)
- .select("a.name-link")
- .attr("href");
+ var stageLink = uiRoot + appBasePath + "/stages/stage/?id=" + stageId +
"&attempt=" + attemptId;
container = svgContainer
.append("a")
.attr("xlink:href", stageLink)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js
b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index 0ba461f..4f8409c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -16,11 +16,16 @@
*/
var uiRoot = "";
+var appBasePath = "";
function setUIRoot(val) {
uiRoot = val;
}
+function setAppBasePath(path) {
+ appBasePath = path;
+}
+
function collapseTablePageLoad(name, table){
if (window.localStorage.getItem(name) == "true") {
// Set it to false so that the click function can revert it
@@ -33,7 +38,7 @@ function collapseTable(thisName, table){
var status = window.localStorage.getItem(thisName) == "true";
status = !status;
- var thisClass = '.' + thisName
+ var thisClass = '.' + thisName;
// Expand the list of additional metrics.
var tableDiv = $(thisClass).parent().find('.' + table);
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index aaaa21a..87ab543 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -289,6 +289,7 @@ private[spark] object UIUtils extends Logging {
<html>
<head>
{commonHeaderNodes(request)}
+ <script>setAppBasePath('{activeTab.basePath}')</script>
{if (showVisualization) vizHeaderNodes(request) else Seq.empty}
{if (useDataTables) dataTablesHeaderNodes(request) else Seq.empty}
<link rel="shortcut icon"
diff --git
a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
index 84f8882..06b6483 100644
--- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -78,6 +78,28 @@ abstract class RealBrowserUISeleniumSuite(val driverProp:
String)
}
}
+ test("SPARK-31882: Link URL for Stage DAGs should not depend on paged
table.") {
+ withSpark(newSparkContext()) { sc =>
+ sc.parallelize(1 to 100).map(v => (v, v)).repartition(10).reduceByKey(_
+ _).collect
+
+ eventually(timeout(10.seconds), interval(50.microseconds)) {
+ val pathWithPagedTable =
+
"/jobs/job/?id=0&completedStage.page=2&completedStage.sort=Stage+Id&" +
+ "completedStage.desc=true&completedStage.pageSize=1#completed"
+ goToUi(sc, pathWithPagedTable)
+
+ // Open DAG Viz.
+ webDriver.findElement(By.id("job-dag-viz")).click()
+ val stages = webDriver.findElements(By.cssSelector("svg[class='job'] >
a"))
+ stages.size() should be (3)
+
+ stages.get(0).getAttribute("href") should include
("/stages/stage/?id=0&attempt=0")
+ stages.get(1).getAttribute("href") should include
("/stages/stage/?id=1&attempt=0")
+ stages.get(2).getAttribute("href") should include
("/stages/stage/?id=2&attempt=0")
+ }
+ }
+ }
+
/**
* Create a test SparkContext with the SparkUI enabled.
* It is safe to `get` the SparkUI directly from the SparkContext returned
here.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
![https://user-images.githubusercontent.com/4736016/83376286-c29f3d00-a40c-11ea-891b-eb8f42afbb27.png"](https://user-images.githubusercontent.com/4736016/83376286-c29f3d00-a40c-11ea-891b-eb8f42afbb27.png"){kind=link}
![https://user-images.githubusercontent.com/4736016/83376288-c3d06a00-a40c-11ea-8bb2-38542e5010c1.png"](https://user-images.githubusercontent.com/4736016/83376288-c3d06a00-a40c-11ea-8bb2-38542e5010c1.png"){kind=link}