Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231932218
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,960 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = true;
+
+$(document).ajaxStop(function () {
+ if (blockUICount) {
+ $.unblockUI();
+ blockUICount = false;
+ }
+});
+
+$(document).ajaxStart(function () {
+ if (blockUICount) {
+ $.blockUI({message: '<h3>Loading Stage Page...</h3>'});
+ }
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+ "duration-pre": ConvertDurationString,
+
+ "duration-asc": function ( a, b ) {
+ a = ConvertDurationString( a );
+ b = ConvertDurationString( b );
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
+
+ "duration-desc": function ( a, b ) {
+ a = ConvertDurationString( a );
+ b = ConvertDurationString( b );
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1
+function stageEndPoint(appId) {
+ var queryString = document.baseURI.split('?');
+ var words = document.baseURI.split('/');
+ var indexOfProxy = words.indexOf("proxy");
+ var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+ if (indexOfProxy > 0) {
+ var appId = words[indexOfProxy + 1];
+ var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+ return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+ }
+ var indexOfHistory = words.indexOf("history");
+ if (indexOfHistory > 0) {
+ var appId = words[indexOfHistory + 1];
+ var appAttemptId = words[indexOfHistory + 2];
+ var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+ if (isNaN(appAttemptId) || appAttemptId == "0") {
+ return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+ } else {
+ return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+ }
+ }
+ return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+ switch(columnKey) {
+ case "executorRunTime":
+ return "Duration";
+ break;
+
+ case "jvmGcTime":
+ return "GC Time";
+ break;
+
+ case "gettingResultTime":
+ return "Getting Result Time";
+ break;
+
+ case "inputMetrics":
+ return "Input Size / Records";
+ break;
+
+ case "outputMetrics":
+ return "Output Size / Records";
+ break;
+
+ case "peakExecutionMemory":
+ return "Peak Execution Memory";
+ break;
+
+ case "resultSerializationTime":
+ return "Result Serialization Time";
+ break;
+
+ case "schedulerDelay":
+ return "Scheduler Delay";
+ break;
+
+ case "diskBytesSpilled":
+ return "Shuffle spill (disk)";
+ break;
+
+ case "memoryBytesSpilled":
+ return "Shuffle spill (memory)";
+ break;
+
+ case "shuffleReadMetrics":
+ return "Shuffle Read Size / Records";
+ break;
+
+ case "shuffleWriteMetrics":
+ return "Shuffle Write Size / Records";
+ break;
+
+ case "executorDeserializeTime":
+ return "Task Deserialization Time";
+ break;
+
+ default:
+ return "NA";
+ }
+}
+
+function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
+ switch(row.columnKey) {
+ case 'inputMetrics':
+ var str = formatBytes(row.data.bytesRead[columnIndex], type) +
" / " +
+ row.data.recordsRead[columnIndex];
+ return str;
+ break;
+
+ case 'outputMetrics':
+ var str = formatBytes(row.data.bytesWritten[columnIndex],
type) + " / " +
+ row.data.recordsWritten[columnIndex];
+ return str;
+ break;
+
+ case 'shuffleReadMetrics':
+ var str = formatBytes(row.data.readBytes[columnIndex], type) +
" / " +
+ row.data.readRecords[columnIndex];
+ return str;
+ break;
+
+ case 'shuffleReadBlockedTime':
+ var str = formatDuration(row.data.fetchWaitTime[columnIndex]);
+ return str;
+ break;
+
+ case 'shuffleRemoteReads':
+ var str = formatBytes(row.data.remoteBytesRead[columnIndex],
type);
+ return str;
+ break;
+
+ case 'shuffleWriteMetrics':
+ var str = formatBytes(row.data.writeBytes[columnIndex], type)
+ " / " +
+ row.data.writeRecords[columnIndex];
+ return str;
+ break;
+
+ default:
+ return (row.columnKey == 'peakExecutionMemory' ||
row.columnKey == 'memoryBytesSpilled'
+ || row.columnKey == 'diskBytesSpilled') ? formatBytes(
+ row.data[columnIndex], type) :
(formatDuration(row.data[columnIndex]));
+
+ }
+}
+
+function
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTable) {
+ var taskMetricsTable = "#summary-metrics-table";
+ if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
+ taskSummaryMetricsDataTable.clear().draw();
+
taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
+ } else {
+ var taskConf = {
+ "data": taskSummaryMetricsTable,
+ "columns": [
+ {data : 'metric'},
+ // Min
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row,
type, 0);
+ }
+ },
+ // 25th percentile
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row,
type, 1);
+ }
+ },
+ // Median
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row,
type, 2);
+ }
+ },
+ // 75th percentile
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row,
type, 3);
+ }
+ },
+ // Max
+ {
+ data: function (row, type) {
+ return displayRowsForSummaryMetricsTable(row,
type, 4);
+ }
+ }
+ ],
+ "columnDefs": [
+ { "type": "duration", "targets": 1 },
+ { "type": "duration", "targets": 2 },
+ { "type": "duration", "targets": 3 },
+ { "type": "duration", "targets": 4 },
+ { "type": "duration", "targets": 5 }
+ ],
+ "paging": false,
+ "searching": false,
+ "order": [[0, "asc"]],
+ "bSort": false,
+ "bAutoWidth": false
+ };
+ taskSummaryMetricsDataTable =
$(taskMetricsTable).DataTable(taskConf);
+ }
+ taskSummaryMetricsTableCurrentStateArray =
taskSummaryMetricsTable.slice();
+}
+
+function createRowMetadataForColumn(columnName, data, checkboxId, colKey) {
+ var row = {
+ "metric": columnName,
+ "data": data,
+ "checkboxId": checkboxId,
+ "columnKey": colKey
+ };
+ return row;
+}
+
+function reselectCheckboxesBasedOnTaskTableState() {
+ var allChecked = true;
+ var taskSummaryMetricsTableCurrentFilteredArray =
taskSummaryMetricsTableCurrentStateArray.slice();
+ if (typeof taskTableSelector !== 'undefined' &&
taskSummaryMetricsTableCurrentStateArray.length > 0) {
+ for (var k = 0; k < optionalColumns.length; k++) {
+ if (taskTableSelector.column(optionalColumns[k]).visible()) {
+ $("#box-"+optionalColumns[k]).prop('checked', true);
+
taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row
=> (row.checkboxId).toString() == optionalColumns[k])[0]);
+ taskSummaryMetricsTableCurrentFilteredArray =
taskSummaryMetricsTableCurrentStateArray.slice();
+ } else {
+ allChecked = false;
+ }
+ }
+ if (allChecked) {
+ $("#box-0").prop('checked', true);
+ }
+
createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
+ }
+}
+
+function getStageAttemptId() {
+ var words = document.baseURI.split('?');
+ var attemptIdStr = words[1].split('&')[1];
+ var digitsRegex = /[0-9]+/;
+ // We are using regex here to extract the stage attempt id as there
might be certain url's with format
+ // like
/proxy/application_1539986433979_27115/stages/stage/?id=0&attempt=0#tasksTitle
+ var stgAttemptId = words[1].split("&").filter(
+ word =>
word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
+ return stgAttemptId;
+}
+
+var taskSummaryMetricsTableArray = [];
+var taskSummaryMetricsTableCurrentStateArray = [];
+var taskSummaryMetricsDataTable;
+var optionalColumns = [11, 12, 13, 14, 15, 16, 17];
+var taskTableSelector;
+
+$(document).ready(function () {
+ setDataTableDefaults();
+
+ $("#showAdditionalMetrics").append(
+ "<div><a id='additionalMetrics'>" +
+ "<span class='expand-input-rate-arrow arrow-closed'
id='arrowtoggle1'></span>" +
+ " Show Additional Metrics" +
+ "</a></div>" +
+ "<div class='container-fluid container-fluid-div'
id='toggle-metrics' hidden>" +
+ "<div><input type='checkbox' class='toggle-vis' id='box-0'
data-column='0'> Select All</div>" +
+ "<div id='scheduler_delay'
class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis'
id='box-11' data-column='11'> Scheduler Delay</div>" +
+ "<div id='task_deserialization_time'
class='task-deserialization-time-checkbox-div'><input type='checkbox'
class='toggle-vis' id='box-12' data-column='12'> Task Deserialization
Time</div>" +
+ "<div id='shuffle_read_blocked_time'
class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox'
class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked
Time</div>" +
+ "<div id='shuffle_remote_reads'
class='shuffle-remote-reads-checkbox-div'><input type='checkbox'
class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" +
+ "<div id='result_serialization_time'
class='result-serialization-time-checkbox-div'><input type='checkbox'
class='toggle-vis' id='box-15' data-column='15'> Result Serialization
Time</div>" +
+ "<div id='getting_result_time'
class='getting-result-time-checkbox-div'><input type='checkbox'
class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" +
+ "<div id='peak_execution_memory'
class='peak-execution-memory-checkbox-div'><input type='checkbox'
class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" +
+ "</div>");
+
+ $('#scheduler_delay').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Scheduler delay includes time to ship the task
from the scheduler to the executor, and time to send " +
+ "the task result from the executor to the scheduler. If
scheduler delay is large, consider decreasing the size of tasks or decreasing
the size of task results.");
+ $('#task_deserialization_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Time spent deserializing the task closure on the
executor, including the time to read the broadcasted task.");
+ $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Time that the task spent blocked waiting for
shuffle data to be read from remote machines.");
+ $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Total shuffle bytes read from remote executors.
This is a subset of the shuffle read bytes; the remaining shuffle data is read
locally. ");
+ $('#result_serialization_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Time spent serializing the task result on the
executor before sending it back to the driver.");
+ $('#getting_result_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Time that the driver spends fetching task
results from workers. If this is large, consider decreasing the amount of data
returned from each task.");
+ $('#peak_execution_memory').attr("data-toggle", "tooltip")
+ .attr("data-placement", "right")
+ .attr("title", "Execution memory refers to the memory used by
internal data structures created during " +
+ "shuffles, aggregations and joins when Tungsten is
enabled. The value of this accumulator " +
+ "should be approximately the sum of the peak sizes across
all such data structures created " +
+ "in this task. For SQL jobs, this only tracks all unsafe
operators, broadcast joins, and " +
+ "external sort.");
+ $('[data-toggle="tooltip"]').tooltip();
+ tasksSummary = $("#parent-container");
+ getStandAloneAppId(function (appId) {
+
+ var endPoint = stageEndPoint(appId);
+ var stageAttemptId = getStageAttemptId();
+ $.getJSON(endPoint + "/" + stageAttemptId, function(response,
status, jqXHR) {
+
+ var responseBody = response;
+ var dataToShow = {};
+ dataToShow.showInputData = responseBody.inputBytes > 0;
+ dataToShow.showOutputData = responseBody.outputBytes > 0;
+ dataToShow.showShuffleReadData = responseBody.shuffleReadBytes
> 0;
+ dataToShow.showShuffleWriteData =
responseBody.shuffleWriteBytes > 0;
+ dataToShow.showBytesSpilledData =
+ (responseBody.diskBytesSpilled > 0 ||
responseBody.memoryBytesSpilled > 0);
+
+ if (!dataToShow.showShuffleReadData) {
+ $('#shuffle_read_blocked_time').remove();
+ $('#shuffle_remote_reads').remove();
+ optionalColumns.splice(2, 2);
+ }
+
+ // prepare data for executor summary table
+ stageExecutorSummaryInfoKeys =
Object.keys(responseBody.executorSummary);
+ $.getJSON(createRESTEndPointForExecutorsPage(appId),
+ function(executorSummaryResponse, status, jqXHR) {
+ var executorDetailsMap = {};
+ executorSummaryResponse.forEach(function (executorDetail) {
+ executorDetailsMap[executorDetail.id] = executorDetail;
+ });
+
+ var executorSummaryTable = [];
+ stageExecutorSummaryInfoKeys.forEach(function
(columnKeyIndex) {
+ var executorSummary =
responseBody.executorSummary[columnKeyIndex];
+ var executorDetail =
executorDetailsMap[columnKeyIndex.toString()];
+ executorSummary.id = columnKeyIndex;
+ executorSummary.executorLogs = {};
+ executorSummary.hostPort = "CANNOT FIND ADDRESS";
+
+ if (executorDetail) {
+ if (executorDetail["executorLogs"]) {
+
responseBody.executorSummary[columnKeyIndex].executorLogs =
+ executorDetail["executorLogs"];
+ }
+ if (executorDetail["hostPort"]) {
+
responseBody.executorSummary[columnKeyIndex].hostPort =
+ executorDetail["hostPort"];
+ }
+ }
+
executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
+ });
+ // building task aggregated metrics by executor table
+ var executorSummaryConf = {
+ "data": executorSummaryTable,
+ "columns": [
+ {data : "id"},
+ {data : "executorLogs", render: formatLogsCells},
+ {data : "hostPort"},
+ {
+ data : function (row, type) {
+ return type === 'display' ?
formatDuration(row.taskTime) : row.taskTime;
+ }
+ },
+ {
+ data : function (row, type) {
+ var totaltasks = row.succeededTasks +
row.failedTasks + row.killedTasks;
+ return type === 'display' ? totaltasks :
totaltasks.toString();
+ }
+ },
+ {data : "failedTasks"},
+ {data : "killedTasks"},
+ {data : "succeededTasks"},
+ {data : "isBlacklistedForStage"},
+ {
+ data : function (row, type) {
+ return row.inputRecords != 0 ?
formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.outputRecords != 0 ?
formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.shuffleReadRecords != 0 ?
formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.shuffleWriteRecords != 0 ?
formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return typeof row.memoryBytesSpilled !=
'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return typeof row.diskBytesSpilled !=
'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
+ }
+ }
+ ],
+ "order": [[0, "asc"]],
+ "bAutoWidth": false
+ }
+ var executorSummaryTableSelector =
+
$("#summary-executor-table").DataTable(executorSummaryConf);
+ $('#parent-container [data-toggle="tooltip"]').tooltip();
+
+
executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
+ if (dataToShow.showInputData) {
+ $('#executor-summary-input').attr("data-toggle",
"tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Bytes and records read from Hadoop
or from Spark storage.");
+ $('#executor-summary-input').tooltip(true);
+ }
+
executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
+ if (dataToShow.showOutputData) {
+ $('#executor-summary-output').attr("data-toggle",
"tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Bytes and records written to
Hadoop.");
+ $('#executor-summary-output').tooltip(true);
+ }
+
executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
+ if (dataToShow.showShuffleReadData) {
+
$('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Total shuffle bytes and records
read (includes both data read locally and data read from remote executors).");
+ $('#executor-summary-shuffle-read').tooltip(true);
+ }
+
executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
+ if (dataToShow.showShuffleWriteData) {
+
$('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
+ .attr("data-placement", "top")
+ .attr("title", "Bytes and records written to disk
in order to be read by a shuffle in a future stage.");
+ $('#executor-summary-shuffle-write').tooltip(true);
+ }
+
executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
+
executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
+ });
+
+ // prepare data for accumulatorUpdates
+ var accumulatorTable =
responseBody.accumulatorUpdates.filter(accumUpdate =>
+ !(accumUpdate.name).toString().includes("internal."));
+
+ // rendering the UI page
+ var data = {"executors": response};
+ $.get(createTemplateURI(appId, "stagespage"),
function(template) {
+
tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html(),
data));
+
+ $("#additionalMetrics").click(function(){
+ $("#arrowtoggle1").toggleClass("arrow-open
arrow-closed");
+ $("#toggle-metrics").toggle();
+ if (window.localStorage) {
+ window.localStorage.setItem("arrowtoggle1class",
$("#arrowtoggle1").attr('class'));
+ }
+ });
+
+ $("#aggregatedMetrics").click(function(){
+ $("#arrowtoggle2").toggleClass("arrow-open
arrow-closed");
+ $("#toggle-aggregatedMetrics").toggle();
+ if (window.localStorage) {
+ window.localStorage.setItem("arrowtoggle2class",
$("#arrowtoggle2").attr('class'));
+ }
+ });
+
+ var quantiles = "0,0.25,0.5,0.75,1.0";
+ $.getJSON(endPoint + "/" + stageAttemptId +
"/taskSummary?quantiles=" + quantiles,
+ function(taskMetricsResponse, status, jqXHR) {
+ var taskMetricKeys = Object.keys(taskMetricsResponse);
+ taskMetricKeys.forEach(function (columnKey) {
+ switch(columnKey) {
+ case "shuffleReadMetrics":
+ var row1 = createRowMetadataForColumn(
+
getColumnNameForTaskMetricSummary(columnKey), taskMetricsResponse[columnKey],
3, columnKey);
--- End diff --
Done
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
