Github user tgravescs commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r218888800
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+ $.blockUI({message: '<h3>Loading Stage Page...</h3>'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+ "file-size-pre": ConvertDurationString,
+
+ "file-size-asc": function ( a, b ) {
+ a = ConvertDurationString( a );
+ b = ConvertDurationString( b );
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
+
+ "file-size-desc": function ( a, b ) {
+ a = ConvertDurationString( a );
+ b = ConvertDurationString( b );
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0&attempt=0
+function stageEndPoint(appId) {
+ var words = document.baseURI.split('/');
+ var words2 = document.baseURI.split('?');
+ var ind = words.indexOf("proxy");
+ if (ind > 0) {
+ var appId = words[ind + 1];
+ var stageIdLen = words2[1].indexOf('&');
+ var stageId = words2[1].substr(3, stageIdLen - 3);
+ var newBaseURI = words.slice(0, ind + 2).join('/');
+ return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+ }
+ ind = words.indexOf("history");
+ if (ind > 0) {
+ var appId = words[ind + 1];
+ var attemptId = words[ind + 2];
+ var stageIdLen = words2[1].indexOf('&');
+ var stageId = words2[1].substr(3, stageIdLen - 3);
+ var newBaseURI = words.slice(0, ind).join('/');
+ if (isNaN(attemptId) || attemptId == "0") {
+ return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+ } else {
+ return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+ }
+ }
+ var stageIdLen = words2[1].indexOf('&');
+ var stageId = words2[1].substr(3, stageIdLen - 3);
+ return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+ switch(columnKey) {
+ case "executorRunTime":
+ return "Duration";
+ break;
+
+ case "jvmGcTime":
+ return "GC Time";
+ break;
+
+ case "gettingResultTime":
+ return "Getting Result Time";
+ break;
+
+ case "inputMetrics":
+ return "Input Size / Records";
+ break;
+
+ case "outputMetrics":
+ return "Output Size / Records";
+ break;
+
+ case "peakExecutionMemory":
+ return "Peak Execution Memory";
+ break;
+
+ case "resultSerializationTime":
+ return "Result Serialization Time";
+ break;
+
+ case "schedulerDelay":
+ return "Scheduler Delay";
+ break;
+
+ case "diskBytesSpilled":
+ return "Shuffle spill (disk)";
+ break;
+
+ case "memoryBytesSpilled":
+ return "Shuffle spill (memory)";
+ break;
+
+ case "shuffleReadMetrics":
+ return "Shuffle Read Size / Records";
+ break;
+
+ case "shuffleWriteMetrics":
+ return "Shuffle Write Size / Records";
+ break;
+
+ case "executorDeserializeTime":
+ return "Task Deserialization Time";
+ break;
+
+ default:
+ return "NA";
+ }
+}
+
+$(document).ready(function () {
+ setDataTableDefaults();
+
+ $("#showAdditionalMetrics").append(
+ "<div><a id='taskMetric'>" +
+ "<span class='expand-input-rate-arrow arrow-closed'
id='arrowtoggle1'></span>" +
+ " Show Additional Metrics" +
+ "</a></div>" +
+ "<div class='container-fluid container-fluid-div'
id='toggle-metrics' hidden>" +
+ "<div><input type='checkbox' class='toggle-vis' id='box-0'
data-column='0'> Select All</div>" +
+ "<div id='scheduler_delay'><input type='checkbox'
class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>" +
+ "<div id='task_deserialization_time'><input type='checkbox'
class='toggle-vis' id='box-12' data-column='12'> Task Deserialization
Time</div>" +
+ "<div id='shuffle_read_blocked_time'><input type='checkbox'
class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked
Time</div>" +
+ "<div id='shuffle_remote_reads'><input type='checkbox'
class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" +
+ "<div id='result_serialization_time'><input type='checkbox'
class='toggle-vis' id='box-15' data-column='15'> Result Serialization
Time</div>" +
+ "<div id='getting_result_time'><input type='checkbox'
class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" +
+ "<div id='peak_execution_memory'><input type='checkbox'
class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" +
+ "</div>");
+
+ $('#scheduler_delay').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Scheduler delay includes time to ship the task
from the scheduler to the executor, and time to send " +
+ "the task result from the executor to the scheduler. If
scheduler delay is large, consider decreasing the size of tasks or decreasing
the size of task results.");
+ $('#task_deserialization_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Time spent deserializing the task closure on the
executor, including the time to read the broadcasted task.");
+ $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Time that the task spent blocked waiting for
shuffle data to be read from remote machines.");
+ $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Total shuffle bytes read from remote executors.
This is a subset of the shuffle read bytes; the remaining shuffle data is read
locally. ");
+ $('#result_serialization_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Time spent serializing the task result on the
executor before sending it back to the driver.");
+ $('#getting_result_time').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Time that the driver spends fetching task
results from workers. If this is large, consider decreasing the amount of data
returned from each task.");
+ $('#peak_execution_memory').attr("data-toggle", "tooltip")
+ .attr("data-placement", "bottom")
+ .attr("title", "Execution memory refers to the memory used by
internal data structures created during " +
+ "shuffles, aggregations and joins when Tungsten is
enabled. The value of this accumulator " +
+ "should be approximately the sum of the peak sizes across
all such data structures created " +
+ "in this task. For SQL jobs, this only tracks all unsafe
operators, broadcast joins, and " +
+ "external sort.");
+ $('#scheduler_delay').tooltip(true);
+ $('#task_deserialization_time').tooltip(true);
+ $('#shuffle_read_blocked_time').tooltip(true);
+ $('#shuffle_remote_reads').tooltip(true);
+ $('#result_serialization_time').tooltip(true);
+ $('#getting_result_time').tooltip(true);
+ $('#peak_execution_memory').tooltip(true);
+ tasksSummary = $("#active-tasks");
+ getStandAloneAppId(function (appId) {
+
+ var endPoint = stageEndPoint(appId);
+ $.getJSON(endPoint, function(response, status, jqXHR) {
+
+ var responseBody = response[0];
+ // prepare data for task aggregated metrics table
+ indices = Object.keys(responseBody.executorSummary);
+ var task_summary_table = [];
+ indices.forEach(function (ix) {
+ responseBody.executorSummary[ix].id = ix;
+ task_summary_table.push(responseBody.executorSummary[ix]);
+ });
+
+ // prepare data for accumulatorUpdates
+ var indices = Object.keys(responseBody.accumulatorUpdates);
+ var accumulator_table_all = [];
+ var accumulator_table = [];
+ indices.forEach(function (ix) {
+
accumulator_table_all.push(responseBody.accumulatorUpdates[ix]);
+ });
+
+ accumulator_table_all.forEach(function (x){
+ var name = (x.name).toString();
+ if(name.includes("internal.") == false){
+ accumulator_table.push(x);
+ }
+ });
+
+ // rendering the UI page
+ var data = {"executors": response};
+ $.get(createTemplateURI(appId, "stagespage"),
function(template) {
+
tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html(),
data));
+
+ $("#taskMetric").click(function(){
+ $("#arrowtoggle1").toggleClass("arrow-open
arrow-closed");
+ $("#toggle-metrics").toggle();
+ });
+
+ $("#aggregatedMetrics").click(function(){
+ $("#arrowtoggle2").toggleClass("arrow-open
arrow-closed");
+ $("#toggle-aggregatedMetrics").toggle();
+ });
+
+ var task_metrics_table = [];
+ var stageAttemptId = getStageAttemptId();
+ var quantiles = "0,0.25,0.5,0.75,1.0";
+ $.getJSON(stageEndPoint(appId) +
"/"+stageAttemptId+"/taskSummary?quantiles="+quantiles,
function(taskMetricsResponse, status, jqXHR) {
+ var taskMetricIndices =
Object.keys(taskMetricsResponse);
+ taskMetricIndices.forEach(function (ix) {
+ var columnName =
getColumnNameForTaskMetricSummary(ix);
+ if (columnName == "Shuffle Read Size / Records") {
+ var row1 = {
+ "metric": columnName,
+ "data": taskMetricsResponse[ix]
+ };
+ var row2 = {
+ "metric": "Shuffle Read Blocked Time",
+ "data": taskMetricsResponse[ix]
+ };
+ var row3 = {
+ "metric": "Shuffle Remote Reads",
+ "data": taskMetricsResponse[ix]
+ };
+ task_metrics_table.push(row1);
+ task_metrics_table.push(row2);
+ task_metrics_table.push(row3);
+ }
+ else if (columnName != "NA") {
+ var row = {
+ "metric": columnName,
+ "data": taskMetricsResponse[ix]
+ };
+ task_metrics_table.push(row);
+ }
+ });
+
+ var taskMetricsTable = "#summary-metrics-table";
+ var task_conf = {
+ "data": task_metrics_table,
+ "columns": [
+ {data : 'metric'},
+ {
+ data: function (row, type) {
+ switch(row.metric) {
+ case 'Input Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesRead));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsRead));
+ var str =
formatBytes(str1arr[0], type) + " / " + str2arr[0];
+ return str;
+ break;
+
+ case 'Output Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesWritten));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsWritten));
+ var str =
formatBytes(str1arr[0], type) + " / " + str2arr[0];
+ return str;
+ break;
+
+ case 'Shuffle Read Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.readBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.readRecords));
+ var str =
formatBytes(str1arr[0], type) + " / " + str2arr[0];
+ return str;
+ break;
+
+ case 'Shuffle Read Blocked Time':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.fetchWaitTime));
+ var str =
formatDuration(str1arr[0]);
+ return str;
+ break;
+
+ case 'Shuffle Remote Reads':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.remoteBytesRead));
+ var str =
formatBytes(str1arr[0], type);
+ return str;
+ break;
+
+ case 'Shuffle Write Size /
Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.writeBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.writeRecords));
+ var str =
formatBytes(str1arr[0], type) + " / " + str2arr[0];
+ return str;
+ break;
+
+ default:
+ return (row.metric == 'Peak
Execution Memory' || row.metric == 'Shuffle spill (memory)'
+ || row.metric ==
'Shuffle spill (disk)') ? formatBytes(row.data[0], type) :
(formatDuration(row.data[0]));
+
+ }
+ }
+ },
+ {
+ data: function (row, type) {
+ switch(row.metric) {
+ case 'Input Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesRead));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsRead));
+ var str =
formatBytes(str1arr[1], type) + " / " + str2arr[1];
+ return str;
+ break;
+
+ case 'Output Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesWritten));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsWritten));
+ var str =
formatBytes(str1arr[1], type) + " / " + str2arr[1];
+ return str;
+ break;
+
+ case 'Shuffle Read Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.readBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.readRecords));
+ var str =
formatBytes(str1arr[1], type) + " / " + str2arr[1];
+ return str;
+ break;
+
+ case 'Shuffle Read Blocked Time':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.fetchWaitTime));
+ var str =
formatDuration(str1arr[1]);
+ return str;
+ break;
+
+ case 'Shuffle Remote Reads':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.remoteBytesRead));
+ var str =
formatBytes(str1arr[1], type);
+ return str;
+ break;
+
+ case 'Shuffle Write Size /
Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.writeBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.writeRecords));
+ var str =
formatBytes(str1arr[1], type) + " / " + str2arr[1];
+ return str;
+ break;
+
+ default:
+ return (row.metric == 'Peak
Execution Memory' || row.metric == 'Shuffle spill (memory)'
+ || row.metric ==
'Shuffle spill (disk)') ? formatBytes(row.data[1], type) :
(formatDuration(row.data[1]));
+
+ }
+ }
+ },
+ {
+ data: function (row, type) {
+ switch(row.metric) {
+ case 'Input Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesRead));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsRead));
+ var str =
formatBytes(str1arr[2], type) + " / " + str2arr[2];
+ return str;
+ break;
+
+ case 'Output Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesWritten));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsWritten));
+ var str =
formatBytes(str1arr[2], type) + " / " + str2arr[2];
+ return str;
+ break;
+
+ case 'Shuffle Read Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.readBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.readRecords));
+ var str =
formatBytes(str1arr[2], type) + " / " + str2arr[2];
+ return str;
+ break;
+
+ case 'Shuffle Read Blocked Time':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.fetchWaitTime));
+ var str =
formatDuration(str1arr[2]);
+ return str;
+ break;
+
+ case 'Shuffle Remote Reads':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.remoteBytesRead));
+ var str =
formatBytes(str1arr[2], type);
+ return str;
+ break;
+
+ case 'Shuffle Write Size /
Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.writeBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.writeRecords));
+ var str =
formatBytes(str1arr[2], type) + " / " + str2arr[2];
+ return str;
+ break;
+
+ default:
+ return (row.metric == 'Peak
Execution Memory' || row.metric == 'Shuffle spill (memory)'
+ || row.metric ==
'Shuffle spill (disk)') ? formatBytes(row.data[2], type) :
(formatDuration(row.data[2]));
+
+ }
+ }
+ },
+ {
+ data: function (row, type) {
+ switch(row.metric) {
+ case 'Input Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesRead));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsRead));
+ var str =
formatBytes(str1arr[3], type) + " / " + str2arr[3];
+ return str;
+ break;
+
+ case 'Output Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesWritten));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsWritten));
+ var str =
formatBytes(str1arr[3], type) + " / " + str2arr[3];
+ return str;
+ break;
+
+ case 'Shuffle Read Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.readBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.readRecords));
+ var str =
formatBytes(str1arr[3], type) + " / " + str2arr[3];
+ return str;
+ break;
+
+ case 'Shuffle Read Blocked Time':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.fetchWaitTime));
+ var str =
formatDuration(str1arr[3]);
+ return str;
+ break;
+
+ case 'Shuffle Remote Reads':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.remoteBytesRead));
+ var str =
formatBytes(str1arr[3], type);
+ return str;
+ break;
+
+ case 'Shuffle Write Size /
Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.writeBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.writeRecords));
+ var str =
formatBytes(str1arr[3], type) + " / " + str2arr[3];
+ return str;
+ break;
+
+ default:
+ return (row.metric == 'Peak
Execution Memory' || row.metric == 'Shuffle spill (memory)'
+ || row.metric ==
'Shuffle spill (disk)') ? formatBytes(row.data[3], type) :
(formatDuration(row.data[3]));
+
+ }
+ }
+ },
+ {
+ data: function (row, type) {
+ switch(row.metric) {
+ case 'Input Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesRead));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsRead));
+ var str =
formatBytes(str1arr[4], type) + " / " + str2arr[4];
+ return str;
+ break;
+
+ case 'Output Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.bytesWritten));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.recordsWritten));
+ var str =
formatBytes(str1arr[4], type) + " / " + str2arr[4];
+ return str;
+ break;
+
+ case 'Shuffle Read Size / Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.readBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.readRecords));
+ var str =
formatBytes(str1arr[4], type) + " / " + str2arr[4];
+ return str;
+ break;
+
+ case 'Shuffle Read Blocked Time':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.fetchWaitTime));
+ var str =
formatDuration(str1arr[4]);
+ return str;
+ break;
+
+ case 'Shuffle Remote Reads':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.remoteBytesRead));
+ var str =
formatBytes(str1arr[4], type);
+ return str;
+ break;
+
+ case 'Shuffle Write Size /
Records':
+ var str1arr =
extractDataFromArrayString(JSON.stringify(row.data.writeBytes));
+ var str2arr =
extractDataFromArrayString(JSON.stringify(row.data.writeRecords));
+ var str =
formatBytes(str1arr[4], type) + " / " + str2arr[4];
+ return str;
+ break;
+
+ default:
+ return (row.metric == 'Peak
Execution Memory' || row.metric == 'Shuffle spill (memory)'
+ || row.metric ==
'Shuffle spill (disk)') ? formatBytes(row.data[4], type) :
(formatDuration(row.data[4]));
+
+ }
+ }
+ }
+ ],
+ "columnDefs": [
+ { "type": "file-size", "targets": 1 },
+ { "type": "file-size", "targets": 2 },
+ { "type": "file-size", "targets": 3 },
+ { "type": "file-size", "targets": 4 },
+ { "type": "file-size", "targets": 5 }
+ ],
+ "paging": false,
+ "searching": false,
+ "order": [[0, "asc"]]
+ };
+ $(taskMetricsTable).DataTable(task_conf);
+ });
+
+ // building task aggregated metric table
+ var tasksSummarytable = "#summary-stages-table";
+ var task_summary_conf = {
+ "data": task_summary_table,
+ "columns": [
+ {data : "id"},
+ {data : "executorLogs", render: formatLogsCells},
+ {data : "hostPort"},
+ {
+ data : function (row, type) {
+ return type === 'display' ?
formatDuration(row.taskTime) : row.taskTime;
+ }
+ },
+ {
+ data : function (row, type) {
+ var totaltasks = row.succeededTasks +
row.failedTasks + row.killedTasks;
+ return type === 'display' ? totaltasks :
totaltasks.toString();
+ }
+ },
+ {data : "failedTasks"},
+ {data : "killedTasks"},
+ {data : "succeededTasks"},
+ {data : "isBlacklistedForStage"},
+ {
+ data : function (row, type) {
+ return row.inputRecords != 0 ?
formatBytes(row.inputBytes, type) + " / " + row.inputRecords : 0;
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.outputRecords != 0 ?
formatBytes(row.outputBytes, type) + " / " + row.outputRecords : 0;
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.shuffleReadRecords != 0 ?
formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : 0;
+ }
+ },
+ {
+ data : function (row, type) {
+ return row.shuffleWriteRecords != 0 ?
formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : 0;
+ }
+ },
+ {
+ data : function (row, type) {
+ return typeof row.memoryBytesSpilled !=
'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
+ }
+ },
+ {
+ data : function (row, type) {
+ return typeof row.diskBytesSpilled !=
'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
+ }
+ }
+ ],
+ "order": [[0, "asc"]]
+ }
+ $(tasksSummarytable).DataTable(task_summary_conf);
+ $('#active-tasks [data-toggle="tooltip"]').tooltip();
+
+ // building accumulator update table
+ var accumulatorTable = "#accumulator-table";
+ var accumulator_conf = {
+ "data": accumulator_table,
+ "columns": [
+ {data : "id"},
+ {data : "name"},
+ {data : "value"}
+ ],
+ "paging": false,
+ "searching": false,
+ "order": [[0, "asc"]]
+ }
+ $(accumulatorTable).DataTable(accumulator_conf);
+
+ // building tasks table that uses server side functionality
+ var taskTable = "#active-tasks-table";
+ var task_conf = {
+ "serverSide": true,
+ "paging": true,
+ "info": true,
+ "processing": true,
+ "ajax": {
+ "url": stageEndPoint(appId) + "/" + stageAttemptId
+ "/taskTable",
+ "data": {
+ "numTasks": responseBody.numTasks
+ },
+ "dataSrc": function ( jsons ) {
+ var jsonStr = JSON.stringify(jsons);
+ var marrr = JSON.parse(jsonStr);
+ return marrr.aaData;
+ }
+ },
+ "columns": [
+ {data: function (row, type) {
+ return type !== 'display' ? (isNaN(row.index)
? 0 : row.index ) : row.index;
+ },
+ name: "Index"
+ },
+ {data : "taskId", name: "ID"},
+ {data : "attempt", name: "Attempt"},
+ {data : "status", name: "Status"},
+ {data : "taskLocality", name: "Locality Level"},
+ {data : "executorId", name: "Executor ID"},
+ {data : "host", name: "Host"},
+ {data : "executorLogs", name: "Logs", render:
formatLogsCells},
+ {data : "launchTime", name: "Launch Time", render:
formatDate},
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ return type === 'display' ?
formatDuration(row.taskMetrics.executorRunTime) :
row.taskMetrics.executorRunTime;
+ } else {
+ return "N/A";
+ }
+ },
+ name: "Duration"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ return type === 'display' ?
formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
+ } else {
+ return "N/A";
+ }
+ },
+ name: "GC Time"
+ },
+ {
+ data : function (row, type) {
+ return type === 'display' ?
formatDuration(row.schedulerDelay) : row.schedulerDelay;
+ },
+ name: "Scheduler Delay"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ return type === 'display' ?
formatDuration(row.taskMetrics.executorDeserializeTime) :
row.taskMetrics.executorDeserializeTime;
+ } else {
+ return "N/A";
+ }
+ },
+ name: "Task Deserialization Time"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if (row.taskMetrics.shuffleReadMetrics
!== 'undefined') {
+ return type === 'display' ?
formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) :
row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Read Blocked Time"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ return type === 'display' ?
formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) :
row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
+ } else {
+ return "N/A";
+ }
+ },
+ name: "Shuffle Remote Reads"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ return type === 'display' ?
formatDuration(row.taskMetrics.resultSerializationTime) :
row.taskMetrics.resultSerializationTime;
+ } else {
+ return "N/A";
+ }
+ },
+ name: "Result Serialization Time"
+ },
+ {
+ data : function (row, type) {
+ return type === 'display' ?
formatDuration(row.gettingResultTime) : row.gettingResultTime;
+ },
+ name: "Getting Result Time"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ return type === 'display' ?
formatBytes(row.taskMetrics.peakExecutionMemory, type) :
row.taskMetrics.peakExecutionMemory;
+ } else {
+ return "N/A";
+ }
+ },
+ name: "Peak Execution Memory"
+ },
+ {
+ data : function (row, type) {
+ if (accumulator_table.length > 0 &&
row.accumulatorUpdates.length > 0) {
+ var accIndex =
row.accumulatorUpdates.length - 1;
+ return
row.accumulatorUpdates[accIndex].name + ' : ' +
row.accumulatorUpdates[accIndex].update;
+ } else {
+ return "";
+ }
+ },
+ name: "Accumulators"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if
(row.taskMetrics.inputMetrics.bytesRead > 0) {
+ if (type === 'display') {
+ return
formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " +
row.taskMetrics.inputMetrics.recordsRead;
+ } else {
+ return
row.taskMetrics.inputMetrics.bytesRead + " / " +
row.taskMetrics.inputMetrics.recordsRead;
+ }
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Input Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if
(row.taskMetrics.outputMetrics.bytesWritten > 0) {
+ if (type === 'display') {
+ return
formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " +
row.taskMetrics.outputMetrics.recordsWritten;
+ } else {
+ return
row.taskMetrics.outputMetrics.bytesWritten + " / " +
row.taskMetrics.outputMetrics.recordsWritten;
+ }
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Output Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if
(row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
+ return type === 'display' ?
formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) /
1000000) : row.taskMetrics.shuffleWriteMetrics.writeTime;
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Write Time"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if
(row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
+ if (type === 'display') {
+ return
formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " +
row.taskMetrics.shuffleWriteMetrics.recordsWritten;
+ } else {
+ return
row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " +
row.taskMetrics.shuffleWriteMetrics.recordsWritten;
+ }
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Write Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if
(row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
+ var totalBytesRead =
parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) +
parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
+ if (type === 'display') {
+ return
formatBytes(totalBytesRead, type) + " / " +
row.taskMetrics.shuffleReadMetrics.recordsRead;
+ } else {
+ return totalBytesRead + " / "
+ row.taskMetrics.shuffleReadMetrics.recordsRead;
+ }
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Read Size / Records"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if (row.taskMetrics.memoryBytesSpilled
> 0) {
+ return type === 'display' ?
formatBytes(row.taskMetrics.memoryBytesSpilled, type) :
row.taskMetrics.memoryBytesSpilled;
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Spill (Memory)"
+ },
+ {
+ data : function (row, type) {
+ if ("taskMetrics" in row) {
+ if (row.taskMetrics.diskBytesSpilled >
0) {
+ return type === 'display' ?
formatBytes(row.taskMetrics.diskBytesSpilled, type) :
row.taskMetrics.diskBytesSpilled;
+ } else {
+ return "";
+ }
+ } else {
+ return "";
+ }
+ },
+ name: "Shuffle Spill (Disk)"
+ },
+ {
+ data : function (row, type) {
+ var msg = row.errorMessage;
+ if (typeof msg === 'undefined'){
+ return "";
+ } else {
+ var form_head = msg.substring(0,
msg.indexOf("at"));
+ var form = "<span
onclick=\"this.parentNode.querySelector('.stacktrace-details').classList.toggle('collapsed')\"
class=\"expand-details\">+details</span>";
+ var form_msg = "<div
class=\"stacktrace-details collapsed\"><pre>" + row.errorMessage +
"</pre></div>";
+ return form_head + form + form_msg;
+ }
+ },
+ name: "Errors"
+ }
+ ],
+ "columnDefs": [
+ { "visible": false, "targets": 11 },
+ { "visible": false, "targets": 12 },
+ { "visible": false, "targets": 13 },
+ { "visible": false, "targets": 14 },
+ { "visible": false, "targets": 15 },
+ { "visible": false, "targets": 16 },
+ { "visible": false, "targets": 17 },
+ { "visible": false, "targets": 18 }
+ ],
+ };
+ var taskTableSelector = $(taskTable).DataTable(task_conf);
+ $('#active-tasks-table_filter input').unbind();
+ $('#active-tasks-table_filter input').bind('keyup',
function(e) {
+ taskTableSelector.search( this.value ).draw();
--- End diff --
search on the task table seems to do query upon typing, not sure that is
what we want, would be better if use completed and hit enter since its doing
server side query
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
