Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r230102635 --- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js --- @@ -0,0 +1,959 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var blockUICount = 0; + +$(document).ajaxStop(function () { + if (blockUICount == 0) { + $.unblockUI(); + blockUICount++; + } +}); + +$(document).ajaxStart(function () { + if (blockUICount == 0) { + $.blockUI({message: '<h3>Loading Stage Page...</h3>'}); + } +}); + +$.extend( $.fn.dataTable.ext.type.order, { + "duration-pre": ConvertDurationString, + + "duration-asc": function ( a, b ) { + a = ConvertDurationString( a ); + b = ConvertDurationString( b ); + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "duration-desc": function ( a, b ) { + a = ConvertDurationString( a ); + b = ConvertDurationString( b ); + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +} ); + +// This function will only parse the URL under certain format +// e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1 +// e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1 +function stageEndPoint(appId) { + var queryString = document.baseURI.split('?'); + var words = document.baseURI.split('/'); + var indexOfProxy = words.indexOf("proxy"); + var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1]; + if (indexOfProxy > 0) { + var appId = words[indexOfProxy + 1]; + var newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/'); + return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId; + } + var indexOfHistory = words.indexOf("history"); + if (indexOfHistory > 0) { + var appId = words[indexOfHistory + 1]; + var appAttemptId = words[indexOfHistory + 2]; + var newBaseURI = words.slice(0, words.indexOf("history")).join('/'); + if (isNaN(appAttemptId) || appAttemptId == "0") { + return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId; + } else { + return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId; + } + } + return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId; +} + +function getColumnNameForTaskMetricSummary(columnKey) { + switch(columnKey) { + case "executorRunTime": + return "Duration"; + break; + + case "jvmGcTime": + return "GC Time"; + break; + + case "gettingResultTime": + return "Getting Result Time"; + break; + + case "inputMetrics": + return "Input Size / Records"; + break; + + case "outputMetrics": + return "Output Size / Records"; + break; + + case "peakExecutionMemory": + return "Peak Execution Memory"; + break; + + case "resultSerializationTime": + return "Result Serialization Time"; + break; + + case "schedulerDelay": + return "Scheduler Delay"; + break; + + case "diskBytesSpilled": + return "Shuffle spill (disk)"; + break; + + case "memoryBytesSpilled": + return "Shuffle spill (memory)"; + break; + + case "shuffleReadMetrics": + return "Shuffle Read Size / Records"; + break; + + case "shuffleWriteMetrics": + return "Shuffle Write Size / Records"; + break; + + case "executorDeserializeTime": + return "Task Deserialization Time"; + break; + + default: + return "NA"; + } +} + +function displayRowsForSummaryMetricsTable(row, type, columnIndex) { + switch(row.metric) { + case 'Input Size / Records': + var str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " + + row.data.recordsRead[columnIndex]; + return str; + break; + + case 'Output Size / Records': + var str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " + + row.data.recordsWritten[columnIndex]; + return str; + break; + + case 'Shuffle Read Size / Records': + var str = formatBytes(row.data.readBytes[columnIndex], type) + " / " + + row.data.readRecords[columnIndex]; + return str; + break; + + case 'Shuffle Read Blocked Time': + var str = formatDuration(row.data.fetchWaitTime[columnIndex]); + return str; + break; + + case 'Shuffle Remote Reads': + var str = formatBytes(row.data.remoteBytesRead[columnIndex], type); + return str; + break; + + case 'Shuffle Write Size / Records': + var str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " + + row.data.writeRecords[columnIndex]; + return str; + break; + + default: + return (row.metric == 'Peak Execution Memory' || row.metric == 'Shuffle spill (memory)' + || row.metric == 'Shuffle spill (disk)') ? formatBytes( + row.data[columnIndex], type) : (formatDuration(row.data[columnIndex])); + + } +} + +function createDataTableForTaskSummaryMetricsTable(task_summary_metrics_table) { + var taskMetricsTable = "#summary-metrics-table"; + if ($.fn.dataTable.isDataTable(taskMetricsTable)) { + taskSummaryMetricsDataTable.clear().draw(); + taskSummaryMetricsDataTable.rows.add(task_summary_metrics_table).draw(); + } else { + var task_conf = { + "data": task_summary_metrics_table, + "columns": [ + {data : 'metric'}, + // Min + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 0); + } + }, + // 25th percentile + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 1); + } + }, + // Median + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 2); + } + }, + // 75th percentile + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 3); + } + }, + // Max + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 4); + } + } + ], + "columnDefs": [ + { "type": "duration", "targets": 1 }, + { "type": "duration", "targets": 2 }, + { "type": "duration", "targets": 3 }, + { "type": "duration", "targets": 4 }, + { "type": "duration", "targets": 5 } + ], + "paging": false, + "searching": false, + "order": [[0, "asc"]], + "bSort": false + }; + taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(task_conf); + } + task_summary_metrics_table_current_state_array = task_summary_metrics_table.slice(); +} + +function createRowMetadataForColumn(columnName, data, checkboxId) { + var row = { + "metric": columnName, + "data": data, + "checkboxId": checkboxId + }; + return row; +} + +function reselectCheckboxesBasedOnTaskTableState() { + var allChecked = true; + var task_summary_metrics_table_current_filtered_array = task_summary_metrics_table_current_state_array.slice(); + if (typeof taskTableSelector !== 'undefined' && task_summary_metrics_table_current_state_array.length > 0) { + for (k = 0; k < optionalColumns.length; k++) { + if (taskTableSelector.column(optionalColumns[k]).visible()) { + $("#box-"+optionalColumns[k]).prop('checked', true); + task_summary_metrics_table_current_state_array.push(task_summary_metrics_table_array.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]); + task_summary_metrics_table_current_filtered_array = task_summary_metrics_table_current_state_array.slice(); + } else { + allChecked = false; + } + } + if (allChecked) { + $("#box-0").prop('checked', true); + } + createDataTableForTaskSummaryMetricsTable(task_summary_metrics_table_current_filtered_array); + } +} + +function getStageAttemptId() { + var words = document.baseURI.split('?'); + var attemptIdStr = words[1].split('&')[1]; + var digitsRegex = /[0-9]+/; + var stgAttemptId = words[1].split("&").filter( + word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex); + return stgAttemptId; +} + +var task_summary_metrics_table_array = []; +var task_summary_metrics_table_current_state_array = []; +var taskSummaryMetricsDataTable; +var optionalColumns = [11, 12, 13, 14, 15, 16, 17]; +var taskTableSelector; + +$(document).ready(function () { + setDataTableDefaults(); + + $("#showAdditionalMetrics").append( + "<div><a id='additionalMetrics'>" + + "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>" + + " Show Additional Metrics" + + "</a></div>" + + "<div class='container-fluid container-fluid-div' id='toggle-metrics' hidden>" + + "<div><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" + + "<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>" + + "<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Task Deserialization Time</div>" + + "<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked Time</div>" + + "<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" + + "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Result Serialization Time</div>" + + "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" + + "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" + + "</div>"); + + $('#scheduler_delay').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " + + "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results."); + $('#task_deserialization_time').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task."); + $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines."); + $('#shuffle_remote_reads').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. "); + $('#result_serialization_time').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver."); + $('#getting_result_time').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task."); + $('#peak_execution_memory').attr("data-toggle", "tooltip") + .attr("data-placement", "right") + .attr("title", "Execution memory refers to the memory used by internal data structures created during " + + "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " + + "should be approximately the sum of the peak sizes across all such data structures created " + + "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " + + "external sort."); + $('#scheduler_delay').tooltip(true); + $('#task_deserialization_time').tooltip(true); + $('#shuffle_read_blocked_time').tooltip(true); + $('#shuffle_remote_reads').tooltip(true); + $('#result_serialization_time').tooltip(true); + $('#getting_result_time').tooltip(true); + $('#peak_execution_memory').tooltip(true); + tasksSummary = $("#parent-container"); + getStandAloneAppId(function (appId) { + + var endPoint = stageEndPoint(appId); + var stageAttemptId = getStageAttemptId(); + $.getJSON(endPoint + "/" + stageAttemptId, function(response, status, jqXHR) { + + var responseBody = response; + var dataToShow = {}; + dataToShow.showInputData = responseBody.inputBytes > 0; + dataToShow.showOutputData = responseBody.outputBytes > 0; + dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0; + dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0; + dataToShow.showBytesSpilledData = + (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0); + + if (!dataToShow.showShuffleReadData) { + $('#shuffle_read_blocked_time').remove(); + $('#shuffle_remote_reads').remove(); + optionalColumns.splice(2, 2); + } + + // prepare data for executor summary table + stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary); + $.getJSON(createRESTEndPointForExecutorsPage(appId), + function(executorSummaryResponse, status, jqXHR) { + var executorSummaryMap = new Map(); + for (var i = 0; i < executorSummaryResponse.length; i++) { + executorSummaryMap.set(executorSummaryResponse[i].id, executorSummaryResponse[i]); + } + var executor_summary_table = []; + stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) { + responseBody.executorSummary[columnKeyIndex].id = columnKeyIndex; + if ("executorLogs" in executorSummaryMap.get(columnKeyIndex.toString()) && + executorSummaryMap.get(columnKeyIndex.toString()).executorLogs != null) { + responseBody.executorSummary[columnKeyIndex].executorLogs = + executorSummaryMap.get(columnKeyIndex.toString()).executorLogs; + } else { + responseBody.executorSummary[columnKeyIndex].executorLogs = {}; + } + if ("hostPort" in executorSummaryMap.get(columnKeyIndex.toString()) && + executorSummaryMap.get(columnKeyIndex.toString()).hostPort != null) { + responseBody.executorSummary[columnKeyIndex].hostPort = + executorSummaryMap.get(columnKeyIndex.toString()).hostPort; + } else { + responseBody.executorSummary[columnKeyIndex].hostPort = "CANNOT FIND ADDRESS"; + } + executor_summary_table.push(responseBody.executorSummary[columnKeyIndex]); + }); + // building task aggregated metrics by executor table + var executorSummaryTable = "#summary-executor-table"; --- End diff -- Done
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org