This is an automated email from the ASF dual-hosted git repository. dbecker pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit c30325e21c5974998aee2b5d3a7bfd1d27565b7a Author: Surya Hebbar <[email protected]> AuthorDate: Wed Oct 18 17:57:33 2023 +0530 IMPALA-12504: Split graphical query timeline script into es6 modules This patch divides the query timeline script into multiple modules for better maintainability and to help writing unit tests. The patch also improves parsing of utilization values from the query profile, as the order and name of counters may change within the profile, to ensure appropriate mappings between plotted readings, initially the profile is traversed to find the correct mapping. Change-Id: Id9d62a799d838876509686a75ecea778b2c72cc6 Reviewed-on: http://gerrit.cloudera.org:8080/20588 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- www/query_timeline.tmpl | 757 +-------------------- www/scripts/query_timeline/chart_commons.js | 110 +++ www/scripts/query_timeline/fragment_diagram.js | 602 ++++++++++++++++ www/scripts/query_timeline/global_members.js | 61 ++ .../query_timeline/host_utilization_diagram.js | 227 ++++++ www/scripts/util.js | 15 +- 6 files changed, 1042 insertions(+), 730 deletions(-) diff --git a/www/query_timeline.tmpl b/www/query_timeline.tmpl index 83fe97cd1..fb3b2c761 100644 --- a/www/query_timeline.tmpl +++ b/www/query_timeline.tmpl @@ -27,7 +27,7 @@ under the License. <div class="container"> -<style id="css"> +<style id="page_export-css"> #fragment_diagram, #phases_header, #timeticks_footer { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif; @@ -52,7 +52,7 @@ under the License. </label> </div> <label> - <input type="checkbox" id="plan_order" onClick="renderFragmentDiagram()"/> + <input type="checkbox" id="plan_order"/> Print tree in plan order (if unchecked, print in fragment order) </label> @@ -93,700 +93,40 @@ under the License. <svg id="timeticks_footer"></svg> </div> </div> -<div id="utilization_diagram"> - <!--Utilization metrics is not available. Please make sure to set query option RESOURCE_TRACE_RATIO=true.--> +<div id="host_utilization_diagram"> + <!--Utilization metrics is not available. Please make sure to set query option + RESOURCE_TRACE_RATIO=true.--> </div> {{/plan_metadata_unavailable}} {{> www/common-footer.tmpl }} -<script> - +<script type="module"> $("#plan-timing-tab").addClass("active"); -var profile = {}; - -var stroke_fill_colors = { black : "#000000", dark_grey : "#505050", - light_grey : "#F0F0F0", transperent : "rgba(0, 0, 0, 0)" }; - -var row_height = 15; -var char_width = 6; -var margin_header_footer = 5; -var margin_chart_end = 60; -var border_stroke_width = 2; -var diagram_width = window.innerWidth - border_stroke_width; // border width - -// #phases_header -var phases = [ - { color: "#C0C0FF", label: "Prepare" } -, { color: "#E0E0E0", label: "Open" } -, { color: "#FFFFC0", label: "Produce First Batch" } -, { color: "#C0FFFF", label: "Send First Batch" } -, { color: "#C0FFC0", label: "Process Remaining Batches" } -, { color: "#FFC0C0", label: "Close" } -]; - -// #fragment_diagram -var fragment_colors = ["#A9A9A9", "#FF8C00", "#8A2BE2", "#A52A2A", "#00008B", "#006400", - "#228B22", "#4B0082", "#DAA520", "#008B8B", "#000000", "#DC143C"]; -var fragments = []; -var all_nodes = []; -var receiver_nodes = []; -var max_namelen = 0; -var frag_name_width; -var name_width; -var chart_width; - -// #timeticks_footer -var ntics = 10; -var integer_part_estimate = 4; -var decimals = 2; -var maxts = 0; -var last_maxts = -1; - -// #utilization_diagram -var utilization_counter_names = ["avg io wait", "avg sys", "avg user"]; -var cpu_utilization_chart; -var cpu_nodes_usage_aggregate; -var sampled_timeseries; -var timeaxis_name = 'timeticks'; -var prev_num_samples = 0; -var max_samples = 64; -var max_samples_available = 0; -var max_samples_collected = 0; -var max_samples_period; -var utilization_style; - -var fragment_events_parse_successful = false; -var utilization_metrics_parse_successful = false; - -// binding DOM elements -var timing_diagram = document.getElementById("timing_diagram"); -var phases_header = document.getElementById("phases_header"); -var fragment_diagram = document.getElementById("fragment_diagram"); -var timeticks_footer = document.getElementById("timeticks_footer"); -var utilization_diagram = document.getElementById("utilization_diagram"); - -var export_filename = document.getElementById("export_filename"); -var export_format = document.getElementById("export_format"); - -export_filename.value = export_filename.value.replace(/\W/g,'_'); - -function get_svg_rect(fill_color, x, y, width, height, dash, stroke_color) { - var rect = document.createElementNS("http://www.w3.org/2000/svg";, "rect"); - rect.setAttribute("x", `${x}px`); - rect.setAttribute("y", `${y}px`); - rect.setAttribute("width", `${width}px`); - rect.setAttribute("height", `${height}px`); - rect.setAttribute("fill", fill_color); - if (dash) { - rect.setAttribute("stroke", stroke_color); - rect.setAttribute("stroke-dasharray", "2 2"); - } - return rect; -} - -function get_svg_text(text, fill_color, x, y, height, container_center, max_width = 0) { - var text_el = document.createElementNS("http://www.w3.org/2000/svg";, "text"); - text_el.appendChild(document.createTextNode(text)); - text_el.setAttribute("x", `${x}px`); - text_el.setAttribute("y", `${y}px`); - text_el.style.fontSize = `${height / 1.5}px`; - if (container_center) { - text_el.setAttribute("dominant-baseline", "middle"); - text_el.setAttribute("text-anchor", "middle"); - } - text_el.setAttribute("fill", fill_color); - if (max_width != 0) { - text_el.setAttribute("textLength", max_width); - text_el.setAttribute("lengthAdjust", "spacingAndGlyphs"); - } - return text_el; -} - -function get_svg_line(stroke_color, x1, y1, x2, y2, dash) { - var line = document.createElementNS("http://www.w3.org/2000/svg";, "line"); - line.setAttribute("x1", `${x1}px`); - line.setAttribute("y1", `${y1}px`); - line.setAttribute("x2", `${x2}px`); - line.setAttribute("y2", `${y2}px`); - line.setAttribute("stroke", stroke_color); - if (dash) { - line.setAttribute("stroke-dasharray", "2 2"); - } - return line; -} - -function DrawBars(svg, rownum, row_height, events, xoffset, px_per_ns) { - var color_idx = 0; - var last_end = xoffset; - bar_height = row_height - 2; - - events.forEach(function(ev) { - if (ev.no_bar == undefined) { - var x = last_end; - var y = rownum * row_height; - - var endts = Math.max.apply(null, ev.tslist); - var width = xoffset + endts * px_per_ns - last_end; - last_end = x + width; - - // Block phase outline - svg.appendChild(get_svg_rect(stroke_fill_colors.black, x, y, width, bar_height, false)); - if (width > 2) { - svg.appendChild(get_svg_rect(phases[color_idx].color, x + 1, y + 1, - width - 2, bar_height - 2, false)); - } - - color_idx++; - - // Grey dividers for other instances that finished earlier - ev.tslist.forEach(function(ts) { - var dx = (endts - ts) * px_per_ns; - var ignore_px = 2; // Don't print tiny skews - if (Math.abs(dx) > ignore_px) { - svg.appendChild(get_svg_line(stroke_fill_colors.dark_grey, last_end - dx, y , last_end - dx, - y + bar_height, false)); - } - }); - } - }); -} - -function collectFragmentEventsFromProfile(ignored_arg) { - rownum = 0; - max_namelen = 0; - fragments = []; - all_nodes = []; - receiver_nodes = []; - var color_idx = 0; - try { - // First pass: compute sizes - profile.child_profiles[2].child_profiles.forEach(function(fp) { - - if (fp.child_profiles != undefined && - fp.child_profiles[0].event_sequences != undefined) { - var cp = fp.child_profiles[0]; - var fevents = fp.child_profiles[0].event_sequences[0].events; - - // Build list of timestamps that spans instances for each event - for (var en = 0; en < fevents.length; ++en) { - if (fevents[en].label.includes("AsyncCodegen")) { - fevents[en].no_bar = true; - continue; - } - fevents[en].tslist = [ fevents[en].timestamp ]; - for (var instance = 1; instance < fp.child_profiles.length; ++instance) { - if (fp.child_profiles[instance].event_sequences != undefined) { - fevents[en].tslist.push( - fp.child_profiles[instance].event_sequences[0].events[en].timestamp); - } - } - } - - fragment = { - name: fp.profile_name, - events: fevents, - nodes: [ ], - color: fragment_colors[color_idx] - } - // Pick a new color for each plan fragment - color_idx = (color_idx + 1) % fragment_colors.length; - maxts = Math.max(maxts, fevents[fevents.length - 1].timestamp); - max_namelen = Math.max(max_namelen, fp.profile_name.length); - var node_path = []; - var node_stack = []; - cp.child_profiles.forEach(function get_plan_nodes(pp, index) { - if (pp.node_metadata != undefined) { - node_path.push(index); - var name_flds = pp.profile_name.split(/[()]/); - var node_type = name_flds[0].trim(); - var node_id = name_flds.length > 1 ? name_flds[1].split(/[=]/)[1] : 0; - node_name = pp.profile_name.replace("_NODE", "").replace("_", " ") - .replace("KrpcDataStreamSender", "SENDER") - .replace("Hash Join Builder", "JOIN BUILD") - .replace("join node_", ""); - if (node_type.indexOf("SCAN_NODE") >= 0) { - var table_name = pp.info_strings.find(({ key }) => key === "Table Name") - .value.split(/[.]/); - node_name = node_name.replace("SCAN", - `SCAN [${table_name[table_name.length - 1]}]`); - } - - var is_receiver = node_type == "EXCHANGE_NODE" || - (node_type == "HASH_JOIN_NODE" && pp.num_children < 3); - - var is_sender = (node_type == "Hash Join Builder" || - node_type == "KrpcDataStreamSender"); - var parent_node; - if (node_type == "PLAN_ROOT_SINK") { - parent_node = undefined; - } else if (pp.node_metadata.data_sink_id != undefined) { - parent_node = receiver_nodes[node_id]; // Exchange sender dst - } else if (pp.node_metadata.join_build_id != undefined) { - parent_node = receiver_nodes[node_id]; // Join sender dst - } else if (node_stack.length > 0) { - parent_node = node_stack[node_stack.length - 1]; - } else if (all_nodes.length) { - parent_node = all_nodes[all_nodes.length - 1]; - } - - max_namelen = Math.max(max_namelen, node_name.length + node_stack.length + 1); - - if (pp.event_sequences != undefined) { - var node_events = pp.event_sequences[0].events; - - // Start the instance event list for each event with timestamps from this instance - for (var en = 0; en < node_events.length; ++en) { - node_events[en].tslist = [ node_events[en].timestamp ]; - if (node_type == "HASH_JOIN_NODE" && (en == 1 || en == 2)) { - node_events[en].no_bar = true; - } - } - } - var node = { - name: node_name, - type: node_type, - node_id: node_id, - num_children: 0, - child_index: 0, - metadata: pp.node_metadata, - parent_node: parent_node, - events: node_events, - path: node_path.slice(0), - is_receiver: is_receiver, - is_sender: is_sender - } - - if (is_sender) { - node.parent_node.sender_frag_index = fragments.length; - } - - if (is_receiver) { - receiver_nodes[node_id] = node; - } - - if (parent_node != undefined) { - node.child_index = parent_node.num_children++; - } - - all_nodes.push(node); - - fragment.nodes.push(node); - - if (pp.child_profiles != undefined) { - node_stack.push(node); - pp.child_profiles.forEach(get_plan_nodes); - node = node_stack.pop(); - } - rownum++; - node_path.pop(); - } - }); - - - // For each node, retrieve the instance timestamps for the remaining instances - for (var ni = 0; ni < fragment.nodes.length; ++ni) { - var node = fragment.nodes[ni]; - for (var cpn = 1; cpn < fp.child_profiles.length; ++cpn) { - var cp = fp.child_profiles[cpn]; - - // Use the saved node path to traverse to the same position in this instance - for (var pi = 0; pi < node.path.length; ++pi) { - cp = cp.child_profiles[node.path[pi]]; - } - console.assert(cp.node_metadata.data_sink_id == undefined || - cp.profile_name.indexOf(`(dst_id=${node.node_id})`)); - console.assert(cp.node_metadata.plan_node_id == undefined || - cp.node_metadata.plan_node_id == node.node_id); - - // Add instance events to this node - if (cp.event_sequences != undefined) { - for (var en = 0; en < node.events.length; ++en) { - node.events[en].tslist.push(cp.event_sequences[0].events[en].timestamp); - } - } - } - } - - fragments.push(fragment); - } - }); - frag_name_width = (Math.max(2, (fragments.length - 1).toString().length) + 3) * char_width; - name_width = max_namelen * char_width + (frag_name_width + 2); - fragment_events_parse_successful = true; - } catch(e) { - fragment_events_parse_successful = false; - console.log(e); - } -} - -function setDimensions(ignored_arg) { - var display_height = Math.min(window.innerHeight - timing_diagram.offsetTop - 50 - - (utilization_metrics_parse_successful? getUtilizationHeight() : 0), rownum * row_height); - - chart_width = diagram_width - name_width - margin_chart_end - border_stroke_width; - - phases_header.style.height = `${row_height}px`; - fragment_diagram.parentElement.style.height = `${display_height}px`; - fragment_diagram.style.height = `${rownum * row_height}px`; - timeticks_footer.style.height = `${row_height}px`; - - fragment_diagram.parentElement.style.width = `${diagram_width}px`; - phases_header.parentElement.style.width = `${diagram_width}px`; - timeticks_footer.parentElement.style.width = `${diagram_width}px`; - timing_diagram.parentElement.style.width = `${diagram_width}px`; - - fragment_diagram.style.width = `${diagram_width}px`; - phases_header.style.width = `${diagram_width}px`; - timeticks_footer.style.width = `${diagram_width}px`; -} - -function clearDOMChildren(element) { - while (element.firstChild) { - element.removeChild(element.firstChild); - } -} - -async function renderPhases() { - clearDOMChildren(phases_header); - var color_idx = 0; - var width = Math.ceil(chart_width / phases.length); - phases.forEach(function(p) { - var x = name_width + Math.ceil(chart_width * color_idx / phases.length); - x = Math.min(x, diagram_width - width); - - phases_header.appendChild(get_svg_rect(stroke_fill_colors.black, x, 0, width, row_height, false)); - phases_header.appendChild(get_svg_rect(phases[color_idx++].color, x + 1, 1, - width - 2, row_height - 2, false)); - phases_header.appendChild(get_svg_text(p.label, stroke_fill_colors.black, x + width / 2, - (row_height + 4) / 2 , row_height, true, Math.min(p.label.length * char_width, width / 1.5))); - }); -} - -async function renderFragmentDiagram() { - clearDOMChildren(fragment_diagram); - last_maxts = maxts; - var plan_order = document.getElementById('plan_order').checked; - var px_per_ns = chart_width / maxts; - var rownum_l = 0; - var max_indent = 0; - var pending_children = 0; - var pending_senders = 0; - var text_y = row_height - 4; - fragments.forEach(function printFragment(fragment) { - if (!fragment.printed) { - fragment.printed = true; - - var pending_fragments = []; - var fevents = fragment.events; - - frag_name = fragment.name.replace("Coordinator ", "").replace("Fragment ", ""); - - fragment_diagram.appendChild(get_svg_text(frag_name, fragment.color, 1, text_y, - row_height, false)); - - // Fragment/sender timing row - DrawBars(fragment_diagram, rownum_l, row_height, fevents, name_width, px_per_ns); - - for (var i = 0; i < fragment.nodes.length; ++i) { - var node = fragment.nodes[i]; +import {renderTimingDiagram, collectFragmentEventsFromProfile, ntics, set_ntics} from + "./www/scripts/query_timeline/fragment_diagram.js"; +import {collectUtilizationFromProfile, toogleUtilizationVisibility} from + "/www/scripts/query_timeline/host_utilization_diagram.js"; +import {profile, set_profile, diagram_width, set_diagram_width, border_stroke_width, + resizeHorizontalAll, maxts} from "/www/scripts/query_timeline/global_members.js"; - if (node.events != undefined) { - // Plan node timing row - DrawBars(fragment_diagram, rownum_l, row_height, node.events, name_width, px_per_ns); - if (node.type == "HASH_JOIN_NODE") { - fragment_diagram.appendChild(get_svg_text("X", stroke_fill_colors.black, - name_width + Math.min.apply(null, node.events[2].tslist) * px_per_ns, - text_y, row_height, false)); - fragment_diagram.appendChild(get_svg_text("O", stroke_fill_colors.black, - name_width + Math.min.apply(null, node.events[2].tslist) * px_per_ns, - text_y, row_height, false)); - } - } +var chart_export_style; +var last_maxts; - if (node.is_receiver) { - pending_senders++; - } else if (node.is_sender) { - pending_senders--; - } - if (!node.is_sender) { - pending_children--; - } - - var label_x = frag_name_width + char_width * pending_children; - var label_width = Math.min(char_width * node.name.length, - name_width - label_x - 2); - fragment_diagram.appendChild(get_svg_text(node.name, fragment.color, - label_x, text_y, row_height, false)); - - if (node.parent_node != undefined) { - var y = row_height * node.parent_node.rendering.rownum; - if (node.is_sender) { - var x = name_width + Math.min.apply(null, fevents[3].tslist) * px_per_ns; - // Dotted horizontal connector to received rows - fragment_diagram.appendChild(get_svg_line(fragment.color, name_width, - y + row_height / 2 - 1, x, y + row_height / 2 - 1, true)); - - // Dotted rectangle for received rows - var x2 = name_width + Math.max.apply(null,fevents[4].tslist) * px_per_ns; - fragment_diagram.appendChild(get_svg_rect(stroke_fill_colors.transperent, - x, y + 4, x2 - x, row_height - 10, true, fragment.color)); - } - - if (node.is_sender && node.parent_node.rendering.rownum != rownum_l - 1) { - // DAG edge on right side to distant sender - var x = name_width - (pending_senders) * char_width - char_width / 2; - fragment_diagram.appendChild(get_svg_line(fragment.color, - node.parent_node.rendering.label_end, - y + row_height / 2, x , y + row_height / 2, false)); - fragment_diagram.appendChild(get_svg_line(fragment.color, x, y + row_height / 2, - x, text_y - row_height / 2 + 3, false)); - fragment_diagram.appendChild(get_svg_line(fragment.color, x, - text_y - row_height / 2 + 3, label_x + label_width, - text_y - row_height / 2 + 3, false)); - - } else { - // DAG edge from parent to immediate child - var x = frag_name_width + (pending_children + 1) * char_width - char_width / 2; - fragment_diagram.appendChild(get_svg_line(fragment.color, x, y + row_height - 3, - x, text_y - row_height + 6, false)); - } - - } - node.rendering = { rownum: rownum_l, label_end: label_x + label_width }; - if (node.num_children) // Scan (leaf) node - pending_children += (node.num_children - node.is_receiver); - text_y += row_height; - rownum_l++; - - if (node.is_receiver) { - if (plan_order) { - printFragment(fragments[node.sender_frag_index]) - } else { - pending_fragments.push(fragments[node.sender_frag_index]); - } - } - - } - - // Visit sender fragments in reverse order to avoid dag edges crossing - pending_fragments.reverse().forEach(printFragment); - - } - }); - fragments.forEach(function(fragment) { - fragment.printed = false; - }); -} - -async function renderTimeticks() { - clearDOMChildren(timeticks_footer); - var sec_per_tic = maxts / ntics / 1e9; - var px_per_tic = chart_width / ntics; - var x = name_width; - var y = 0; - var text_y = row_height - 4; - var timetick_label; - for (var i = 1; i <= ntics; ++i) { - timeticks_footer.appendChild(get_svg_rect(stroke_fill_colors.black, x, y, px_per_tic, - row_height, false)); - timeticks_footer.appendChild(get_svg_rect(stroke_fill_colors.light_grey, x + 1, y + 1, px_per_tic - 2, - row_height - 2, false)); - timetick_label = (i * sec_per_tic).toFixed(decimals); - timeticks_footer.appendChild(get_svg_text(timetick_label, stroke_fill_colors.black, - x + px_per_tic - timetick_label.length * char_width + 2, text_y, row_height, false)); - x += px_per_tic; - } -} - -function getUtilizationHeight() { - return window.innerHeight * 0.3; -} - -function toogleUtilizationVisibility() { - if (utilization_metrics_parse_successful) { - utilization_diagram.style.display = 'flex'; - } else { - utilization_diagram.style.display = 'none'; - } -} - -function initializeUtilizationChart() { - try { - cpu_utilization_chart.destroy(); - } catch (e) {} - cpu_utilization_chart = c3.generate({ - bindto : "#utilization_diagram", - data : { - columns : [[timeaxis_name, 0]], - type : 'area', - groups : [ utilization_counter_names ], - order : 'asc', - x : timeaxis_name - }, size : { - height : getUtilizationHeight(), - width : diagram_width - }, padding : { - left : name_width, - right : margin_chart_end - }, axis : { - x : - { - padding : { - left : 0, - right : 0 - }, - tick : { - format : function (x) { return x.toFixed(decimals); } - } - } - }, legend : { - show : false - }, tooltip : { - format : { - value : function (value, ratio, id, index) { return value.toFixed(decimals) + '%'; } - } - } - }); - cpu_utilization_chart.load({ - unload : true - }); -} - -function initializeUtilizationMetrics() { - // user, sys, io and sampled timeticks - cpu_nodes_usage_aggregate = new Array(3); - for (var i = 0; i < 3; ++i) { - cpu_nodes_usage_aggregate[i] = new Array(max_samples + 2).fill(0); - cpu_nodes_usage_aggregate[i][0] = utilization_counter_names[i]; - } - sampled_timeseries = new Array(max_samples + 2).fill(null); - sampled_timeseries[0] = timeaxis_name; -} - -function accumulateUtilization(utilization_array, time_series_counter) { - var utilization_samples = time_series_counter.data.split(",").map(el => parseInt(el) / 100); - if (utilization_samples.length <= 1) return; - for (var j = 0; j < utilization_samples.length; ++j) { - utilization_array[j + 1] += utilization_samples[j]; - } - if (time_series_counter.num > max_samples_collected) { - max_samples_available = utilization_samples.length; - max_samples_period = time_series_counter.period; - max_samples_collected = time_series_counter.num; - } -} - -function clearTimesamples(timesamples_array) { - for (var j = 1; j <= max_samples_available + 1; ++j) { - timesamples_array[j] = null; - } -} - -function clearUtilization(utilization_array) { - for (var j = 1; j <= max_samples_available + 1; ++j) { - utilization_array[j] = 0; - } -} - -function collectUtilizationFromProfile(ignored_arg) { - try { - var per_node_profiles = profile.child_profiles[2].child_profiles[0]; - if (cpu_utilization_chart == undefined) { - initializeUtilizationMetrics(); - initializeUtilizationChart(); - } - // Update the plot, only when number of samples in SummaryStatsCounter is updated - if (profile.child_profiles[1].summary_stats_counters[0].num_of_samples != prev_num_samples) { - // For each node's profile - per_node_profiles.child_profiles.forEach(function (cpu_node, cpu_node_i) { - for (var i = 0; i < 3; ++i) {// usr, sys and iowait - accumulateUtilization(cpu_nodes_usage_aggregate[i], - cpu_node.time_series_counters[i]); - } - }); - if (max_samples_available <= 2) { - cpu_utilization_chart.destroy(); - cpu_utilization_chart = undefined; - clearDOMChildren(utilization_diagram); - utilization_diagram.style = ""; - utilization_diagram.className = ""; - utilization_diagram.style.height = `${getUtilizationHeight()}px`; - utilization_diagram.innerHTML = `<span style="color:#ff0f0f;font-size:20px;"> - Warning: not enough samples for CPU utilization plot. Please decrease the - value of starting flag variable <b></i>periodic_counter_update_period_ms - </b></i> to increase the granularity of CPU utilization plot.</span>`; - utilization_diagram.style.justifyContent = "center"; - utilization_diagram.style.alignItems = "center"; - utilization_metrics_parse_successful = true; - return; - } - cpu_nodes_usage_aggregate.forEach(function (acc_usage) { - for (var i = 1; i <= max_samples_available; ++i) { - acc_usage[i] = acc_usage[i] / per_node_profiles.child_profiles.length; - } - }); - var avg_period = max_samples_collected * max_samples_period / max_samples_available; - avg_period = avg_period / 1000; - for (var k = 0; k <= max_samples_available; k++) { - sampled_timeseries[k + 1] = (k * avg_period); - } - if (maxts / 1e9 > sampled_timeseries[max_samples_available + 1]) { - sampled_timeseries[max_samples_available + 1] = maxts / 1e9; - } else { - maxts = sampled_timeseries[max_samples_available + 1] * 1e9; - } - cpu_utilization_chart.load({ - columns : [...cpu_nodes_usage_aggregate, sampled_timeseries] - }); - cpu_nodes_usage_aggregate.forEach(function (acc_usage) { - clearUtilization(acc_usage); - }); - clearTimesamples(sampled_timeseries); - prev_num_samples = profile.child_profiles[1].summary_stats_counters[0].num_of_samples; - } - utilization_metrics_parse_successful = true; - } catch (e) { - utilization_metrics_parse_successful = false; - console.log(e); - } -} - -async function renderUtilization() { - cpu_utilization_chart.resize({ - height: getUtilizationHeight(), - width: diagram_width - }); - utilization_diagram.style.width = `${diagram_width}px`; -} - -function renderAll() { - if (fragment_events_parse_successful) { - setDimensions(); - renderPhases(); - renderFragmentDiagram(); - renderTimeticks(); - } - if (utilization_metrics_parse_successful) { - renderUtilization(); - } -} - -function refresh() { +window.onload = function refresh() { var req = new XMLHttpRequest(); req.onload = function() { if (req.status == 200) { - profile = JSON.parse(req.responseText)["profile_json"]; + set_profile(JSON.parse(req.responseText)["profile_json"]); collectFragmentEventsFromProfile(); collectUtilizationFromProfile(); - toogleUtilizationVisibility(); - if (maxts == last_maxts) { - renderUtilization(); - } else { - renderAll(); + if (last_maxts != maxts) { + renderTimingDiagram(); + last_maxts = maxts; } + toogleUtilizationVisibility(); if (profile.child_profiles[0].info_strings.find(({key}) => key === "End Time").value == "") { setTimeout(refresh, 1000); @@ -797,10 +137,15 @@ function refresh() { req.send(); } +window.addEventListener('resize', function(event) { + set_diagram_width(Math.max(window.innerWidth, diagram_width - border_stroke_width)); + resizeHorizontalAll(); +}, true); + // Attaches a SVG blob of the complete timeline to the associated link -export_link.addEventListener('click', function(event) { +export_link.addEventListener('click', function (event) { if (export_format.value == ".html") { - var export_style = document.getElementById("css"); + var export_style = document.getElementById("page_export-css"); // Deep clone 'parentNode's as wrappers to SVG components var fragment_diagram_wrapper = fragment_diagram.parentNode.cloneNode(true); @@ -810,9 +155,9 @@ export_link.addEventListener('click', function(event) { var html_blob = new Blob(['<!DOCTYPE html><body>', '<h1 style=\"font-family:monospace;\">Query {{query_id}}</h1>', - `<style>${export_style.innerHTML} ${utilization_style}</style>`, + `<style>${export_style.innerHTML} ${chart_export_style}</style>`, phases_header.parentNode.outerHTML, fragment_diagram_wrapper.outerHTML, - timeticks_footer.parentElement.outerHTML, utilization_diagram.outerHTML, + timeticks_footer.parentElement.outerHTML, host_utilization_diagram.outerHTML, '</body></html>'], {type: 'text/html;charset=utf-8'}); export_link.href = URL.createObjectURL(html_blob); } @@ -820,45 +165,9 @@ export_link.addEventListener('click', function(event) { export_link.click(); }); -timeticks_footer.addEventListener('wheel', function(e) { - var rendering_constraint = char_width * (decimals + integer_part_estimate) >= chart_width / ntics; - if (e.shiftKey) { - if (e.altKey) { - if (decimals <= 1 && e.wheelDelta <= 0) return; - if (e.wheelDelta >= 0) { - if (rendering_constraint) return; - ++decimals; - } else { - --decimals; - } - } else { - if (ntics <= 10 && e.wheelDelta <= 0) return; - if (rendering_constraint && e.wheelDelta >= 0) return; - ntics += Math.round(e.wheelDelta / 200); - } - renderTimeticks(); - } -}); +fetch(make_url("/www/c3/c3.v7.min.css")) + .then(res => res.text()) + .then(style => chart_export_style = style); -fragment_diagram.addEventListener('wheel', function(e) { - var rendering_constraint = char_width * (decimals + integer_part_estimate) >= chart_width / ntics; - if (e.shiftKey) { - if (diagram_width <= window.innerWidth && e.wheelDelta <= 0) return; - if (rendering_constraint && e.wheelDelta <= 0) return; - diagram_width += Math.round(e.wheelDelta * 6 / 10); - renderAll(); - } -}); - -window.addEventListener('resize', function(event) { - diagram_width = Math.max(window.innerWidth, diagram_width - border_stroke_width); - renderAll(); -}, true); - -window.onload = function() { - refresh(); - fetch(make_url("/www/c3/c3.v7.min.css")) - .then(res => res.text()) - .then(style => utilization_style = style); -} +export_filename.value = export_filename.value.replace(/\W/g,'_'); </script> diff --git a/www/scripts/query_timeline/chart_commons.js b/www/scripts/query_timeline/chart_commons.js new file mode 100644 index 000000000..11a087596 --- /dev/null +++ b/www/scripts/query_timeline/chart_commons.js @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {maxts, set_maxts, clearDOMChildren} from "./global_members.js"; +import {name_width} from "./fragment_diagram.js"; + +function accumulateTimeseriesValues(values_array, time_series_counter, max_samples) { + var samples = time_series_counter.data.split(",").map(el => parseInt(el)); + var max_traverse_len = Math.min(samples.length, values_array.length - 2); + if (isNaN(samples[0])) return; + for (var j = 0; j < max_traverse_len; ++j) { + values_array[j + 2] += samples[j]; + } + if (time_series_counter.num > max_samples.collected || time_series_counter.period + > max_samples.period) { + max_samples.available = samples.length; + max_samples.period = time_series_counter.period; + max_samples.collected = time_series_counter.num; + } +} + +export function generateTimesamples(timesamples_array, max_samples, extend) { + var avg_period = max_samples.collected * max_samples.period / max_samples.available; + avg_period = avg_period / 1000; + var max_traverse_len = Math.min(max_samples.available, timesamples_array.length - 2); + for (var k = 0; k <= max_traverse_len; ++k) { + timesamples_array[k + 1] = (k * avg_period); + } + if (maxts / 1e9 > timesamples_array[max_samples.available + 1]) { + // extend by one additional point or correct the final point + if (extend) { + // extend + timesamples_array[max_samples.available + 2] = maxts / 1e9; + } else { + // correct + timesamples_array[max_samples.available + 1] = maxts / 1e9; + } + } else { + set_maxts(timesamples_array[max_samples.available + 1] * 1e9); + } + var j = max_samples.available + (extend ? 3 : 2); + for (; j < timesamples_array.length && timesamples_array[j] != null; ++j) { + timesamples_array[j] = null; + } +} + +export function mapTimeseriesCounters(time_series_counters, counters) { + for (var i = 0; i < counters.length; i++) { + for (var j = 0; j < time_series_counters.length; j++) { + if (time_series_counters[j].counter_name == counters[i][0]) { + counters[i][2] = j; + } + } + } +} + +export function clearTimeseriesValues(values_array, max_samples) { + var max_traverse_len = Math.min(max_samples.available + 1, values_array.length - 1); + for (var j = 2; j <= max_traverse_len; ++j) { + values_array[j] = null; + } +} + +export function aggregateProfileTimeseries(parent_profile, aggregate_array, + counters, max_samples) { + parent_profile.child_profiles.forEach(function (time_series_profiles) { + for (var i = 0; i < aggregate_array.length; ++i) { + accumulateTimeseriesValues(aggregate_array[i], + time_series_profiles.time_series_counters[counters[i][2]], + max_samples); + } + }); +} + +export function displayWarning(parent_element, msg, diagram_width, margin_l, margin_r) { + clearDOMChildren(parent_element); + parent_element.style.display = "flex"; + parent_element.style.width = `${diagram_width - margin_l - margin_r}px`; + parent_element.style.height = "200px"; + parent_element.style.justifyContent = "center"; + parent_element.style.alignItems = "center"; + parent_element.style.padding = "20px"; + parent_element.style.textAlign = "center"; + parent_element.style.marginLeft = `${margin_l}px`; + parent_element.style.border = "1px solid black"; + parent_element.innerHTML = `<span style="color:#ff0f0f;font-size:20px;">${msg}</span>`; +} + +export function destroyChart(chart, chart_dom_obj) { + try { + chart.destroy(); + } catch (e) { + } + clearDOMChildren(chart_dom_obj); + return null; +} diff --git a/www/scripts/query_timeline/fragment_diagram.js b/www/scripts/query_timeline/fragment_diagram.js new file mode 100644 index 000000000..74f8b899b --- /dev/null +++ b/www/scripts/query_timeline/fragment_diagram.js @@ -0,0 +1,602 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {profile, set_maxts, maxts, decimals, set_decimals, diagram_width, + set_diagram_width, diagram_controls_height, diagram_min_height, + margin_header_footer, border_stroke_width, margin_chart_end, clearDOMChildren, + resizeHorizontalAll} from "./global_members.js"; +import {host_utilization_chart, getUtilizationHeight} from + "./host_utilization_diagram.js"; + +export var name_width; +export var page_additional_height; + +var stroke_fill_colors = { black : "#000000", dark_grey : "#505050", + light_grey : "#F0F0F0", transperent : "rgba(0, 0, 0, 0)" }; +var rownum; +var row_height = 15; +var integer_part_estimate = 4; +var char_width = 6; +var fragment_events_parse_successful = false; + +// #phases_header +var phases = [ + { color: "#C0C0FF", label: "Prepare" }, + { color: "#E0E0E0", label: "Open" }, + { color: "#FFFFC0", label: "Produce First Batch" }, + { color: "#C0FFFF", label: "Send First Batch" }, + { color: "#C0FFC0", label: "Process Remaining Batches" }, + { color: "#FFC0C0", label: "Close" } +]; + +// #fragment_diagram +var fragment_colors = ["#A9A9A9", "#FF8C00", "#8A2BE2", "#A52A2A", "#00008B", "#006400", + "#228B22", "#4B0082", "#DAA520", "#008B8B", "#000000", "#DC143C"]; +var fragments = []; +var fragment_diagram_title = getSvgTitle(""); +var all_nodes = []; +var receiver_nodes = []; +var max_namelen = 0; +var frag_name_width; +var chart_width; + +// #timeticks_footer +export var ntics = 10; + +export function set_ntics(val) { + ntics = val; +} + +function removeChildIfExists(parentElement, childElement) { + try { + parentElement.removeChild(childElement); + } catch(e) { + } +} + +function getSvgRect(fill_color, x, y, width, height, dash, stroke_color) { + var rect = document.createElementNS("http://www.w3.org/2000/svg";, "rect"); + rect.setAttribute("x", `${x}px`); + rect.setAttribute("y", `${y}px`); + rect.setAttribute("width", `${width}px`); + rect.setAttribute("height", `${height}px`); + rect.setAttribute("fill", fill_color); + rect.setAttribute("stroke", stroke_color); + if (dash) { + rect.setAttribute("stroke-dasharray", "2 2"); + } + return rect; +} + +function getSvgText(text, fill_color, x, y, height, container_center, max_width = 0) { + var text_el = document.createElementNS("http://www.w3.org/2000/svg";, "text"); + text_el.appendChild(document.createTextNode(text)); + text_el.setAttribute("x", `${x}px`); + text_el.setAttribute("y", `${y}px`); + text_el.style.fontSize = `${height / 1.5}px`; + if (container_center) { + text_el.setAttribute("dominant-baseline", "middle"); + text_el.setAttribute("text-anchor", "middle"); + } + text_el.setAttribute("fill", fill_color); + if (max_width != 0) { + text_el.setAttribute("textLength", max_width); + text_el.setAttribute("lengthAdjust", "spacingAndGlyphs"); + } + return text_el; +} + +function getSvgLine(stroke_color, x1, y1, x2, y2, dash) { + var line = document.createElementNS("http://www.w3.org/2000/svg";, "line"); + line.setAttribute("x1", `${x1}px`); + line.setAttribute("y1", `${y1}px`); + line.setAttribute("x2", `${x2}px`); + line.setAttribute("y2", `${y2}px`); + line.setAttribute("stroke", stroke_color); + if (dash) { + line.setAttribute("stroke-dasharray", "2 2"); + } + return line; +} + +function getSvgTitle(text) { + var title = document.createElementNS("http://www.w3.org/2000/svg";, "title"); + title.textContent = text; + return title; +} + +function getSvgGroup() { + var group = document.createElementNS("http://www.w3.org/2000/svg";, "g"); + return group; +} + +function DrawBars(svg, rownum, row_height, events, xoffset, px_per_ns) { + var color_idx = 0; + var last_end = xoffset; + var bar_height = row_height - 2; + var plan_node = getSvgGroup(); + plan_node.classList.add("plan_node"); + events.forEach(function(ev) { + if (ev.no_bar == undefined) { + var x = last_end; + var y = rownum * row_height; + + var endts = Math.max.apply(null, ev.tslist); + var width = xoffset + endts * px_per_ns - last_end; + last_end = x + width; + + // Block phase outline + plan_node.appendChild(getSvgRect(phases[color_idx].color, x, y, width, bar_height, + false, stroke_fill_colors.black)); + color_idx++; + + // Grey dividers for other instances that finished earlier + ev.tslist.forEach(function(ts) { + var dx = (endts - ts) * px_per_ns; + var ignore_px = 2; // Don't print tiny skews + if (Math.abs(dx) > ignore_px) { + plan_node.appendChild(getSvgLine(stroke_fill_colors.dark_grey, last_end - dx, + y, last_end - dx, y + bar_height, false)); + } + }); + svg.appendChild(plan_node); + } + }); +} + +async function renderPhases() { + clearDOMChildren(phases_header); + var color_idx = 0; + var width = Math.ceil(chart_width / phases.length); + phases.forEach(function(p) { + var x = name_width + Math.ceil(chart_width * color_idx / phases.length); + x = Math.min(x, diagram_width - width); + + phases_header.appendChild(getSvgRect(stroke_fill_colors.black, x, 0, width, + row_height, false)); + phases_header.appendChild(getSvgRect(phases[color_idx++].color, x + 1, 1, + width - 2, row_height - 2, false)); + phases_header.appendChild(getSvgText(p.label, stroke_fill_colors.black, x + width + / 2, (row_height + 4) / 2 , row_height, true, + Math.min(p.label.length * char_width, width / 1.5))); + }); +} + +async function renderFragmentDiagram() { + clearDOMChildren(fragment_diagram); + var px_per_ns = chart_width / maxts; + var rownum_l = 0; + var max_indent = 0; + var pending_children = 0; + var pending_senders = 0; + var text_y = row_height - 4; + fragment_diagram.appendChild(fragment_diagram_title); + fragments.forEach(function printFragment(fragment) { + if (!fragment.printed) { + fragment.printed = true; + + var pending_fragments = []; + var fevents = fragment.events; + + var frag_name = fragment.name.replace("Coordinator ", "").replace("Fragment ", ""); + + fragment_diagram.appendChild(getSvgText(frag_name, fragment.color, 1, text_y, + row_height, false)); + + // Fragment/sender timing row + DrawBars(fragment_diagram, rownum_l, row_height, fevents, name_width, px_per_ns); + + for (var i = 0; i < fragment.nodes.length; ++i) { + var node = fragment.nodes[i]; + + if (node.events != undefined) { + // Plan node timing row + DrawBars(fragment_diagram, rownum_l, row_height, node.events, name_width, + px_per_ns); + if (node.type == "HASH_JOIN_NODE") { + fragment_diagram.appendChild(getSvgText("X", stroke_fill_colors.black, + name_width + Math.min.apply(null, node.events[2].tslist) * px_per_ns, + text_y, row_height, false)); + fragment_diagram.appendChild(getSvgText("O", stroke_fill_colors.black, + name_width + Math.min.apply(null, node.events[2].tslist) * px_per_ns, + text_y, row_height, false)); + } + } + + if (node.is_receiver) { + pending_senders++; + } else if (node.is_sender) { + pending_senders--; + } + if (!node.is_sender) { + pending_children--; + } + + var label_x = frag_name_width + char_width * pending_children; + var label_width = Math.min(char_width * node.name.length, + name_width - label_x - 2); + fragment_diagram.appendChild(getSvgText(node.name, fragment.color, + label_x, text_y, row_height, false)); + + if (node.parent_node != undefined) { + var y = row_height * node.parent_node.rendering.rownum; + if (node.is_sender) { + var x = name_width + Math.min.apply(null, fevents[3].tslist) * px_per_ns; + // Dotted horizontal connector to received rows + fragment_diagram.appendChild(getSvgLine(fragment.color, name_width, + y + row_height / 2 - 1, x, y + row_height / 2 - 1, true)); + + // Dotted rectangle for received rows + var x2 = name_width + Math.max.apply(null, fevents[4].tslist) * px_per_ns; + fragment_diagram.appendChild(getSvgRect(stroke_fill_colors.transperent, + x, y + 4, x2 - x, row_height - 10, true, fragment.color)); + } + + if (node.is_sender && node.parent_node.rendering.rownum != rownum_l - 1) { + // DAG edge on right side to distant sender + var x = name_width - (pending_senders) * char_width - char_width / 2; + fragment_diagram.appendChild(getSvgLine(fragment.color, + node.parent_node.rendering.label_end, + y + row_height / 2, x , y + row_height / 2, false)); + fragment_diagram.appendChild(getSvgLine(fragment.color, x, + y + row_height / 2, x, text_y - row_height / 2 + 3, false)); + fragment_diagram.appendChild(getSvgLine(fragment.color, x, + text_y - row_height / 2 + 3, label_x + label_width, + text_y - row_height / 2 + 3, false)); + + } else { + // DAG edge from parent to immediate child + var x = frag_name_width + (pending_children + 1) * char_width - + char_width / 2; + fragment_diagram.appendChild(getSvgLine(fragment.color, x, y + + row_height - 3, x, text_y - row_height + 6, false)); + } + } + node.rendering = { rownum: rownum_l, label_end: label_x + label_width }; + if (node.num_children) // Scan (leaf) node + pending_children += (node.num_children - node.is_receiver); + text_y += row_height; + rownum_l++; + + if (node.is_receiver) { + if (plan_order.checked) { + printFragment(fragments[node.sender_frag_index]) + } else { + pending_fragments.push(fragments[node.sender_frag_index]); + } + } + + } + + // Visit sender fragments in reverse order to avoid dag edges crossing + pending_fragments.reverse().forEach(printFragment); + + } + }); + fragments.forEach(function(fragment) { + fragment.printed = false; + }); +} + +async function renderTimeticks() { + clearDOMChildren(timeticks_footer); + var sec_per_tic = maxts / ntics / 1e9; + var px_per_tic = chart_width / ntics; + var x = name_width; + var y = 0; + var text_y = row_height - 4; + var timetick_label; + for (var i = 1; i <= ntics; ++i) { + timeticks_footer.appendChild(getSvgRect(stroke_fill_colors.black, x, y, px_per_tic, + row_height, false)); + timeticks_footer.appendChild(getSvgRect(stroke_fill_colors.light_grey, x + 1, + y + 1, px_per_tic - 2, row_height - 2, false)); + timetick_label = (i * sec_per_tic).toFixed(decimals); + timeticks_footer.appendChild(getSvgText(timetick_label, stroke_fill_colors.black, + x + px_per_tic - timetick_label.length * char_width + 2, text_y, row_height, + false)); + x += px_per_tic; + } +} + +export function collectFragmentEventsFromProfile() { + rownum = 0; + max_namelen = 0; + fragments = []; + all_nodes = []; + receiver_nodes = []; + var color_idx = 0; + try { + // First pass: compute sizes + var execution_profile = profile.child_profiles[2]; + var execution_profile_name = execution_profile.profile_name.split(" ").slice(0,2) + .join(" "); + console.assert(execution_profile_name == "Execution Profile"); + execution_profile.child_profiles.forEach(function(fp) { + + if (fp.child_profiles != undefined && + fp.child_profiles[0].event_sequences != undefined) { + var cp = fp.child_profiles[0]; + var fevents = fp.child_profiles[0].event_sequences[0].events; + + // Build list of timestamps that spans instances for each event + for (var en = 0; en < fevents.length; ++en) { + if (fevents[en].label.includes("AsyncCodegen")) { + fevents[en].no_bar = true; + continue; + } + fevents[en].tslist = [ fevents[en].timestamp ]; + } + for (var instance = 1; instance < fp.child_profiles.length; ++instance) { + if (fp.child_profiles[instance].event_sequences != undefined) { + if (fp.child_profiles[instance].event_sequences[0].events.length == + fevents.length) { + for (var en = 0; en < fevents.length; ++en) { + if (fevents[en].no_bar) continue; + fevents[en].tslist.push( + fp.child_profiles[instance].event_sequences[0].events[en].timestamp); + } + } else { + var en = 0, i = 0; + while(i < fevents.length && en < fp.child_profiles[instance] + .event_sequences[0].events.length) { + if (fevents[i].no_bar) { + if (fevents[i].label == fp.child_profiles[instance] + .event_sequences[0].events[en].label) { en++; } + } else if (fp.child_profiles[instance].event_sequences[0] + .events[en].label == fevents[i].label) { + fevents[i].tslist.push(fp.child_profiles[instance].event_sequences[0] + .events[en].timestamp); + ++en; + } + ++i; + } + } + } + } + + var fragment = { + name: fp.profile_name, + events: fevents, + nodes: [ ], + color: fragment_colors[color_idx] + } + // Pick a new color for each plan fragment + color_idx = (color_idx + 1) % fragment_colors.length; + set_maxts(Math.max(maxts, fevents[fevents.length - 1].timestamp)); + max_namelen = Math.max(max_namelen, fp.profile_name.length); + var node_path = []; + var node_stack = []; + var node_name; + cp.child_profiles.forEach(function get_plan_nodes(pp, index) { + if (pp.node_metadata != undefined) { + node_path.push(index); + var name_flds = pp.profile_name.split(/[()]/); + var node_type = name_flds[0].trim(); + var node_id = name_flds.length > 1 ? name_flds[1].split(/[=]/)[1] : 0; + node_name = pp.profile_name.replace("_NODE", "").replace("_", " ") + .replace("KrpcDataStreamSender", "SENDER") + .replace("Hash Join Builder", "JOIN BUILD") + .replace("join node_", ""); + if (node_type.indexOf("SCAN_NODE") >= 0) { + var table_name = pp.info_strings.find(({ key }) => key === "Table Name") + .value.split(/[.]/); + node_name = node_name.replace("SCAN", + `SCAN [${table_name[table_name.length - 1]}]`); + } + + var is_receiver = node_type == "EXCHANGE_NODE" || + (node_type == "HASH_JOIN_NODE" && pp.num_children < 3); + + var is_sender = (node_type == "Hash Join Builder" || + node_type == "KrpcDataStreamSender"); + var parent_node; + if (node_type == "PLAN_ROOT_SINK") { + parent_node = undefined; + } else if (pp.node_metadata.data_sink_id != undefined) { + parent_node = receiver_nodes[node_id]; // Exchange sender dst + } else if (pp.node_metadata.join_build_id != undefined) { + parent_node = receiver_nodes[node_id]; // Join sender dst + } else if (node_stack.length > 0) { + parent_node = node_stack[node_stack.length - 1]; + } else if (all_nodes.length) { + parent_node = all_nodes[all_nodes.length - 1]; + } + + max_namelen = Math.max(max_namelen, node_name.length + node_stack.length + 1); + + if (pp.event_sequences != undefined) { + var node_events = pp.event_sequences[0].events; + + // Start the instance event list for each event with timestamps + // from this instance + for (var en = 0; en < node_events.length; ++en) { + node_events[en].tslist = [ node_events[en].timestamp ]; + if (node_type == "HASH_JOIN_NODE" && (en == 1 || en == 2)) { + node_events[en].no_bar = true; + } + } + } + var node = { + name: node_name, + type: node_type, + node_id: node_id, + num_children: 0, + child_index: 0, + metadata: pp.node_metadata, + parent_node: parent_node, + events: node_events, + path: node_path.slice(0), + is_receiver: is_receiver, + is_sender: is_sender + } + + if (is_sender) { + node.parent_node.sender_frag_index = fragments.length; + } + + if (is_receiver) { + receiver_nodes[node_id] = node; + } + + if (parent_node != undefined) { + node.child_index = parent_node.num_children++; + } + + all_nodes.push(node); + + fragment.nodes.push(node); + + if (pp.child_profiles != undefined) { + node_stack.push(node); + pp.child_profiles.forEach(get_plan_nodes); + node = node_stack.pop(); + } + rownum++; + node_path.pop(); + } + }); + + // For each node, retrieve the instance timestamps for the remaining instances + for (var ni = 0; ni < fragment.nodes.length; ++ni) { + var node = fragment.nodes[ni]; + for (var cpn = 1; cpn < fp.child_profiles.length; ++cpn) { + var cp = fp.child_profiles[cpn]; + + // Use the saved node path to traverse to the same position in this instance + for (var pi = 0; pi < node.path.length; ++pi) { + cp = cp.child_profiles[node.path[pi]]; + } + console.assert(cp.node_metadata.data_sink_id == undefined || + cp.profile_name.indexOf(`(dst_id=${node.node_id})`)); + console.assert(cp.node_metadata.plan_node_id == undefined || + cp.node_metadata.plan_node_id == node.node_id); + + // Add instance events to this node + if (cp.event_sequences != undefined) { + if (node.events.length == cp.event_sequences[0].events.length) { + for (var en = 0; en < node.events.length; ++en) { + node.events[en].tslist.push(cp.event_sequences[0].events[en].timestamp); + } + } else { + var i = 0, en = 0; + while(i < node.events.length && en < cp.event_sequences[0].events.length) { + if (node.events[i].label == cp.event_sequences[0].events[en].label) { + node.events[i].tslist.push(cp.event_sequences[0].events[en].timestamp); + ++en; ++i; + } else { + ++i; + } + } + } + } + } + } + fragments.push(fragment); + } + }); + frag_name_width = (Math.max(2, (fragments.length - 1).toString().length) + 3) + * char_width; + name_width = max_namelen * char_width + (frag_name_width + 2); + fragment_events_parse_successful = true; + } catch(e) { + fragment_events_parse_successful = false; + console.log(e); + } +} + +export function setTimingDiagramDimensions(ignored_arg) { + // phashes header and timeticks footer with margin and border by considering the offset + page_additional_height = timing_diagram.offsetTop + (row_height + margin_header_footer + + border_stroke_width * 4) * 2; + var fragment_diagram_initial_height = window.innerHeight - page_additional_height; + var remaining_height = fragment_diagram_initial_height - getUtilizationHeight(); + var display_height = Math.max(diagram_min_height, Math.min(remaining_height, + rownum * row_height)); + chart_width = diagram_width - name_width - margin_chart_end - border_stroke_width; + + phases_header.style.height = `${row_height}px`; + fragment_diagram.parentElement.style.height = `${display_height}px`; + fragment_diagram.style.height = `${Math.max(diagram_min_height, + rownum * row_height)}px`; + timeticks_footer.style.height = `${row_height}px`; + + fragment_diagram.parentElement.style.width = `${diagram_width}px`; + phases_header.parentElement.style.width = `${diagram_width}px`; + timeticks_footer.parentElement.style.width = `${diagram_width}px`; + timing_diagram.parentElement.style.width = `${diagram_width}px`; + + fragment_diagram.style.width = `${diagram_width}px`; + phases_header.style.width = `${diagram_width}px`; + timeticks_footer.style.width = `${diagram_width}px`; +} + +export function renderTimingDiagram() { + if (fragment_events_parse_successful) { + setTimingDiagramDimensions(); + renderPhases(); + renderFragmentDiagram(); + renderTimeticks(); + } +} + +fragment_diagram.addEventListener('wheel', function(e) { + if (e.shiftKey) { + if (e.wheelDelta <= 0 && diagram_width <= window.innerWidth) return; + var next_diagram_width = diagram_width + Math.round(e.wheelDelta); + if (next_diagram_width <= window.innerWidth) { + next_diagram_width = window.innerWidth; + } + var next_chart_width = next_diagram_width - name_width - margin_chart_end + - border_stroke_width; + var next_ntics = (next_diagram_width - diagram_width) * 10 / window.innerWidth; + next_ntics = ntics + Math.round(next_ntics); + var rendering_constraint = char_width * (decimals + integer_part_estimate) >= + next_chart_width / next_ntics; + if (rendering_constraint) return; + ntics = next_ntics; + set_diagram_width(next_diagram_width); + resizeHorizontalAll(); + } +}); + +timeticks_footer.addEventListener('wheel', function(e) { + if (e.shiftKey) { + if (e.altKey) { + if (e.wheelDelta <= 0) { + if (decimals <= 1) return; + set_decimals(decimals - 1); + } else { + var rendering_constraint = char_width * ((decimals + 1) + integer_part_estimate) + >= chart_width / ntics; + if (rendering_constraint) return; + set_decimals(decimals + 1); + } + } else { + if (e.wheelDelta <= 0 && ntics <= 10) return; + var next_ntics = ntics + Math.round(e.wheelDelta / 200); + if (next_ntics <= 10) return; + var rendering_constraint = char_width * (decimals + integer_part_estimate) + >= chart_width / next_ntics; + if (rendering_constraint) return; + ntics = next_ntics; + } + renderTimeticks(); + } +}); + +plan_order.addEventListener('click', renderFragmentDiagram); diff --git a/www/scripts/query_timeline/global_members.js b/www/scripts/query_timeline/global_members.js new file mode 100644 index 000000000..17608a986 --- /dev/null +++ b/www/scripts/query_timeline/global_members.js @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {renderTimingDiagram, setTimingDiagramDimensions} from "./fragment_diagram.js"; +import {resizeUtilizationChart} from "./host_utilization_diagram.js"; + +export var profile = {}; +export var maxts = 0; +export var decimals = 2; +export var border_stroke_width = 2; +export var diagram_width = window.innerWidth - border_stroke_width; // border width +export var margin_header_footer = 5; +export var margin_chart_end = 60; +export var diagram_min_height = 200; +export var diagram_controls_height = Math.max(28, window.innerHeight * 0.025); + +export function set_profile(val) { + profile = val; +} + +export function set_decimals(val) { + decimals = val; +} + +export function set_maxts(val) { + maxts = val; +} + +export function set_diagram_width(val) { + diagram_width = val; +} + +export function clearDOMChildren(element) { + while (element.firstChild) { + element.removeChild(element.firstChild); + } +} + +export function resizeHorizontalAll() { + renderTimingDiagram(); + resizeUtilizationChart(); +} + +export function resizeVerticalAll() { + setTimingDiagramDimensions(); + resizeUtilizationChart(); +} diff --git a/www/scripts/query_timeline/host_utilization_diagram.js b/www/scripts/query_timeline/host_utilization_diagram.js new file mode 100644 index 000000000..7d29746e0 --- /dev/null +++ b/www/scripts/query_timeline/host_utilization_diagram.js @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import {profile, set_maxts, maxts, diagram_width, diagram_controls_height, + resizeVerticalAll, diagram_min_height, margin_chart_end, decimals} + from "./global_members.js"; +import {name_width, page_additional_height, setTimingDiagramDimensions} + from "./fragment_diagram.js"; +import {aggregateProfileTimeseries, generateTimesamples, clearTimeseriesValues, + mapTimeseriesCounters, displayWarning, destroyChart} from "./chart_commons.js"; + +// #host_utilization_diagram +export var host_utilization_visible = true; +export var host_utilization_chart = null; +export var utilization_metrics_parse_successful = false; + +var cpu_utilization_counters = [ + ["HostCpuIoWaitPercentage", "avg io wait", 0], + ["HostCpuSysPercentage", "avg sys", 0], + ["HostCpuUserPercentage", "avg user", 0] +]; +var cpu_nodes_usage_aggregate; +var sampled_utilization_timeseries; +var utilization_timeaxis_name = "utilization timeticks"; +var prev_utilization_num_samples = 0; +var max_samples_utilization = { + allocated : 64, + available : 0, + collected : 0, + period : null +}; +var host_utilization_resize_factor = 0.2; + +function initializeUtilizationChart() { + var axis_mappings = {}; + for(var i = 0; i < cpu_utilization_counters.length; ++i) { + axis_mappings[cpu_utilization_counters[i][1]] = "y"; + } + var cpu_utilization_counter_group = new Array(cpu_utilization_counters.length); + for (var i = 0; i < cpu_utilization_counters.length; i++){ + cpu_utilization_counter_group[i] = cpu_utilization_counters[i][1]; + } + host_utilization_diagram.style = null; + host_utilization_chart = c3.generate({ + bindto : "#host_utilization_diagram", + data : { + columns : [[utilization_timeaxis_name, 0]], + axes: axis_mappings, + type : "area", + groups : [ cpu_utilization_counter_group ], + order : "asc", + x : utilization_timeaxis_name + }, size : { + height : getUtilizationHeight(), + width : diagram_width + }, padding : { + left : name_width, + right : margin_chart_end + }, axis : { + x : + { + padding : { + left : 0, + right : 0 + }, + tick : { + format : function (x) { return x.toFixed(decimals); } + } + }, + y : + { + tick : { + format : function (y) { return y + '%'; } + } + } + }, legend : { + show : false + }, tooltip : { + format : { + value : function (value, ratio, id, index) { + if (cpu_utilization_counter_group.includes(id)){ + return value.toFixed(decimals) + '%'; + } else { + return `${getReadableSize(value, decimals)}/s`; + } + }, + title : function (x, index) { + return x.toFixed(decimals) + "s"; + } + } + } + }); + host_utilization_chart.load({ + unload : true + }); + var chart_width = diagram_width - margin_chart_end - name_width; +} + +function initializeUtilizationMetrics(parent_profile, counters_y1, max_samples, + timeaxis_name) { + console.assert(parent_profile.profile_name == "Per Node Profiles"); + // user, sys, io and sampled timeticks + var cpu_nodes_usage_aggregate = new Array(counters_y1.length); + max_samples.available = 0; + max_samples.period = 0; + for (var i = 0; i < counters_y1.length; ++i) { + cpu_nodes_usage_aggregate[i] = new Array(max_samples.allocated + 2) + .fill(null); + cpu_nodes_usage_aggregate[i][0] = counters_y1[i][1]; + cpu_nodes_usage_aggregate[i][1] = 0; + } + var sampled_utilization_timeseries = new Array(max_samples.allocated + 2) + .fill(null); + sampled_utilization_timeseries[0] = timeaxis_name; + mapTimeseriesCounters(parent_profile.child_profiles[0].time_series_counters, counters_y1); + return {cpu_nodes_usage_aggregate, sampled_utilization_timeseries}; +} + +export function getUtilizationHeight() { + return Math.max(diagram_min_height, window.innerHeight * + host_utilization_resize_factor); +} + +export function toogleUtilizationVisibility() { + if (utilization_metrics_parse_successful && host_utilization_visible) { + host_utilization_diagram.style.display = "initial"; + } else { + host_utilization_diagram.style.display = "none"; + } +} + +export async function resizeUtilizationChart() { + if (host_utilization_chart == null) return; + var chart_width = diagram_width - margin_chart_end - name_width; + host_utilization_chart.resize({ + height : getUtilizationHeight(), + width : diagram_width + }); +} + +export function collectUtilizationFromProfile() { + // do not collect metrics, in case host utilization is not visible + if (!host_utilization_visible){ + utilization_metrics_parse_successful = false; + return; + } + + // try parsing the metrics from JSON, to seek case of errors in expected format + // and do not render the chart unless attributes are parsed accurately + try { + // initialize the collection arrays for subsequent collections + // arrays are overwritten without further re-allocation to reduce memory usage + var per_node_profiles = profile.child_profiles[2].child_profiles[0]; + console.assert(per_node_profiles.profile_name == "Per Node Profiles"); + if (host_utilization_chart == null) { + ({cpu_nodes_usage_aggregate, sampled_utilization_timeseries} = + initializeUtilizationMetrics(per_node_profiles, cpu_utilization_counters, + max_samples_utilization, utilization_timeaxis_name)); + initializeUtilizationChart(); + } + var impala_server_profile = profile.child_profiles[1]; + console.assert(impala_server_profile.profile_name == "ImpalaServer"); + // Update the plot, only when number of samples in SummaryStatsCounter is updated + if (impala_server_profile.summary_stats_counters[0].num_of_samples == + prev_utilization_num_samples) { + utilization_metrics_parse_successful = true; + return; + } + + // For each node's profile seperately aggregate into two different units of metrics + aggregateProfileTimeseries(per_node_profiles, cpu_nodes_usage_aggregate, + cpu_utilization_counters, max_samples_utilization); + + // display warnings in case less samples are available without plotting + if (max_samples_utilization.available <= 2) { + var utilization_samples_message = `Warning: Not enough samples for + CPU utilization plot. Please decrease the value of starting flag + variable <b><i>periodic_counter_update_period_ms </b></i> to + increase the granularity of CPU utilization plot.`; + host_utilization_chart = destroyChart(host_utilization_chart, + host_utilization_diagram); + displayWarning(host_utilization_diagram, utilization_samples_message, + diagram_width, name_width, margin_chart_end); + utilization_metrics_parse_successful = true; + return; + } + // average the aggregated metrics + cpu_nodes_usage_aggregate.forEach(function (acc_usage) { + for (var i = 2; i < max_samples_utilization.available + 2; ++i) { + acc_usage[i] = acc_usage[i] / (100 * per_node_profiles.child_profiles.length); + } + }); + // generate timestamps for utilization values and decide on last timestamp value + generateTimesamples(sampled_utilization_timeseries, max_samples_utilization, false); + + // load the utilization values to the chart + host_utilization_chart.load({ + columns : [...cpu_nodes_usage_aggregate, sampled_utilization_timeseries] + }); + // clear utilization value and timestamp samples arrays + cpu_nodes_usage_aggregate.forEach(function (acc_usage) { + clearTimeseriesValues(acc_usage, max_samples_utilization); + }); + prev_utilization_num_samples = profile.child_profiles[1].summary_stats_counters[0] + .num_of_samples; + utilization_metrics_parse_successful = true; + } catch (e) { + utilization_metrics_parse_successful = false; + host_utilization_chart = destroyChart(host_utilization_chart, + host_utilization_diagram); + console.log(e); + } +} diff --git a/www/scripts/util.js b/www/scripts/util.js index 6263c24eb..ea470132a 100644 --- a/www/scripts/util.js +++ b/www/scripts/util.js @@ -15,11 +15,14 @@ // specific language governing permissions and limitations // under the License. -function getReadableSize(bytes) { - var units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB']; - if (bytes <= 0) return bytes + ' B'; - var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); - return (bytes / Math.pow(1024, i)).toFixed(2) + ' ' + units[i]; +function getReadableSize(value, precision = 2) { + var units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; + var current_unit = 0; + while( value >= 1e3 ) { + value = value / 1e3; + ++current_unit; + } + return value.toFixed(precision) + ' ' + units[current_unit]; } function getReadableTimeNS(value) { @@ -91,4 +94,4 @@ function renderTime(data, type, row) { return getReadableTimeNS(data); } return data; -} \ No newline at end of file +}
