Repository: drill Updated Branches: refs/heads/master 9a6cb59b9 -> da241134f
DRILL-6279: Indicate operators that spilled in-memory data to disk on Web UI As part of this coomit added Bootstrap's Glyphicons. closes #1197 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cc440af0 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cc440af0 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cc440af0 Branch: refs/heads/master Commit: cc440af0397d0ec8b52227db9b7e09513b5ff157 Parents: 9a6cb59 Author: Kunal Khatua <kkha...@maprtech.com> Authored: Sun Mar 25 22:05:06 2018 -0700 Committer: Arina Ielchiieva <arina.yelchiy...@gmail.com> Committed: Fri Apr 6 11:52:22 2018 +0300 ---------------------------------------------------------------------- .../server/rest/profile/OperatorWrapper.java | 89 ++++++++++++++++++- .../exec/server/rest/profile/TableBuilder.java | 14 ++- .../src/main/resources/rest/profile/profile.ftl | 12 ++- .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes .../fonts/glyphicons-halflings-regular.woff2 | Bin 0 -> 18028 bytes 7 files changed, 109 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java index afccbb7..3a60c71 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.rest.profile; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -38,7 +39,17 @@ import com.google.common.base.Preconditions; * Wrapper class for profiles of ALL operator instances of the same operator type within a major fragment. */ public class OperatorWrapper { + @SuppressWarnings("unused") + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorWrapper.class); + + private static final String HTML_ATTRIB_SPILLS = "spills"; + private static final String HTML_ATTRIB_CLASS = "class"; + private static final String HTML_ATTRIB_STYLE = "style"; + private static final String HTML_ATTRIB_TITLE = "title"; + private static final DecimalFormat DECIMAL_FORMATTER = new DecimalFormat("#.##"); private static final String UNKNOWN_OPERATOR = "UNKNOWN_OPERATOR"; + //Negative valued constant used for denoting invalid index to indicate absence of metric + private static final int NO_SPILL_METRIC_INDEX = Integer.MIN_VALUE; private final int major; private final List<ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String>> opsAndHosts; // [(operatorProfile --> minorFragment number,host), ...] private final OperatorProfile firstProfile; @@ -146,6 +157,11 @@ public class OperatorWrapper { tb.appendCell(path, null, null, opTblBgColor); tb.appendCell(operatorName); + //Check if spill information is available + int spillCycleMetricIndex = getSpillCycleMetricIndex(operatorType); + boolean isSpillableOp = (spillCycleMetricIndex != NO_SPILL_METRIC_INDEX); + boolean hasSpilledToDisk = false; + //Get MajorFragment Busy+Wait Time Tally long majorBusyNanos = majorFragmentBusyTally.get(new OperatorPathBuilder().setMajor(major).build()); @@ -153,6 +169,8 @@ public class OperatorWrapper { double processSum = 0.0; double waitSum = 0.0; double memSum = 0.0; + double spillCycleSum = 0.0; + long spillCycleMax = 0L; long recordSum = 0L; //Construct list for sorting purposes (using legacy Comparators) @@ -168,6 +186,22 @@ public class OperatorWrapper { recordSum += sp.getRecords(); } opList.add(ip.getLeft()); + + //Capture Spill Info + //Check to ensure index < #metrics (old profiles have less metrics); else reset isSpillableOp + if (isSpillableOp) { + //NOTE: We get non-zero value for non-existent metrics, so we can't use getMetric(index) + //profile.getMetric(spillCycleMetricIndex).getLongValue(); + //Forced to iterate list + for (MetricValue metricVal : profile.getMetricList()) { + if (metricVal.getMetricId() == spillCycleMetricIndex) { + long spillCycles = metricVal.getLongValue(); + spillCycleMax = Math.max(spillCycles, spillCycleMax); + spillCycleSum += spillCycles; + hasSpilledToDisk = (spillCycleSum > 0.0); + } + } + } } final ImmutablePair<OperatorProfile, Integer> longSetup = Collections.max(opList, Comparators.setupTime); @@ -190,8 +224,59 @@ public class OperatorWrapper { tb.appendFormattedInteger(recordSum); final ImmutablePair<OperatorProfile, Integer> peakMem = Collections.max(opList, Comparators.operatorPeakMemory); - tb.appendBytes(Math.round(memSum / size)); - tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated()); + + //Inject spill-to-disk attributes + Map<String, String> avgSpillMap = null; + Map<String, String> maxSpillMap = null; + if (hasSpilledToDisk) { + avgSpillMap = new HashMap<>(); + //Average SpillCycle + double avgSpillCycle = spillCycleSum/size; + avgSpillMap.put(HTML_ATTRIB_TITLE, DECIMAL_FORMATTER.format(avgSpillCycle) + " spills on average"); + avgSpillMap.put(HTML_ATTRIB_STYLE, "cursor:help;" + spillCycleMax); + avgSpillMap.put(HTML_ATTRIB_CLASS, "spill-tag"); //JScript will inject Icon + avgSpillMap.put(HTML_ATTRIB_SPILLS, DECIMAL_FORMATTER.format(avgSpillCycle)); //JScript will inject Count + maxSpillMap = new HashMap<>(); + maxSpillMap.put(HTML_ATTRIB_TITLE, "Most # spills: " + spillCycleMax); + maxSpillMap.put(HTML_ATTRIB_STYLE, "cursor:help;" + spillCycleMax); + maxSpillMap.put(HTML_ATTRIB_CLASS, "spill-tag"); //JScript will inject Icon + maxSpillMap.put(HTML_ATTRIB_SPILLS, String.valueOf(spillCycleMax)); //JScript will inject Count + } + + tb.appendBytes(Math.round(memSum / size), avgSpillMap); + tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated(), maxSpillMap); + } + + /** + * Returns index of Spill Count/Cycle metric + * @param operatorType + * @return index of spill metric + */ + private int getSpillCycleMetricIndex(CoreOperatorType operatorType) { + String metricName; + + switch (operatorType) { + case EXTERNAL_SORT: + metricName = "SPILL_COUNT"; + break; + case HASH_AGGREGATE: + case HASH_JOIN: + metricName = "SPILL_CYCLE"; + break; + default: + return NO_SPILL_METRIC_INDEX; + } + + int metricIndex = 0; //Default + String[] metricNames = OperatorMetricRegistry.getMetricNames(operatorType.getNumber()); + for (String name : metricNames) { + if (name.equalsIgnoreCase(metricName)) { + return metricIndex; + } + metricIndex++; + } + //Backward compatibility with rendering older profiles. Ideally we should never touch this if an expected metric is not there + return NO_SPILL_METRIC_INDEX; } public String getMetricsTable() { http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java index 3833f51..0b65254 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/TableBuilder.java @@ -201,7 +201,7 @@ public class TableBuilder { } public void appendBytes(final long l) { - appendBytes(l, null); + appendBytes(l, null, null, null); } public void appendBytes(final long l, final String link) { @@ -209,8 +209,18 @@ public class TableBuilder { } public void appendBytes(final long l, final String link, final String tooltip) { + appendBytes(l, link, tooltip, null); + } + + public void appendBytes(final long l, Map<String, String> attributeMap) { + appendBytes(l, null, null, attributeMap); + } + + public void appendBytes(final long l, final String link, final String tooltip, Map<String, String> attributeMap) { //Embedding dataTable's data-order attribute - Map<String, String> attributeMap = new HashMap<String, String>(); + if (attributeMap == null) { + attributeMap = new HashMap<>(); + } attributeMap.put("data-order", String.valueOf(l)); appendCell(bytePrint(l), link, tooltip, null, attributeMap); } http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/resources/rest/profile/profile.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/profile/profile.ftl b/exec/java-exec/src/main/resources/rest/profile/profile.ftl index 7f4474c..671a57b 100644 --- a/exec/java-exec/src/main/resources/rest/profile/profile.ftl +++ b/exec/java-exec/src/main/resources/rest/profile/profile.ftl @@ -372,6 +372,16 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de </div> <br> <script> + //Inject Spilled Tags + $(window).on('load', function () { + var spillLabel = document.getElementsByClassName("spill-tag"); + var i; + for (i = 0; i < spillLabel.length; i++) { + var content = spillLabel[i].innerHTML; + spillLabel[i].innerHTML = "<span class=\"glyphicon glyphicon-download-alt\"> </span>"+content; + } + }); + //Configuration for Query Viewer in Profile ace.require("ace/ext/language_tools"); var viewer = ace.edit("query-text"); @@ -400,9 +410,7 @@ table.sortable thead .sorting_desc { background-image: url("/static/img/black-de enableSnippets: false, enableLiveAutocompletion: false }); - </script> - <script> //Configuration for Query Editor in Profile ace.require("ace/ext/language_tools"); var editor = ace.edit("query-editor"); http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.eot ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.eot b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.eot new file mode 100644 index 0000000..b93a495 Binary files /dev/null and b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.eot differ http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.ttf ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.ttf b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.ttf new file mode 100644 index 0000000..1413fc6 Binary files /dev/null and b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.ttf differ http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff new file mode 100644 index 0000000..9e61285 Binary files /dev/null and b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff differ http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff2 ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff2 b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff2 new file mode 100644 index 0000000..64539b5 Binary files /dev/null and b/exec/java-exec/src/main/resources/rest/static/fonts/glyphicons-halflings-regular.woff2 differ
