[
https://issues.apache.org/jira/browse/DRILL-5195?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15881157#comment-15881157
]
ASF GitHub Bot commented on DRILL-5195:
---------------------------------------
Github user paul-rogers commented on a diff in the pull request:
https://github.com/apache/drill/pull/756#discussion_r102803614
--- Diff:
exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java
---
@@ -83,56 +84,77 @@ public String getContent() {
maxRecords = Math.max(sp.getRecords(), maxRecords);
}
- builder.appendFormattedInteger(maxBatches, null);
- builder.appendFormattedInteger(maxRecords, null);
- builder.appendBytes(op.getPeakLocalMemoryAllocated(), null);
+ builder.appendFormattedInteger(maxBatches, null, null);
+ builder.appendFormattedInteger(maxRecords, null, null);
+ builder.appendBytes(op.getPeakLocalMemoryAllocated(), null, null);
}
return builder.build();
}
- public static final String[] OPERATORS_OVERVIEW_COLUMNS = {"Operator
ID", "Type", "Min Setup Time", "Avg Setup Time",
- "Max Setup Time", "Min Process Time", "Avg Process Time", "Max Process
Time", "Min Wait Time", "Avg Wait Time",
- "Max Wait Time", "Avg Peak Memory", "Max Peak Memory"};
+ public static final String[] OPERATORS_OVERVIEW_COLUMNS = {"Operator
ID", "Type",
+ "Avg Setup Time", "Max Setup Time",
+ "Avg Process Time", "Max Process Time", "Min Wait Time", "Avg Wait
Time", "Max Wait Time",
+ "% Fragment Time", "% Query Time", "Rows",
+ "Avg Peak Memory", "Max Peak Memory"};
- public void addSummary(TableBuilder tb) {
+ public static final String[] OPERATORS_OVERVIEW_COLUMNS_TOOLTIP = {null,
null,
+ null, null, null, null, null, null, null,
+ "% of time that the major fragment spent in operator", "% of Query
time spent in operator", "Records emitted by the operator",
+ null, null};
+ //Palette to help shade operators sharing a common major fragment
+ private static final String[] OPERATOR_OVERVIEW_BGCOLOR_PALETTE =
{"#ffffff","#f2f2f2"};
+
+ public void addSummary(TableBuilder tb, HashMap<String, Long>
majorFragmentBusyTally, long majorFragmentBusyTallyTotal) {
+ //Select background color from palette
+ String opTblBgColor = OPERATOR_OVERVIEW_BGCOLOR_PALETTE[major%2];
String path = new
OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build();
- tb.appendCell(path, null);
- tb.appendCell(operatorName, null);
+ tb.appendCellWithBgColor(path, null, null, opTblBgColor);
+ tb.appendCell(operatorName, null, null);
+
+ //Get MajorFragment Busy+Wait Time Tally
+ long majorBusyNanos = majorFragmentBusyTally.get(new
OperatorPathBuilder().setMajor(major).build());
double setupSum = 0.0;
double processSum = 0.0;
double waitSum = 0.0;
double memSum = 0.0;
+ long recordSum = 0L;
for (ImmutablePair<OperatorProfile, Integer> ip : ops) {
OperatorProfile profile = ip.getLeft();
setupSum += profile.getSetupNanos();
processSum += profile.getProcessNanos();
waitSum += profile.getWaitNanos();
memSum += profile.getPeakLocalMemoryAllocated();
+ for (final StreamProfile sp : profile.getInputProfileList()) {
+ recordSum += sp.getRecords();
+ }
}
- final ImmutablePair<OperatorProfile, Integer> shortSetup =
Collections.min(ops, Comparators.setupTime);
final ImmutablePair<OperatorProfile, Integer> longSetup =
Collections.max(ops, Comparators.setupTime);
- tb.appendNanos(shortSetup.getLeft().getSetupNanos(), null);
- tb.appendNanos(Math.round(setupSum / size), null);
- tb.appendNanos(longSetup.getLeft().getSetupNanos(), null);
+ tb.appendNanos(Math.round(setupSum / size), null, null);
+ tb.appendNanos(longSetup.getLeft().getSetupNanos(), null, null);
- final ImmutablePair<OperatorProfile, Integer> shortProcess =
Collections.min(ops, Comparators.processTime);
final ImmutablePair<OperatorProfile, Integer> longProcess =
Collections.max(ops, Comparators.processTime);
- tb.appendNanos(shortProcess.getLeft().getProcessNanos(), null);
- tb.appendNanos(Math.round(processSum / size), null);
- tb.appendNanos(longProcess.getLeft().getProcessNanos(), null);
+ tb.appendNanos(Math.round(processSum / size), null, null);
+ tb.appendNanos(longProcess.getLeft().getProcessNanos(), null, null);
final ImmutablePair<OperatorProfile, Integer> shortWait =
Collections.min(ops, Comparators.waitTime);
final ImmutablePair<OperatorProfile, Integer> longWait =
Collections.max(ops, Comparators.waitTime);
- tb.appendNanos(shortWait.getLeft().getWaitNanos(), null);
- tb.appendNanos(Math.round(waitSum / size), null);
- tb.appendNanos(longWait.getLeft().getWaitNanos(), null);
+ tb.appendNanos(shortWait.getLeft().getWaitNanos(), null, null);
+ tb.appendNanos(Math.round(waitSum / size), null, null);
+ tb.appendNanos(longWait.getLeft().getWaitNanos(), null, null);
+
+ tb.appendPercent(processSum / majorBusyNanos, null, String.format(
+ "∑OperatorBusy: %,.2fs / ∑MajorBusy: %,.2fs",
processSum/1E9, majorBusyNanos/1E9));
--- End diff --
Not sure where we use this, but seems a bit to "developery" for an end-use
UI. Maybe something like "Total operator time" and "Total major fragment time".
Note also rounding issue above, please use `Math.round()`.
Ah, I see, this is a tool tip. Not sure this really needs to appear on
every cell. Instead, in the tool tip for the column header, just explain that
this is "<title of one col> / <title of another col>". This works if the
numbers already appear on the page.
> Publish Operator and MajorFragment Stats in Profile page
> --------------------------------------------------------
>
> Key: DRILL-5195
> URL: https://issues.apache.org/jira/browse/DRILL-5195
> Project: Apache Drill
> Issue Type: Improvement
> Components: Web Server
> Affects Versions: 1.9.0
> Reporter: Kunal Khatua
> Assignee: Kunal Khatua
> Attachments: dbit_complete.png, dbit_inflight.png, dbit_opOverview.png
>
>
> Currently, we show runtimes for major fragments, and min,max,avg times for
> setup, processing and waiting for various operators.
> It would be worthwhile to have additional stats for the following:
> MajorFragment
> %Busy - % of the active time for all the minor fragments within each major
> fragment that they were busy.
> Operator Profile
> %Busy - % of the active time for all the fragments within each operator
> that they were busy.
> Records - Total number of records propagated out by that operator.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
