[
https://issues.apache.org/jira/browse/DRILL-5195?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15881147#comment-15881147
]
ASF GitHub Bot commented on DRILL-5195:
---------------------------------------
Github user paul-rogers commented on a diff in the pull request:
https://github.com/apache/drill/pull/756#discussion_r102800590
--- Diff:
exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/FragmentWrapper.java
---
@@ -49,58 +51,135 @@ public String getId() {
return String.format("fragment-%s", major.getMajorFragmentId());
}
- public static final String[] FRAGMENT_OVERVIEW_COLUMNS = {"Major
Fragment", "Minor Fragments Reporting",
- "First Start", "Last Start", "First End", "Last End", "Min Runtime",
"Avg Runtime", "Max Runtime", "Last Update",
- "Last Progress", "Max Peak Memory"};
+ public static final String[] ACTIVE_FRAGMENT_OVERVIEW_COLUMNS = {"Major
Fragment", "Minor Fragments Reporting",
+ "First Start", "Last Start", "First End", "Last End", "Min Runtime",
"Avg Runtime", "Max Runtime", "% Busy",
+ "Last Update", "Last Progress", "Max Peak Memory"};
+
+ public static final String[] ACTIVE_FRAGMENT_OVERVIEW_COLUMNS_TOOLTIP =
{null, "# Minor Fragments Spawned",
+ null, null, null, null, "Shortest duration of a fragment", "Avg
duration of a fragment", "Longest duration of a fragment", "%time Fragments
were Busy",
+ "Last time a running fragment's status was updated", "Last time we
heard from a running fragment", null};
// Not including Major Fragment ID and Minor Fragments Reporting
- public static final int NUM_NULLABLE_OVERVIEW_COLUMNS =
FRAGMENT_OVERVIEW_COLUMNS.length - 2;
+ public static final int NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS =
ACTIVE_FRAGMENT_OVERVIEW_COLUMNS.length - 2;
public void addSummary(TableBuilder tb) {
// Use only minor fragments that have complete profiles
// Complete iff the fragment profile has at least one operator
profile, and start and end times.
final List<MinorFragmentProfile> complete = new ArrayList<>(
Collections2.filter(major.getMinorFragmentProfileList(),
Filters.hasOperatorsAndTimes));
- tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null);
- tb.appendCell(complete.size() + " / " +
major.getMinorFragmentProfileCount(), null);
+ tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null,
null);
+ tb.appendCell(complete.size() + " / " +
major.getMinorFragmentProfileCount(), null, null);
// If there are no stats to aggregate, create an empty row
if (complete.size() < 1) {
- tb.appendRepeated("", null, NUM_NULLABLE_OVERVIEW_COLUMNS);
+ tb.appendRepeated("", null, NUM_NULLABLE_ACTIVE_OVERVIEW_COLUMNS,
null);
return;
}
final MinorFragmentProfile firstStart = Collections.min(complete,
Comparators.startTime);
final MinorFragmentProfile lastStart = Collections.max(complete,
Comparators.startTime);
- tb.appendMillis(firstStart.getStartTime() - start, null);
- tb.appendMillis(lastStart.getStartTime() - start, null);
+ tb.appendMillis(firstStart.getStartTime() - start, null, null);
+ tb.appendMillis(lastStart.getStartTime() - start, null, null);
final MinorFragmentProfile firstEnd = Collections.min(complete,
Comparators.endTime);
final MinorFragmentProfile lastEnd = Collections.max(complete,
Comparators.endTime);
- tb.appendMillis(firstEnd.getEndTime() - start, null);
- tb.appendMillis(lastEnd.getEndTime() - start, null);
+ tb.appendMillis(firstEnd.getEndTime() - start, null, null);
+ tb.appendMillis(lastEnd.getEndTime() - start, null, null);
- long total = 0;
+ long totalDuration = 0L;
+ double totalProcessInMillis = 0.0d;
+ double totalWaitInMillis = 0.0d;
for (final MinorFragmentProfile p : complete) {
- total += p.getEndTime() - p.getStartTime();
+ totalDuration += p.getEndTime() - p.getStartTime();
+ //Capture Busy & Wait Time
+ List<OperatorProfile> opProfileList = p.getOperatorProfileList();
+ for (OperatorProfile operatorProfile : opProfileList) {
+ totalProcessInMillis += operatorProfile.getProcessNanos()/1E6;
+ totalWaitInMillis += operatorProfile.getWaitNanos()/1E6;
+ }
}
final MinorFragmentProfile shortRun = Collections.min(complete,
Comparators.runTime);
final MinorFragmentProfile longRun = Collections.max(complete,
Comparators.runTime);
- tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null);
- tb.appendMillis(total / complete.size(), null);
- tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null);
+ tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null,
null);
+ tb.appendMillis(totalDuration / complete.size(), null, null);
+ tb.appendMillis(longRun.getEndTime() - longRun.getStartTime(), null,
null);
+
+ tb.appendPercent(totalProcessInMillis / (totalProcessInMillis +
totalWaitInMillis), null,
+ String.format("∑Busy: %,.2fs + ∑Wait: %,.2fs",
totalProcessInMillis/1E3, totalWaitInMillis/1E3));
final MinorFragmentProfile lastUpdate = Collections.max(complete,
Comparators.lastUpdate);
- tb.appendTime(lastUpdate.getLastUpdate(), null);
+ tb.appendMillis(System.currentTimeMillis()-lastUpdate.getLastUpdate(),
null, new SimpleDateFormat().format(new Date(lastUpdate.getLastUpdate())));
final MinorFragmentProfile lastProgress = Collections.max(complete,
Comparators.lastProgress);
- tb.appendTime(lastProgress.getLastProgress(), null);
+
tb.appendMillis(System.currentTimeMillis()-lastProgress.getLastProgress(),
null, new SimpleDateFormat().format(new Date(lastProgress.getLastProgress())));
+
+ // TODO(DRILL-3494): Names (maxMem, getMaxMemoryUsed) are misleading;
the value is peak memory allocated to fragment
+ final MinorFragmentProfile maxMem = Collections.max(complete,
Comparators.fragmentPeakMemory);
+ tb.appendBytes(maxMem.getMaxMemoryUsed(), null, null);
+ }
+
+ public static final String[] COMPLETED_FRAGMENT_OVERVIEW_COLUMNS =
{"Major Fragment", "Minor Fragments Reporting",
+ "First Start", "Last Start", "First End", "Last End", "Min Runtime",
"Avg Runtime", "Max Runtime", "% Busy", "Max Peak Memory"};
+
+ public static final String[] COMPLETED_FRAGMENT_OVERVIEW_COLUMNS_TOOLTIP
= {null, "# Minor Fragments Spawned",
+ null, null, null, null, "Shortest duration of a fragment", "Avg
duration of a fragment", "Longest duration of a fragment", "%time Fragments
were Busy", null};
+
+ //Not including Major Fragment ID and Minor Fragments Reporting
+ public static final int NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS =
COMPLETED_FRAGMENT_OVERVIEW_COLUMNS.length - 2;
+
+ public void addFinalSummary(TableBuilder tb) {
+
+ // Use only minor fragments that have complete profiles
+ // Complete iff the fragment profile has at least one operator
profile, and start and end times.
+ final List<MinorFragmentProfile> complete = new ArrayList<>(
+ Collections2.filter(major.getMinorFragmentProfileList(),
Filters.hasOperatorsAndTimes));
+
+ tb.appendCell(new OperatorPathBuilder().setMajor(major).build(), null,
null);
+ tb.appendCell(complete.size() + " / " +
major.getMinorFragmentProfileCount(), null, null);
+
+ // If there are no stats to aggregate, create an empty row
+ if (complete.size() < 1) {
+ tb.appendRepeated("", null, NUM_NULLABLE_COMPLETED_OVERVIEW_COLUMNS,
null);
+ return;
+ }
+
+ final MinorFragmentProfile firstStart = Collections.min(complete,
Comparators.startTime);
+ final MinorFragmentProfile lastStart = Collections.max(complete,
Comparators.startTime);
+ tb.appendMillis(firstStart.getStartTime() - start, null, null);
+ tb.appendMillis(lastStart.getStartTime() - start, null, null);
+
+ final MinorFragmentProfile firstEnd = Collections.min(complete,
Comparators.endTime);
+ final MinorFragmentProfile lastEnd = Collections.max(complete,
Comparators.endTime);
+ tb.appendMillis(firstEnd.getEndTime() - start, null, null);
+ tb.appendMillis(lastEnd.getEndTime() - start, null, null);
+
+ long totalDuration = 0L;
+ double totalProcessInMillis = 0.0d;
+ double totalWaitInMillis = 0.0d;
+ for (final MinorFragmentProfile p : complete) {
+ totalDuration += p.getEndTime() - p.getStartTime();
+ //Capture Busy & Wait Time
+ List<OperatorProfile> opProfileList = p.getOperatorProfileList();
+ for (OperatorProfile operatorProfile : opProfileList) {
+ totalProcessInMillis += operatorProfile.getProcessNanos()/1E6;
+ totalWaitInMillis += operatorProfile.getWaitNanos()/1E6;
+ }
+ }
+
+ final MinorFragmentProfile shortRun = Collections.min(complete,
Comparators.runTime);
+ final MinorFragmentProfile longRun = Collections.max(complete,
Comparators.runTime);
+ tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime(), null,
null);
--- End diff --
Here and above. What happens if the fragment has not yet ended? Or, if we
catch a queued fragment that has not yet started? Will this subtraction have
meaning?
Should we move the calculation into a method or function that handles these
situations?
> Publish Operator and MajorFragment Stats in Profile page
> --------------------------------------------------------
>
> Key: DRILL-5195
> URL: https://issues.apache.org/jira/browse/DRILL-5195
> Project: Apache Drill
> Issue Type: Improvement
> Components: Web Server
> Affects Versions: 1.9.0
> Reporter: Kunal Khatua
> Assignee: Kunal Khatua
> Attachments: dbit_complete.png, dbit_inflight.png, dbit_opOverview.png
>
>
> Currently, we show runtimes for major fragments, and min,max,avg times for
> setup, processing and waiting for various operators.
> It would be worthwhile to have additional stats for the following:
> MajorFragment
> %Busy - % of the active time for all the minor fragments within each major
> fragment that they were busy.
> Operator Profile
> %Busy - % of the active time for all the fragments within each operator
> that they were busy.
> Records - Total number of records propagated out by that operator.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
