Repository: tez Updated Branches: refs/heads/branch-0.7 e969a8f70 -> ca0ecab2c
TEZ-3089. TaskConcurrencyAnalyzer can return negative task count with very large jobs (rbalamohan) (cherry picked from commit 89bc6abf6c3caddb6224a19b7b47010b7bca4eff) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ca0ecab2 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ca0ecab2 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ca0ecab2 Branch: refs/heads/branch-0.7 Commit: ca0ecab2c82a4e52c5adc58d2551fe45e084c5d0 Parents: e969a8f Author: Rajesh Balamohan <[email protected]> Authored: Tue Feb 2 03:56:46 2016 -0800 Committer: Rajesh Balamohan <[email protected]> Committed: Thu Feb 4 19:00:51 2016 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../plugins/TaskConcurrencyAnalyzer.java | 24 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ca0ecab2/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index a1c31e2..0d7433c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES TEZ-2972. Avoid task rescheduling when a node turns unhealthy ALL CHANGES + TEZ-3089. TaskConcurrencyAnalyzer can return negative task count with very large jobs TEZ-2307. Possible wrong error message when submitting new dag TEZ-3076. Reduce merge memory overhead to support large number of in-memory mapoutputs TEZ-3066. TaskAttemptFinishedEvent ConcurrentModificationException in recovery or history logging services. http://git-wip-us.apache.org/repos/asf/tez/blob/ca0ecab2/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java ---------------------------------------------------------------------- diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java index 070294f..72f3b36 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java @@ -80,8 +80,28 @@ public class TaskConcurrencyAnalyzer extends TezAnalyzerBase implements Analyzer */ TreeMultiset<TimeInfo> timeInfoSet = TreeMultiset.create(new Comparator<TimeInfo>() { @Override public int compare(TimeInfo o1, TimeInfo o2) { - return (o1.timestamp < o2.timestamp) ? -1 : - ((o1.timestamp == o2.timestamp) ? 0 : 1); + if (o1.timestamp < o2.timestamp) { + return -1; + } + + if (o1.timestamp > o2.timestamp) { + return 1; + } + + if (o1.timestamp == o2.timestamp) { + //check event type + if (o1.eventType.equals(o2.eventType)) { + return 0; + } + + if (o1.eventType.equals(EventType.START) + && o2.eventType.equals(EventType.FINISH)) { + return -1; + } else { + return 1; + } + } + return 0; } });
