MAPREDUCE-6720. Inconsistent values of counters across tasks and job reported to timeline service. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf87eed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf87eed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf87eed Branch: refs/heads/trunk Commit: 8bf87eede2b1735993abc0f2cc49c971f9f8e222 Parents: 6d94303 Author: Naganarasimha <[email protected]> Authored: Wed Jun 22 23:32:19 2016 +0530 Committer: Sangjin Lee <[email protected]> Committed: Sun Jul 10 08:46:04 2016 -0700 ---------------------------------------------------------------------- .../mapreduce/jobhistory/JobHistoryEventHandler.java | 4 ++++ .../hadoop/mapreduce/jobhistory/JobFinishedEvent.java | 12 ++++++------ .../hadoop/mapreduce/util/JobHistoryEventUtils.java | 7 ++++++- 3 files changed, 16 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf87eed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index a09c17b..9b59676 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -1125,6 +1125,10 @@ public class JobHistoryEventHandler extends AbstractService org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); entity.setId(taskId); + if (event.getEventType() == EventType.TASK_STARTED) { + entity.addInfo("TASK_TYPE", + ((TaskStartedEvent)event).getTaskType().toString()); + } entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); return entity; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf87eed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java index 76d6897..ea21f60 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java @@ -158,12 +158,12 @@ public class JobFinishedEvent implements HistoryEvent { @Override public Set<TimelineMetric> getTimelineMetrics() { - Set<TimelineMetric> jobMetrics = JobHistoryEventUtils - .countersToTimelineMetric(getMapCounters(), finishTime); - jobMetrics.addAll(JobHistoryEventUtils - .countersToTimelineMetric(getReduceCounters(), finishTime)); - jobMetrics.addAll(JobHistoryEventUtils - .countersToTimelineMetric(getTotalCounters(), finishTime)); + Set<TimelineMetric> jobMetrics = JobHistoryEventUtils. + countersToTimelineMetric(getTotalCounters(), finishTime); + jobMetrics.addAll(JobHistoryEventUtils. + countersToTimelineMetric(getMapCounters(), finishTime, "MAP:")); + jobMetrics.addAll(JobHistoryEventUtils. + countersToTimelineMetric(getReduceCounters(), finishTime, "REDUCE:")); return jobMetrics; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf87eed/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java index 35d066c..89abf0e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java @@ -61,11 +61,16 @@ public final class JobHistoryEventUtils { public static Set<TimelineMetric> countersToTimelineMetric(Counters counters, long timestamp) { + return countersToTimelineMetric(counters, timestamp, ""); + } + + public static Set<TimelineMetric> countersToTimelineMetric(Counters counters, + long timestamp, String groupNamePrefix) { Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>(); for (CounterGroup g : counters) { String groupName = g.getName(); for (Counter c : g) { - String name = groupName + ":" + c.getName(); + String name = groupNamePrefix + groupName + ":" + c.getName(); TimelineMetric metric = new TimelineMetric(); metric.setId(name); metric.addValue(timestamp, c.getValue()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
