YARN-5243. fix several rebase and other miscellaneous issues before merge. (Sangjin Lee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ff6833b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ff6833b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ff6833b Branch: refs/heads/trunk Commit: 1ff6833bbacf5c4eeaff5e70553ac083a691bb21 Parents: 32b033d Author: Varun Saxena <[email protected]> Authored: Wed Jun 15 22:10:56 2016 +0530 Committer: Sangjin Lee <[email protected]> Committed: Sun Jul 10 08:46:03 2016 -0700 ---------------------------------------------------------------------- .../jobhistory/JobHistoryEventHandler.java | 42 ++++++++++---------- .../v2/app/rm/RMContainerAllocator.java | 3 +- .../hadoop/mapreduce/jobhistory/TestEvents.java | 2 +- .../mapreduce/jobhistory/AMStartedEvent.java | 2 +- .../mapreduce/jobhistory/HistoryEvent.java | 2 +- .../mapreduce/jobhistory/JobFinishedEvent.java | 2 +- .../jobhistory/JobInfoChangeEvent.java | 2 +- .../mapreduce/jobhistory/JobInitedEvent.java | 2 +- .../jobhistory/JobPriorityChangeEvent.java | 2 +- .../jobhistory/JobQueueChangeEvent.java | 2 +- .../jobhistory/JobStatusChangedEvent.java | 2 +- .../mapreduce/jobhistory/JobSubmittedEvent.java | 2 +- .../JobUnsuccessfulCompletionEvent.java | 2 +- .../jobhistory/MapAttemptFinishedEvent.java | 2 +- .../jobhistory/NormalizedResourceEvent.java | 2 +- .../jobhistory/ReduceAttemptFinishedEvent.java | 2 +- .../jobhistory/TaskAttemptFinishedEvent.java | 2 +- .../jobhistory/TaskAttemptStartedEvent.java | 2 +- .../TaskAttemptUnsuccessfulCompletionEvent.java | 2 +- .../mapreduce/jobhistory/TaskFailedEvent.java | 2 +- .../mapreduce/jobhistory/TaskFinishedEvent.java | 2 +- .../mapreduce/jobhistory/TaskStartedEvent.java | 2 +- .../mapreduce/jobhistory/TaskUpdatedEvent.java | 2 +- .../mapred/TestMRTimelineEventHandling.java | 28 ++++++------- .../apache/hadoop/test/MapredTestDriver.java | 2 +- .../dev-support/findbugs-exclude.xml | 5 --- .../distributedshell/ApplicationMaster.java | 2 +- .../distributedshell/TestDistributedShell.java | 2 +- .../client/api/impl/TimelineClientImpl.java | 4 +- .../src/main/resources/yarn-default.xml | 2 +- .../yarn/util/TestTimelineServiceHelper.java | 20 +++++----- .../impl/pb/NodeHeartbeatRequestPBImpl.java | 4 +- .../hadoop/yarn/server/nodemanager/Context.java | 2 +- .../yarn/server/nodemanager/NodeManager.java | 13 ++++-- .../application/ApplicationImpl.java | 2 +- .../monitor/ContainersMonitorImpl.java | 6 +-- .../nodemanager/webapp/TestNMWebServer.java | 2 - .../hadoop-yarn-server-resourcemanager/pom.xml | 2 +- .../resourcemanager/RMActiveServiceContext.java | 32 +-------------- .../yarn/server/resourcemanager/RMContext.java | 4 +- .../server/resourcemanager/RMContextImpl.java | 2 +- .../server/resourcemanager/ResourceManager.java | 34 +++++----------- .../server/resourcemanager/rmapp/RMApp.java | 14 +++++-- .../server/resourcemanager/rmapp/RMAppImpl.java | 8 ++++ 44 files changed, 123 insertions(+), 154 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 887533d..a09c17b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -137,7 +137,7 @@ public class JobHistoryEventHandler extends AbstractService protected volatile boolean forceJobCompletion = false; protected TimelineClient timelineClient; - + private boolean timelineServiceV2Enabled = false; private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB"; @@ -263,16 +263,16 @@ public class JobHistoryEventHandler extends AbstractService MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD, MRJobConfig.DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD); - // TODO replace MR specific configurations on timeline service with getting - // configuration from RM through registerApplicationMaster() in - // ApplicationMasterProtocol with return value for timeline service + // TODO replace MR specific configurations on timeline service with getting + // configuration from RM through registerApplicationMaster() in + // ApplicationMasterProtocol with return value for timeline service // configuration status: off, on_with_v1 or on_with_v2. if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) { LOG.info("Emitting job history data to the timeline service is enabled"); if (YarnConfiguration.timelineServiceEnabled(conf)) { - timelineClient = + timelineClient = ((MRAppMaster.RunningAppContext)context).getTimelineClient(); timelineClient.init(conf); timelineServiceV2Enabled = @@ -1062,11 +1062,11 @@ public class JobHistoryEventHandler extends AbstractService + "Server", ex); } } - - // create JobEntity from HistoryEvent with adding other info, like: + + // create JobEntity from HistoryEvent with adding other info, like: // jobId, timestamp and entityType. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity - createJobEntity(HistoryEvent event, long timestamp, JobId jobId, + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + createJobEntity(HistoryEvent event, long timestamp, JobId jobId, String entityType, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = @@ -1093,16 +1093,16 @@ public class JobHistoryEventHandler extends AbstractService return entity; } - // create BaseEntity from HistoryEvent with adding other info, like: + // create BaseEntity from HistoryEvent with adding other info, like: // timestamp and entityType. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createBaseEntity(HistoryEvent event, long timestamp, String entityType, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent = event.toTimelineEvent(); tEvent.setTimestamp(timestamp); - - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.addEvent(tEvent); entity.setType(entityType); @@ -1115,10 +1115,10 @@ public class JobHistoryEventHandler extends AbstractService } return entity; } - - // create TaskEntity from HistoryEvent with adding other info, like: + + // create TaskEntity from HistoryEvent with adding other info, like: // taskId, jobId, timestamp, entityType and relatedJobEntity. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskEntity(HistoryEvent event, long timestamp, String taskId, String entityType, String relatedJobEntity, JobId jobId, boolean setCreatedTime) { @@ -1128,12 +1128,12 @@ public class JobHistoryEventHandler extends AbstractService entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); return entity; } - - // create TaskAttemptEntity from HistoryEvent with adding other info, like: + + // create TaskAttemptEntity from HistoryEvent with adding other info, like: // timestamp, taskAttemptId, entityType, relatedTaskEntity and taskId. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity - createTaskAttemptEntity(HistoryEvent event, long timestamp, - String taskAttemptId, String entityType, String relatedTaskEntity, + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + createTaskAttemptEntity(HistoryEvent event, long timestamp, + String taskAttemptId, String entityType, String relatedTaskEntity, String taskId, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index e1fc84f..ea2046b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -822,8 +822,9 @@ public class RMContainerAllocator extends RMContainerRequestor handleUpdatedNodes(response); handleJobPriorityChange(response); + // handle receiving the timeline collector address for this app String collectorAddr = response.getCollectorAddr(); - MRAppMaster.RunningAppContext appContext = + MRAppMaster.RunningAppContext appContext = (MRAppMaster.RunningAppContext)this.getContext(); if (collectorAddr != null && !collectorAddr.isEmpty() && appContext.getTimelineClient() != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java index f5f4a3c..ac510b3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java @@ -418,7 +418,7 @@ public class TestEvents { public Set<TimelineMetric> getTimelineMetrics() { return null; } - + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java index 82db9f3..a1447c5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java @@ -170,7 +170,7 @@ public class AMStartedEvent implements HistoryEvent { public EventType getEventType() { return EventType.AM_STARTED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java index cdafe46..1d59ebe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java @@ -41,7 +41,7 @@ public interface HistoryEvent { /** Set the Avro datum wrapped by this. */ void setDatum(Object datum); - + /** Map HistoryEvent to TimelineEvent */ TimelineEvent toTimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java index 2efbc57..76d6897 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java @@ -139,7 +139,7 @@ public class JobFinishedEvent implements HistoryEvent { public Counters getReduceCounters() { return reduceCounters; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java index eb9ad0d..f5941aa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java @@ -66,7 +66,7 @@ public class JobInfoChangeEvent implements HistoryEvent { public EventType getEventType() { return EventType.JOB_INFO_CHANGED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java index d91208c..784267f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java @@ -77,7 +77,7 @@ public class JobInitedEvent implements HistoryEvent { } /** Get whether the job's map and reduce stages were combined */ public boolean getUberized() { return datum.getUberized(); } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java index ac4943e..1616dd1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java @@ -66,7 +66,7 @@ public class JobPriorityChangeEvent implements HistoryEvent { public EventType getEventType() { return EventType.JOB_PRIORITY_CHANGED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java index b12006b..66f3781 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java @@ -64,7 +64,7 @@ public class JobQueueChangeEvent implements HistoryEvent { } return null; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java index 0b33b4f..0963b45 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java @@ -62,7 +62,7 @@ public class JobStatusChangedEvent implements HistoryEvent { public EventType getEventType() { return EventType.JOB_STATUS_CHANGED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java index 7d05571..e394f5b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java @@ -259,7 +259,7 @@ public class JobSubmittedEvent implements HistoryEvent { tEvent.addInfo("WORKFLOW_ADJACENCIES", getWorkflowAdjacencies()); tEvent.addInfo("WORKFLOW_TAGS", getWorkflowTags()); - + return tEvent; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java index 5d3c45f..ce6fa32 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java @@ -123,7 +123,7 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent { final CharSequence diagnostics = datum.getDiagnostics(); return diagnostics == null ? NODIAGS : diagnostics.toString(); } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java index 1d28335..3121c4e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java @@ -247,5 +247,5 @@ public class MapAttemptFinishedEvent implements HistoryEvent { .countersToTimelineMetric(getCounters(), finishTime); return metrics; } - + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java index d90ff93..7adae23 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java @@ -76,7 +76,7 @@ public class NormalizedResourceEvent implements HistoryEvent { public void setDatum(Object datum) { throw new UnsupportedOperationException("Not a seriable object"); } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java index 5ddd839..9c0f09b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java @@ -228,7 +228,7 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent { public int[] getPhysMemKbytes() { return physMemKbytes; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java index 30d2b09..a931ca2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java @@ -141,7 +141,7 @@ public class TaskAttemptFinishedEvent implements HistoryEvent { ? EventType.MAP_ATTEMPT_FINISHED : EventType.REDUCE_ATTEMPT_FINISHED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java index 99213fd..d09d5ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java @@ -136,7 +136,7 @@ public class TaskAttemptStartedEvent implements HistoryEvent { } return null; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java index 8826c96..1732d91 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java @@ -251,7 +251,7 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { public int[] getPhysMemKbytes() { return physMemKbytes; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java index 9a29b81..d14350d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java @@ -142,7 +142,7 @@ public class TaskFailedEvent implements HistoryEvent { public EventType getEventType() { return EventType.TASK_FAILED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java index fd003cc..0bc4383 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java @@ -130,7 +130,7 @@ public class TaskFinishedEvent implements HistoryEvent { tEvent.addInfo("FINISH_TIME", getFinishTime()); tEvent.addInfo("STATUS", TaskStatus.State.SUCCEEDED.toString()); tEvent.addInfo("SUCCESSFUL_TASK_ATTEMPT_ID", - getSuccessfulTaskAttemptId() == null ? "" : + getSuccessfulTaskAttemptId() == null ? "" : getSuccessfulTaskAttemptId().toString()); return tEvent; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java index 6792ce7..9d2fc0e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java @@ -76,7 +76,7 @@ public class TaskStartedEvent implements HistoryEvent { public EventType getEventType() { return EventType.TASK_STARTED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java index 0722207..010129d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java @@ -62,7 +62,7 @@ public class TaskUpdatedEvent implements HistoryEvent { public EventType getEventType() { return EventType.TASK_UPDATED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index 27ce48c..6b9f27e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -65,7 +65,7 @@ public class TestMRTimelineEventHandling { private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector"; private static final Log LOG = LogFactory.getLog(TestMRTimelineEventHandling.class); - + @Test public void testTimelineServiceStartInMiniCluster() throws Exception { Configuration conf = new YarnConfiguration(); @@ -168,7 +168,7 @@ public class TestMRTimelineEventHandling { } } } - + @Test public void testMRNewTimelineServiceEventHandling() throws Exception { LOG.info("testMRNewTimelineServiceEventHandling start."); @@ -184,7 +184,7 @@ public class TestMRTimelineEventHandling { conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME); conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME + ".class", PerNodeTimelineCollectorsAuxService.class.getName()); - + conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true); MiniMRYarnCluster cluster = null; @@ -215,9 +215,9 @@ public class TestMRTimelineEventHandling { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(cluster.getConfig())); yarnClient.start(); - EnumSet<YarnApplicationState> appStates = + EnumSet<YarnApplicationState> appStates = EnumSet.allOf(YarnApplicationState.class); - + ApplicationId firstAppId = null; List<ApplicationReport> apps = yarnClient.getApplications(appStates); Assert.assertEquals(apps.size(), 1); @@ -230,7 +230,7 @@ public class TestMRTimelineEventHandling { job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.FAILED, job.getJobStatus().getState().getValue()); - + apps = yarnClient.getApplications(appStates); Assert.assertEquals(apps.size(), 2); @@ -250,10 +250,10 @@ public class TestMRTimelineEventHandling { if(testRootFolder.isDirectory()) { FileUtils.deleteDirectory(testRootFolder); } - + } } - + private void checkNewTimelineEvent(ApplicationId appId, ApplicationReport appReport) throws IOException { String tmpRoot = @@ -261,7 +261,7 @@ public class TestMRTimelineEventHandling { + "/entities/"; File tmpRootFolder = new File(tmpRoot); - + Assert.assertTrue(tmpRootFolder.isDirectory()); String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" + UserGroupInformation.getCurrentUser().getShortUserName() + @@ -319,7 +319,7 @@ public class TestMRTimelineEventHandling { Assert.assertTrue("Task output directory: " + outputDirTask + " does not exist.", taskFolder.isDirectory()); - + String taskEventFileName = appId.toString().replaceAll("application", "task") + "_m_000000" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; @@ -330,15 +330,15 @@ public class TestMRTimelineEventHandling { taskEventFile.exists()); verifyEntity(taskEventFile, EventType.TASK_FINISHED.name(), true, false, null); - + // check for task attempt event file String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/"; File taskAttemptFolder = new File(outputDirTaskAttempt); - Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt + + Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt + " does not exist.", taskAttemptFolder.isDirectory()); - + String taskAttemptEventFileName = appId.toString().replaceAll( - "application", "attempt") + "_m_000000_0" + + "application", "attempt") + "_m_000000_0" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; String taskAttemptEventFilePath = outputDirTaskAttempt + http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java index 4c42bd3..76198b8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java @@ -93,7 +93,7 @@ public class MapredTestDriver { pgd.addClass("timelineperformance", TimelineServicePerformance.class, "A job that launches mappers to test timline service " + "performance."); - pgd.addClass("nnbench", NNBench.class, + pgd.addClass("nnbench", NNBench.class, "A benchmark that stresses the namenode w/ MR."); pgd.addClass("nnbenchWithoutMR", NNBenchWithoutMR.class, "A benchmark that stresses the namenode w/o MR."); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 08c6ba2..7c19c5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -552,9 +552,4 @@ <Package name="org.apache.hadoop.yarn.api.records.impl.pb" /> <Bug pattern="NP_BOOLEAN_RETURN_NULL" /> </Match> - <!-- Object cast is based on the event type --> - <Match> - <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" /> - <Bug pattern="BC_UNCONFIRMED_CAST" /> - </Match> </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 2e34ef8..907d09e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -631,7 +631,7 @@ public class ApplicationMaster { DSEvent.DS_APP_ATTEMPT_START); } else { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), - DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi); + DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 74da919..a01d21b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -149,7 +149,7 @@ public class TestDistributedShell { // Enable ContainersMonitorImpl conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, LinuxResourceCalculatorPlugin.class.getName()); - conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, + conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, ProcfsBasedProcessTree.class.getName()); conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index ef8838e..478efc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -495,8 +495,8 @@ public class TimelineClientImpl extends TimelineClient { } if (resp == null || resp.getClientResponseStatus() != ClientResponse.Status.OK) { - String msg = "Response from the timeline server is " + - ((resp == null) ? "null": + String msg = "Response from the timeline server is " + + ((resp == null) ? "null": "not successful," + " HTTP error code: " + resp.getStatus() + ", Server response:\n" + resp.getEntity(String.class)); LOG.error(msg); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 944c7d6..ed220c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -784,7 +784,7 @@ <property> <description>The setting that controls whether yarn system metrics is - published to the Timeline server (version one) or not, by RM. + published to the Timeline server (version one) or not, by RM. This configuration is now deprecated in favor of yarn.system-metrics-publisher.enabled.</description> <name>yarn.resourcemanager.system-metrics-publisher.enabled</name> http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java index f852df0..34b9497 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java @@ -27,49 +27,49 @@ import org.junit.Assert; import org.junit.Test; public class TestTimelineServiceHelper { - + @Test public void testMapCastToHashMap() { // Test null map be casted to null Map<String, String> nullMap = null; Assert.assertNull(TimelineServiceHelper.mapCastToHashMap(nullMap)); - + // Test empty hashmap be casted to a empty hashmap Map<String, String> emptyHashMap = new HashMap<String, String>(); Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0); - + // Test empty non-hashmap be casted to a empty hashmap Map<String, String> emptyTreeMap = new TreeMap<String, String>(); Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0); - + // Test non-empty hashmap be casted to hashmap correctly Map<String, String> firstHashMap = new HashMap<String, String>(); String key = "KEY"; String value = "VALUE"; firstHashMap.put(key, value); Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap); - + // Test non-empty non-hashmap is casted correctly. Map<String, String> firstTreeMap = new TreeMap<String, String>(); firstTreeMap.put(key, value); - HashMap<String, String> alternateHashMap = + HashMap<String, String> alternateHashMap = TimelineServiceHelper.mapCastToHashMap(firstTreeMap); Assert.assertEquals(firstTreeMap.size(), alternateHashMap.size()); Assert.assertEquals(alternateHashMap.get(key), value); - + // Test complicated hashmap be casted correctly Map<String, Set<String>> complicatedHashMap = new HashMap<String, Set<String>>(); Set<String> hashSet = new HashSet<String>(); hashSet.add(value); complicatedHashMap.put(key, hashSet); - Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap), + Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap), complicatedHashMap); - + // Test complicated non-hashmap get casted correctly Map<String, Set<String>> complicatedTreeMap = new TreeMap<String, Set<String>>(); complicatedTreeMap.put(key, hashSet); - Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key), + Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key), hashSet); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java index 066abfc..912a7a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java @@ -59,7 +59,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { private List<LogAggregationReport> logAggregationReportsForApps = null; Map<ApplicationId, String> registeredCollectors = null; - + public NodeHeartbeatRequestPBImpl() { builder = NodeHeartbeatRequestProto.newBuilder(); } @@ -157,7 +157,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { LogAggregationReport value) { return ((LogAggregationReportPBImpl) value).getProto(); } - + private void addRegisteredCollectorsToProto() { maybeInitBuilder(); builder.clearRegisteredCollectors(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index e53c0b0..131eaa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -102,7 +102,7 @@ public interface Context { NMStateStoreService getNMStateStore(); boolean getDecommissioned(); - + Configuration getConf(); void setDecommissioned(boolean isDecommissioned); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index d48a00c..0c84be0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -471,9 +471,9 @@ public class NodeManager extends CompositeService public static class NMContext implements Context { private NodeId nodeId = null; - + private Configuration conf = null; - + protected final ConcurrentMap<ApplicationId, Application> applications = new ConcurrentHashMap<ApplicationId, Application>(); @@ -551,7 +551,7 @@ public class NodeManager extends CompositeService public ConcurrentMap<ApplicationId, Application> getApplications() { return this.applications; } - + @Override public Configuration getConf() { return this.conf; @@ -684,7 +684,12 @@ public class NodeManager extends CompositeService public void addRegisteredCollectors( Map<ApplicationId, String> newRegisteredCollectors) { - this.registeredCollectors.putAll(newRegisteredCollectors); + if (registeredCollectors != null) { + this.registeredCollectors.putAll(newRegisteredCollectors); + } else { + LOG.warn("collectors are added when the registered collectors are " + + "initialized"); + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 22779bb..5a5d30c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -184,7 +184,7 @@ public class ApplicationImpl implements Application { public ApplicationId getAppId() { return appId; } - + @Override public ApplicationState getApplicationState() { this.readLock.lock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 6df7211..4167ece 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -243,7 +243,6 @@ public class ContainersMonitorImpl extends AbstractService implements ; } } - super.serviceStop(); } @@ -422,7 +421,6 @@ public class ContainersMonitorImpl extends AbstractService implements .entrySet()) { ContainerId containerId = entry.getKey(); ProcessTreeInfo ptInfo = entry.getValue(); - try { String pId = ptInfo.getPID(); @@ -478,7 +476,7 @@ public class ContainersMonitorImpl extends AbstractService implements float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore / resourceCalculatorPlugin.getNumProcessors(); - + // Multiply by 1000 to avoid losing data when converting to int int milliVcoresUsed = (int) (cpuUsageTotalCoresPercentage * 1000 * maxVCoresAllottedForContainers /nodeCpuPercentageForYARN); @@ -514,7 +512,7 @@ public class ContainersMonitorImpl extends AbstractService implements containerMetricsUnregisterDelayMs).recordCpuUsage ((int)cpuUsagePercentPerCore, milliVcoresUsed); } - + boolean isMemoryOverLimit = false; String msg = ""; int containerExitStatus = ContainerExitStatus.INVALID; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index 9923f40..41037f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -111,7 +111,6 @@ public class TestNMWebServer { return true; } }; - conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); @@ -175,7 +174,6 @@ public class TestNMWebServer { return true; } }; - conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 7b64ba0..834dc3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -177,7 +177,7 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-timelineservice</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 12daa95..caa0ff13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -30,10 +30,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; -import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -95,8 +93,6 @@ public class RMActiveServiceContext { private NodesListManager nodesListManager; private ResourceTrackerService resourceTrackerService; private ApplicationMasterService applicationMasterService; - private RMApplicationHistoryWriter rmApplicationHistoryWriter; - private SystemMetricsPublisher systemMetricsPublisher; private RMTimelineCollectorManager timelineCollectorManager; private RMNodeLabelsManager nodeLabelManager; @@ -375,12 +371,6 @@ public class RMActiveServiceContext { @Private @Unstable - public RMApplicationHistoryWriter getRMApplicationHistoryWriter() { - return rmApplicationHistoryWriter; - } - - @Private - @Unstable public RMTimelineCollectorManager getRMTimelineCollectorManager() { return timelineCollectorManager; } @@ -394,26 +384,6 @@ public class RMActiveServiceContext { @Private @Unstable - public void setSystemMetricsPublisher( - SystemMetricsPublisher metricsPublisher) { - this.systemMetricsPublisher = metricsPublisher; - } - - @Private - @Unstable - public SystemMetricsPublisher getSystemMetricsPublisher() { - return systemMetricsPublisher; - } - - @Private - @Unstable - public void setRMApplicationHistoryWriter( - RMApplicationHistoryWriter rmApplicationHistoryWriter) { - this.rmApplicationHistoryWriter = rmApplicationHistoryWriter; - } - - @Private - @Unstable public long getEpoch() { return this.epoch; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 3b8a5b4..2ba445c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -113,10 +113,10 @@ public interface RMContext { void setSystemMetricsPublisher(SystemMetricsPublisher systemMetricsPublisher); SystemMetricsPublisher getSystemMetricsPublisher(); - + void setRMTimelineCollectorManager( RMTimelineCollectorManager timelineCollectorManager); - + RMTimelineCollectorManager getRMTimelineCollectorManager(); ConfigurationProvider getConfigurationProvider(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 12a5577..e1e3298 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -380,7 +380,7 @@ public class RMContextImpl implements RMContext { public RMTimelineCollectorManager getRMTimelineCollectorManager() { return activeServiceContext.getRMTimelineCollectorManager(); } - + @Override public void setSystemMetricsPublisher( SystemMetricsPublisher metricsPublisher) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 9fa10d2..0c1df33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -22,11 +22,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.net.InetSocketAddress; +import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -123,15 +123,6 @@ import org.apache.hadoop.yarn.webapp.WebApps.Builder; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.security.PrivilegedExceptionAction; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.List; import com.google.common.annotations.VisibleForTesting; /** @@ -312,6 +303,15 @@ public class ResourceManager extends CompositeService implements Recoverable { addService(rmApplicationHistoryWriter); rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter); + // initialize the RM timeline collector first so that the system metrics + // publisher can bind to it + if (YarnConfiguration.timelineServiceV2Enabled(this.conf)) { + RMTimelineCollectorManager timelineCollectorManager = + createRMTimelineCollectorManager(); + addService(timelineCollectorManager); + rmContext.setRMTimelineCollectorManager(timelineCollectorManager); + } + SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher(); addIfService(systemMetricsPublisher); @@ -602,18 +602,6 @@ public class ResourceManager extends CompositeService implements Recoverable { rmContext.setDelegationTokenRenewer(delegationTokenRenewer); } - RMApplicationHistoryWriter rmApplicationHistoryWriter = - createRMApplicationHistoryWriter(); - addService(rmApplicationHistoryWriter); - rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter); - - if (YarnConfiguration.timelineServiceV2Enabled(configuration)) { - RMTimelineCollectorManager timelineCollectorManager = - createRMTimelineCollectorManager(); - addService(timelineCollectorManager); - rmContext.setRMTimelineCollectorManager(timelineCollectorManager); - } - // Register event handler for NodesListManager nodesListManager = new NodesListManager(rmContext); rmDispatcher.register(NodesListManagerEventType.class, nodesListManager); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 8a795cd..98cbd92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -177,19 +177,25 @@ public interface RMApp extends EventHandler<RMAppEvent> { String getTrackingUrl(); /** - * The collector address for the application. - * @return the address for the application's collector. + * The collector address for the application. It should be used only if the + * timeline service v.2 is enabled. + * + * @return the address for the application's collector, or null if the + * timeline service v.2 is not enabled. */ String getCollectorAddr(); /** - * Set collector address for the application + * Set collector address for the application. It should be used only if the + * timeline service v.2 is enabled. + * * @param collectorAddr the address of collector */ void setCollectorAddr(String collectorAddr); /** - * Remove collector address when application is finished or killed. + * Remove collector address when application is finished or killed. It should + * be used only if the timeline service v.2 is enabled. */ void removeCollectorAddr(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff6833b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 6e448f7..45ff79c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -526,6 +526,10 @@ public class RMAppImpl implements RMApp, Recoverable { } } + /** + * Starts the application level timeline collector for this app. This should + * be used only if the timeline service v.2 is enabled. + */ public void startTimelineCollector() { AppLevelTimelineCollector collector = new AppLevelTimelineCollector(applicationId); @@ -533,6 +537,10 @@ public class RMAppImpl implements RMApp, Recoverable { applicationId, collector); } + /** + * Stops the application level timeline collector for this app. This should be + * used only if the timeline service v.2 is enabled. + */ public void stopTimelineCollector() { rmContext.getRMTimelineCollectorManager().remove(applicationId); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
