YARN-3987. Send AM container completed msg to NM once AM finishes. Contributed by sandflee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a030546 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a030546 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a030546 Branch: refs/heads/HDFS-7285 Commit: 0a030546e24c55662a603bb63c9029ad0ccf43fc Parents: 7a445fc Author: Jian He <jia...@apache.org> Authored: Thu Aug 13 16:20:36 2015 -0700 Committer: Jian He <jia...@apache.org> Committed: Thu Aug 13 16:22:53 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 14 ++++++++++++++ 2 files changed, 17 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a030546/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a4c16b1..c451320 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -769,6 +769,9 @@ Release 2.8.0 - UNRELEASED YARN-4047. ClientRMService getApplications has high scheduler lock contention. (Jason Lowe via jianhe) + YARN-3987. Send AM container completed msg to NM once AM finishes. + (sandflee via jianhe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a030546/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 0914022..80f5eb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1658,6 +1658,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } + // Ack NM to remove finished AM container, not waiting for + // new appattempt to pull am container complete msg, new appattempt + // may launch fail and leaves too many completed container in NM + private void sendFinishedAMContainerToNM(NodeId nodeId, + ContainerId containerId) { + List<ContainerId> containerIdList = new ArrayList<ContainerId>(); + containerIdList.add(containerId); + eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent( + nodeId, containerIdList)); + } // Ack NM to remove finished containers from context. private void sendFinishedContainersToNM() { @@ -1686,9 +1696,13 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { new ArrayList<ContainerStatus>()); appAttempt.finishedContainersSentToAM.get(nodeId).add( containerFinishedEvent.getContainerStatus()); + if (!appAttempt.getSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { appAttempt.sendFinishedContainersToNM(); + } else { + appAttempt.sendFinishedAMContainerToNM(nodeId, + containerFinishedEvent.getContainerStatus().getContainerId()); } }
