Repository: hadoop Updated Branches: refs/heads/branch-2.6 4feb17ea2 -> 2dc43a205
YARN-5262. Optimize sending RMNodeFinishedContainersPulledByAMEvent for every AM heartbeat. Contributed by Rohith Sharma K S (cherry picked from commit 26b5e6116f392b6be91dca57968259f87554ce33) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2dc43a20 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2dc43a20 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2dc43a20 Branch: refs/heads/branch-2.6 Commit: 2dc43a20549dbfe788e8284c93fc26a60acfdd8a Parents: 4feb17e Author: Jason Lowe <[email protected]> Authored: Tue Aug 9 22:18:24 2016 +0000 Committer: Jason Lowe <[email protected]> Committed: Tue Aug 9 22:20:49 2016 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/rmapp/attempt/RMAppAttemptImpl.java | 9 ++++++--- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 7 +++++++ 3 files changed, 16 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dc43a20/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 64d9401..26daf07 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -13,6 +13,9 @@ Release 2.6.5 - UNRELEASED YARN-4690. Skip object allocation in FSAppAttempt#getResourceUsage when possible (Ming Ma via sjlee) + YARN-5262. Optimize sending RMNodeFinishedContainersPulledByAMEvent for + every AM heartbeat (Rohith Sharma K S via jlowe) + BUG FIXES YARN-2046. Out of band heartbeats are sent only on container kill and http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dc43a20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index a19f5ba..c76a1e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -712,9 +712,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - finishedContainersSentToAM.putIfAbsent(nodeId, new ArrayList - <ContainerStatus>()); - finishedContainersSentToAM.get(nodeId).addAll(finishedContainers); + if (!finishedContainers.isEmpty()) { + finishedContainersSentToAM.putIfAbsent(nodeId, + new ArrayList<ContainerStatus>()); + finishedContainersSentToAM.get(nodeId).addAll(finishedContainers); + } } return returnList; @@ -1629,6 +1631,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, containerIdList)); } + this.finishedContainersSentToAM.clear(); } // Add am container to the list so that am container instance will be http://git-wip-us.apache.org/repos/asf/hadoop/blob/2dc43a20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index ccd2613..97e8887 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -1317,6 +1317,13 @@ public class TestRMAppAttemptTransitions { Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty()); Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt) .size()); + + // verify if no containers to acknowledge to NM then event should not be + // triggered. Number of times event invoked is 1 i.e on second pull + containerStatuses = applicationAttempt.pullJustFinishedContainers(); + Assert.assertEquals(0, containerStatuses.size()); + Mockito.verify(rmnodeEventHandler, times(1)) + .handle(Mockito.any(RMNodeEvent.class)); } private static List<ContainerStatus> getFinishedContainersSentToAM( --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
