YARN-4005. Completed container whose app is finished is possibly not removed from NMStateStore. Contributed by Jun Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38aed1a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38aed1a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38aed1a9 Branch: refs/heads/YARN-1197 Commit: 38aed1a94ed7b6da62e2445b5610bc02b1cddeeb Parents: ae57d60 Author: Jian He <jia...@apache.org> Authored: Thu Aug 13 14:46:08 2015 -0700 Committer: Jian He <jia...@apache.org> Committed: Thu Aug 13 14:46:08 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodemanager/NodeStatusUpdaterImpl.java | 8 ++--- .../nodemanager/TestNodeStatusUpdater.java | 34 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/38aed1a9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9745d9d..3d19734 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -763,6 +763,9 @@ Release 2.8.0 - UNRELEASED YARN-3992. TestApplicationPriority.testApplicationPriorityAllocation fails intermittently. (Contributed by Sunil G) + YARN-4005. Completed container whose app is finished is possibly not + removed from NMStateStore. (Jun Gong via jianhe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/38aed1a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 30a2bd5..7c5c28b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -474,12 +474,12 @@ public class NodeStatusUpdaterImpl extends AbstractService implements } else { if (!isContainerRecentlyStopped(containerId)) { pendingCompletedContainers.put(containerId, containerStatus); - // Adding to finished containers cache. Cache will keep it around at - // least for #durationToTrackStoppedContainers duration. In the - // subsequent call to stop container it will get removed from cache. - addCompletedContainer(containerId); } } + // Adding to finished containers cache. Cache will keep it around at + // least for #durationToTrackStoppedContainers duration. In the + // subsequent call to stop container it will get removed from cache. + addCompletedContainer(containerId); } else { containerStatuses.add(containerStatus); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/38aed1a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index bc48adf..a9ef72f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -994,6 +994,40 @@ public class TestNodeStatusUpdater { Assert.assertTrue(containerIdSet.contains(runningContainerId)); } + @Test(timeout = 10000) + public void testCompletedContainersIsRecentlyStopped() throws Exception { + NodeManager nm = new NodeManager(); + nm.init(conf); + NodeStatusUpdaterImpl nodeStatusUpdater = + (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater(); + ApplicationId appId = ApplicationId.newInstance(0, 0); + Application completedApp = mock(Application.class); + when(completedApp.getApplicationState()).thenReturn( + ApplicationState.FINISHED); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 0); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); + Token containerToken = + BuilderUtils.newContainerToken(containerId, "host", 1234, "user", + BuilderUtils.newResource(1024, 1), 0, 123, + "password".getBytes(), 0); + Container completedContainer = new ContainerImpl(conf, null, + null, null, null, null, + BuilderUtils.newContainerTokenIdentifier(containerToken)) { + @Override + public ContainerState getCurrentState() { + return ContainerState.COMPLETE; + } + }; + + nm.getNMContext().getApplications().putIfAbsent(appId, completedApp); + nm.getNMContext().getContainers().put(containerId, completedContainer); + + Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size()); + Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped( + containerId)); + } + @Test public void testCleanedupApplicationContainerCleanup() throws IOException { NodeManager nm = new NodeManager();
