YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06d22eb2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06d22eb2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06d22eb2 Branch: refs/heads/YARN-6592 Commit: 06d22eb2784d4e2386b50926286409adb8b47973 Parents: 3663239 Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 23:01:43 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:54:37 2018 -0800 ---------------------------------------------------------------------- .../rmcontainer/RMContainerImpl.java | 8 +++---- .../rmcontainer/TestRMContainerImpl.java | 25 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06d22eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index a504221..541621b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer { RMContainerEvent event) { NMContainerStatus report = ((RMContainerRecoverEvent) event).getContainerReport(); - // Set the allocation tags from the + // Set the allocation tags from the NMContainerStatus container.setAllocationTags(report.getAllocationTags()); // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().addContainer( @@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer { // Something wrong happened, kill the container LOG.warn("Something wrong happened, container size reported by NM" + " is not expected, ContainerID=" + container.getContainerId() - + " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:" + + " rm-size-resource:" + rmContainerResource + " nm-size-resource:" + nmContainerResource); container.eventHandler.handle(new RMNodeCleanContainerEvent( container.nodeId, container.getContainerId())); @@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer { @Override public void transition(RMContainerImpl container, RMContainerEvent event) { - // Notify placementManager + // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().removeContainer( container.getNodeId(), container.getContainerId(), container.getAllocationTags()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06d22eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 2bf6a21..27c5fbd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -171,7 +172,7 @@ public class TestRMContainerImpl { assertEquals(containerStatus, cfEvent.getContainerStatus()); assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType()); - // In RELEASED state. A FINIHSED event may come in. + // In RELEASED state. A FINISHED event may come in. rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); @@ -375,7 +376,7 @@ public class TestRMContainerImpl { } @Test - public void testContainerTransitionNotifyPlacementTagsManager() + public void testContainerTransitionNotifyAllocationTagsManager() throws Exception { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock( @@ -494,5 +495,25 @@ public class TestRMContainerImpl { Assert.assertEquals(0, tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + + /* Fourth container: NEW -> RECOVERED */ + rmContainer = new RMContainerImpl(container, + SchedulerRequestKey.extractFrom(container), appAttemptId, nodeId, + "user", rmContext); + rmContainer.setAllocationTags(ImmutableSet.of("mapper")); + + Assert.assertEquals(0, + tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); + + NMContainerStatus containerStatus = NMContainerStatus + .newInstance(containerId, 0, ContainerState.NEW, + Resource.newInstance(1024, 1), "recover container", 0, + Priority.newInstance(0), 0); + containerStatus.setAllocationTags(ImmutableSet.of("mapper")); + rmContainer + .handle(new RMContainerRecoverEvent(containerId, containerStatus)); + + Assert.assertEquals(1, + tagsManager.getNodeCardinalityByOp(nodeId, appId, null, Long::max)); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org