Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Aug 19 23:49:39 2014 @@ -41,13 +41,14 @@ import java.util.List; import java.util.Map; import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -114,7 +115,7 @@ public class TestLeafQueue { setupQueueConfiguration(csConf, newRoot); YarnConfiguration conf = new YarnConfiguration(); cs.setConf(conf); - + csContext = mock(CapacitySchedulerContext.class); when(csContext.getConfiguration()).thenReturn(csConf); when(csContext.getConf()).thenReturn(conf); @@ -122,7 +123,7 @@ public class TestLeafQueue { thenReturn(Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()). thenReturn(Resources.createResource(16*GB, 32)); - when(csContext.getClusterResources()). + when(csContext.getClusterResource()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); @@ -142,7 +143,9 @@ public class TestLeafQueue { queues, queues, TestUtils.spyHook); - cs.reinitialize(csConf, rmContext); + cs.setRMContext(rmContext); + cs.init(csConf); + cs.start(); } private static final String A = "a"; @@ -350,8 +353,7 @@ public class TestLeafQueue { // Submit applications final ApplicationAttemptId appAttemptId_0 = TestUtils .getMockApplicationAttemptId(0, 1); - FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null, - rmContext); + AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(), a.getQueueName(), user_0); @@ -497,8 +499,11 @@ public class TestLeafQueue { // Release each container from app_0 for (RMContainer rmContainer : app_0.getLiveContainers()) { - a.completedContainer(clusterResource, app_0, node_0, rmContainer, - null, RMContainerEventType.KILL, null); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); } assertEquals(1*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -508,8 +513,11 @@ public class TestLeafQueue { // Release each container from app_1 for (RMContainer rmContainer : app_1.getLiveContainers()) { - a.completedContainer(clusterResource, app_1, node_0, rmContainer, - null, RMContainerEventType.KILL, null); + a.completedContainer(clusterResource, app_1, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); } assertEquals(0*GB, a.getUsedResources().getMemory()); @@ -687,7 +695,7 @@ public class TestLeafQueue { assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G - assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G + assertEquals(0*GB, app_1.getHeadroom().getMemory()); // User limit = 2G // Again one to user_0 since he hasn't exceeded user limit yet a.assignContainers(clusterResource, node_0); @@ -695,7 +703,7 @@ public class TestLeafQueue { assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G - assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G + assertEquals(0*GB, app_1.getHeadroom().getMemory()); // 3G - 2G // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); @@ -869,8 +877,11 @@ public class TestLeafQueue { // 8. Release each container from app_0 for (RMContainer rmContainer : app_0.getLiveContainers()) { - a.completedContainer(clusterResource, app_0, node_0, rmContainer, - null, RMContainerEventType.KILL, null); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); } assertEquals(5*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -880,8 +891,11 @@ public class TestLeafQueue { // 9. Release each container from app_2 for (RMContainer rmContainer : app_2.getLiveContainers()) { - a.completedContainer(clusterResource, app_2, node_0, rmContainer, - null, RMContainerEventType.KILL, null); + a.completedContainer(clusterResource, app_2, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); } assertEquals(2*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -891,8 +905,11 @@ public class TestLeafQueue { // 10. Release each container from app_3 for (RMContainer rmContainer : app_3.getLiveContainers()) { - a.completedContainer(clusterResource, app_3, node_0, rmContainer, - null, RMContainerEventType.KILL, null); + a.completedContainer(clusterResource, app_3, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); } assertEquals(0*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -978,9 +995,12 @@ public class TestLeafQueue { assertEquals(2*GB, a.getMetrics().getAllocatedMB()); // Now free 1 container from app_0 i.e. 1G - a.completedContainer(clusterResource, app_0, node_0, - app_0.getLiveContainers().iterator().next(), - null, RMContainerEventType.KILL, null); + RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); a.assignContainers(clusterResource, node_0); assertEquals(5*GB, a.getUsedResources().getMemory()); assertEquals(1*GB, app_0.getCurrentConsumption().getMemory()); @@ -991,9 +1011,12 @@ public class TestLeafQueue { assertEquals(1*GB, a.getMetrics().getAllocatedMB()); // Now finish another container from app_0 and fulfill the reservation - a.completedContainer(clusterResource, app_0, node_0, - app_0.getLiveContainers().iterator().next(), - null, RMContainerEventType.KILL, null); + rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); a.assignContainers(clusterResource, node_0); assertEquals(4*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -1093,9 +1116,12 @@ public class TestLeafQueue { assertEquals(6*GB, a.getMetrics().getAllocatedMB()); // Now free 1 container from app_0 and try to assign to node_0 - a.completedContainer(clusterResource, app_0, node_0, - app_0.getLiveContainers().iterator().next(), - null, RMContainerEventType.KILL, null); + RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); a.assignContainers(clusterResource, node_0); assertEquals(8*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -1185,9 +1211,12 @@ public class TestLeafQueue { assertEquals(2*GB, node_0.getUsedResource().getMemory()); // Now free 1 container from app_0 i.e. 1G, and re-reserve it - a.completedContainer(clusterResource, app_0, node_0, - app_0.getLiveContainers().iterator().next(), - null, RMContainerEventType.KILL, null); + RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); a.assignContainers(clusterResource, node_0); assertEquals(5*GB, a.getUsedResources().getMemory()); assertEquals(1*GB, app_0.getCurrentConsumption().getMemory()); @@ -1217,9 +1246,12 @@ public class TestLeafQueue { assertEquals(2, app_1.getReReservations(priority)); // Now finish another container from app_0 and see the reservation cancelled - a.completedContainer(clusterResource, app_0, node_0, - app_0.getLiveContainers().iterator().next(), - null, RMContainerEventType.KILL, null); + rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null); CSAssignment assignment = a.assignContainers(clusterResource, node_0); assertEquals(8*GB, a.getUsedResources().getMemory()); assertEquals(0*GB, app_0.getCurrentConsumption().getMemory()); @@ -1652,7 +1684,7 @@ public class TestLeafQueue { newQueues, queues, TestUtils.spyHook); queues = newQueues; - root.reinitialize(newRoot, cs.getClusterResources()); + root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization assertEquals(3, e.activeApplications.size()); @@ -1677,7 +1709,7 @@ public class TestLeafQueue { newQueues, queues, TestUtils.spyHook); queues = newQueues; - root.reinitialize(newRoot, cs.getClusterResources()); + root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization assertEquals(60, e.getNodeLocalityDelay()); @@ -2071,7 +2103,7 @@ public class TestLeafQueue { when(csContext.getConfiguration()).thenReturn(csConf); when(csContext.getConf()).thenReturn(new YarnConfiguration()); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); - when(csContext.getClusterResources()).thenReturn(clusterResource); + when(csContext.getClusterResource()).thenReturn(clusterResource); when(csContext.getMinimumResourceCapability()).thenReturn( Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( @@ -2081,5 +2113,8 @@ public class TestLeafQueue { @After public void tearDown() throws Exception { + if (cs != null) { + cs.stop(); + } } }
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Tue Aug 19 23:49:39 2014 @@ -86,7 +86,7 @@ public class TestParentQueue { Resources.createResource(GB, 1)); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(16*GB, 32)); - when(csContext.getClusterResources()). + when(csContext.getClusterResource()). thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32)); when(csContext.getApplicationComparator()). thenReturn(CapacityScheduler.applicationComparator); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Tue Aug 19 23:49:39 2014 @@ -43,11 +43,15 @@ public class TestQueueParsing { YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); + RMContextImpl rmContext = new RMContextImpl(null, null, + null, null, null, null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null); capacityScheduler.setConf(conf); - capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, - null, null, null, null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), - new ClientToAMTokenSecretManagerInRM(), null)); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(conf); + capacityScheduler.start(); + capacityScheduler.reinitialize(conf, rmContext); CSQueue a = capacityScheduler.getQueue("a"); Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); @@ -62,6 +66,7 @@ public class TestQueueParsing { Assert.assertEquals(0.7 * 0.5 * 0.45, c12.getAbsoluteCapacity(), DELTA); Assert.assertEquals(0.7 * 0.55 * 0.7, c12.getAbsoluteMaximumCapacity(), DELTA); + capacityScheduler.stop(); } private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { @@ -142,7 +147,10 @@ public class TestQueueParsing { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(new YarnConfiguration()); + capacityScheduler.init(conf); + capacityScheduler.start(); capacityScheduler.reinitialize(conf, null); + capacityScheduler.stop(); } public void testMaxCapacity() throws Exception { @@ -164,6 +172,8 @@ public class TestQueueParsing { try { capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(new YarnConfiguration()); + capacityScheduler.init(conf); + capacityScheduler.start(); capacityScheduler.reinitialize(conf, null); } catch (IllegalArgumentException iae) { fail = true; @@ -176,6 +186,8 @@ public class TestQueueParsing { // Now this should work capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(new YarnConfiguration()); + capacityScheduler.init(conf); + capacityScheduler.start(); capacityScheduler.reinitialize(conf, null); fail = false; @@ -187,6 +199,7 @@ public class TestQueueParsing { } Assert.assertTrue("Didn't throw IllegalArgumentException for wrong " + "setMaxCap", fail); + capacityScheduler.stop(); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Tue Aug 19 23:49:39 2014 @@ -86,13 +86,12 @@ public class TestUtils { Configuration conf = new Configuration(); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); - RMContext rmContext = + RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null, - new AMRMTokenSecretManager(conf), + new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), writer); - return rmContext; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java Tue Aug 19 23:49:39 2014 @@ -21,16 +21,18 @@ package org.apache.hadoop.yarn.server.re import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; /** * Dummy implementation of Schedulable for unit testing. */ -public class FakeSchedulable extends Schedulable { +public class FakeSchedulable implements Schedulable { private Resource usage; private Resource minShare; private Resource maxShare; + private Resource fairShare; private ResourceWeights weights; private Priority priority; private long startTime; @@ -84,6 +86,26 @@ public class FakeSchedulable extends Sch } @Override + public RMContainer preemptContainer() { + return null; + } + + @Override + public Resource getFairShare() { + return this.fairShare; + } + + @Override + public void setFairShare(Resource fairShare) { + this.fairShare = fairShare; + } + + @Override + public boolean isActive() { + return true; + } + + @Override public Resource getDemand() { return null; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java Tue Aug 19 23:49:39 2014 @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileWriter; @@ -28,6 +27,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.Clock; @@ -99,9 +99,12 @@ public class TestAllocationFileLoaderSer assertEquals(1, rules.size()); assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); - assertEquals(2, allocConf.getQueueNames().size()); - assertTrue(allocConf.getQueueNames().contains("root.queueA")); - assertTrue(allocConf.getQueueNames().contains("root.queueB")); + assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .size()); + assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueA")); + assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueB")); confHolder.allocConf = null; @@ -114,6 +117,9 @@ public class TestAllocationFileLoaderSer out.println(" </queue>"); out.println(" <queuePlacementPolicy>"); out.println(" <rule name='specified' />"); + out.println(" <rule name='nestedUserQueue' >"); + out.println(" <rule name='primaryGroup' />"); + out.println(" </rule>"); out.println(" <rule name='default' />"); out.println(" </queuePlacementPolicy>"); out.println("</allocations>"); @@ -131,12 +137,18 @@ public class TestAllocationFileLoaderSer allocConf = confHolder.allocConf; policy = allocConf.getPlacementPolicy(); rules = policy.getRules(); - assertEquals(2, rules.size()); + assertEquals(3, rules.size()); assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); - assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); + assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1) + .getClass()); + assertEquals(QueuePlacementRule.PrimaryGroup.class, + ((NestedUserQueue) (rules.get(1))).nestedRule.getClass()); + assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); - assertEquals(1, allocConf.getQueueNames().size()); - assertTrue(allocConf.getQueueNames().contains("root.queueB")); + assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .size()); + assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueB")); } @Test @@ -162,18 +174,29 @@ public class TestAllocationFileLoaderSer out.println("<queue name=\"queueC\">"); out.println("<aclSubmitApps>alice,bob admins</aclSubmitApps>"); out.println("</queue>"); - // Give queue D a limit of 3 running apps + // Give queue D a limit of 3 running apps and 0.4f maxAMShare out.println("<queue name=\"queueD\">"); out.println("<maxRunningApps>3</maxRunningApps>"); + out.println("<maxAMShare>0.4</maxAMShare>"); out.println("</queue>"); // Give queue E a preemption timeout of one minute out.println("<queue name=\"queueE\">"); out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>"); out.println("</queue>"); + //Make queue F a parent queue without configured leaf queues using the 'type' attribute + out.println("<queue name=\"queueF\" type=\"parent\" >"); + out.println("</queue>"); + //Create hierarchical queues G,H + out.println("<queue name=\"queueG\">"); + out.println(" <queue name=\"queueH\">"); + out.println(" </queue>"); + out.println("</queue>"); // Set default limit of apps per queue to 15 out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>"); // Set default limit of apps per user to 5 out.println("<userMaxAppsDefault>5</userMaxAppsDefault>"); + // Set default limit of AMResourceShare to 0.5f + out.println("<queueMaxAMShareDefault>0.5f</queueMaxAMShareDefault>"); // Give user1 a limit of 10 jobs out.println("<user name=\"user1\">"); out.println("<maxRunningApps>10</maxRunningApps>"); @@ -194,7 +217,7 @@ public class TestAllocationFileLoaderSer allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; - assertEquals(5, queueConf.getQueueNames().size()); + assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), @@ -220,6 +243,13 @@ public class TestAllocationFileLoaderSer assertEquals(10, queueConf.getUserMaxApps("user1")); assertEquals(5, queueConf.getUserMaxApps("user2")); + assertEquals(.5f, queueConf.getQueueMaxAMShare("root." + YarnConfiguration.DEFAULT_QUEUE_NAME), 0.01); + assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueA"), 0.01); + assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueB"), 0.01); + assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueC"), 0.01); + assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01); + assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01); + // Root should get * ACL assertEquals("*", queueConf.getQueueAcl("root", QueueACL.ADMINISTER_QUEUE).getAclString()); @@ -250,6 +280,14 @@ public class TestAllocationFileLoaderSer assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE")); assertEquals(300000, queueConf.getFairSharePreemptionTimeout()); + assertTrue(queueConf.getConfiguredQueues() + .get(FSQueueType.PARENT) + .contains("root.queueF")); + assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) + .contains("root.queueG")); + assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueG.queueH")); + // Verify existing queues have default scheduling policy assertEquals(DominantResourceFairnessPolicy.NAME, queueConf.getSchedulingPolicy("root").getName()); @@ -315,7 +353,7 @@ public class TestAllocationFileLoaderSer allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; - assertEquals(5, queueConf.getQueueNames().size()); + assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java Tue Aug 19 23:49:39 2014 @@ -48,6 +48,8 @@ public class TestFSLeafQueue { ResourceManager resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); String queueName = "root.queue1"; @@ -60,7 +62,7 @@ public class TestFSLeafQueue { @Test public void testUpdateDemand() { - AppSchedulable app = mock(AppSchedulable.class); + FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(maxResource); schedulable.addAppSchedulable(app);
