Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Nov 30 19:58:09 2012 @@ -31,6 +31,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import javax.xml.parsers.ParserConfigurationException; + +import junit.framework.Assert; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.Clock; @@ -47,7 +51,7 @@ import org.apache.hadoop.yarn.factories. import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -61,6 +65,7 @@ import org.apache.hadoop.yarn.server.res import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.xml.sax.SAXException; public class TestFairScheduler { @@ -97,7 +102,7 @@ public class TestFairScheduler { Configuration conf = createConfiguration(); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); - Store store = StoreFactory.getStore(conf); + RMStateStore store = StoreFactory.getStore(conf); resourceManager = new ResourceManager(store); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); @@ -195,15 +200,64 @@ public class TestFairScheduler { scheduler.update(); - Collection<FSQueue> queues = scheduler.getQueueManager().getQueues(); + Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); - for (FSQueue p : queues) { - if (p.getName() != "default") { - assertEquals(5120, p.getQueueSchedulable().getFairShare().getMemory()); + for (FSLeafQueue p : queues) { + if (!p.getName().equals("root.default")) { + assertEquals(5120, p.getFairShare().getMemory()); } } } + + @Test + public void testSimpleHierarchicalFairShareCalculation() { + // Add one big node (only care about aggregate capacity) + int capacity = 10 * 24; + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity)); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Have two queues which want entire cluster capacity + createSchedulingRequest(10 * 1024, "queue1", "user1"); + createSchedulingRequest(10 * 1024, "parent.queue2", "user1"); + createSchedulingRequest(10 * 1024, "parent.queue3", "user1"); + + scheduler.update(); + + QueueManager queueManager = scheduler.getQueueManager(); + Collection<FSLeafQueue> queues = queueManager.getLeafQueues(); + assertEquals(4, queues.size()); + + FSLeafQueue queue1 = queueManager.getLeafQueue("queue1"); + FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2"); + FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3"); + assertEquals(capacity / 2, queue1.getFairShare().getMemory()); + assertEquals(capacity / 4, queue2.getFairShare().getMemory()); + assertEquals(capacity / 4, queue3.getFairShare().getMemory()); + } + + @Test + public void testHierarchicalQueuesSimilarParents() { + QueueManager queueManager = scheduler.getQueueManager(); + FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child"); + Assert.assertEquals(2, queueManager.getLeafQueues().size()); + Assert.assertNotNull(leafQueue); + Assert.assertEquals("root.parent.child", leafQueue.getName()); + + FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent"); + Assert.assertNull(leafQueue2); + Assert.assertEquals(2, queueManager.getLeafQueues().size()); + + FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild"); + Assert.assertNull(leafQueue3); + Assert.assertEquals(2, queueManager.getLeafQueues().size()); + + FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister"); + Assert.assertNotNull(leafQueue4); + Assert.assertEquals("root.parent.sister", leafQueue4.getName()); + Assert.assertEquals(3, queueManager.getLeafQueues().size()); + } @Test public void testSimpleContainerAllocation() { @@ -228,14 +282,14 @@ public class TestFairScheduler { // Asked for less than min_allocation. assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, scheduler.getQueueManager().getQueue("queue1"). - getQueueSchedulable().getResourceUsage().getMemory()); + getResourceUsage().getMemory()); NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2, new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>()); scheduler.handle(updateEvent2); assertEquals(1024, scheduler.getQueueManager().getQueue("queue1"). - getQueueSchedulable().getResourceUsage().getMemory()); + getResourceUsage().getMemory()); } @Test @@ -254,7 +308,7 @@ public class TestFairScheduler { // Make sure queue 1 is allocated app capacity assertEquals(1024, scheduler.getQueueManager().getQueue("queue1"). - getQueueSchedulable().getResourceUsage().getMemory()); + getResourceUsage().getMemory()); // Now queue 2 requests likewise ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user1", 1); @@ -263,7 +317,7 @@ public class TestFairScheduler { // Make sure queue 2 is waiting with a reservation assertEquals(0, scheduler.getQueueManager().getQueue("queue2"). - getQueueSchedulable().getResourceUsage().getMemory()); + getResourceUsage().getMemory()); assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory()); // Now another node checks in with capacity @@ -276,7 +330,7 @@ public class TestFairScheduler { // Make sure this goes to queue 2 assertEquals(1024, scheduler.getQueueManager().getQueue("queue2"). - getQueueSchedulable().getResourceUsage().getMemory()); + getResourceUsage().getMemory()); // The old reservation should still be there... assertEquals(1024, scheduler.applications.get(attId).getCurrentReservation().getMemory()); @@ -294,17 +348,22 @@ public class TestFairScheduler { AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( createAppAttemptId(1, 1), "default", "user1"); scheduler.handle(appAddedEvent); - assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size()); - assertEquals(0, scheduler.getQueueManager().getQueue("default").getApplications().size()); + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + .getAppSchedulables().size()); + assertEquals(0, scheduler.getQueueManager().getLeafQueue("default") + .getAppSchedulables().size()); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); scheduler.reinitialize(conf, resourceManager.getRMContext()); AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( createAppAttemptId(2, 1), "default", "user2"); scheduler.handle(appAddedEvent2); - assertEquals(1, scheduler.getQueueManager().getQueue("user1").getApplications().size()); - assertEquals(1, scheduler.getQueueManager().getQueue("default").getApplications().size()); - assertEquals(0, scheduler.getQueueManager().getQueue("user2").getApplications().size()); + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + .getAppSchedulables().size()); + assertEquals(1, scheduler.getQueueManager().getLeafQueue("default") + .getAppSchedulables().size()); + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2") + .getAppSchedulables().size()); } @Test @@ -338,18 +397,17 @@ public class TestFairScheduler { scheduler.update(); - Collection<FSQueue> queues = scheduler.getQueueManager().getQueues(); + Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); - for (FSQueue p : queues) { - if (p.getName().equals("queueA")) { - assertEquals(1024, p.getQueueSchedulable().getFairShare().getMemory()); + for (FSLeafQueue p : queues) { + if (p.getName().equals("root.queueA")) { + assertEquals(1024, p.getFairShare().getMemory()); } - else if (p.getName().equals("queueB")) { - assertEquals(2048, p.getQueueSchedulable().getFairShare().getMemory()); + else if (p.getName().equals("root.queueB")) { + assertEquals(2048, p.getFairShare().getMemory()); } } - } /** @@ -358,11 +416,11 @@ public class TestFairScheduler { @Test public void testQueueDemandCalculation() throws Exception { ApplicationAttemptId id11 = createAppAttemptId(1, 1); - scheduler.addApplication(id11, "queue1", "user1"); + scheduler.addApplication(id11, "root.queue1", "user1"); ApplicationAttemptId id21 = createAppAttemptId(2, 1); - scheduler.addApplication(id21, "queue2", "user1"); + scheduler.addApplication(id21, "root.queue2", "user1"); ApplicationAttemptId id22 = createAppAttemptId(2, 2); - scheduler.addApplication(id22, "queue2", "user1"); + scheduler.addApplication(id22, "root.queue2", "user1"); int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB; @@ -388,10 +446,10 @@ public class TestFairScheduler { scheduler.update(); - assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("queue1") - .getQueueSchedulable().getDemand().getMemory()); + assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1") + .getDemand().getMemory()); assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler - .getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand() + .getQueueManager().getQueue("root.queue2").getDemand() .getMemory()); } @@ -401,11 +459,12 @@ public class TestFairScheduler { createAppAttemptId(1, 1), "default", "user1"); scheduler.handle(appAddedEvent1); - // Scheduler should have one queue (the default) - assertEquals(1, scheduler.getQueueManager().getQueues().size()); + // Scheduler should have two queues (the default and the one created for user1) + assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); // That queue should have one app - assertEquals(1, scheduler.getQueueManager().getQueue("default").getApplications().size()); + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + .getAppSchedulables().size()); AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( createAppAttemptId(1, 1), RMAppAttemptState.FINISHED); @@ -413,8 +472,9 @@ public class TestFairScheduler { // Now remove app scheduler.handle(appRemovedEvent1); - // Default queue should have no apps - assertEquals(0, scheduler.getQueueManager().getQueue("default").getApplications().size()); + // Queue should have no apps + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1") + .getAppSchedulables().size()); } @Test @@ -466,61 +526,99 @@ public class TestFairScheduler { QueueManager queueManager = scheduler.getQueueManager(); queueManager.initialize(); - assertEquals(6, queueManager.getQueues().size()); // 5 in file + default queue + assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue assertEquals(Resources.createResource(0), - queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME)); + queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), - queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME)); + queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(1024), - queueManager.getMinResources("queueA")); + queueManager.getMinResources("root.queueA")); assertEquals(Resources.createResource(2048), - queueManager.getMinResources("queueB")); + queueManager.getMinResources("root.queueB")); assertEquals(Resources.createResource(0), - queueManager.getMinResources("queueC")); + queueManager.getMinResources("root.queueC")); assertEquals(Resources.createResource(0), - queueManager.getMinResources("queueD")); + queueManager.getMinResources("root.queueD")); assertEquals(Resources.createResource(0), - queueManager.getMinResources("queueE")); + queueManager.getMinResources("root.queueE")); - assertEquals(15, queueManager.getQueueMaxApps(YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(15, queueManager.getQueueMaxApps("queueA")); - assertEquals(15, queueManager.getQueueMaxApps("queueB")); - assertEquals(15, queueManager.getQueueMaxApps("queueC")); - assertEquals(3, queueManager.getQueueMaxApps("queueD")); - assertEquals(15, queueManager.getQueueMaxApps("queueE")); + assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(15, queueManager.getQueueMaxApps("root.queueA")); + assertEquals(15, queueManager.getQueueMaxApps("root.queueB")); + assertEquals(15, queueManager.getQueueMaxApps("root.queueC")); + assertEquals(3, queueManager.getQueueMaxApps("root.queueD")); + assertEquals(15, queueManager.getQueueMaxApps("root.queueE")); assertEquals(10, queueManager.getUserMaxApps("user1")); assertEquals(5, queueManager.getUserMaxApps("user2")); // Unspecified queues should get default ACL - Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA"); + Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA"); assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE)); assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString()); assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS)); assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString()); // Queue B ACL - Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("queueB"); + Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB"); assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE)); assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString()); // Queue c ACL - Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("queueC"); + Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC"); assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS)); assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString()); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout( + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueB")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueC")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueD")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA")); - assertEquals(60000, queueManager.getMinSharePreemptionTimeout("queueE")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE")); assertEquals(300000, queueManager.getFairSharePreemptionTimeout()); } @Test + public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException, + AllocationConfigurationException, ParserConfigurationException { + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"queueA\">"); + out.println("<minResources>2048</minResources>"); + out.println("</queue>"); + out.println("<queue name=\"queueB\">"); + out.println("<minResources>2048</minResources>"); + out.println("<queue name=\"queueC\">"); + out.println("<minResources>2048</minResources>"); + out.println("</queue>"); + out.println("<queue name=\"queueD\">"); + out.println("<minResources>2048</minResources>"); + out.println("</queue>"); + out.println("</queue>"); + out.println("</allocations>"); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + Collection<FSLeafQueue> leafQueues = queueManager.getLeafQueues(); + Assert.assertEquals(4, leafQueues.size()); + Assert.assertNotNull(queueManager.getLeafQueue("queueA")); + Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC")); + Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD")); + Assert.assertNotNull(queueManager.getLeafQueue("default")); + // Make sure querying for queues didn't create any new ones: + Assert.assertEquals(4, leafQueues.size()); + } + + @Test public void testBackwardsCompatibleAllocationFileParsing() throws Exception { Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); @@ -569,29 +667,29 @@ public class TestFairScheduler { QueueManager queueManager = scheduler.getQueueManager(); queueManager.initialize(); - assertEquals(6, queueManager.getQueues().size()); // 5 in file + default queue + assertEquals(6, queueManager.getLeafQueues().size()); // 5 in file + default queue assertEquals(Resources.createResource(0), - queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME)); + queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), - queueManager.getMinResources(YarnConfiguration.DEFAULT_QUEUE_NAME)); + queueManager.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(1024), - queueManager.getMinResources("queueA")); + queueManager.getMinResources("root.queueA")); assertEquals(Resources.createResource(2048), - queueManager.getMinResources("queueB")); + queueManager.getMinResources("root.queueB")); assertEquals(Resources.createResource(0), - queueManager.getMinResources("queueC")); + queueManager.getMinResources("root.queueC")); assertEquals(Resources.createResource(0), - queueManager.getMinResources("queueD")); + queueManager.getMinResources("root.queueD")); assertEquals(Resources.createResource(0), - queueManager.getMinResources("queueE")); + queueManager.getMinResources("root.queueE")); - assertEquals(15, queueManager.getQueueMaxApps(YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(15, queueManager.getQueueMaxApps("queueA")); - assertEquals(15, queueManager.getQueueMaxApps("queueB")); - assertEquals(15, queueManager.getQueueMaxApps("queueC")); - assertEquals(3, queueManager.getQueueMaxApps("queueD")); - assertEquals(15, queueManager.getQueueMaxApps("queueE")); + assertEquals(15, queueManager.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(15, queueManager.getQueueMaxApps("root.queueA")); + assertEquals(15, queueManager.getQueueMaxApps("root.queueB")); + assertEquals(15, queueManager.getQueueMaxApps("root.queueC")); + assertEquals(3, queueManager.getQueueMaxApps("root.queueD")); + assertEquals(15, queueManager.getQueueMaxApps("root.queueE")); assertEquals(10, queueManager.getUserMaxApps("user1")); assertEquals(5, queueManager.getUserMaxApps("user2")); @@ -603,23 +701,23 @@ public class TestFairScheduler { assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString()); // Queue B ACL - Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("queueB"); + Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB"); assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE)); assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString()); // Queue c ACL - Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("queueC"); + Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC"); assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS)); assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString()); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout( + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueB")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueC")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueD")); - assertEquals(120000, queueManager.getMinSharePreemptionTimeout("queueA")); - assertEquals(60000, queueManager.getMinSharePreemptionTimeout("queueE")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueB")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueC")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueD")); + assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA")); + assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE")); assertEquals(300000, queueManager.getFairSharePreemptionTimeout()); } @@ -659,25 +757,25 @@ public class TestFairScheduler { // Queue B arrives and wants 1 * 1024 createSchedulingRequest(1 * 1024, "queueB", "user1"); scheduler.update(); - Collection<FSQueue> queues = scheduler.getQueueManager().getQueues(); + Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Queue A should be above min share, B below. - for (FSQueue p : queues) { - if (p.getName().equals("queueA")) { - assertEquals(false, scheduler.isStarvedForMinShare(p.getQueueSchedulable())); + for (FSLeafQueue p : queues) { + if (p.getName().equals("root.queueA")) { + assertEquals(false, scheduler.isStarvedForMinShare(p)); } - else if (p.getName().equals("queueB")) { - assertEquals(true, scheduler.isStarvedForMinShare(p.getQueueSchedulable())); + else if (p.getName().equals("root.queueB")) { + assertEquals(true, scheduler.isStarvedForMinShare(p)); } } // Node checks in again, should allocate for B scheduler.handle(nodeEvent2); // Now B should have min share ( = demand here) - for (FSQueue p : queues) { - if (p.getName().equals("queueB")) { - assertEquals(false, scheduler.isStarvedForMinShare(p.getQueueSchedulable())); + for (FSLeafQueue p : queues) { + if (p.getName().equals("root.queueB")) { + assertEquals(false, scheduler.isStarvedForMinShare(p)); } } } @@ -718,16 +816,16 @@ public class TestFairScheduler { // Queue B arrives and wants 1 * 1024 createSchedulingRequest(1 * 1024, "queueB", "user1"); scheduler.update(); - Collection<FSQueue> queues = scheduler.getQueueManager().getQueues(); + Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Queue A should be above fair share, B below. - for (FSQueue p : queues) { - if (p.getName().equals("queueA")) { - assertEquals(false, scheduler.isStarvedForFairShare(p.getQueueSchedulable())); + for (FSLeafQueue p : queues) { + if (p.getName().equals("root.queueA")) { + assertEquals(false, scheduler.isStarvedForFairShare(p)); } - else if (p.getName().equals("queueB")) { - assertEquals(true, scheduler.isStarvedForFairShare(p.getQueueSchedulable())); + else if (p.getName().equals("root.queueB")) { + assertEquals(true, scheduler.isStarvedForFairShare(p)); } } @@ -735,9 +833,9 @@ public class TestFairScheduler { scheduler.handle(nodeEvent2); // B should not be starved for fair share, since entire demand is // satisfied. - for (FSQueue p : queues) { - if (p.getName().equals("queueB")) { - assertEquals(false, scheduler.isStarvedForFairShare(p.getQueueSchedulable())); + for (FSLeafQueue p : queues) { + if (p.getName().equals("root.queueB")) { + assertEquals(false, scheduler.isStarvedForFairShare(p)); } } } @@ -845,7 +943,7 @@ public class TestFairScheduler { // We should be able to claw back one container from A and B each. // Make sure it is lowest priority container. - scheduler.preemptResources(scheduler.getQueueSchedulables(), + scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), Resources.createResource(2 * 1024)); assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size()); assertEquals(1, scheduler.applications.get(app2).getLiveContainers().size()); @@ -856,7 +954,7 @@ public class TestFairScheduler { // We should be able to claw back another container from A and B each. // Make sure it is lowest priority container. - scheduler.preemptResources(scheduler.getQueueSchedulables(), + scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), Resources.createResource(2 * 1024)); assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size()); assertEquals(0, scheduler.applications.get(app2).getLiveContainers().size()); @@ -866,7 +964,7 @@ public class TestFairScheduler { assertEquals(0, scheduler.applications.get(app6).getLiveContainers().size()); // Now A and B are below fair share, so preemption shouldn't do anything - scheduler.preemptResources(scheduler.getQueueSchedulables(), + scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(), Resources.createResource(2 * 1024)); assertEquals(1, scheduler.applications.get(app1).getLiveContainers().size()); assertEquals(0, scheduler.applications.get(app2).getLiveContainers().size()); @@ -977,10 +1075,10 @@ public class TestFairScheduler { scheduler.update(); - FSQueueSchedulable schedC = - scheduler.getQueueManager().getQueue("queueC").getQueueSchedulable(); - FSQueueSchedulable schedD = - scheduler.getQueueManager().getQueue("queueD").getQueueSchedulable(); + FSLeafQueue schedC = + scheduler.getQueueManager().getLeafQueue("queueC"); + FSLeafQueue schedD = + scheduler.getQueueManager().getLeafQueue("queueD"); assertTrue(Resources.equals( Resources.none(), scheduler.resToPreempt(schedC, clock.getTime())));
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Fri Nov 30 19:58:09 2012 @@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; @@ -59,7 +59,7 @@ public class TestFifoScheduler { @Before public void setUp() throws Exception { - Store store = StoreFactory.getStore(new Configuration()); + RMStateStore store = StoreFactory.getStore(new Configuration()); resourceManager = new ResourceManager(store); Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, @@ -91,7 +91,7 @@ public class TestFifoScheduler { @Test public void testAppAttemptMetrics() throws Exception { AsyncDispatcher dispatcher = new InlineDispatcher(); - RMContext rmContext = new RMContextImpl(null, dispatcher, null, + RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null); FifoScheduler schedular = new FifoScheduler(); Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Fri Nov 30 19:58:09 2012 @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -160,7 +159,7 @@ public class TestRMWebApp { for (RMNode node : deactivatedNodes) { deactivatedNodesMap.put(node.getHostName(), node); } - return new RMContextImpl(new MemStore(), null, null, null, null, + return new RMContextImpl(null, null, null, null, null, null, null, null) { @Override public ConcurrentMap<ApplicationId, RMApp> getRMApps() { @@ -201,7 +200,7 @@ public class TestRMWebApp { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new ClientToAMTokenSecretManagerInRM())); return cs; Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Fri Nov 30 19:58:09 2012 @@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.service.CompositeService; @@ -154,7 +154,7 @@ public class MiniYARNCluster extends Com getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":0"); } - Store store = StoreFactory.getStore(getConfig()); + RMStateStore store = StoreFactory.getStore(getConfig()); resourceManager = new ResourceManager(store) { @Override protected void doSecureLogin() throws IOException { Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Nov 30 19:58:09 2012 @@ -53,6 +53,22 @@ Hadoop MapReduce Next Generation - Fair capacity between the running apps. queues can also be given weights to share the cluster non-proportionally in the config file. + The fair scheduler supports hierarchical queues. All queues descend from a + queue named "root". Available resources are distributed among the children + of the root queue in the typical fair scheduling fashion. Then, the children + distribute the resources assigned to them to their children in the same + fashion. Applications may only be scheduled on leaf queues. Queues can be + specified as children of other queues by placing them as sub-elements of + their parents in the fair scheduler configuration file. + + A queue's name starts with the names of its parents, with periods as + separators. So a queue named "queue1" under the root named, would be + referred to as "root.queue1", and a queue named "queue2" under a queue + named "parent1" would be referred to as "root.parent1.queue2". When + referring to queues, the root part of the name is optional, so queue1 could + be referred to as just "queue1", and a queue2 could be referred to as just + "parent1.queue2". + In addition to providing fair sharing, the Fair Scheduler allows assigning guaranteed minimum shares to queues, which is useful for ensuring that certain users, groups or production applications always get sufficient @@ -163,11 +179,14 @@ Allocation file format <?xml version="1.0"?> <allocations> <queue name="sample_queue"> - <minResources>100000</minResources> - <maxResources>900000</maxResources> + <minResources>10000</minResources> + <maxResources>90000</maxResources> <maxRunningApps>50</maxRunningApps> <weight>2.0</weight> <schedulingMode>fair</schedulingMode> + <queue name="sample_sub_queue"> + <minResources>5000</minResources> + </queue> </queue> <user name="sample_user"> <maxRunningApps>30</maxRunningApps> Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm Fri Nov 30 19:58:09 2012 @@ -56,6 +56,8 @@ History Server REST API's. *---------------+--------------+-------------------------------+ || Item || Data Type || Description | *---------------+--------------+-------------------------------+ +| startedOn | long | The time the history server was started (in ms since epoch)| +*---------------+--------------+-------------------------------+ | hadoopVersion | string | Version of hadoop common | *---------------+--------------+-------------------------------+ | hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | @@ -87,6 +89,7 @@ History Server REST API's. +---+ { "historyInfo" : { + "startedOn":1353512830963, "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012", "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7", "hadoopVersion" : "0.23.1-SNAPSHOT" @@ -117,6 +120,7 @@ History Server REST API's. +---+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?> <historyInfo> + <startedOn>1353512830963</startedOn> <hadoopVersion>0.23.1-SNAPSHOT</hadoopVersion> <hadoopBuildVersion>0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7</hadoopBuildVersion> <hadoopVersionBuiltOn>Wed Jan 11 21:18:36 UTC 2012</hadoopVersionBuiltOn> Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm Fri Nov 30 19:58:09 2012 @@ -49,7 +49,7 @@ MapReduce NextGen aka YARN aka MRv2 * {{{./WebApplicationProxy.html}Web Application Proxy}} - * {{{./CLIMiniCluster.html}CLI MiniCluster}} + * {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}} - * {{{./EncryptedShuffle.html}Encrypted Shuffle}} + * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html}Encrypted Shuffle}}
