Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Mon May 12 12:43:59 2014 @@ -350,8 +350,7 @@ public class TestLeafQueue { // Submit applications final ApplicationAttemptId appAttemptId_0 = TestUtils .getMockApplicationAttemptId(0, 1); - FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null, - rmContext); + AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(), a.getQueueName(), user_0); @@ -687,7 +686,7 @@ public class TestLeafQueue { assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G - assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G + assertEquals(0*GB, app_1.getHeadroom().getMemory()); // User limit = 2G // Again one to user_0 since he hasn't exceeded user limit yet a.assignContainers(clusterResource, node_0); @@ -695,7 +694,7 @@ public class TestLeafQueue { assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemory()); assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G - assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G + assertEquals(0*GB, app_1.getHeadroom().getMemory()); // 3G - 2G // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java Mon May 12 12:43:59 2014 @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileWriter; @@ -28,6 +27,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.Clock; @@ -99,9 +99,12 @@ public class TestAllocationFileLoaderSer assertEquals(1, rules.size()); assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass()); assertEquals(1, allocConf.getQueueMaxApps("root.queueA")); - assertEquals(2, allocConf.getQueueNames().size()); - assertTrue(allocConf.getQueueNames().contains("root.queueA")); - assertTrue(allocConf.getQueueNames().contains("root.queueB")); + assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .size()); + assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueA")); + assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueB")); confHolder.allocConf = null; @@ -114,6 +117,9 @@ public class TestAllocationFileLoaderSer out.println(" </queue>"); out.println(" <queuePlacementPolicy>"); out.println(" <rule name='specified' />"); + out.println(" <rule name='nestedUserQueue' >"); + out.println(" <rule name='primaryGroup' />"); + out.println(" </rule>"); out.println(" <rule name='default' />"); out.println(" </queuePlacementPolicy>"); out.println("</allocations>"); @@ -131,12 +137,18 @@ public class TestAllocationFileLoaderSer allocConf = confHolder.allocConf; policy = allocConf.getPlacementPolicy(); rules = policy.getRules(); - assertEquals(2, rules.size()); + assertEquals(3, rules.size()); assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); - assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); + assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1) + .getClass()); + assertEquals(QueuePlacementRule.PrimaryGroup.class, + ((NestedUserQueue) (rules.get(1))).nestedRule.getClass()); + assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass()); assertEquals(3, allocConf.getQueueMaxApps("root.queueB")); - assertEquals(1, allocConf.getQueueNames().size()); - assertTrue(allocConf.getQueueNames().contains("root.queueB")); + assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .size()); + assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueB")); } @Test @@ -170,6 +182,14 @@ public class TestAllocationFileLoaderSer out.println("<queue name=\"queueE\">"); out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>"); out.println("</queue>"); + //Make queue F a parent queue without configured leaf queues using the 'type' attribute + out.println("<queue name=\"queueF\" type=\"parent\" >"); + out.println("</queue>"); + //Create hierarchical queues G,H + out.println("<queue name=\"queueG\">"); + out.println(" <queue name=\"queueH\">"); + out.println(" </queue>"); + out.println("</queue>"); // Set default limit of apps per queue to 15 out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>"); // Set default limit of apps per user to 5 @@ -194,7 +214,7 @@ public class TestAllocationFileLoaderSer allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; - assertEquals(5, queueConf.getQueueNames().size()); + assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), @@ -250,6 +270,14 @@ public class TestAllocationFileLoaderSer assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE")); assertEquals(300000, queueConf.getFairSharePreemptionTimeout()); + assertTrue(queueConf.getConfiguredQueues() + .get(FSQueueType.PARENT) + .contains("root.queueF")); + assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) + .contains("root.queueG")); + assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.queueG.queueH")); + // Verify existing queues have default scheduling policy assertEquals(DominantResourceFairnessPolicy.NAME, queueConf.getSchedulingPolicy("root").getName()); @@ -315,7 +343,7 @@ public class TestAllocationFileLoaderSer allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; - assertEquals(5, queueConf.getQueueNames().size()); + assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(0), Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon May 12 12:43:59 2014 @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -43,7 +44,6 @@ import java.util.Set; import javax.xml.parsers.ParserConfigurationException; import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -737,8 +738,11 @@ public class TestFairScheduler { rules.add(new QueuePlacementRule.Default().initialize(true, null)); Set<String> queues = Sets.newHashSet("root.user1", "root.user3group", "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2"); + Map<FSQueueType, Set<String>> configuredQueues = new HashMap<FSQueueType, Set<String>>(); + configuredQueues.put(FSQueueType.LEAF, queues); + configuredQueues.put(FSQueueType.PARENT, new HashSet<String>()); scheduler.getAllocationConfiguration().placementPolicy = - new QueuePlacementPolicy(rules, queues, conf); + new QueuePlacementPolicy(rules, configuredQueues, conf); appId = createSchedulingRequest(1024, "somequeue", "user1"); assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName()); appId = createSchedulingRequest(1024, "default", "user1"); @@ -758,7 +762,7 @@ public class TestFairScheduler { rules.add(new QueuePlacementRule.Specified().initialize(true, null)); rules.add(new QueuePlacementRule.Default().initialize(true, null)); scheduler.getAllocationConfiguration().placementPolicy = - new QueuePlacementPolicy(rules, queues, conf); + new QueuePlacementPolicy(rules, configuredQueues, conf); appId = createSchedulingRequest(1024, "somequeue", "user1"); assertEquals("root.user1", scheduler.getSchedulerApp(appId).getQueueName()); appId = createSchedulingRequest(1024, "somequeue", "otheruser"); @@ -809,7 +813,89 @@ public class TestFairScheduler { } } } + + @Test + public void testNestedUserQueue() throws IOException { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + SimpleGroupsMapping.class, GroupMappingServiceProvider.class); + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"user1group\" type=\"parent\">"); + out.println("<minResources>1024mb,0vcores</minResources>"); + out.println("</queue>"); + out.println("<queuePlacementPolicy>"); + out.println("<rule name=\"specified\" create=\"false\" />"); + out.println("<rule name=\"nestedUserQueue\">"); + out.println(" <rule name=\"primaryGroup\" create=\"false\" />"); + out.println("</rule>"); + out.println("<rule name=\"default\" />"); + out.println("</queuePlacementPolicy>"); + out.println("</allocations>"); + out.close(); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); + RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); + + FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default", + "user1"); + + assertEquals("root.user1group.user1", user1Leaf.getName()); + } + + @Test + public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"parentq\" type=\"parent\">"); + out.println("<minResources>1024mb,0vcores</minResources>"); + out.println("</queue>"); + out.println("<queuePlacementPolicy>"); + out.println("<rule name=\"nestedUserQueue\">"); + out.println(" <rule name=\"specified\" create=\"false\" />"); + out.println("</rule>"); + out.println("<rule name=\"default\" />"); + out.println("</queuePlacementPolicy>"); + out.println("</allocations>"); + out.close(); + + RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); + RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); + + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + int capacity = 16 * 1024; + // create node with 16 G + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity), + 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // user1,user2 submit their apps to parentq and create user queues + scheduler.assignToQueue(rmApp1, "root.parentq", "user1"); + scheduler.assignToQueue(rmApp2, "root.parentq", "user2"); + + scheduler.update(); + + Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager() + .getLeafQueues(); + + for (FSLeafQueue leaf : leafQueues) { + if (leaf.getName().equals("root.parentq.user1") + || leaf.getName().equals("root.parentq.user2")) { + // assert that the fair share is 1/4th node1's capacity + assertEquals(capacity / 4, leaf.getFairShare().getMemory()); + // assert weights are equal for both the user queues + assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0); + } + } + } + /** * Make allocation requests and ensure they are reflected in queue demand. */ Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java Mon May 12 12:43:59 2014 @@ -17,8 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; import static org.mockito.Mockito.*; import java.util.HashSet; @@ -57,45 +56,77 @@ public class TestQueueManager { @Test public void testReloadTurnsLeafQueueIntoParent() throws Exception { - updateConfiguredQueues(queueManager, "queue1"); + updateConfiguredLeafQueues(queueManager, "queue1"); // When no apps are running in the leaf queue, should be fine turning it // into a parent. - updateConfiguredQueues(queueManager, "queue1.queue2"); + updateConfiguredLeafQueues(queueManager, "queue1.queue2"); assertNull(queueManager.getLeafQueue("queue1", false)); assertNotNull(queueManager.getLeafQueue("queue1.queue2", false)); // When leaf queues are empty, should be ok deleting them and // turning parent into a leaf. - updateConfiguredQueues(queueManager, "queue1"); + updateConfiguredLeafQueues(queueManager, "queue1"); assertNull(queueManager.getLeafQueue("queue1.queue2", false)); assertNotNull(queueManager.getLeafQueue("queue1", false)); // When apps exist in leaf queue, we shouldn't be able to create // children under it, but things should work otherwise. notEmptyQueues.add(queueManager.getLeafQueue("queue1", false)); - updateConfiguredQueues(queueManager, "queue1.queue2"); + updateConfiguredLeafQueues(queueManager, "queue1.queue2"); assertNull(queueManager.getLeafQueue("queue1.queue2", false)); assertNotNull(queueManager.getLeafQueue("queue1", false)); // When apps exist in leaf queues under a parent queue, shouldn't be // able to turn it into a leaf queue, but things should work otherwise. notEmptyQueues.clear(); - updateConfiguredQueues(queueManager, "queue1.queue2"); + updateConfiguredLeafQueues(queueManager, "queue1.queue2"); notEmptyQueues.add(queueManager.getQueue("root.queue1")); - updateConfiguredQueues(queueManager, "queue1"); + updateConfiguredLeafQueues(queueManager, "queue1"); assertNotNull(queueManager.getLeafQueue("queue1.queue2", false)); assertNull(queueManager.getLeafQueue("queue1", false)); // Should never to be able to create a queue under the default queue - updateConfiguredQueues(queueManager, "default.queue3"); + updateConfiguredLeafQueues(queueManager, "default.queue3"); assertNull(queueManager.getLeafQueue("default.queue3", false)); assertNotNull(queueManager.getLeafQueue("default", false)); } - private void updateConfiguredQueues(QueueManager queueMgr, String... confQueues) { + @Test + public void testReloadTurnsLeafToParentWithNoLeaf() { + AllocationConfiguration allocConf = new AllocationConfiguration(conf); + // Create a leaf queue1 + allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1"); + queueManager.updateAllocationConfiguration(allocConf); + assertNotNull(queueManager.getLeafQueue("root.queue1", false)); + + // Lets say later on admin makes queue1 a parent queue by + // specifying "type=parent" in the alloc xml and lets say apps running in + // queue1 + notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false)); + allocConf = new AllocationConfiguration(conf); + allocConf.configuredQueues.get(FSQueueType.PARENT) + .add("root.queue1"); + + // When allocs are reloaded queue1 shouldn't be converter to parent + queueManager.updateAllocationConfiguration(allocConf); + assertNotNull(queueManager.getLeafQueue("root.queue1", false)); + assertNull(queueManager.getParentQueue("root.queue1", false)); + + // Now lets assume apps completed and there are no apps in queue1 + notEmptyQueues.clear(); + // We should see queue1 transform from leaf queue to parent queue. + queueManager.updateAllocationConfiguration(allocConf); + assertNull(queueManager.getLeafQueue("root.queue1", false)); + assertNotNull(queueManager.getParentQueue("root.queue1", false)); + // this parent should not have any children + assertTrue(queueManager.getParentQueue("root.queue1", false) + .getChildQueues().isEmpty()); + } + + private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) { AllocationConfiguration allocConf = new AllocationConfiguration(conf); - allocConf.queueNames = Sets.newHashSet(confQueues); + allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues)); queueMgr.updateAllocationConfiguration(allocConf); } } Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java Mon May 12 12:43:59 2014 @@ -17,8 +17,11 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Set; import javax.xml.parsers.DocumentBuilder; @@ -28,16 +31,15 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import com.google.common.collect.Sets; - public class TestQueuePlacementPolicy { private final static Configuration conf = new Configuration(); - private final static Set<String> configuredQueues = Sets.newHashSet("root.someuser"); + private Map<FSQueueType, Set<String>> configuredQueues; @BeforeClass public static void setup() { @@ -45,6 +47,14 @@ public class TestQueuePlacementPolicy { SimpleGroupsMapping.class, GroupMappingServiceProvider.class); } + @Before + public void initTest() { + configuredQueues = new HashMap<FSQueueType, Set<String>>(); + for (FSQueueType type : FSQueueType.values()) { + configuredQueues.put(type, new HashSet<String>()); + } + } + @Test public void testSpecifiedUserPolicy() throws Exception { StringBuffer sb = new StringBuffer(); @@ -53,9 +63,12 @@ public class TestQueuePlacementPolicy { sb.append(" <rule name='user' />"); sb.append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq",policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); - assertEquals("root.otheruser", policy.assignAppToQueue("default", "otheruser")); + assertEquals("root.specifiedq", + policy.assignAppToQueue("specifiedq", "someuser")); + assertEquals("root.someuser", + policy.assignAppToQueue("default", "someuser")); + assertEquals("root.otheruser", + policy.assignAppToQueue("default", "otheruser")); } @Test @@ -66,6 +79,8 @@ public class TestQueuePlacementPolicy { sb.append(" <rule name='user' create=\"false\" />"); sb.append(" <rule name='default' />"); sb.append("</queuePlacementPolicy>"); + + configuredQueues.get(FSQueueType.LEAF).add("root.someuser"); QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser")); assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); @@ -81,7 +96,8 @@ public class TestQueuePlacementPolicy { sb.append(" <rule name='reject' />"); sb.append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser")); + assertEquals("root.specifiedq", + policy.assignAppToQueue("specifiedq", "someuser")); assertEquals(null, policy.assignAppToQueue("default", "someuser")); } @@ -117,10 +133,188 @@ public class TestQueuePlacementPolicy { parse(sb.toString()); } + @Test + public void testNestedUserQueueParsingErrors() { + // No nested rule specified in hierarchical user queue + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' />"); + sb.append(" <rule name='nestedUserQueue'/>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + assertIfExceptionThrown(sb); + + // Specified nested rule is not a QueuePlacementRule + sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='unknownRule'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + assertIfExceptionThrown(sb); + } + + private void assertIfExceptionThrown(StringBuffer sb) { + Throwable th = null; + try { + parse(sb.toString()); + } catch (Exception e) { + th = e; + } + + assertTrue(th instanceof AllocationConfigurationException); + } + + @Test + public void testNestedUserQueueParsing() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='primaryGroup'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + Throwable th = null; + try { + parse(sb.toString()); + } catch (Exception e) { + th = e; + } + + assertNull(th); + } + + @Test + public void testNestedUserQueuePrimaryGroup() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' create='false' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='primaryGroup'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + // User queue would be created under primary group queue + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.user1group.user1", + policy.assignAppToQueue("root.default", "user1")); + // Other rules above and below hierarchical user queue rule should work as + // usual + configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq"); + // test if specified rule(above nestedUserQueue rule) works ok + assertEquals("root.specifiedq", + policy.assignAppToQueue("root.specifiedq", "user2")); + + // test if default rule(below nestedUserQueue rule) works + configuredQueues.get(FSQueueType.LEAF).add("root.user3group"); + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user3")); + } + + @Test + public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { + // Primary group rule has create='false' + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='primaryGroup' create='false'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + QueuePlacementPolicy policy = parse(sb.toString()); + + // Should return root.default since primary group 'root.user1group' is not + // configured + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user1")); + + // Let's configure primary group and check if user queue is created + configuredQueues.get(FSQueueType.PARENT).add("root.user1group"); + policy = parse(sb.toString()); + assertEquals("root.user1group.user1", + policy.assignAppToQueue("root.default", "user1")); + + // Both Primary group and nestedUserQueue rule has create='false' + sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue' create='false'>"); + sb.append(" <rule name='primaryGroup' create='false'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + // Should return root.default since primary group and user queue for user 2 + // are not configured. + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user2")); + + // Now configure both primary group and the user queue for user2 + configuredQueues.get(FSQueueType.PARENT).add("root.user2group"); + configuredQueues.get(FSQueueType.LEAF).add("root.user2group.user2"); + policy = parse(sb.toString()); + + assertEquals("root.user2group.user2", + policy.assignAppToQueue("root.default", "user2")); + } + + @Test + public void testNestedUserQueueSecondaryGroup() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='secondaryGroupExistingQueue'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + QueuePlacementPolicy policy = parse(sb.toString()); + // Should return root.default since secondary groups are not configured + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user1")); + + // configure secondary group for user1 + configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1"); + policy = parse(sb.toString()); + // user queue created should be created under secondary group + assertEquals("root.user1subgroup1.user1", + policy.assignAppToQueue("root.default", "user1")); + } + + @Test + public void testNestedUserQueueSpecificRule() throws Exception { + // This test covers the use case where users can specify different parent + // queues and want user queues under those. + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='specified' create='false'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + // Let's create couple of parent queues + configuredQueues.get(FSQueueType.PARENT).add("root.parent1"); + configuredQueues.get(FSQueueType.PARENT).add("root.parent2"); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.parent1.user1", + policy.assignAppToQueue("root.parent1", "user1")); + assertEquals("root.parent2.user2", + policy.assignAppToQueue("root.parent2", "user2")); + } + private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. - DocumentBuilderFactory docBuilderFactory = - DocumentBuilderFactory.newInstance(); + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory + .newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(IOUtils.toInputStream(str)); Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Mon May 12 12:43:59 2014 @@ -37,24 +37,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Mon May 12 12:43:59 2014 @@ -43,24 +43,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1593948&r1=1593947&r2=1593948&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon May 12 12:43:59 2014 @@ -206,8 +206,10 @@ Allocation file format The allocation file must be in XML format. The format contains five types of elements: - * <<Queue elements>>, which represent queues. Each may contain the following - properties: + * <<Queue elements>>, which represent queues. Queue elements can take an optional + attribute âtypeâ,which when set to âparentâ makes it a parent queue. This is useful + when we want to create a parent queue without configuring any leaf queues. + Each queue element may contain the following properties: * minResources: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores @@ -299,6 +301,15 @@ Allocation file format that matches a secondary group of the user who submitted it. The first secondary group that matches a configured queue will be selected. + * nestedUserQueue : the app is placed into a queue with the name of the user + under the queue suggested by the nested rule. This is similar to âuserâ + rule,the difference being in ânestedUserQueueâ rule,user queues can be created + under any parent queue, while âuserâ rule creates user queues only under root queue. + Note that nestedUserQueue rule would be applied only if the nested rule returns a + parent queue.One can configure a parent queue either by setting âtypeâ attribute of queue + to âparentâ or by configuring at least one leaf under that queue which makes it a parent. + See example allocation for a sample use case. + * default: the app is placed into the queue named "default". * reject: the app is rejected. @@ -319,6 +330,12 @@ Allocation file format <minResources>5000 mb,0vcores</minResources> </queue> </queue> + + <!â- Queue âsecondary_group_queueâ is a parent queue and may have + user queues under it -â> + <queue name=âsecondary_group_queueâ type=âparentâ> + <weight>3.0</weight> + </queue> <user name="sample_user"> <maxRunningApps>30</maxRunningApps> @@ -328,6 +345,9 @@ Allocation file format <queuePlacementPolicy> <rule name="specified" /> <rule name="primaryGroup" create="false" /> + <rule name=ânestedUserQueueâ> + <rule name=âsecondaryGroupExistingQueueâ create=âfalseâ /> + </rule> <rule name="default" /> </queuePlacementPolicy> </allocations>
