Queue ACL validations should validate parent queue ACLs before auto-creating leaf queues. Contributed by Suma Shivaprasad.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bb83a8f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bb83a8f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bb83a8f Branch: refs/heads/HDFS-7240 Commit: 8bb83a8f625953e2a45db4bbbfb95cd41bac6af5 Parents: 5c87fb2 Author: Sunil G <[email protected]> Authored: Tue Dec 12 15:20:59 2017 +0530 Committer: Sunil G <[email protected]> Committed: Tue Dec 12 15:20:59 2017 +0530 ---------------------------------------------------------------------- .../server/resourcemanager/RMAppManager.java | 10 +- .../server/resourcemanager/TestAppManager.java | 112 ++++++++++++++++--- 2 files changed, 108 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bb83a8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3ad2c9a..b21fb73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -410,7 +410,15 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, String queueName = submissionContext.getQueue(); String appName = submissionContext.getApplicationName(); CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName); - if (null != csqueue + + if (csqueue == null && placementContext != null) { + //could be an auto created queue through queue mapping. Validate + // parent queue exists and has valid acls + String parentQueueName = placementContext.getParentQueue(); + csqueue = ((CapacityScheduler) scheduler).getQueue(parentQueueName); + } + + if (csqueue != null && !authorizer.checkPermission( new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS), http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bb83a8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index f56de0f..7c2ce64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.isA; +import static org.mockito.Matchers.matches; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -87,6 +88,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity + .CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity + .ManagedParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -279,24 +284,30 @@ public class TestAppManager{ YarnConfiguration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - conf.set(PREFIX + "root.queues", "default,test"); - conf.setFloat(PREFIX + "root.default.capacity", 50.0f); - conf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f); + CapacitySchedulerConfiguration csConf = new + CapacitySchedulerConfiguration(conf, false); + csConf.set(PREFIX + "root.queues", "default,test"); - conf.setFloat(PREFIX + "root.test.capacity", 50.0f); - conf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f); + csConf.setFloat(PREFIX + "root.default.capacity", 50.0f); + csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f); - conf.set(PREFIX + "root.acl_submit_applications", " "); - conf.set(PREFIX + "root.acl_administer_queue", " "); + csConf.setFloat(PREFIX + "root.test.capacity", 50.0f); + csConf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f); - conf.set(PREFIX + "root.default.acl_submit_applications", " "); - conf.set(PREFIX + "root.default.acl_administer_queue", " "); + csConf.set(PREFIX + "root.acl_submit_applications", " "); + csConf.set(PREFIX + "root.acl_administer_queue", " "); - conf.set(PREFIX + "root.test.acl_submit_applications", "test"); - conf.set(PREFIX + "root.test.acl_administer_queue", "test"); + csConf.set(PREFIX + "root.default.acl_submit_applications", " "); + csConf.set(PREFIX + "root.default.acl_administer_queue", " "); - conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); + csConf.set(PREFIX + "root.test.acl_submit_applications", "test"); + csConf.set(PREFIX + "root.test.acl_administer_queue", "test"); + + csConf.set(PREFIX + "root.test.acl_submit_applications", "test"); + csConf.set(PREFIX + "root.test.acl_administer_queue", "test"); + + csConf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); // Setup a PlacementManager returns a new queue PlacementManager placementMgr = mock(PlacementManager.class); @@ -309,7 +320,7 @@ public class TestAppManager{ } }).when(placementMgr).placeApplication( - any(ApplicationSubmissionContext.class), any(String.class)); + any(ApplicationSubmissionContext.class), matches("test")); asContext.setQueue("oldQueue"); @@ -331,12 +342,87 @@ public class TestAppManager{ try { //should fail since user does not have permission to submit to queue // 'test' + asContext.setApplicationId(appId = MockApps.newAppID(2)); newAppMonitor.submitApplication(asContext, "test1"); } catch(YarnException e) { assertTrue(e.getCause() instanceof AccessControlException); } } + @Test + public void + testQueueSubmitWithACLsEnabledWithQueueMappingForAutoCreatedQueue() + throws IOException, YarnException, InterruptedException { + YarnConfiguration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true"); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration( + conf, false); + csConf.set(PREFIX + "root.queues", "default,managedparent"); + + csConf.setFloat(PREFIX + "root.default.capacity", 50.0f); + csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f); + + csConf.set(PREFIX + "root.acl_submit_applications", " "); + csConf.set(PREFIX + "root.acl_administer_queue", " "); + + csConf.set(PREFIX + "root.default.acl_submit_applications", " "); + csConf.set(PREFIX + "root.default.acl_administer_queue", " "); + + csConf.set(PREFIX + "root.managedparent.acl_administer_queue", "admin"); + csConf.set(PREFIX + "root.managedparent.acl_submit_applications", "user1"); + + csConf.setAutoCreateChildQueueEnabled("root.managedparent", true); + csConf.setAutoCreatedLeafQueueConfigCapacity("root.managedparent", 30f); + csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.managedparent", 100f); + + // Setup a PlacementManager returns a new queue + PlacementManager placementMgr = mock(PlacementManager.class); + doAnswer(new Answer<ApplicationPlacementContext>() { + + @Override + public ApplicationPlacementContext answer(InvocationOnMock invocation) + throws Throwable { + return new ApplicationPlacementContext("user1", "managedparent"); + } + + }).when(placementMgr).placeApplication( + any(ApplicationSubmissionContext.class), matches("user1|user2")); + + asContext.setQueue("oldQueue"); + + MockRM newMockRM = new MockRM(conf); + CapacityScheduler cs = + ((CapacityScheduler) newMockRM.getResourceScheduler()); + ManagedParentQueue managedParentQueue = new ManagedParentQueue(cs, + "managedparent", cs.getQueue("root"), null); + cs.getCapacitySchedulerQueueManager().addQueue("managedparent", + managedParentQueue); + + RMContext newMockRMContext = newMockRM.getRMContext(); + newMockRMContext.setQueuePlacementManager(placementMgr); + ApplicationMasterService masterService = new ApplicationMasterService( + newMockRMContext, newMockRMContext.getScheduler()); + + TestRMAppManager newAppMonitor = new TestRMAppManager(newMockRMContext, + new ClientToAMTokenSecretManagerInRM(), newMockRMContext.getScheduler(), + masterService, new ApplicationACLsManager(conf), conf); + + //only user test has permission to submit to 'test' queue + newAppMonitor.submitApplication(asContext, "user1"); + + try { + //should fail since user does not have permission to submit to queue + // 'test' + asContext.setApplicationId(appId = MockApps.newAppID(2)); + newAppMonitor.submitApplication(asContext, "user2"); + } catch (YarnException e) { + assertTrue(e.getCause() instanceof AccessControlException); + } + } + @After public void tearDown() { setAppEventType(RMAppEventType.KILL); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
