Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java Tue Aug 19 23:49:39 2014 @@ -51,6 +51,8 @@ public class TestFairSchedulerEventLog { resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); } @@ -69,7 +71,13 @@ public class TestFairSchedulerEventLog { public void tearDown() { logFile.delete(); logFile.getParentFile().delete(); // fairscheduler/ - scheduler = null; - resourceManager = null; + if (scheduler != null) { + scheduler.stop(); + scheduler = null; + } + if (resourceManager != null) { + resourceManager.stop(); + resourceManager = null; + } } }
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Tue Aug 19 23:49:39 2014 @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.junit.Before; @@ -40,12 +41,14 @@ public class TestMaxRunningAppsEnforcer private MaxRunningAppsEnforcer maxAppsEnforcer; private int appNum; private TestFairScheduler.MockClock clock; + private RMContext rmContext; + private FairScheduler scheduler; @Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); - FairScheduler scheduler = mock(FairScheduler.class); + scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); @@ -59,13 +62,16 @@ public class TestMaxRunningAppsEnforcer userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; + rmContext = mock(RMContext.class); + when(rmContext.getEpoch()).thenReturn(0); } - private FSSchedulerApp addApp(FSLeafQueue queue, String user) { + private FSAppAttempt addApp(FSLeafQueue queue, String user) { ApplicationId appId = ApplicationId.newInstance(0l, appNum++); ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0); boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user); - FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null, null); + FSAppAttempt app = new FSAppAttempt(scheduler, attId, user, queue, null, + rmContext); queue.addApp(app, runnable); if (runnable) { maxAppsEnforcer.trackRunnableApp(app); @@ -75,7 +81,7 @@ public class TestMaxRunningAppsEnforcer return app; } - private void removeApp(FSSchedulerApp app) { + private void removeApp(FSAppAttempt app) { app.getQueue().removeApp(app); maxAppsEnforcer.untrackRunnableApp(app); maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue()); @@ -88,7 +94,7 @@ public class TestMaxRunningAppsEnforcer queueMaxApps.put("root", 2); queueMaxApps.put("root.queue1", 1); queueMaxApps.put("root.queue2", 1); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); @@ -105,7 +111,7 @@ public class TestMaxRunningAppsEnforcer FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); assertEquals(1, leaf1.getRunnableAppSchedulables().size()); @@ -123,7 +129,7 @@ public class TestMaxRunningAppsEnforcer FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true); queueMaxApps.put("root.queue1.leaf1", 2); userMaxApps.put("user1", 1); - FSSchedulerApp app1 = addApp(leaf1, "user1"); + FSAppAttempt app1 = addApp(leaf1, "user1"); addApp(leaf1, "user2"); addApp(leaf1, "user3"); addApp(leaf2, "user1"); @@ -142,7 +148,7 @@ public class TestMaxRunningAppsEnforcer FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); clock.tick(20); @@ -162,7 +168,7 @@ public class TestMaxRunningAppsEnforcer FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true); FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true); queueMaxApps.put("root.queue1", 2); - FSSchedulerApp app1 = addApp(leaf1, "user"); + FSAppAttempt app1 = addApp(leaf1, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); addApp(leaf2, "user"); @@ -177,21 +183,18 @@ public class TestMaxRunningAppsEnforcer @Test public void testMultiListStartTimeIteratorEmptyAppLists() { - List<List<AppSchedulable>> lists = new ArrayList<List<AppSchedulable>>(); - lists.add(Arrays.asList(mockAppSched(1))); - lists.add(Arrays.asList(mockAppSched(2))); - Iterator<FSSchedulerApp> iter = + List<List<FSAppAttempt>> lists = new ArrayList<List<FSAppAttempt>>(); + lists.add(Arrays.asList(mockAppAttempt(1))); + lists.add(Arrays.asList(mockAppAttempt(2))); + Iterator<FSAppAttempt> iter = new MaxRunningAppsEnforcer.MultiListStartTimeIterator(lists); - assertEquals(1, iter.next().getAppSchedulable().getStartTime()); - assertEquals(2, iter.next().getAppSchedulable().getStartTime()); + assertEquals(1, iter.next().getStartTime()); + assertEquals(2, iter.next().getStartTime()); } - private AppSchedulable mockAppSched(long startTime) { - AppSchedulable appSched = mock(AppSchedulable.class); - when(appSched.getStartTime()).thenReturn(startTime); - FSSchedulerApp schedApp = mock(FSSchedulerApp.class); - when(schedApp.getAppSchedulable()).thenReturn(appSched); - when(appSched.getApp()).thenReturn(schedApp); - return appSched; + private FSAppAttempt mockAppAttempt(long startTime) { + FSAppAttempt schedApp = mock(FSAppAttempt.class); + when(schedApp.getStartTime()).thenReturn(startTime); + return schedApp; } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java Tue Aug 19 23:49:39 2014 @@ -17,8 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; import static org.mockito.Mockito.*; import java.util.HashSet; @@ -57,45 +56,77 @@ public class TestQueueManager { @Test public void testReloadTurnsLeafQueueIntoParent() throws Exception { - updateConfiguredQueues(queueManager, "queue1"); + updateConfiguredLeafQueues(queueManager, "queue1"); // When no apps are running in the leaf queue, should be fine turning it // into a parent. - updateConfiguredQueues(queueManager, "queue1.queue2"); + updateConfiguredLeafQueues(queueManager, "queue1.queue2"); assertNull(queueManager.getLeafQueue("queue1", false)); assertNotNull(queueManager.getLeafQueue("queue1.queue2", false)); // When leaf queues are empty, should be ok deleting them and // turning parent into a leaf. - updateConfiguredQueues(queueManager, "queue1"); + updateConfiguredLeafQueues(queueManager, "queue1"); assertNull(queueManager.getLeafQueue("queue1.queue2", false)); assertNotNull(queueManager.getLeafQueue("queue1", false)); // When apps exist in leaf queue, we shouldn't be able to create // children under it, but things should work otherwise. notEmptyQueues.add(queueManager.getLeafQueue("queue1", false)); - updateConfiguredQueues(queueManager, "queue1.queue2"); + updateConfiguredLeafQueues(queueManager, "queue1.queue2"); assertNull(queueManager.getLeafQueue("queue1.queue2", false)); assertNotNull(queueManager.getLeafQueue("queue1", false)); // When apps exist in leaf queues under a parent queue, shouldn't be // able to turn it into a leaf queue, but things should work otherwise. notEmptyQueues.clear(); - updateConfiguredQueues(queueManager, "queue1.queue2"); + updateConfiguredLeafQueues(queueManager, "queue1.queue2"); notEmptyQueues.add(queueManager.getQueue("root.queue1")); - updateConfiguredQueues(queueManager, "queue1"); + updateConfiguredLeafQueues(queueManager, "queue1"); assertNotNull(queueManager.getLeafQueue("queue1.queue2", false)); assertNull(queueManager.getLeafQueue("queue1", false)); // Should never to be able to create a queue under the default queue - updateConfiguredQueues(queueManager, "default.queue3"); + updateConfiguredLeafQueues(queueManager, "default.queue3"); assertNull(queueManager.getLeafQueue("default.queue3", false)); assertNotNull(queueManager.getLeafQueue("default", false)); } - private void updateConfiguredQueues(QueueManager queueMgr, String... confQueues) { + @Test + public void testReloadTurnsLeafToParentWithNoLeaf() { + AllocationConfiguration allocConf = new AllocationConfiguration(conf); + // Create a leaf queue1 + allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1"); + queueManager.updateAllocationConfiguration(allocConf); + assertNotNull(queueManager.getLeafQueue("root.queue1", false)); + + // Lets say later on admin makes queue1 a parent queue by + // specifying "type=parent" in the alloc xml and lets say apps running in + // queue1 + notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false)); + allocConf = new AllocationConfiguration(conf); + allocConf.configuredQueues.get(FSQueueType.PARENT) + .add("root.queue1"); + + // When allocs are reloaded queue1 shouldn't be converter to parent + queueManager.updateAllocationConfiguration(allocConf); + assertNotNull(queueManager.getLeafQueue("root.queue1", false)); + assertNull(queueManager.getParentQueue("root.queue1", false)); + + // Now lets assume apps completed and there are no apps in queue1 + notEmptyQueues.clear(); + // We should see queue1 transform from leaf queue to parent queue. + queueManager.updateAllocationConfiguration(allocConf); + assertNull(queueManager.getLeafQueue("root.queue1", false)); + assertNotNull(queueManager.getParentQueue("root.queue1", false)); + // this parent should not have any children + assertTrue(queueManager.getParentQueue("root.queue1", false) + .getChildQueues().isEmpty()); + } + + private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) { AllocationConfiguration allocConf = new AllocationConfiguration(conf); - allocConf.queueNames = Sets.newHashSet(confQueues); + allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues)); queueMgr.updateAllocationConfiguration(allocConf); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java Tue Aug 19 23:49:39 2014 @@ -17,8 +17,11 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Set; import javax.xml.parsers.DocumentBuilder; @@ -28,16 +31,15 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.GroupMappingServiceProvider; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; -import com.google.common.collect.Sets; - public class TestQueuePlacementPolicy { private final static Configuration conf = new Configuration(); - private final static Set<String> configuredQueues = Sets.newHashSet("root.someuser"); + private Map<FSQueueType, Set<String>> configuredQueues; @BeforeClass public static void setup() { @@ -45,6 +47,14 @@ public class TestQueuePlacementPolicy { SimpleGroupsMapping.class, GroupMappingServiceProvider.class); } + @Before + public void initTest() { + configuredQueues = new HashMap<FSQueueType, Set<String>>(); + for (FSQueueType type : FSQueueType.values()) { + configuredQueues.put(type, new HashSet<String>()); + } + } + @Test public void testSpecifiedUserPolicy() throws Exception { StringBuffer sb = new StringBuffer(); @@ -53,9 +63,12 @@ public class TestQueuePlacementPolicy { sb.append(" <rule name='user' />"); sb.append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq",policy.assignAppToQueue("specifiedq", "someuser")); - assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); - assertEquals("root.otheruser", policy.assignAppToQueue("default", "otheruser")); + assertEquals("root.specifiedq", + policy.assignAppToQueue("specifiedq", "someuser")); + assertEquals("root.someuser", + policy.assignAppToQueue("default", "someuser")); + assertEquals("root.otheruser", + policy.assignAppToQueue("default", "otheruser")); } @Test @@ -66,6 +79,8 @@ public class TestQueuePlacementPolicy { sb.append(" <rule name='user' create=\"false\" />"); sb.append(" <rule name='default' />"); sb.append("</queuePlacementPolicy>"); + + configuredQueues.get(FSQueueType.LEAF).add("root.someuser"); QueuePlacementPolicy policy = parse(sb.toString()); assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser")); assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser")); @@ -81,7 +96,8 @@ public class TestQueuePlacementPolicy { sb.append(" <rule name='reject' />"); sb.append("</queuePlacementPolicy>"); QueuePlacementPolicy policy = parse(sb.toString()); - assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser")); + assertEquals("root.specifiedq", + policy.assignAppToQueue("specifiedq", "someuser")); assertEquals(null, policy.assignAppToQueue("default", "someuser")); } @@ -117,10 +133,223 @@ public class TestQueuePlacementPolicy { parse(sb.toString()); } + @Test + public void testDefaultRuleWithQueueAttribute() throws Exception { + // This test covers the use case where we would like default rule + // to point to a different queue by default rather than root.default + configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue"); + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' create='false' />"); + sb.append(" <rule name='default' queue='root.someDefaultQueue'/>"); + sb.append("</queuePlacementPolicy>"); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.someDefaultQueue", + policy.assignAppToQueue("root.default", "user1")); + } + + @Test + public void testNestedUserQueueParsingErrors() { + // No nested rule specified in hierarchical user queue + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' />"); + sb.append(" <rule name='nestedUserQueue'/>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + assertIfExceptionThrown(sb); + + // Specified nested rule is not a QueuePlacementRule + sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='unknownRule'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + assertIfExceptionThrown(sb); + } + + private void assertIfExceptionThrown(StringBuffer sb) { + Throwable th = null; + try { + parse(sb.toString()); + } catch (Exception e) { + th = e; + } + + assertTrue(th instanceof AllocationConfigurationException); + } + + @Test + public void testNestedUserQueueParsing() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='primaryGroup'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + Throwable th = null; + try { + parse(sb.toString()); + } catch (Exception e) { + th = e; + } + + assertNull(th); + } + + @Test + public void testNestedUserQueuePrimaryGroup() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' create='false' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='primaryGroup'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + // User queue would be created under primary group queue + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.user1group.user1", + policy.assignAppToQueue("root.default", "user1")); + // Other rules above and below hierarchical user queue rule should work as + // usual + configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq"); + // test if specified rule(above nestedUserQueue rule) works ok + assertEquals("root.specifiedq", + policy.assignAppToQueue("root.specifiedq", "user2")); + + // test if default rule(below nestedUserQueue rule) works + configuredQueues.get(FSQueueType.LEAF).add("root.user3group"); + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user3")); + } + + @Test + public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception { + // Primary group rule has create='false' + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='primaryGroup' create='false'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + QueuePlacementPolicy policy = parse(sb.toString()); + + // Should return root.default since primary group 'root.user1group' is not + // configured + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user1")); + + // Let's configure primary group and check if user queue is created + configuredQueues.get(FSQueueType.PARENT).add("root.user1group"); + policy = parse(sb.toString()); + assertEquals("root.user1group.user1", + policy.assignAppToQueue("root.default", "user1")); + + // Both Primary group and nestedUserQueue rule has create='false' + sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue' create='false'>"); + sb.append(" <rule name='primaryGroup' create='false'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + // Should return root.default since primary group and user queue for user 2 + // are not configured. + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user2")); + + // Now configure both primary group and the user queue for user2 + configuredQueues.get(FSQueueType.PARENT).add("root.user2group"); + configuredQueues.get(FSQueueType.LEAF).add("root.user2group.user2"); + policy = parse(sb.toString()); + + assertEquals("root.user2group.user2", + policy.assignAppToQueue("root.default", "user2")); + } + + @Test + public void testNestedUserQueueSecondaryGroup() throws Exception { + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='secondaryGroupExistingQueue'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + QueuePlacementPolicy policy = parse(sb.toString()); + // Should return root.default since secondary groups are not configured + assertEquals("root.default", + policy.assignAppToQueue("root.default", "user1")); + + // configure secondary group for user1 + configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1"); + policy = parse(sb.toString()); + // user queue created should be created under secondary group + assertEquals("root.user1subgroup1.user1", + policy.assignAppToQueue("root.default", "user1")); + } + + @Test + public void testNestedUserQueueSpecificRule() throws Exception { + // This test covers the use case where users can specify different parent + // queues and want user queues under those. + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='specified' create='false'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + // Let's create couple of parent queues + configuredQueues.get(FSQueueType.PARENT).add("root.parent1"); + configuredQueues.get(FSQueueType.PARENT).add("root.parent2"); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.parent1.user1", + policy.assignAppToQueue("root.parent1", "user1")); + assertEquals("root.parent2.user2", + policy.assignAppToQueue("root.parent2", "user2")); + } + + @Test + public void testNestedUserQueueDefaultRule() throws Exception { + // This test covers the use case where we would like user queues to be + // created under a default parent queue + configuredQueues.get(FSQueueType.PARENT).add("root.parentq"); + StringBuffer sb = new StringBuffer(); + sb.append("<queuePlacementPolicy>"); + sb.append(" <rule name='specified' create='false' />"); + sb.append(" <rule name='nestedUserQueue'>"); + sb.append(" <rule name='default' queue='root.parentq'/>"); + sb.append(" </rule>"); + sb.append(" <rule name='default' />"); + sb.append("</queuePlacementPolicy>"); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.parentq.user1", + policy.assignAppToQueue("root.default", "user1")); + } + private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. - DocumentBuilderFactory docBuilderFactory = - DocumentBuilderFactory.newInstance(); + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory + .newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(IOUtils.toInputStream(str)); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Tue Aug 19 23:49:39 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.re import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; - import static org.mockito.Mockito.mock; import java.io.IOException; @@ -30,8 +29,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -60,13 +57,13 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -78,6 +75,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -147,9 +145,13 @@ public class TestFifoScheduler { RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, writer); - FifoScheduler schedular = new FifoScheduler(); - schedular.reinitialize(new Configuration(), rmContext); - QueueMetrics metrics = schedular.getRootQueueMetrics(); + FifoScheduler scheduler = new FifoScheduler(); + Configuration conf = new Configuration(); + scheduler.setRMContext(rmContext); + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, rmContext); + QueueMetrics metrics = scheduler.getRootQueueMetrics(); int beforeAppsSubmitted = metrics.getAppsSubmitted(); ApplicationId appId = BuilderUtils.newApplicationId(200, 1); @@ -157,18 +159,19 @@ public class TestFifoScheduler { appId, 1); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user"); - schedular.handle(appEvent); + scheduler.handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false); - schedular.handle(attemptEvent); + scheduler.handle(attemptEvent); appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false); - schedular.handle(attemptEvent2); + scheduler.handle(attemptEvent2); int afterAppsSubmitted = metrics.getAppsSubmitted(); Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted); + scheduler.stop(); } @Test(timeout=2000) @@ -186,6 +189,9 @@ public class TestFifoScheduler { null, containerTokenSecretManager, nmTokenSecretManager, null, writer); FifoScheduler scheduler = new FifoScheduler(); + scheduler.setRMContext(rmContext); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.newNodeInfo(1, @@ -234,6 +240,7 @@ public class TestFifoScheduler { //Also check that the containers were scheduled SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId); Assert.assertEquals(3, info.getLiveContainers().size()); + scheduler.stop(); } @Test(timeout=2000) @@ -256,6 +263,9 @@ public class TestFifoScheduler { return nodes; } }; + scheduler.setRMContext(rmContext); + scheduler.init(conf); + scheduler.start(); scheduler.reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1"); @@ -594,9 +604,12 @@ public class TestFifoScheduler { public void testAddAndRemoveAppFromFiFoScheduler() throws Exception { Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); MockRM rm = new MockRM(conf); - FifoScheduler fs = (FifoScheduler)rm.getResourceScheduler(); + @SuppressWarnings("unchecked") + AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs = + (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm + .getResourceScheduler(); TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( fs.getSchedulerApplications(), fs, "queue"); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java Tue Aug 19 23:49:39 2014 @@ -18,30 +18,34 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; +import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Collection; - -import javax.crypto.SecretKey; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; @@ -50,7 +54,9 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; +import org.apache.hadoop.yarn.server.security.MasterKeyData; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -65,6 +71,8 @@ public class TestAMRMTokens { private final Configuration conf; private static final int maxWaitAttempts = 50; + private static final int rolling_interval_sec = 13; + private static final long am_expire_ms = 4000; @Parameters public static Collection<Object[]> configs() { @@ -180,8 +188,8 @@ public class TestAMRMTokens { // The exception will still have the earlier appAttemptId as it picks it // up from the token. Assert.assertTrue(t.getCause().getMessage().contains( - "Password not found for ApplicationAttempt " + - applicationAttemptId.toString())); + applicationAttemptId.toString() + + " not found in AMRMTokenSecretManager.")); } } finally { @@ -201,15 +209,22 @@ public class TestAMRMTokens { @Test public void testMasterKeyRollOver() throws Exception { + conf.setLong( + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + rolling_interval_sec); + conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); MyContainerManager containerManager = new MyContainerManager(); final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager); rm.start(); - + Long startTime = System.currentTimeMillis(); final Configuration conf = rm.getConfig(); final YarnRPC rpc = YarnRPC.create(conf); ApplicationMasterProtocol rmClient = null; - + AMRMTokenSecretManager appTokenSecretManager = + rm.getRMContext().getAMRMTokenSecretManager(); + MasterKeyData oldKey = appTokenSecretManager.getMasterKey(); + Assert.assertNotNull(oldKey); try { MockNM nm1 = rm.registerNode("localhost:1234", 5120); @@ -218,7 +233,7 @@ public class TestAMRMTokens { nm1.nodeHeartbeat(true); int waitCount = 0; - while (containerManager.containerTokens == null && waitCount++ < 20) { + while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) { LOG.info("Waiting for AM Launch to happen.."); Thread.sleep(1000); } @@ -250,21 +265,65 @@ public class TestAMRMTokens { Assert.assertTrue( rmClient.allocate(allocateRequest).getAMCommand() == null); - // Simulate a master-key-roll-over - AMRMTokenSecretManager appTokenSecretManager = - rm.getRMContext().getAMRMTokenSecretManager(); - SecretKey oldKey = appTokenSecretManager.getMasterKey(); - appTokenSecretManager.rollMasterKey(); - SecretKey newKey = appTokenSecretManager.getMasterKey(); + // Wait for enough time and make sure the roll_over happens + // At mean time, the old AMRMToken should continue to work + while(System.currentTimeMillis() - startTime < rolling_interval_sec*1000) { + rmClient.allocate(allocateRequest); + Thread.sleep(500); + } + + MasterKeyData newKey = appTokenSecretManager.getMasterKey(); + Assert.assertNotNull(newKey); Assert.assertFalse("Master key should have changed!", oldKey.equals(newKey)); + // Another allocate call with old AMRMToken. Should continue to work. + rpc.stopProxy(rmClient, conf); // To avoid using cached client + rmClient = createRMClient(rm, conf, rpc, currentUser); + Assert + .assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null); + + waitCount = 0; + while(waitCount++ <= maxWaitAttempts) { + if (appTokenSecretManager.getCurrnetMasterKeyData() != oldKey) { + break; + } + try { + rmClient.allocate(allocateRequest); + } catch (Exception ex) { + break; + } + Thread.sleep(200); + } + // active the nextMasterKey, and replace the currentMasterKey + Assert.assertTrue(appTokenSecretManager.getCurrnetMasterKeyData().equals(newKey)); + Assert.assertTrue(appTokenSecretManager.getMasterKey().equals(newKey)); + Assert.assertTrue(appTokenSecretManager.getNextMasterKeyData() == null); + + // Create a new Token + Token<AMRMTokenIdentifier> newToken = + appTokenSecretManager.createAndGetAMRMToken(applicationAttemptId); + SecurityUtil.setTokenService(newToken, rmBindAddress); + currentUser.addToken(newToken); // Another allocate call. Should continue to work. rpc.stopProxy(rmClient, conf); // To avoid using cached client rmClient = createRMClient(rm, conf, rpc, currentUser); allocateRequest = Records.newRecord(AllocateRequest.class); - Assert.assertTrue( - rmClient.allocate(allocateRequest).getAMCommand() == null); + Assert + .assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null); + + // Should not work by using the old AMRMToken. + rpc.stopProxy(rmClient, conf); // To avoid using cached client + try { + currentUser.addToken(amRMToken); + rmClient = createRMClient(rm, conf, rpc, currentUser); + allocateRequest = Records.newRecord(AllocateRequest.class); + Assert + .assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null); + Assert.fail("The old Token should not work"); + } catch (Exception ex) { + // expect exception + } } finally { rm.stop(); if (rmClient != null) { @@ -273,6 +332,51 @@ public class TestAMRMTokens { } } + @Test (timeout = 20000) + public void testAMRMMasterKeysUpdate() throws Exception { + MockRM rm = new MockRM(conf) { + @Override + protected void doSecureLogin() throws IOException { + // Skip the login. + } + }; + rm.start(); + MockNM nm = rm.registerNode("127.0.0.1:1234", 8000); + RMApp app = rm.submitApp(200); + MockAM am = MockRM.launchAndRegisterAM(app, rm, nm); + + // Do allocate. Should not update AMRMToken + AllocateResponse response = + am.allocate(Records.newRecord(AllocateRequest.class)); + Assert.assertNull(response.getAMRMToken()); + + // roll over the master key + // Do allocate again. the AM should get the latest AMRMToken + rm.getRMContext().getAMRMTokenSecretManager().rollMasterKey(); + response = am.allocate(Records.newRecord(AllocateRequest.class)); + Assert.assertNotNull(response.getAMRMToken()); + + Token<AMRMTokenIdentifier> amrmToken = + ConverterUtils.convertFromYarn(response.getAMRMToken(), new Text( + response.getAMRMToken().getService())); + + Assert.assertEquals(amrmToken.decodeIdentifier().getKeyId(), rm + .getRMContext().getAMRMTokenSecretManager().getMasterKey().getMasterKey() + .getKeyId()); + + // Do allocate again. The master key does not update. + // AM should not update its AMRMToken either + response = am.allocate(Records.newRecord(AllocateRequest.class)); + Assert.assertNull(response.getAMRMToken()); + + // Activate the next master key. Since there is new master key generated + // in AMRMTokenSecretManager. The AMRMToken will not get updated for AM + rm.getRMContext().getAMRMTokenSecretManager().activateNextMasterKey(); + response = am.allocate(Records.newRecord(AllocateRequest.class)); + Assert.assertNull(response.getAMRMToken()); + rm.stop(); + } + private ApplicationMasterProtocol createRMClient(final MockRM rm, final Configuration conf, final YarnRPC rpc, UserGroupInformation currentUser) { Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java Tue Aug 19 23:49:39 2014 @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -673,7 +674,40 @@ public class TestDelegationTokenRenewer Thread.sleep(200); } } - + + @Test(timeout=20000) + public void testDTRonAppSubmission() + throws IOException, InterruptedException, BrokenBarrierException { + final Credentials credsx = new Credentials(); + final Token<?> tokenx = mock(Token.class); + credsx.addToken(new Text("token"), tokenx); + doReturn(true).when(tokenx).isManaged(); + doThrow(new IOException("boom")) + .when(tokenx).renew(any(Configuration.class)); + // fire up the renewer + final DelegationTokenRenewer dtr = + createNewDelegationTokenRenewer(conf, counter); + RMContext mockContext = mock(RMContext.class); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + dtr.setRMContext(mockContext); + when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); + dtr.init(conf); + dtr.start(); + + try { + dtr.addApplicationSync(mock(ApplicationId.class), credsx, false); + fail("Catch IOException on app submission"); + } catch (IOException e){ + Assert.assertTrue(e.getMessage().contains(tokenx.toString())); + Assert.assertTrue(e.getCause().toString().contains("boom")); + } + + } + @Test(timeout=20000) public void testConcurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException { Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java Tue Aug 19 23:49:39 2014 @@ -48,8 +48,8 @@ public class TestNodesPage { // Number of Actual Table Headers for NodesPage.NodesBlock might change in // future. In that case this value should be adjusted to the new value. - final int numberOfThInMetricsTable = 13; - final int numberOfActualTableHeaders = 10; + final int numberOfThInMetricsTable = 16; + final int numberOfActualTableHeaders = 12; private Injector injector; Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Tue Aug 19 23:49:39 2014 @@ -203,10 +203,11 @@ public class TestRMWebApp { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + cs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); + cs.init(conf); return cs; } @@ -269,19 +270,21 @@ public class TestRMWebApp { ResourceManager rm = mock(ResourceManager.class); RMContext rmContext = mockRMContext(apps, racks, nodes, mbsPerNode); - ResourceScheduler rs = mockFifoScheduler(); + ResourceScheduler rs = mockFifoScheduler(rmContext); when(rm.getResourceScheduler()).thenReturn(rs); when(rm.getRMContext()).thenReturn(rmContext); return rm; } - public static FifoScheduler mockFifoScheduler() throws Exception { + public static FifoScheduler mockFifoScheduler(RMContext rmContext) + throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupFifoQueueConfiguration(conf); FifoScheduler fs = new FifoScheduler(); fs.setConf(new YarnConfiguration()); - fs.reinitialize(conf, null); + fs.setRMContext(rmContext); + fs.init(conf); return fs; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Tue Aug 19 23:49:39 2014 @@ -389,6 +389,10 @@ public class TestRMWebServices extends J WebServicesTestUtils.getXmlInt(element, "reservedMB"), WebServicesTestUtils.getXmlInt(element, "availableMB"), WebServicesTestUtils.getXmlInt(element, "allocatedMB"), + WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"), WebServicesTestUtils.getXmlInt(element, "containersAllocated"), WebServicesTestUtils.getXmlInt(element, "totalMB"), WebServicesTestUtils.getXmlInt(element, "totalNodes"), @@ -404,11 +408,13 @@ public class TestRMWebServices extends J Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 19, clusterinfo.length()); + assertEquals("incorrect number of elements", 23, clusterinfo.length()); verifyClusterMetrics( clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), clusterinfo.getInt("allocatedMB"), + clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"), + clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"), clusterinfo.getInt("containersAllocated"), clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"), clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"), @@ -418,7 +424,9 @@ public class TestRMWebServices extends J public void verifyClusterMetrics(int submittedApps, int completedApps, int reservedMB, int availableMB, - int allocMB, int containersAlloc, int totalMB, int totalNodes, + int allocMB, int reservedVirtualCores, int availableVirtualCores, + int allocVirtualCores, int totalVirtualCores, + int containersAlloc, int totalMB, int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int rebootedNodes, int activeNodes) throws JSONException, Exception { @@ -428,7 +436,8 @@ public class TestRMWebServices extends J long totalMBExpect = metrics.getAvailableMB() + metrics.getAllocatedMB(); - + long totalVirtualCoresExpect = + metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores(); assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), submittedApps); assertEquals("appsCompleted doesn't match", @@ -439,6 +448,12 @@ public class TestRMWebServices extends J metrics.getAvailableMB(), availableMB); assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); + assertEquals("reservedVirtualCores doesn't match", + metrics.getReservedVirtualCores(), reservedVirtualCores); + assertEquals("availableVirtualCores doesn't match", + metrics.getAvailableVirtualCores(), availableVirtualCores); + assertEquals("allocatedVirtualCores doesn't match", + totalVirtualCoresExpect, allocVirtualCores); assertEquals("containersAllocated doesn't match", 0, containersAlloc); assertEquals("totalMB doesn't match", totalMBExpect, totalMB); assertEquals( Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Tue Aug 19 23:49:39 2014 @@ -1310,33 +1310,44 @@ public class TestRMWebServicesApps exten WebServicesTestUtils.getXmlString(element, "amContainerLogs"), WebServicesTestUtils.getXmlInt(element, "allocatedMB"), WebServicesTestUtils.getXmlInt(element, "allocatedVCores"), - WebServicesTestUtils.getXmlInt(element, "runningContainers")); + WebServicesTestUtils.getXmlInt(element, "runningContainers"), + WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"), + WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"), + WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"), + WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted")); } } public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - // 20 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 20, info.length()); + // 28 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 24, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), - info.getString("name"), info.getString("applicationType"), info.getString("queue"), - info.getString("state"), info.getString("finalStatus"), - (float) info.getDouble("progress"), info.getString("trackingUI"), - info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), - info.getString("amContainerLogs"), info.getInt("allocatedMB"), - info.getInt("allocatedVCores"), info.getInt("runningContainers")); + info.getString("name"), info.getString("applicationType"), + info.getString("queue"), info.getString("state"), + info.getString("finalStatus"), (float) info.getDouble("progress"), + info.getString("trackingUI"), info.getString("diagnostics"), + info.getLong("clusterId"), info.getLong("startedTime"), + info.getLong("finishedTime"), info.getLong("elapsedTime"), + info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), + info.getInt("allocatedMB"), info.getInt("allocatedVCores"), + info.getInt("runningContainers"), + info.getInt("preemptedResourceMB"), + info.getInt("preemptedResourceVCores"), + info.getInt("numNonAMContainerPreempted"), + info.getInt("numAMContainerPreempted")); } public void verifyAppInfoGeneric(RMApp app, String id, String user, - String name, String applicationType, String queue, String state, String finalStatus, - float progress, String trackingUI, String diagnostics, long clusterId, - long startedTime, long finishedTime, long elapsedTime, - String amHostHttpAddress, String amContainerLogs, int allocatedMB, - int allocatedVCores, int numContainers) throws JSONException, + String name, String applicationType, String queue, String state, + String finalStatus, float progress, String trackingUI, + String diagnostics, long clusterId, long startedTime, long finishedTime, + long elapsedTime, String amHostHttpAddress, String amContainerLogs, + int allocatedMB, int allocatedVCores, int numContainers, + int preemptedResourceMB, int preemptedResourceVCores, + int numNonAMContainerPreempted, int numAMContainerPreempted) throws JSONException, Exception { WebServicesTestUtils.checkStringMatch("id", app.getApplicationId() @@ -1371,6 +1382,18 @@ public class TestRMWebServicesApps exten assertEquals("allocatedMB doesn't match", 1024, allocatedMB); assertEquals("allocatedVCores doesn't match", 1, allocatedVCores); assertEquals("numContainers doesn't match", 1, numContainers); + assertEquals("preemptedResourceMB doesn't match", app + .getRMAppMetrics().getResourcePreempted().getMemory(), + preemptedResourceMB); + assertEquals("preemptedResourceVCores doesn't match", app + .getRMAppMetrics().getResourcePreempted().getVirtualCores(), + preemptedResourceVCores); + assertEquals("numNonAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumNonAMContainersPreempted(), + numNonAMContainerPreempted); + assertEquals("numAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumAMContainersPreempted(), + numAMContainerPreempted); } @Test Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Tue Aug 19 23:49:39 2014 @@ -656,13 +656,15 @@ public class TestRMWebServicesNodes exte WebServicesTestUtils.getXmlInt(element, "numContainers"), WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"), WebServicesTestUtils.getXmlLong(element, "availMemoryMB"), + WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"), + WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"), WebServicesTestUtils.getXmlString(element, "version")); } } public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm) throws JSONException, Exception { - assertEquals("incorrect number of elements", 11, nodeInfo.length()); + assertEquals("incorrect number of elements", 13, nodeInfo.length()); verifyNodeInfoGeneric(nm, nodeInfo.getString("state"), nodeInfo.getString("rack"), @@ -671,6 +673,7 @@ public class TestRMWebServicesNodes exte nodeInfo.getLong("lastHealthUpdate"), nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"), nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"), + nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"), nodeInfo.getString("version")); } @@ -678,7 +681,8 @@ public class TestRMWebServicesNodes exte public void verifyNodeInfoGeneric(MockNM nm, String state, String rack, String id, String nodeHostName, String nodeHTTPAddress, long lastHealthUpdate, String healthReport, - int numContainers, long usedMemoryMB, long availMemoryMB, String version) + int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, + long availVirtualCores, String version) throws JSONException, Exception { RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); @@ -712,6 +716,10 @@ public class TestRMWebServicesNodes exte .getUsedResource().getMemory(), usedMemoryMB); assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report .getAvailableResource().getMemory(), availMemoryMB); + assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report + .getUsedResource().getVirtualCores(), usedVirtualCores); + assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report + .getAvailableResource().getVirtualCores(), availVirtualCores); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Tue Aug 19 23:49:39 2014 @@ -37,24 +37,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Tue Aug 19 23:49:39 2014 @@ -25,6 +25,7 @@ import java.net.UnknownHostException; import java.util.Collection; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -56,8 +57,7 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -69,6 +69,8 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; +import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -504,12 +506,6 @@ public class MiniYARNCluster extends Com String logDirsString = prepareDirs("log", numLogDirs); config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString); - File remoteLogDir = - new File(testWorkDir, MiniYARNCluster.this.getName() - + "-remoteLogDir-nm-" + index); - remoteLogDir.mkdir(); - config.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - remoteLogDir.getAbsolutePath()); // By default AM + 2 containers config.setInt(YarnConfiguration.NM_PMEM_MB, 4*1024); config.set(YarnConfiguration.NM_ADDRESS, @@ -658,12 +654,14 @@ public class MiniYARNCluster extends Com */ public boolean waitForNodeManagersToConnect(long timeout) throws YarnException, InterruptedException { - ResourceManager rm = getResourceManager(); GetClusterMetricsRequest req = GetClusterMetricsRequest.newInstance(); - for (int i = 0; i < timeout / 100; i++) { - if (nodeManagers.length == rm.getClientRMService().getClusterMetrics(req) - .getClusterMetrics().getNumNodeManagers()) { + ResourceManager rm = getResourceManager(); + if (rm == null) { + throw new YarnException("Can not find the active RM."); + } + else if (nodeManagers.length == rm.getClientRMService() + .getClusterMetrics(req).getClusterMetrics().getNumNodeManagers()) { return true; } Thread.sleep(100); @@ -723,6 +721,7 @@ public class MiniYARNCluster extends Com if (appHistoryServer != null) { appHistoryServer.stop(); } + AHSWebApp.resetInstance(); super.serviceStop(); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Tue Aug 19 23:49:39 2014 @@ -29,8 +29,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -52,6 +50,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; @@ -60,15 +59,18 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -200,8 +202,6 @@ public class TestContainerManagerSecurit ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId validAppAttemptId = ApplicationAttemptId.newInstance(appId, 1); - ApplicationAttemptId invalidAppAttemptId = - ApplicationAttemptId.newInstance(appId, 2); ContainerId validContainerId = ContainerId.newInstance(validAppAttemptId, 0); @@ -215,7 +215,11 @@ public class TestContainerManagerSecurit org.apache.hadoop.yarn.api.records.Token validContainerToken = containerTokenSecretManager.createContainerToken(validContainerId, - validNode, user, r); + validNode, user, r, Priority.newInstance(10), 1234); + ContainerTokenIdentifier identifier = + BuilderUtils.newContainerTokenIdentifier(validContainerToken); + Assert.assertEquals(Priority.newInstance(10), identifier.getPriority()); + Assert.assertEquals(1234, identifier.getCreationTime()); StringBuilder sb; // testInvalidNMToken ... creating NMToken using different secret manager. @@ -263,27 +267,14 @@ public class TestContainerManagerSecurit testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, invalidNMToken, true))); - // using appAttempt-2 token for launching container for appAttempt-1. - invalidNMToken = - nmTokenSecretManagerRM.createNMToken(invalidAppAttemptId, validNode, - user); - sb = new StringBuilder("\nNMToken for application attempt : "); - sb.append(invalidAppAttemptId.toString()) - .append(" was used for starting container with container token") - .append(" issued for application attempt : ") - .append(validAppAttemptId.toString()); - Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode, - validContainerToken, invalidNMToken, true).contains(sb.toString())); - // using correct tokens. nmtoken for app attempt should get saved. conf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, 4 * 60 * 1000); validContainerToken = containerTokenSecretManager.createContainerToken(validContainerId, - validNode, user, r); - - testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, - validNMToken, false); + validNode, user, r, Priority.newInstance(0), 0); + Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode, + validContainerToken, validNMToken, false).isEmpty()); Assert.assertTrue(nmTokenSecretManagerNM .isAppAttemptNMTokenKeyPresent(validAppAttemptId)); @@ -325,6 +316,18 @@ public class TestContainerManagerSecurit Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken, false).contains(sb.toString())); + // using appAttempt-1 NMtoken for launching container for appAttempt-2 should + // succeed. + ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(appId, 2); + Token attempt1NMToken = + nmTokenSecretManagerRM + .createNMToken(validAppAttemptId, validNode, user); + org.apache.hadoop.yarn.api.records.Token newContainerToken = + containerTokenSecretManager.createContainerToken( + ContainerId.newInstance(attempt2, 1), validNode, user, r, + Priority.newInstance(0), 0); + Assert.assertTrue(testStartContainer(rpc, attempt2, validNode, + newContainerToken, attempt1NMToken, false).isEmpty()); } private void waitForContainerToFinishOnNM(ContainerId containerId) { @@ -590,7 +593,7 @@ public class TestContainerManagerSecurit // Creating modified containerToken Token containerToken = tamperedContainerTokenSecretManager.createContainerToken(cId, nodeId, - user, r); + user, r, Priority.newInstance(0), 0); Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.create(conf); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Tue Aug 19 23:49:39 2014 @@ -43,24 +43,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java Tue Aug 19 23:49:39 2014 @@ -25,9 +25,12 @@ import org.apache.hadoop.service.Service import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.net.InetSocketAddress; + public class TestWebAppProxyServer { private WebAppProxyServer webAppProxy = null; private final String proxyAddress = "0.0.0.0:8888"; @@ -56,4 +59,14 @@ public class TestWebAppProxyServer { } assertEquals(STATE.STARTED, webAppProxy.getServiceState()); } + + @Test + public void testBindAddress() { + YarnConfiguration conf = new YarnConfiguration(); + + InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf); + Assert.assertEquals("Web Proxy default bind address port is incorrect", + YarnConfiguration.DEFAULT_PROXY_PORT, + defaultBindAddress.getPort()); + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Tue Aug 19 23:49:39 2014 @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Capacity Scheduler - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Purpose} Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Tue Aug 19 23:49:39 2014 @@ -18,8 +18,6 @@ Hadoop MapReduce Next Generation - Fair Scheduler - \[ {{{./index.html}Go Back}} \] - %{toc|section=1|fromDepth=0} * {Purpose} @@ -156,6 +154,12 @@ Properties that can be placed in yarn-si * Whether to use preemption. Note that preemption is experimental in the current version. Defaults to false. + * <<<yarn.scheduler.fair.preemption.cluster-utilization-threshold>>> + + * The utilization threshold after which preemption kicks in. The + utilization is computed as the maximum ratio of usage to capacity among + all resources. Defaults to 0.8f. + * <<<yarn.scheduler.fair.sizebasedweight>>> * Whether to assign shares to individual apps based on their size, rather than @@ -201,13 +205,21 @@ Properties that can be placed in yarn-si instead. Defaults to true. If a queue placement policy is given in the allocations file, this property is ignored. + * <<<yarn.scheduler.fair.update-interval-ms>>> + + * The interval at which to lock the scheduler and recalculate fair shares, + recalculate demand, and check whether anything is due for preemption. + Defaults to 500 ms. + Allocation file format The allocation file must be in XML format. The format contains five types of elements: - * <<Queue elements>>, which represent queues. Each may contain the following - properties: + * <<Queue elements>>, which represent queues. Queue elements can take an optional + attribute âtypeâ,which when set to âparentâ makes it a parent queue. This is useful + when we want to create a parent queue without configuring any leaf queues. + Each queue element may contain the following properties: * minResources: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores @@ -231,6 +243,12 @@ Allocation file format * maxRunningApps: limit the number of apps from the queue to run at once + * maxAMShare: limit the fraction of the queue's fair share that can be used + to run application masters. This property can only be used for leaf queues. + For example, if set to 1.0f, then AMs in the leaf queue can take up to 100% + of both the memory and CPU fair share. The default value is -1.0f, which + means that this check is disabled. + * weight: to share the cluster non-proportionally with other queues. Weights default to 1, and a queue with weight 2 should receive approximately twice as many resources as a queue with the default weight. @@ -273,6 +291,9 @@ Allocation file format * <<A queueMaxAppsDefault element>>, which sets the default running app limit for queues; overriden by maxRunningApps element in each queue. + * <<A queueMaxAMShareDefault element>>, which sets the default AM resource + limit for queue; overriden by maxAMShare element in each queue. + * <<A defaultQueueSchedulingPolicy element>>, which sets the default scheduling policy for queues; overriden by the schedulingPolicy element in each queue if specified. Defaults to "fair". @@ -299,7 +320,17 @@ Allocation file format that matches a secondary group of the user who submitted it. The first secondary group that matches a configured queue will be selected. - * default: the app is placed into the queue named "default". + * nestedUserQueue : the app is placed into a queue with the name of the user + under the queue suggested by the nested rule. This is similar to âuserâ + rule,the difference being in ânestedUserQueueâ rule,user queues can be created + under any parent queue, while âuserâ rule creates user queues only under root queue. + Note that nestedUserQueue rule would be applied only if the nested rule returns a + parent queue.One can configure a parent queue either by setting âtypeâ attribute of queue + to âparentâ or by configuring at least one leaf under that queue which makes it a parent. + See example allocation for a sample use case. + + * default: the app is placed into the queue specified in the âqueueâ attribute of the + default rule. If âqueueâ attribute is not specified, the app is placed into âroot.defaultâ queue. * reject: the app is rejected. @@ -312,6 +343,7 @@ Allocation file format <minResources>10000 mb,0vcores</minResources> <maxResources>90000 mb,0vcores</maxResources> <maxRunningApps>50</maxRunningApps> + <maxAMShare>0.1</maxAMShare> <weight>2.0</weight> <schedulingPolicy>fair</schedulingPolicy> <queue name="sample_sub_queue"> @@ -319,6 +351,14 @@ Allocation file format <minResources>5000 mb,0vcores</minResources> </queue> </queue> + + <queueMaxAMShareDefault>0.5</queueMaxAMShareDefault> + + <!â- Queue âsecondary_group_queueâ is a parent queue and may have + user queues under it -â> + <queue name=âsecondary_group_queueâ type=âparentâ> + <weight>3.0</weight> + </queue> <user name="sample_user"> <maxRunningApps>30</maxRunningApps> @@ -328,7 +368,10 @@ Allocation file format <queuePlacementPolicy> <rule name="specified" /> <rule name="primaryGroup" create="false" /> - <rule name="default" /> + <rule name=ânestedUserQueueâ> + <rule name=âsecondaryGroupExistingQueueâ create=âfalseâ /> + </rule> + <rule name="default" queue=âsample_queueâ /> </queuePlacementPolicy> </allocations> ---
