Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Fri Feb 7 02:43:04 2014 @@ -230,7 +230,7 @@ public class TestRMAppTransitions { RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), @@ -639,6 +639,13 @@ public class TestRMAppTransitions { RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + + assertAppState(RMAppState.KILLING, application); + RMAppEvent appAttemptKilled = + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED); + application.handle(appAttemptKilled); + assertAppState(RMAppState.FINAL_SAVING, application); sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Feb 7 02:43:04 2014 @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -56,10 +57,12 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -621,7 +624,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -647,7 +650,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -1765,7 +1768,7 @@ public class TestFairScheduler { RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); @@ -2547,4 +2550,138 @@ public class TestFairScheduler { TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( scheduler.getSchedulerApplications(), scheduler, "default"); } + + @Test + public void testMoveRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + ApplicationId appId = appAttId.getApplicationId(); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage()); + scheduler.update(); + assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand()); + + scheduler.moveApplication(appId, "queue2"); + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertSame(targetQueue, app.getQueue()); + assertFalse(oldQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage()); + assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage()); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + + scheduler.update(); + assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand()); + assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand()); + } + + @Test + public void testMoveNonRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + assertEquals(0, oldQueue.getNumRunnableApps()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(0, targetQueue.getNumRunnableApps()); + assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test + public void testMoveMakesAppRunnable() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertTrue(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertFalse(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertFalse(targetQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxAppsConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2", + Resource.newInstance(1024, 1)); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveToNonexistentQueue() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + scheduler.getQueueManager().getLeafQueue("queue1", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } } Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Fri Feb 7 02:43:04 2014 @@ -77,7 +77,8 @@ public class TestMaxRunningAppsEnforcer private void removeApp(FSSchedulerApp app) { app.getQueue().removeApp(app); - maxAppsEnforcer.updateRunnabilityOnAppRemoval(app); + maxAppsEnforcer.untrackRunnableApp(app); + maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue()); } @Test Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Fri Feb 7 02:43:04 2014 @@ -1317,8 +1317,8 @@ public class TestRMWebServicesApps exten public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - // 15 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 19, info.length()); + // 20 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 20, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), @@ -1389,7 +1389,7 @@ public class TestRMWebServicesApps exten rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - MockAM am = MockRM.launchAM(app1, rm, amNodeManager); + MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -1405,7 +1405,7 @@ public class TestRMWebServicesApps exten } // wait for app to start a new attempt. rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); - am = MockRM.launchAM(app1, rm, amNodeManager); + am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); numAttempt++; } assertEquals("incorrect number of attempts", maxAppAttempts, Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Feb 7 02:43:04 2014 @@ -349,16 +349,20 @@ Queue Access Control Lists (ACLs) * {Administration} - The fair scheduler provides support for administration at runtime through two mechanisms: + The fair scheduler provides support for administration at runtime through a few mechanisms: - * It is possible to modify minimum shares, limits, weights, preemption timeouts - and queue scheduling policies at runtime by editing the allocation file. The - scheduler will reload this file 10-15 seconds after it sees that it was - modified. - - * Current applications, queues, and fair shares can be examined through the - ResourceManager's web interface, at - http://<ResourceManager URL>/cluster/scheduler. +Modifying configuration at runtime + + It is possible to modify minimum shares, limits, weights, preemption timeouts + and queue scheduling policies at runtime by editing the allocation file. The + scheduler will reload this file 10-15 seconds after it sees that it was + modified. + +Monitoring through web UI + + Current applications, queues, and fair shares can be examined through the + ResourceManager's web interface, at + http://<ResourceManager URL>/cluster/scheduler. The following fields can be seen for each queue on the web interface: @@ -382,3 +386,17 @@ Queue Access Control Lists (ACLs) In addition to the information that the ResourceManager normally displays about each application, the web interface includes the application's fair share. +Moving applications between queues + + The Fair Scheduler supports moving a running application to a different queue. + This can be useful for moving an important application to a higher priority + queue, or for moving an unimportant application to a lower priority queue. + Apps can be moved by running "yarn application -movetoqueue appID -queue + targetQueueName". + + When an application is moved to a queue, its existing allocations become + counted with the new queue's allocations instead of the old for purposes + of determining fairness. An attempt to move an application to a queue will + fail if the addition of the app's resources to that queue would violate the + its maxRunningApps or maxResources constraints. + Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1565519&r1=1565518&r2=1565519&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original) +++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Fri Feb 7 02:43:04 2014 @@ -1123,6 +1123,7 @@ ResourceManager REST API's. * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch * applicationTypes - applications matching the given application types, specified as a comma-separated list. + * applicationTags - applications matching any of the given application tags, specified as a comma-separated list. ------ ** Elements of the <apps> (Applications) object
