Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java Fri Feb 7 01:57:21 2014 @@ -110,6 +110,7 @@ public class AppBlock extends HtmlBlock _("User:", app.getUser()). _("Name:", app.getName()). _("Application Type:", app.getApplicationType()). + _("Application Tags:", app.getApplicationTags()). _("State:", app.getState()). _("FinalStatus:", app.getFinalStatus()). _("Started:", Times.format(app.getStartTime())).
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Fri Feb 7 01:57:21 2014 @@ -261,12 +261,14 @@ public class RMWebServices { @QueryParam("startedTimeEnd") String startedEnd, @QueryParam("finishedTimeBegin") String finishBegin, @QueryParam("finishedTimeEnd") String finishEnd, - @QueryParam("applicationTypes") Set<String> applicationTypes) { + @QueryParam("applicationTypes") Set<String> applicationTypes, + @QueryParam("applicationTags") Set<String> applicationTags) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; boolean checkAppStates = false; + boolean checkAppTags = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -327,6 +329,11 @@ public class RMWebServices { checkAppTypes = true; } + Set<String> appTags = parseQueries(applicationTags, false); + if (!appTags.isEmpty()) { + checkAppTags = true; + } + // stateQuery is deprecated. if (stateQuery != null && !stateQuery.isEmpty()) { statesQuery.add(stateQuery); @@ -354,6 +361,10 @@ public class RMWebServices { request.setApplicationTypes(appTypes); } + if (checkAppTags) { + request.setApplicationTags(appTags); + } + if (checkAppStates) { request.setApplicationStates(appStates); } Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java Fri Feb 7 01:57:21 2014 @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAcce import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import com.google.common.base.Joiner; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Times; @@ -67,6 +67,7 @@ public class AppInfo { protected String diagnostics; protected long clusterId; protected String applicationType; + protected String applicationTags = ""; // these are only allowed if acls allow protected long startedTime; @@ -117,6 +118,9 @@ public class AppInfo { if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; } + if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { + this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); + } this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { @@ -239,6 +243,10 @@ public class AppInfo { public String getApplicationType() { return this.applicationType; } + + public String getApplicationTags() { + return this.applicationTags; + } public int getRunningContainers() { return this.runningContainers; Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Feb 7 01:57:21 2014 @@ -482,6 +482,13 @@ public class MockRM extends ResourceMana RMAppAttempt attempt = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); + rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); + return am; + } + + public static MockAM launchAndRegisterAM(RMApp app, MockRM rm, MockNM nm) + throws Exception { + MockAM am = launchAM(app, rm, nm); am.registerAppAttempt(); rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); return am; Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Fri Feb 7 01:57:21 2014 @@ -41,8 +41,10 @@ import java.util.concurrent.BrokenBarrie import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Sets; import junit.framework.Assert; +import org.apache.commons.lang.math.LongRange; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +53,7 @@ import org.apache.hadoop.security.UserGr import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; @@ -58,6 +61,7 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -71,6 +75,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; @@ -232,6 +237,20 @@ public class TestClientRMService { "application " + request.getApplicationId()); } } + + @Test (expected = ApplicationNotFoundException.class) + public void testMoveAbsentApplication() throws YarnException { + RMContext rmContext = mock(RMContext.class); + when(rmContext.getRMApps()).thenReturn( + new ConcurrentHashMap<ApplicationId, RMApp>()); + ClientRMService rmService = new ClientRMService(rmContext, null, null, + null, null, null); + ApplicationId applicationId = + BuilderUtils.newApplicationId(System.currentTimeMillis(), 0); + MoveApplicationAcrossQueuesRequest request = + MoveApplicationAcrossQueuesRequest.newInstance(applicationId, "newqueue"); + rmService.moveApplicationAcrossQueues(request); + } @Test public void testGetQueueInfo() throws Exception { @@ -450,6 +469,7 @@ public class TestClientRMService { {MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName()}; ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102), getApplicationId(103)}; + List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3"); // Submit applications for (int i = 0; i < appIds.length; i++) { @@ -457,7 +477,8 @@ public class TestClientRMService { when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true); SubmitApplicationRequest submitRequest = mockSubmitAppRequest( - appId, appNames[i], queues[i % queues.length]); + appId, appNames[i], queues[i % queues.length], + new HashSet<String>(tags.subList(0, i + 1))); rmService.submitApplication(submitRequest); } @@ -498,6 +519,41 @@ public class TestClientRMService { userSet.add(UserGroupInformation.getCurrentUser().getShortUserName()); assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size()); + + // Check tags + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.ALL, null, null, null, null, null, null, + null, null); + Set<String> tagSet = new HashSet<String>(); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 6, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(0)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 3, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(1)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 2, + rmService.getApplications(request).getApplicationList().size()); + + tagSet = Sets.newHashSet(tags.get(2)); + request.setApplicationTags(tagSet); + assertEquals("Incorrect number of matching tags", 1, + rmService.getApplications(request).getApplicationList().size()); + + // Check scope + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.VIEWABLE); + assertEquals("Incorrect number of applications for the scope", 6, + rmService.getApplications(request).getApplicationList().size()); + + request = GetApplicationsRequest.newInstance( + ApplicationsRequestScope.OWN); + assertEquals("Incorrect number of applications for the scope", 3, + rmService.getApplications(request).getApplicationList().size()); } @Test(timeout=4000) @@ -568,6 +624,11 @@ public class TestClientRMService { private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue) { + return mockSubmitAppRequest(appId, name, queue, null); + } + + private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, + String name, String queue, Set<String> tags) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( @@ -581,6 +642,7 @@ public class TestClientRMService { submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); + submissionContext.setApplicationTags(tags); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); @@ -649,7 +711,7 @@ public class TestClientRMService { when(asContext.getMaxAppAttempts()).thenReturn(1); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null , System - .currentTimeMillis(), "YARN")); + .currentTimeMillis(), "YARN", null)); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId3, 1); RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, false); Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java Fri Feb 7 01:57:21 2014 @@ -18,6 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.spy; + import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -33,6 +37,7 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.Container; @@ -44,9 +49,17 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -54,7 +67,9 @@ import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Test; +import org.mockito.ArgumentMatcher; +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestRM { private static final Log LOG = LogFactory.getLog(TestRM.class); @@ -397,19 +412,19 @@ public class TestRM { MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); - MockAM am1 = MockRM.launchAM(app1, rm1, nm1); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); MockRM.finishApplicationMaster(app1, rm1, nm1, am1); // a failed app RMApp app2 = rm1.submitApp(200); - MockAM am2 = MockRM.launchAM(app2, rm1, nm1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED); // a killed app RMApp app3 = rm1.submitApp(200); - MockAM am3 = MockRM.launchAM(app3, rm1, nm1); + MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1); rm1.killApp(app3.getApplicationId()); rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED); rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED); @@ -449,7 +464,7 @@ public class TestRM { // a failed app RMApp app2 = rm1.submitApp(200); - MockAM am2 = MockRM.launchAM(app2, rm1, nm1); + MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1); nm1 .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE); am2.waitForState(RMAppAttemptState.FAILED); @@ -466,10 +481,88 @@ public class TestRM { Assert.assertEquals(-1, report1.getRpcPort()); } + /** + * Validate killing an application when it is at accepted state. + * @throws Exception exception + */ + @Test (timeout = 60000) + public void testApplicationKillAtAcceptedState() throws Exception { + + YarnConfiguration conf = new YarnConfiguration(); + final Dispatcher dispatcher = new AsyncDispatcher() { + @Override + public EventHandler getEventHandler() { + + class EventArgMatcher extends ArgumentMatcher<AbstractEvent> { + @Override + public boolean matches(Object argument) { + if (argument instanceof RMAppAttemptEvent) { + if (((RMAppAttemptEvent) argument).getType().equals( + RMAppAttemptEventType.KILL)) { + return true; + } + } + return false; + } + } + + EventHandler handler = spy(super.getEventHandler()); + doNothing().when(handler).handle(argThat(new EventArgMatcher())); + return handler; + } + }; + + MockRM rm = new MockRM(conf) { + @Override + protected Dispatcher createDispatcher() { + return dispatcher; + } + }; + + rm.start(); + MockNM nm1 = + new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); + nm1.registerNode(); + + // a failed app + RMApp application = rm.submitApp(200); + MockAM am = MockRM.launchAM(application, rm, nm1); + am.waitForState(RMAppAttemptState.LAUNCHED); + nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING); + rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED); + + // Now kill the application before new attempt is launched, the app report + // returns the invalid AM host and port. + KillApplicationRequest request = + KillApplicationRequest.newInstance(application.getApplicationId()); + rm.getClientRMService().forceKillApplication(request); + + // Specific test for YARN-1689 follows + // Now let's say a race causes AM to register now. This should not crash RM. + am.registerAppAttempt(false); + + // We explicitly intercepted the kill-event to RMAppAttempt, so app should + // still be in KILLING state. + rm.waitForState(application.getApplicationId(), RMAppState.KILLING); + // AM should now be in running + rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING); + + // Simulate that appAttempt is killed. + rm.getRMContext().getDispatcher().getEventHandler().handle( + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED)); + rm.waitForState(application.getApplicationId(), RMAppState.KILLED); + } + public static void main(String[] args) throws Exception { TestRM t = new TestRM(); t.testGetNewAppId(); t.testAppWithNoContainers(); t.testAppOnMultiNode(); + t.testNMToken(); + t.testActivatingApplicationAfterAddingNM(); + t.testInvalidateAMHostPortWhenAMFailedOrKilled(); + t.testInvalidatedAMHostPortOnAMRestart(); + t.testApplicationKillAtAcceptedState(); } } Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Fri Feb 7 01:57:21 2014 @@ -36,6 +36,8 @@ import org.junit.Test; import java.io.IOException; +import junit.framework.Assert; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -48,12 +50,15 @@ public class TestRMHA { private static final String STATE_ERR = "ResourceManager is in wrong HA state"; - private static final String RM1_ADDRESS = "0.0.0.0:0"; + private static final String RM1_ADDRESS = "1.1.1.1:1"; private static final String RM1_NODE_ID = "rm1"; - private static final String RM2_ADDRESS = "1.1.1.1:1"; + private static final String RM2_ADDRESS = "0.0.0.0:0"; private static final String RM2_NODE_ID = "rm2"; + private static final String RM3_ADDRESS = "2.2.2.2:2"; + private static final String RM3_NODE_ID = "rm3"; + @Before public void setUp() throws Exception { configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); @@ -61,8 +66,8 @@ public class TestRMHA { for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } - configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); } private void checkMonitorHealth() throws IOException { @@ -278,6 +283,36 @@ public class TestRMHA { rm.stop(); } + @Test + public void testHAIDLookup() { + //test implicitly lookup HA-ID + Configuration conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM2_NODE_ID); + + //test explicitly lookup HA-ID + configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + conf = new YarnConfiguration(configuration); + rm = new MockRM(conf); + rm.init(conf); + assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM1_NODE_ID); + + //test if RM_HA_ID can not be found + configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID+ "," + RM3_NODE_ID); + configuration.unset(YarnConfiguration.RM_HA_ID); + conf = new YarnConfiguration(configuration); + try { + rm = new MockRM(conf); + rm.init(conf); + fail("Should get an exception here."); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Invalid configuration! Can not find valid RM_HA_ID.")); + } + } + @SuppressWarnings("rawtypes") class MyCountingDispatcher extends AbstractService implements Dispatcher { Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Fri Feb 7 01:57:21 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.MockApps; @@ -140,6 +141,11 @@ public abstract class MockAsm extends Mo } @Override + public Set<String> getApplicationTags() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override public void setQueue(String name) { throw new UnsupportedOperationException("Not supported yet."); } @@ -235,7 +241,11 @@ public abstract class MockAsm extends Mo public int getMaxAppAttempts() { return maxAppAttempts; } - + + @Override + public Set<String> getApplicationTags() { + return null; + } }; } Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Fri Feb 7 01:57:21 2014 @@ -69,7 +69,7 @@ public class TestAMRestart { new MockNM("127.0.0.1:2351", 4089, rm1.getResourceTrackerService()); nm2.registerNode(); - MockAM am1 = MockRM.launchAM(app1, rm1, nm1); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); int NUM_CONTAINERS = 3; // allocate NUM_CONTAINERS containers am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Fri Feb 7 01:57:21 2014 @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -218,6 +219,11 @@ public class MockRMApp implements RMApp } @Override + public Set<String> getApplicationTags() { + return null; + } + + @Override public boolean isAppSafeToTerminate() { return true; } Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Fri Feb 7 01:57:21 2014 @@ -230,7 +230,7 @@ public class TestRMAppTransitions { RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), @@ -639,6 +639,13 @@ public class TestRMAppTransitions { RMAppEventType.KILL); application.handle(event); rmDispatcher.await(); + + assertAppState(RMAppState.KILLING, application); + RMAppEvent appAttemptKilled = + new RMAppEvent(application.getApplicationId(), + RMAppEventType.ATTEMPT_KILLED); + application.handle(appAttemptKilled); + assertAppState(RMAppState.FINAL_SAVING, application); sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Fri Feb 7 01:57:21 2014 @@ -73,7 +73,7 @@ public class TestQueueMetrics { checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); - metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15)); + metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); @@ -81,7 +81,7 @@ public class TestQueueMetrics { metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); - metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2)); + metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); @@ -171,7 +171,7 @@ public class TestQueueMetrics { metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10)); - metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15)); + metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); @@ -181,7 +181,7 @@ public class TestQueueMetrics { checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); checkApps(userSource, 1, 0, 1, 0, 0, 0, true); - metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2)); + metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); checkResources(userSource, 6*GB, 6, 3, 3, 0, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); @@ -232,7 +232,7 @@ public class TestQueueMetrics { metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10)); metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10)); - metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15)); + metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); checkResources(parentQueueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); @@ -242,7 +242,7 @@ public class TestQueueMetrics { checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); checkApps(userSource, 1, 0, 1, 0, 0, 0, true); - metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2)); + metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); metrics.reserveResource(user, Resources.createResource(3*GB, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Feb 7 01:57:21 2014 @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -56,10 +57,12 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -621,7 +624,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -647,7 +650,7 @@ public class TestFairScheduler { ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, - null, null, null, false, false, 0, null, null), null, null, 0, null); + null, null, null, false, false, 0, null, null), null, null, 0, null, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = @@ -1765,7 +1768,7 @@ public class TestFairScheduler { RMApp application = new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN"); + System.currentTimeMillis(), "YARN", null); resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application); application.handle(new RMAppEvent(applicationId, RMAppEventType.START)); @@ -2547,4 +2550,138 @@ public class TestFairScheduler { TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( scheduler.getSchedulerApplications(), scheduler, "default"); } + + @Test + public void testMoveRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + ApplicationId appId = appAttId.getApplicationId(); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage()); + scheduler.update(); + assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand()); + + scheduler.moveApplication(appId, "queue2"); + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertSame(targetQueue, app.getQueue()); + assertFalse(oldQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage()); + assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage()); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + + scheduler.update(); + assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand()); + assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand()); + } + + @Test + public void testMoveNonRunnableApp() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + assertEquals(0, oldQueue.getNumRunnableApps()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertEquals(0, oldQueue.getNumRunnableApps()); + assertEquals(0, targetQueue.getNumRunnableApps()); + assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test + public void testMoveMakesAppRunnable() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + FSSchedulerApp app = scheduler.getSchedulerApp(appAttId); + assertTrue(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + assertFalse(oldQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertFalse(targetQueue.getNonRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertTrue(targetQueue.getRunnableAppSchedulables() + .contains(app.getAppSchedulable())); + assertEquals(1, targetQueue.getNumRunnableApps()); + assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps()); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxAppsConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); + queueMgr.getLeafQueue("queue2", true); + scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2", + Resource.newInstance(1024, 1)); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.handle(updateEvent); + scheduler.handle(updateEvent); + + assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } + + @Test (expected = YarnException.class) + public void testMoveToNonexistentQueue() throws Exception { + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + scheduler.getQueueManager().getLeafQueue("queue1", true); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1024, 1, "queue1", "user1", 3); + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); + } } Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Fri Feb 7 01:57:21 2014 @@ -77,7 +77,8 @@ public class TestMaxRunningAppsEnforcer private void removeApp(FSSchedulerApp app) { app.getQueue().removeApp(app); - maxAppsEnforcer.updateRunnabilityOnAppRemoval(app); + maxAppsEnforcer.untrackRunnableApp(app); + maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue()); } @Test Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Fri Feb 7 01:57:21 2014 @@ -1317,8 +1317,8 @@ public class TestRMWebServicesApps exten public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - // 15 because trackingUrl not assigned yet - assertEquals("incorrect number of elements", 19, info.length()); + // 20 because trackingUrl not assigned yet + assertEquals("incorrect number of elements", 20, info.length()); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), @@ -1389,7 +1389,7 @@ public class TestRMWebServicesApps exten rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); - MockAM am = MockRM.launchAM(app1, rm, amNodeManager); + MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); int maxAppAttempts = rm.getConfig().getInt( YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -1405,7 +1405,7 @@ public class TestRMWebServicesApps exten } // wait for app to start a new attempt. rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); - am = MockRM.launchAM(app1, rm, amNodeManager); + am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); numAttempt++; } assertEquals("incorrect number of attempts", maxAppAttempts, Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Feb 7 01:57:21 2014 @@ -349,16 +349,20 @@ Queue Access Control Lists (ACLs) * {Administration} - The fair scheduler provides support for administration at runtime through two mechanisms: + The fair scheduler provides support for administration at runtime through a few mechanisms: - * It is possible to modify minimum shares, limits, weights, preemption timeouts - and queue scheduling policies at runtime by editing the allocation file. The - scheduler will reload this file 10-15 seconds after it sees that it was - modified. - - * Current applications, queues, and fair shares can be examined through the - ResourceManager's web interface, at - http://<ResourceManager URL>/cluster/scheduler. +Modifying configuration at runtime + + It is possible to modify minimum shares, limits, weights, preemption timeouts + and queue scheduling policies at runtime by editing the allocation file. The + scheduler will reload this file 10-15 seconds after it sees that it was + modified. + +Monitoring through web UI + + Current applications, queues, and fair shares can be examined through the + ResourceManager's web interface, at + http://<ResourceManager URL>/cluster/scheduler. The following fields can be seen for each queue on the web interface: @@ -382,3 +386,17 @@ Queue Access Control Lists (ACLs) In addition to the information that the ResourceManager normally displays about each application, the web interface includes the application's fair share. +Moving applications between queues + + The Fair Scheduler supports moving a running application to a different queue. + This can be useful for moving an important application to a higher priority + queue, or for moving an unimportant application to a lower priority queue. + Apps can be moved by running "yarn application -movetoqueue appID -queue + targetQueueName". + + When an application is moved to a queue, its existing allocations become + counted with the new queue's allocations instead of the old for purposes + of determining fairness. An attempt to move an application to a queue will + fail if the addition of the app's resources to that queue would violate the + its maxRunningApps or maxResources constraints. + Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1565515&r1=1565514&r2=1565515&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original) +++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Fri Feb 7 01:57:21 2014 @@ -1123,6 +1123,7 @@ ResourceManager REST API's. * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch * applicationTypes - applications matching the given application types, specified as a comma-separated list. + * applicationTags - applications matching any of the given application tags, specified as a comma-separated list. ------ ** Elements of the <apps> (Applications) object
