Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Thu Jan 3 21:23:58 2013 @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationMaster; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationStatus; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -153,6 +154,11 @@ public abstract class MockAsm extends Mo } @Override + public ApplicationSubmissionContext getApplicationSubmissionContext() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override public String getName() { throw new UnsupportedOperationException("Not supported yet."); }
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java Thu Jan 3 21:23:58 2013 @@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.factory.pr import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.junit.After; import org.junit.Before; Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java Thu Jan 3 21:23:58 2013 @@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Thu Jan 3 21:23:58 2013 @@ -24,9 +24,11 @@ import java.util.LinkedHashMap; import java.util.Map; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -67,6 +69,11 @@ public class MockRMApp implements RMApp public ApplicationId getApplicationId() { return id; } + + @Override + public ApplicationSubmissionContext getApplicationSubmissionContext() { + return new ApplicationSubmissionContextPBImpl(); + } @Override public RMAppState getState() { @@ -118,7 +125,9 @@ public class MockRMApp implements RMApp public Map<ApplicationAttemptId, RMAppAttempt> getAppAttempts() { Map<ApplicationAttemptId, RMAppAttempt> attempts = new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>(); - attempts.put(attempt.getAppAttemptId(), attempt); + if(attempt != null) { + attempts.put(attempt.getAppAttemptId(), attempt); + } return attempts; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Jan 3 21:23:58 2013 @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; @@ -167,6 +169,9 @@ public class TestRMAppAttemptTransitions new RMContainerTokenSecretManager(conf), new ClientToAMTokenSecretManagerInRM()); + RMStateStore store = mock(RMStateStore.class); + ((RMContextImpl) rmContext).setStateStore(store); + scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); applicationMasterLauncher = mock(ApplicationMasterLauncher.class); @@ -295,6 +300,14 @@ public class TestRMAppAttemptTransitions assertEquals(0, applicationAttempt.getRanNodes().size()); assertNull(applicationAttempt.getFinalApplicationStatus()); } + + /** + * {@link RMAppAttemptState#RECOVERED} + */ + private void testAppAttemptRecoveredState() { + assertEquals(RMAppAttemptState.RECOVERED, + applicationAttempt.getAppAttemptState()); + } /** * {@link RMAppAttemptState#SCHEDULED} @@ -438,6 +451,15 @@ public class TestRMAppAttemptTransitions new RMAppAttemptEvent( applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.APP_ACCEPTED)); + + if(unmanagedAM){ + assertEquals(RMAppAttemptState.LAUNCHED_UNMANAGED_SAVING, + applicationAttempt.getAppAttemptState()); + applicationAttempt.handle( + new RMAppAttemptStoredEvent( + applicationAttempt.getAppAttemptId(), null)); + } + testAppAttemptScheduledState(); } @@ -463,6 +485,12 @@ public class TestRMAppAttemptTransitions applicationAttempt.getAppAttemptId(), container)); + assertEquals(RMAppAttemptState.ALLOCATED_SAVING, + applicationAttempt.getAppAttemptState()); + applicationAttempt.handle( + new RMAppAttemptStoredEvent( + applicationAttempt.getAppAttemptId(), null)); + testAppAttemptAllocatedState(container); return container; @@ -555,6 +583,15 @@ public class TestRMAppAttemptTransitions } @Test + public void testNewToRecovered() { + applicationAttempt.handle( + new RMAppAttemptEvent( + applicationAttempt.getAppAttemptId(), + RMAppAttemptEventType.RECOVER)); + testAppAttemptRecoveredState(); + } + + @Test public void testSubmittedToFailed() { submitApplicationAttempt(); String message = "Rejected"; @@ -604,7 +641,7 @@ public class TestRMAppAttemptTransitions diagnostics)); testAppAttemptFailedState(amContainer, diagnostics); } - + @Test public void testRunningToFailed() { Container amContainer = allocateApplicationAttempt(); Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Thu Jan 3 21:23:58 2013 @@ -27,7 +27,6 @@ import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -40,8 +39,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -76,8 +73,7 @@ public class TestCapacityScheduler { @Before public void setUp() throws Exception { - RMStateStore store = StoreFactory.getStore(new Configuration()); - resourceManager = new ResourceManager(store); + resourceManager = new ResourceManager(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java Thu Jan 3 21:23:58 2013 @@ -29,8 +29,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.Before; @@ -47,8 +45,7 @@ public class TestFSLeafQueue { Configuration conf = createConfiguration(); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); - RMStateStore store = StoreFactory.getStore(conf); - ResourceManager resourceManager = new ResourceManager(store); + ResourceManager resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Jan 3 21:23:58 2013 @@ -51,8 +51,6 @@ import org.apache.hadoop.yarn.factories. import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.MockNodes; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -102,8 +100,7 @@ public class TestFairScheduler { Configuration conf = createConfiguration(); // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); - RMStateStore store = StoreFactory.getStore(conf); - resourceManager = new ResourceManager(store); + resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); @@ -164,6 +161,13 @@ public class TestFairScheduler { scheduler.allocate(id, ask, new ArrayList<ContainerId>()); return id; } + + private void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) { + List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); + ResourceRequest request = createResourceRequest(memory, "*", priority, 1); + ask.add(request); + scheduler.allocate(attId, ask, new ArrayList<ContainerId>()); + } // TESTS @@ -1101,4 +1105,86 @@ public class TestFairScheduler { assertTrue(Resources.equals( Resources.createResource(1536), scheduler.resToPreempt(schedD, clock.getTime()))); } + + @Test + public void testMultipleContainersWaitingForReservation() { + // Add a node + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Request full capacity of node + createSchedulingRequest(1024, "queue1", "user1", 1); + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, + new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>()); + scheduler.handle(updateEvent); + + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1); + ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1); + + scheduler.update(); + scheduler.handle(updateEvent); + + // One container should get reservation and the other should get nothing + assertEquals(1024, + scheduler.applications.get(attId1).getCurrentReservation().getMemory()); + assertEquals(0, + scheduler.applications.get(attId2).getCurrentReservation().getMemory()); + } + + @Test + public void testUserMaxRunningApps() throws Exception { + // Set max running apps + Configuration conf = createConfiguration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<user name=\"user1\">"); + out.println("<maxRunningApps>1</maxRunningApps>"); + out.println("</user>"); + out.println("</allocations>"); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + // Add a node + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8192)); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Request for app 1 + ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", + "user1", 1); + + scheduler.update(); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, + new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>()); + scheduler.handle(updateEvent); + + // App 1 should be running + assertEquals(1, scheduler.applications.get(attId1).getLiveContainers().size()); + + ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1", + "user1", 1); + + scheduler.update(); + scheduler.handle(updateEvent); + + // App 2 should not be running + assertEquals(0, scheduler.applications.get(attId2).getLiveContainers().size()); + + // Request another container for app 1 + createSchedulingRequestExistingApplication(1024, 1, attId1); + + scheduler.update(); + scheduler.handle(updateEvent); + + // Request should be fulfilled + assertEquals(2, scheduler.applications.get(attId1).getLiveContainers().size()); + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java Thu Jan 3 21:23:58 2013 @@ -27,8 +27,6 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.After; import org.junit.Before; @@ -50,8 +48,7 @@ public class TestFairSchedulerEventLog { // All tests assume only one assignment per node update conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); - RMStateStore store = StoreFactory.getStore(conf); - resourceManager = new ResourceManager(store); + resourceManager = new ResourceManager(); resourceManager.init(conf); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Thu Jan 3 21:23:58 2013 @@ -38,8 +38,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -59,8 +57,7 @@ public class TestFifoScheduler { @Before public void setUp() throws Exception { - RMStateStore store = StoreFactory.getStore(new Configuration()); - resourceManager = new ResourceManager(store); + resourceManager = new ResourceManager(); Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java Thu Jan 3 21:23:58 2013 @@ -357,6 +357,27 @@ public class TestDelegationTokenRenewer } } + @Test + public void testInvalidDTWithAddApplication() throws Exception { + MyFS dfs = (MyFS)FileSystem.get(conf); + LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode()); + + MyToken token = dfs.getDelegationToken(new Text("user1")); + token.cancelToken(); + + Credentials ts = new Credentials(); + ts.addToken(token.getKind(), token); + + // register the tokens for renewal + ApplicationId appId = BuilderUtils.newApplicationId(0, 0); + try { + delegationTokenRenewer.addApplication(appId, ts, true); + fail("App submission with a cancelled token should have failed"); + } catch (InvalidToken e) { + // expected + } + } + /** * Basic idea of the test: * 1. register a token for 2 seconds with no cancel at the end Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Thu Jan 3 21:23:58 2013 @@ -48,8 +48,6 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.service.CompositeService; @@ -154,8 +152,7 @@ public class MiniYARNCluster extends Com getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":0"); } - RMStateStore store = StoreFactory.getStore(getConfig()); - resourceManager = new ResourceManager(store) { + resourceManager = new ResourceManager() { @Override protected void doSecureLogin() throws IOException { // Don't try to login using keytab in the testcase. Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java Thu Jan 3 21:23:58 2013 @@ -47,7 +47,7 @@ public class TestRMNMSecretKeys { // intervene final DrainDispatcher dispatcher = new DrainDispatcher(); - ResourceManager rm = new ResourceManager(null) { + ResourceManager rm = new ResourceManager() { @Override protected void doSecureLogin() throws IOException { // Do nothing. Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java Thu Jan 3 21:23:58 2013 @@ -24,11 +24,13 @@ import java.io.UnsupportedEncodingExcept import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.TrackingUriPlugin; public class ProxyUriUtils { @SuppressWarnings("unused") @@ -143,4 +145,28 @@ public class ProxyUriUtils { throws URISyntaxException { return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl); } + + /** + * Returns the first valid tracking link, if any, from the given id from the + * given list of plug-ins, if any. + * + * @param id the id of the application for which the tracking link is desired + * @param trackingUriPlugins list of plugins from which to get the tracking link + * @return the desired link if possible, otherwise null + * @throws URISyntaxException + */ + public static URI getUriFromTrackingPlugins(ApplicationId id, + List<TrackingUriPlugin> trackingUriPlugins) + throws URISyntaxException { + URI toRet = null; + for(TrackingUriPlugin plugin : trackingUriPlugins) + { + toRet = plugin.getTrackingUri(id); + if (toRet != null) + { + return toRet; + } + } + return null; + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java Thu Jan 3 21:23:58 2013 @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.Enumeration; import java.util.HashSet; +import java.util.List; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServlet; @@ -48,8 +49,10 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.StringHelper; +import org.apache.hadoop.yarn.util.TrackingUriPlugin; import org.apache.hadoop.yarn.webapp.MimeType; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; @@ -61,8 +64,9 @@ public class WebAppProxyServlet extends "Accept-Language", "Accept-Charset")); public static final String PROXY_USER_COOKIE_NAME = "proxy-user"; - - + + private final List<TrackingUriPlugin> trackingUriPlugins; + private static class _ implements Hamlet._ { //Empty } @@ -78,6 +82,18 @@ public class WebAppProxyServlet extends } /** + * Default constructor + */ + public WebAppProxyServlet() + { + super(); + YarnConfiguration conf = new YarnConfiguration(); + this.trackingUriPlugins = + conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR, + TrackingUriPlugin.class); + } + + /** * Output 404 with appropriate message. * @param resp the http response. * @param message the message to include on the page. @@ -238,11 +254,14 @@ public class WebAppProxyServlet extends if(securityEnabled) { String cookieName = getCheckCookieName(id); - for(Cookie c: req.getCookies()) { - if(cookieName.equals(c.getName())) { - userWasWarned = true; - userApproved = userApproved || Boolean.valueOf(c.getValue()); - break; + Cookie[] cookies = req.getCookies(); + if (cookies != null) { + for (Cookie c : cookies) { + if (cookieName.equals(c.getName())) { + userWasWarned = true; + userApproved = userApproved || Boolean.valueOf(c.getValue()); + break; + } } } } @@ -253,6 +272,16 @@ public class WebAppProxyServlet extends if(applicationReport == null) { LOG.warn(req.getRemoteUser()+" Attempting to access "+id+ " that was not found"); + + URI toFetch = + ProxyUriUtils + .getUriFromTrackingPlugins(id, this.trackingUriPlugins); + if (toFetch != null) + { + resp.sendRedirect(resp.encodeRedirectURL(toFetch.toString())); + return; + } + notFound(resp, "Application "+appId+" could not be found, " + "please try the history server"); return; Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java?rev=1428601&r1=1428600&r2=1428601&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java Thu Jan 3 21:23:58 2013 @@ -21,11 +21,16 @@ package org.apache.hadoop.yarn.server.we import static org.junit.Assert.*; import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.TrackingUriPlugin; import org.junit.Test; +import com.google.common.collect.Lists; + public class TestProxyUriUtils { @Test public void testGetPathApplicationId() { @@ -83,4 +88,36 @@ public class TestProxyUriUtils { URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id); assertEquals(expected, result); } + + @Test + public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins() + throws URISyntaxException { + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); + List<TrackingUriPlugin> list = + Lists.newArrayListWithExpectedSize(0); + assertNull(ProxyUriUtils.getUriFromTrackingPlugins(id, list)); + } + + @Test + public void testGetProxyUriFromPluginsReturnsValidUriWhenAble() + throws URISyntaxException { + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); + List<TrackingUriPlugin> list = + Lists.newArrayListWithExpectedSize(2); + // Insert a plugin that returns null. + list.add(new TrackingUriPlugin() { + public URI getTrackingUri(ApplicationId id) throws URISyntaxException { + return null; + } + }); + // Insert a plugin that returns a valid URI. + list.add(new TrackingUriPlugin() { + public URI getTrackingUri(ApplicationId id) throws URISyntaxException { + return new URI("http://history.server.net/";); + } + }); + URI result = ProxyUriUtils.getUriFromTrackingPlugins(id, list); + assertNotNull(result); + + } }
