Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Sat Mar 30 03:50:03 2013 @@ -24,8 +24,8 @@ import static org.junit.Assert.assertTru import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -55,13 +55,12 @@ import org.apache.hadoop.yarn.conf.YarnC import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -138,6 +137,7 @@ public class TestLeafQueue { private static final String C = "c"; private static final String C1 = "c1"; private static final String D = "d"; + private static final String E = "e"; private void setupQueueConfiguration( CapacitySchedulerConfiguration conf, final String newRoot) { @@ -148,7 +148,7 @@ public class TestLeafQueue { conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot; - conf.setQueues(Q_newRoot, new String[] {A, B, C, D}); + conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E}); conf.setCapacity(Q_newRoot, 100); conf.setMaximumCapacity(Q_newRoot, 100); conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); @@ -174,10 +174,14 @@ public class TestLeafQueue { conf.setCapacity(Q_C1, 100); final String Q_D = Q_newRoot + "." + D; - conf.setCapacity(Q_D, 10); + conf.setCapacity(Q_D, 9); conf.setMaximumCapacity(Q_D, 11); conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d"); + final String Q_E = Q_newRoot + "." + E; + conf.setCapacity(Q_E, 1); + conf.setMaximumCapacity(Q_E, 1); + conf.setAcl(Q_E, QueueACL.SUBMIT_APPLICATIONS, "user_e"); } static LeafQueue stubLeafQueue(LeafQueue queue) { @@ -282,7 +286,7 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); // Start testing... @@ -404,11 +408,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Start testing... @@ -537,11 +541,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -630,11 +634,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -669,7 +673,7 @@ public class TestLeafQueue { // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, priority, recordFactory))); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); @@ -686,7 +690,7 @@ public class TestLeafQueue { // Check headroom for app_2 LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 0, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, priority, recordFactory))); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); @@ -747,11 +751,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); /** @@ -781,11 +785,11 @@ public class TestLeafQueue { // Submit resource requests for other apps now to 'activate' them app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 3*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, priority, recordFactory))); app_3.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Now allocations should goto app_2 since @@ -909,11 +913,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1011,7 +1015,7 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); // Setup app_1 to request a 4GB container on host_0 and @@ -1022,7 +1026,7 @@ public class TestLeafQueue { priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(DEFAULT_RACK, 4*GB, 1, priority, recordFactory)); - appRequests_1.add(TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 2, + appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, priority, recordFactory)); app_1.updateResourceRequests(appRequests_1); @@ -1117,11 +1121,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1243,7 +1247,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1308,7 +1312,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); assertEquals(2, app_0.getTotalRequiredResources(priority)); @@ -1386,7 +1390,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority_1, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); // P2 @@ -1398,7 +1402,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_2, 2*GB, 1, priority_2, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority_2, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1523,7 +1527,7 @@ public class TestLeafQueue { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1546,7 +1550,7 @@ public class TestLeafQueue { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1567,6 +1571,102 @@ public class TestLeafQueue { } + @Test (timeout = 30000) + public void testActivateApplicationAfterQueueRefresh() throws Exception { + + // Manipulate queue 'e' + LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); + + // Users + final String user_e = "user_e"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = + new FiCaSchedulerApp(appAttemptId_0, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_0, user_e, E); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = + new FiCaSchedulerApp(appAttemptId_1, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_1, user_e, E); // same user + + final ApplicationAttemptId appAttemptId_2 = + TestUtils.getMockApplicationAttemptId(2, 0); + FiCaSchedulerApp app_2 = + new FiCaSchedulerApp(appAttemptId_2, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_2, user_e, E); // same user + + // before reinitialization + assertEquals(2, e.activeApplications.size()); + assertEquals(1, e.pendingApplications.size()); + + csConf.setDouble(CapacitySchedulerConfiguration + .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + CapacitySchedulerConfiguration + .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); + Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); + CSQueue newRoot = + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, + newQueues, queues, + TestUtils.spyHook); + queues = newQueues; + root.reinitialize(newRoot, cs.getClusterResources()); + + // after reinitialization + assertEquals(3, e.activeApplications.size()); + assertEquals(0, e.pendingApplications.size()); + } + + @Test (timeout = 30000) + public void testActivateApplicationByUpdatingClusterResource() + throws Exception { + + // Manipulate queue 'e' + LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); + + // Users + final String user_e = "user_e"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = + new FiCaSchedulerApp(appAttemptId_0, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_0, user_e, E); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = + new FiCaSchedulerApp(appAttemptId_1, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_1, user_e, E); // same user + + final ApplicationAttemptId appAttemptId_2 = + TestUtils.getMockApplicationAttemptId(2, 0); + FiCaSchedulerApp app_2 = + new FiCaSchedulerApp(appAttemptId_2, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_2, user_e, E); // same user + + // before updating cluster resource + assertEquals(2, e.activeApplications.size()); + assertEquals(1, e.pendingApplications.size()); + + e.updateClusterResource(Resources.createResource(200 * 16 * GB, 100 * 32)); + + // after updating cluster resource + assertEquals(3, e.activeApplications.size()); + assertEquals(0, e.pendingApplications.size()); + } + public boolean hasQueueACL(List<QueueUserACLInfo> aclInfos, QueueACL acl) { for (QueueUserACLInfo aclInfo : aclInfos) { if (aclInfo.getUserAcls().contains(acl)) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java Sat Mar 30 03:50:03 2013 @@ -24,6 +24,7 @@ import java.util.List; import junit.framework.Assert; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FairSchedulingMode; import org.junit.Before; import org.junit.Test; @@ -32,10 +33,12 @@ import org.junit.Test; */ public class TestComputeFairShares { private List<Schedulable> scheds; + private SchedulingMode schedulingMode; @Before public void setUp() throws Exception { scheds = new ArrayList<Schedulable>(); + schedulingMode = new FairSchedulingMode(); } /** @@ -48,7 +51,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50)); scheds.add(new FakeSchedulable(30)); scheds.add(new FakeSchedulable(20)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(10, 10, 10, 10); } @@ -65,7 +69,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50)); scheds.add(new FakeSchedulable(11)); scheds.add(new FakeSchedulable(3)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(13, 13, 11, 3); } @@ -83,7 +88,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(10, 20)); scheds.add(new FakeSchedulable(10, 0)); scheds.add(new FakeSchedulable(3, 2)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(20, 10, 7, 3); } @@ -97,7 +103,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50, 0, 1.0)); scheds.add(new FakeSchedulable(30, 0, 1.0)); scheds.add(new FakeSchedulable(20, 0, 0.5)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + schedulingMode.computeShares(scheds, + Resources.createResource(45)); verifyShares(20, 10, 10, 5); } @@ -114,7 +121,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(11, 0, 1.0)); scheds.add(new FakeSchedulable(30, 0, 1.0)); scheds.add(new FakeSchedulable(20, 0, 0.5)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + schedulingMode.computeShares(scheds, + Resources.createResource(45)); verifyShares(10, 11, 16, 8); } @@ -131,7 +139,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(11, 0, 1.0)); scheds.add(new FakeSchedulable(30, 5, 1.0)); scheds.add(new FakeSchedulable(20, 15, 0.5)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + schedulingMode.computeShares(scheds, + Resources.createResource(45)); verifyShares(10, 10, 10, 15); } @@ -146,7 +155,9 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50 * million)); scheds.add(new FakeSchedulable(30 * million)); scheds.add(new FakeSchedulable(20 * million)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40 * million)); + schedulingMode + .computeShares(scheds, + Resources.createResource(40 * million)); verifyShares(10 * million, 10 * million, 10 * million, 10 * million); } @@ -159,7 +170,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50)); scheds.add(new FakeSchedulable(30)); scheds.add(new FakeSchedulable(0)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(30)); + schedulingMode.computeShares(scheds, + Resources.createResource(30)); verifyShares(10, 10, 10, 0); } @@ -168,7 +180,8 @@ public class TestComputeFairShares { */ @Test public void testEmptyList() { - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(); } Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Sat Mar 30 03:50:03 2013 @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -72,6 +71,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -141,7 +141,8 @@ public class TestFairScheduler { } - private ResourceRequest createResourceRequest(int memory, String host, int priority, int numContainers) { + private ResourceRequest createResourceRequest(int memory, String host, + int priority, int numContainers) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); request.setHostName(host); @@ -156,33 +157,59 @@ public class TestFairScheduler { * Creates a single container priority-1 request and submits to * scheduler. */ - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId) { return createSchedulingRequest(memory, queueId, userId, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers) { return createSchedulingRequest(memory, queueId, userId, numContainers, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers, int priority) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id, queueId, userId); List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); - ResourceRequest request = createResourceRequest(memory, "*", priority, numContainers); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, numContainers); ask.add(request); scheduler.allocate(id, ask, new ArrayList<ContainerId>()); return id; } - private void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) { + private void createSchedulingRequestExistingApplication(int memory, int priority, + ApplicationAttemptId attId) { List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); - ResourceRequest request = createResourceRequest(memory, "*", priority, 1); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, 1); ask.add(request); scheduler.allocate(attId, ask, new ArrayList<ContainerId>()); } // TESTS + @Test(timeout=2000) + public void testLoadConfigurationOnInitialize() throws IOException { + Configuration conf = createConfiguration(); + conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + conf.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 3); + conf.setBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, true); + conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, .5); + conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + Assert.assertEquals(true, scheduler.assignMultiple); + Assert.assertEquals(3, scheduler.maxAssign); + Assert.assertEquals(true, scheduler.sizeBasedWeight); + Assert.assertEquals(.5, scheduler.nodeLocalityThreshold, .01); + Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01); + Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory()); + Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory()); + } + @Test public void testAggregateCapacityTracking() throws Exception { // Add a node @@ -438,7 +465,8 @@ public class TestFairScheduler { // First ask, queue1 requests 1 large (minReqSize * 2). List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>(); - ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1); + ResourceRequest request1 = + createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList<ContainerId>()); @@ -452,7 +480,8 @@ public class TestFairScheduler { // Third ask, queue2 requests 1 large List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>(); - ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1); + ResourceRequest request4 = + createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1); ask3.add(request4); scheduler.allocate(id22, ask3, new ArrayList<ContainerId>()); @@ -1290,7 +1319,7 @@ public class TestFairScheduler { asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1)); asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1)); asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1)); - asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2)); + asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2)); scheduler.allocate(appId, asks, new ArrayList<ContainerId>()); @@ -1325,7 +1354,7 @@ public class TestFairScheduler { FSSchedulerApp app2 = scheduler.applications.get(attId2); FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1"); - queue1.setSchedulingMode(SchedulingMode.FIFO); + queue1.setSchedulingMode(new FifoSchedulingMode()); scheduler.update(); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Sat Mar 30 03:50:03 2013 @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -200,15 +200,15 @@ public class TestFifoScheduler { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); Task task_1_2 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_2); Task task_1_3 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_3); application_1.schedule(); @@ -222,7 +222,7 @@ public class TestFifoScheduler { application_0.addTask(task_0_2); Task task_0_3 = new Task(application_0, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_0.addTask(task_0_3); application_0.schedule(); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Sat Mar 30 03:50:03 2013 @@ -83,7 +83,7 @@ public class TestRMWebServicesApps exten bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration conf = new Configuration(); - conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 2); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rm = new MockRM(conf); @@ -871,9 +871,9 @@ public class TestRMWebServicesApps exten MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); - int maxRetries = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_RETRIES, - YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES); - int retriesLeft = maxRetries; + int maxAppAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + int retriesLeft = maxAppAttempts; while (--retriesLeft > 0) { RMAppEvent event = new RMAppFailedAttemptEvent(app1.getApplicationId(), @@ -882,7 +882,7 @@ public class TestRMWebServicesApps exten rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); amNodeManager.nodeHeartbeat(true); } - assertEquals("incorrect number of attempts", maxRetries, + assertEquals("incorrect number of attempts", maxAppAttempts, app1.getAppAttempts().values().size()); testAppAttemptsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Sat Mar 30 03:50:03 2013 @@ -361,8 +361,7 @@ public class MiniYARNCluster extends Com NodeHeartbeatResponse response = recordFactory.newRecordInstance( NodeHeartbeatResponse.class); try { - response.setHeartbeatResponse(rt.nodeHeartbeat(request) - .getHeartbeatResponse()); + response = rt.nodeHeartbeat(request); } catch (IOException ioe) { LOG.info("Exception in heartbeat from node " + request.getNodeStatus().getNodeId(), ioe); @@ -378,9 +377,7 @@ public class MiniYARNCluster extends Com RegisterNodeManagerResponse response = recordFactory. newRecordInstance(RegisterNodeManagerResponse.class); try { - response.setRegistrationResponse(rt - .registerNodeManager(request) - .getRegistrationResponse()); + response = rt.registerNodeManager(request); } catch (IOException ioe) { LOG.info("Exception in node registration from " + request.getNodeId().toString(), ioe); Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Sat Mar 30 03:50:03 2013 @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -70,11 +69,8 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -89,7 +85,6 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -486,8 +481,8 @@ public class TestContainerManagerSecurit // Request a container allocation. List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); - ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(1024, 1), 1)); + ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(1024, 1), 1)); AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest( BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask, Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java?rev=1462698&r1=1462697&r2=1462698&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java Sat Mar 30 03:50:03 2013 @@ -27,9 +27,9 @@ import org.apache.hadoop.security.UserGr import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -62,12 +62,12 @@ public class TestRMNMSecretKeys { rm.start(); MockNM nm = new MockNM("host:1234", 3072, rm.getResourceTrackerService()); - RegistrationResponse registrationResponse = nm.registerNode(); + RegisterNodeManagerResponse registrationResponse = nm.registerNode(); MasterKey masterKey = registrationResponse.getMasterKey(); Assert.assertNotNull("Registration should cause a key-update!", masterKey); dispatcher.await(); - HeartbeatResponse response = nm.nodeHeartbeat(true); + NodeHeartbeatResponse response = nm.nodeHeartbeat(true); Assert.assertNull( "First heartbeat after registration shouldn't get any key updates!", response.getMasterKey());
