http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index d45f756..a74274c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -191,6 +191,8 @@ public class TestLeafQueue { CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32), + new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32))); ResourceUsage queueResUsage = root.getQueueResourceUsage(); when(csContext.getClusterResourceUsage()) @@ -307,13 +309,11 @@ public class TestLeafQueue { // Verify the value for getAMResourceLimit for queues with < .1 maxcap Resource clusterResource = Resource.newInstance(50 * GB, 50); - a.updateClusterResource(clusterResource, + root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); assertEquals(Resource.newInstance(1 * GB, 1), a.calculateAndGetAMResourceLimit()); - b.updateClusterResource(clusterResource, - new ResourceLimits(clusterResource)); assertEquals(Resource.newInstance(5 * GB, 1), b.calculateAndGetAMResourceLimit()); } @@ -358,6 +358,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -556,6 +558,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -630,6 +634,8 @@ public class TestLeafQueue { // Test max-capacity // Now - no more allocs since we are at max-cap a.setMaxCapacity(0.5f); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), @@ -699,6 +705,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (80 * GB), numNodes * 100); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Set user-limit. Need a small queue within a large cluster. b.setUserLimit(50); @@ -779,6 +787,8 @@ public class TestLeafQueue { Resources.createResource(numNodes * (8 * GB), numNodes * 100); when(csContext.getNumClusterNodes()).thenReturn(numNodes); when(csContext.getClusterResource()).thenReturn(clusterResource); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests so that one application is memory dominant // and other application is vcores dominant @@ -891,6 +901,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -915,6 +927,8 @@ public class TestLeafQueue { // Set user-limit a.setUserLimit(50); a.setUserLimitFactor(2); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // There're two active users assertEquals(2, a.getAbstractUsersManager().getNumActiveUsers()); @@ -940,7 +954,7 @@ public class TestLeafQueue { assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); // Allocate one container to app_0, before allocating this container, - // user-limit = ceil((4 + 1) / 2) = 3G. app_0's used resource (3G) <= + // user-limit = floor((5 + 1) / 2) = 3G. app_0's used resource (3G) <= // user-limit. applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_1, @@ -1068,15 +1082,9 @@ public class TestLeafQueue { a.assignContainers(clusterResource, node_1, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps); - assertEquals(9*GB, a.getUsedResources().getMemorySize()); - assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); - - assertEquals(4*GB, - app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize()); - - assertEquals(1*GB, - app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize()); + assertEquals(12*GB, a.getUsedResources().getMemorySize()); + assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize()); + assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize()); } @SuppressWarnings({ "unchecked", "rawtypes" }) @@ -1100,6 +1108,8 @@ public class TestLeafQueue { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); CapacitySchedulerQueueManager mockCapacitySchedulerQueueManager = mock(CapacitySchedulerQueueManager.class); @@ -1122,6 +1132,8 @@ public class TestLeafQueue { qb.setUserLimit(100); qb.setUserLimitFactor(1); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = @@ -1256,106 +1268,6 @@ public class TestLeafQueue { } @Test - public void testUserHeadroomMultiApp() throws Exception { - // Mock the queue - LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); - //unset maxCapacity - a.setMaxCapacity(1.0f); - - // Users - final String user_0 = "user_0"; - final String user_1 = "user_1"; - - // Submit applications - final ApplicationAttemptId appAttemptId_0 = - TestUtils.getMockApplicationAttemptId(0, 0); - FiCaSchedulerApp app_0 = - new FiCaSchedulerApp(appAttemptId_0, user_0, a, - a.getAbstractUsersManager(), spyRMContext); - a.submitApplicationAttempt(app_0, user_0); - - final ApplicationAttemptId appAttemptId_1 = - TestUtils.getMockApplicationAttemptId(1, 0); - FiCaSchedulerApp app_1 = - new FiCaSchedulerApp(appAttemptId_1, user_0, a, - a.getAbstractUsersManager(), spyRMContext); - a.submitApplicationAttempt(app_1, user_0); // same user - - final ApplicationAttemptId appAttemptId_2 = - TestUtils.getMockApplicationAttemptId(2, 0); - FiCaSchedulerApp app_2 = - new FiCaSchedulerApp(appAttemptId_2, user_1, a, - a.getAbstractUsersManager(), spyRMContext); - a.submitApplicationAttempt(app_2, user_1); - - // Setup some nodes - String host_0 = "127.0.0.1"; - FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, - 0, 16*GB); - String host_1 = "127.0.0.2"; - FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, - 0, 16*GB); - - Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of( - app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(), - app_1, app_2.getApplicationAttemptId(), app_2); - Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(), - node_0, node_1.getNodeID(), node_1); - - final int numNodes = 2; - Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1); - when(csContext.getNumClusterNodes()).thenReturn(numNodes); - - Priority priority = TestUtils.createMockPriority(1); - - app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, - priority, recordFactory))); - - applyCSAssignment(clusterResource, - a.assignContainers(clusterResource, node_0, - new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps); - assertEquals(1*GB, a.getUsedResources().getMemorySize()); - assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize()); - assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize()); - //Now, headroom is the same for all apps for a given user + queue combo - //and a change to any app's headroom is reflected for all the user's apps - //once those apps are active/have themselves calculated headroom for - //allocation at least one time - assertEquals(2*GB, app_0.getHeadroom().getMemorySize()); - assertEquals(0*GB, app_1.getHeadroom().getMemorySize());//not yet active - assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active - - app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); - - applyCSAssignment(clusterResource, - a.assignContainers(clusterResource, node_0, - new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps); - assertEquals(2*GB, a.getUsedResources().getMemorySize()); - assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); - assertEquals(1*GB, app_1.getHeadroom().getMemorySize());//now active - assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active - - //Complete container and verify that headroom is updated, for both apps - //for the user - RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); - a.completedContainer(clusterResource, app_0, node_0, rmContainer, - ContainerStatus.newInstance(rmContainer.getContainerId(), - ContainerState.COMPLETE, "", - ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), - RMContainerEventType.KILL, null, true); - - assertEquals(2*GB, app_0.getHeadroom().getMemorySize()); - assertEquals(2*GB, app_1.getHeadroom().getMemorySize()); - } - - @Test public void testHeadroomWithMaxCap() throws Exception { // Mock the queue LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); @@ -1403,7 +1315,12 @@ public class TestLeafQueue { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); - + + ParentQueue root = (ParentQueue) queues + .get(CapacitySchedulerConfiguration.ROOT); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( @@ -1454,6 +1371,8 @@ public class TestLeafQueue { // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, priority, recordFactory))); @@ -1542,6 +1461,8 @@ public class TestLeafQueue { Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); when(csContext.getClusterResource()).thenReturn(clusterResource); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -1624,6 +1545,8 @@ public class TestLeafQueue { // Test max-capacity // Now - no more allocs since we are at max-cap a.setMaxCapacity(0.5f); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), @@ -1638,6 +1561,8 @@ public class TestLeafQueue { // Now, allocations should goto app_3 since it's under user-limit a.setMaxCapacity(1.0f); a.setUserLimitFactor(1); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), @@ -1743,6 +1668,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (4*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -1880,6 +1807,8 @@ public class TestLeafQueue { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (4*GB), numNodes * 16); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(4*GB, 16)); @@ -2051,6 +1980,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit Priority priority = TestUtils.createMockPriority(1); @@ -2237,11 +2168,10 @@ public class TestLeafQueue { CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); - queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); // Manipulate queue 'b' - LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); + LeafQueue a = stubLeafQueue((LeafQueue) newQueues.get(B)); // Check locality parameters. assertEquals(2, a.getNodeLocalityDelay()); @@ -2277,6 +2207,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8 * GB), numNodes * 16); when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes); + newRoot.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit Priority priority = TestUtils.createMockPriority(1); @@ -2412,6 +2344,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>(); @@ -2545,6 +2479,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource( numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit Priority priority = TestUtils.createMockPriority(1); @@ -2660,17 +2596,14 @@ public class TestLeafQueue { assertEquals(2, e.getNumActiveApplications()); assertEquals(1, e.getNumPendingApplications()); - csConf.setDouble(CapacitySchedulerConfiguration - .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, - CapacitySchedulerConfiguration - .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); + csConf.setDouble( + CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT + * 2); Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); - CSQueue newRoot = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, - CapacitySchedulerConfiguration.ROOT, - newQueues, queues, - TestUtils.spyHook); - queues = newQueues; + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, + TestUtils.spyHook); root.reinitialize(newRoot, csContext.getClusterResource()); // after reinitialization @@ -2697,7 +2630,6 @@ public class TestLeafQueue { CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); - queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization @@ -2745,7 +2677,7 @@ public class TestLeafQueue { assertEquals(1, e.getNumPendingApplications()); Resource clusterResource = Resources.createResource(200 * 16 * GB, 100 * 32); - e.updateClusterResource(clusterResource, + root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); // after updating cluster resource @@ -2837,6 +2769,9 @@ public class TestLeafQueue { numNodes * (8*GB), numNodes * 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + // Setup resource-requests // resourceName: <priority, memory, #containers, relaxLocality> // host_0_0: < 1, 1GB, 1, true > @@ -3036,36 +2971,44 @@ public class TestLeafQueue { @Test public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() throws Exception { + Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - Resource clusterResource = Resources - .createResource(100 * 16 * GB, 100 * 32); + final String newRootName = "root" + System.currentTimeMillis(); + setupQueueConfiguration(csConf, newRootName); + + Resource clusterResource = Resources.createResource(100 * 16 * GB, + 100 * 32); CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource); when(csContext.getRMContext()).thenReturn(rmContext); - csConf.setFloat(CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f); - ParentQueue root = new ParentQueue(csContext, - CapacitySchedulerConfiguration.ROOT, null, null); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80); - LeafQueue a = new LeafQueue(csContext, A, root, null); - assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f); - assertEquals(a.calculateAndGetAMResourceLimit(), - Resources.createResource(160 * GB, 1)); - - csConf.setFloat(CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); - LeafQueue newA = new LeafQueue(csContext, A, root, null); - a.reinitialize(newA, clusterResource); - assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f); - assertEquals(a.calculateAndGetAMResourceLimit(), - Resources.createResource(320 * GB, 1)); + csConf.setFloat( + CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + 0.1f); + + CSQueue root; + root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); - Resource newClusterResource = Resources.createResource(100 * 20 * GB, - 100 * 32); - a.updateClusterResource(newClusterResource, - new ResourceLimits(newClusterResource)); - // 100 * 20 * 0.2 = 400 - assertEquals(a.calculateAndGetAMResourceLimit(), - Resources.createResource(400 * GB, 1)); + // Manipulate queue 'a' + LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B)); + assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f); + assertEquals(b.calculateAndGetAMResourceLimit(), + Resources.createResource(159 * GB, 1)); + + csConf.setFloat( + CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + 0.2f); + clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32); + Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>(); + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, + TestUtils.spyHook); + root.reinitialize(newRoot, clusterResource); + + b = stubLeafQueue((LeafQueue) newQueues.get(B)); + assertEquals(b.calculateAndGetAMResourceLimit(), + Resources.createResource(320 * GB, 1)); } @Test @@ -3142,6 +3085,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (16 * GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); String user_0 = "user_0"; @@ -3308,6 +3253,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource( numNodes * (16*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); String user_0 = "user_0"; @@ -3435,6 +3382,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit // App0 has node local request for host_0/host_1, and app1 has node local @@ -3533,6 +3482,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (100*GB), numNodes * 128); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Pending resource requests for app_0 and app_1 total 5GB. Priority priority = TestUtils.createMockPriority(1); @@ -3699,6 +3650,8 @@ public class TestLeafQueue { Resource clusterResource = Resources.createResource(numNodes * (100*GB), numNodes * 128); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Pending resource requests for user_0: app_0 and app_1 total 3GB. Priority priority = TestUtils.createMockPriority(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index cdbbc51..a2318f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -248,6 +248,8 @@ public class TestParentQueue { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing LeafQueue a = (LeafQueue)queues.get(A); @@ -486,6 +488,8 @@ public class TestParentQueue { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing CSQueue a = queues.get(A); @@ -695,6 +699,8 @@ public class TestParentQueue { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing LeafQueue a = (LeafQueue)queues.get(A); @@ -771,6 +777,8 @@ public class TestParentQueue { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing LeafQueue b3 = (LeafQueue)queues.get(B3); http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 5e6548b..b0f6c73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -267,6 +267,8 @@ public class TestReservations { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -454,6 +456,8 @@ public class TestReservations { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -600,6 +604,8 @@ public class TestReservations { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -782,6 +788,8 @@ public class TestReservations { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -898,6 +906,8 @@ public class TestReservations { 8 * GB); Resource clusterResource = Resources.createResource(2 * 8 * GB); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority p = TestUtils.createMockPriority(5); @@ -1072,6 +1082,8 @@ public class TestReservations { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -1260,6 +1272,8 @@ public class TestReservations { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -1422,6 +1436,9 @@ public class TestReservations { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java index e3c108a..b9d5b82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableTable; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.junit.Assert; @@ -52,6 +53,8 @@ public class TestPriorityUtilizationQueueOrderingPolicy { when(q.getQueueCapacities()).thenReturn(qc); when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i])); + QueueResourceQuotas qr = new QueueResourceQuotas(); + when(q.getQueueResourceQuotas()).thenReturn(qr); list.add(q); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 1108f1a..0132348 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -354,10 +354,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 18; + int numExpectedElements = 20; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 31; + numExpectedElements = 33; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
