http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java index 078b8fd..c80469b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java @@ -18,26 +18,108 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.apache.hadoop.yarn.api.records.ContainerUpdateType.INCREASE_RESOURCE; +import static org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils.RESOURCE_OUTSIDE_ALLOWED_RANGE; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.UpdateContainerError; +import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; +import org.apache.hadoop.yarn.api.records.impl.pb.UpdateContainerRequestPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - public class TestRMServerUtils { + + @Test + public void testValidateAndSplitUpdateResourceRequests() { + List<UpdateContainerRequest> updateRequests = new ArrayList<>(); + int containerVersion = 10; + int resource = 10; + Resource maxAllocation = Resource.newInstance(resource, resource); + + UpdateContainerRequestPBImpl updateContainerRequestPBFail = + new UpdateContainerRequestPBImpl(); + updateContainerRequestPBFail.setContainerVersion(containerVersion); + updateContainerRequestPBFail + .setCapability(Resource.newInstance(resource + 1, resource + 1)); + updateContainerRequestPBFail + .setContainerId(Mockito.mock(ContainerId.class)); + + ContainerId containerIdOk = Mockito.mock(ContainerId.class); + Resource capabilityOk = Resource.newInstance(resource - 1, resource - 1); + UpdateContainerRequestPBImpl updateContainerRequestPBOk = + new UpdateContainerRequestPBImpl(); + updateContainerRequestPBOk.setContainerVersion(containerVersion); + updateContainerRequestPBOk.setCapability(capabilityOk); + updateContainerRequestPBOk.setContainerUpdateType(INCREASE_RESOURCE); + updateContainerRequestPBOk.setContainerId(containerIdOk); + + updateRequests.add(updateContainerRequestPBOk); + updateRequests.add(updateContainerRequestPBFail); + + Dispatcher dispatcher = Mockito.mock(Dispatcher.class); + RMContext rmContext = Mockito.mock(RMContext.class); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher); + + RMContainer rmContainer = Mockito.mock(RMContainer.class); + Mockito.when(scheduler.getRMContainer(Mockito.any())) + .thenReturn(rmContainer); + Container container = Mockito.mock(Container.class); + Mockito.when(container.getVersion()).thenReturn(containerVersion); + Mockito.when(rmContainer.getContainer()).thenReturn(container); + Mockito.when(scheduler.getNormalizedResource(capabilityOk, maxAllocation)) + .thenReturn(capabilityOk); + + AllocateRequest allocateRequest = + AllocateRequest.newInstance(1, 0.5f, new ArrayList<ResourceRequest>(), + new ArrayList<ContainerId>(), updateRequests, null); + + List<UpdateContainerError> updateErrors = new ArrayList<>(); + ContainerUpdates containerUpdates = + RMServerUtils.validateAndSplitUpdateResourceRequests(rmContext, + allocateRequest, maxAllocation, updateErrors); + Assert.assertEquals(1, updateErrors.size()); + Assert.assertEquals(resource + 1, updateErrors.get(0) + .getUpdateContainerRequest().getCapability().getMemorySize()); + Assert.assertEquals(resource + 1, updateErrors.get(0) + .getUpdateContainerRequest().getCapability().getVirtualCores()); + Assert.assertEquals(RESOURCE_OUTSIDE_ALLOWED_RANGE, + updateErrors.get(0).getReason()); + + Assert.assertEquals(1, containerUpdates.getIncreaseRequests().size()); + UpdateContainerRequest increaseRequest = + containerUpdates.getIncreaseRequests().get(0); + Assert.assertEquals(capabilityOk.getVirtualCores(), + increaseRequest.getCapability().getVirtualCores()); + Assert.assertEquals(capabilityOk.getMemorySize(), + increaseRequest.getCapability().getMemorySize()); + Assert.assertEquals(containerIdOk, increaseRequest.getContainerId()); + } + @Test public void testGetApplicableNodeCountForAMLocality() throws Exception { List<NodeId> rack1Nodes = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 2ec2de2..6969242 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -106,6 +106,7 @@ import org.junit.Test; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.junit.rules.ExpectedException; +import org.mockito.Mockito; public class TestSchedulerUtils { @@ -271,7 +272,7 @@ public class TestSchedulerUtils { public void testValidateResourceRequestWithErrorLabelsPermission() throws IOException { // mock queue and scheduler - YarnScheduler scheduler = mock(YarnScheduler.class); + ResourceScheduler scheduler = mock(ResourceScheduler.class); Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); QueueInfo queueInfo = mock(QueueInfo.class); when(queueInfo.getQueueName()).thenReturn("queue"); @@ -280,6 +281,8 @@ public class TestSchedulerUtils { when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) .thenReturn(queueInfo); + when(rmContext.getScheduler()).thenReturn(scheduler); + Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); @@ -298,20 +301,20 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression("y"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression(""); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression(" "); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is a subset of queue labels"); @@ -332,8 +335,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { @@ -354,8 +357,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("z"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { @@ -379,8 +382,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x && y"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { @@ -399,16 +402,16 @@ public class TestSchedulerUtils { YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression(""); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression(" "); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when request labels is empty"); @@ -428,8 +431,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidLabelResourceRequestException e) { invalidlabelexception = true; @@ -456,16 +459,16 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression("y"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); resReq.setNodeLabelExpression("z"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { e.printStackTrace(); fail("Should be valid when queue can access any labels"); @@ -486,8 +489,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { } @@ -507,8 +510,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { @@ -532,8 +535,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), "rack", resource, 1); resReq.setNodeLabelExpression("x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { } finally { @@ -545,8 +548,8 @@ public class TestSchedulerUtils { YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq1 = BuilderUtils .newResourceRequest(mock(Priority.class), "*", resource, 1, "x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq1, "queue", + scheduler, rmContext, maxResource); fail("Should fail"); } catch (InvalidResourceRequestException e) { assertEquals("Invalid label resource request, cluster do not contain , " @@ -560,8 +563,8 @@ public class TestSchedulerUtils { YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq1 = BuilderUtils .newResourceRequest(mock(Priority.class), "*", resource, 1, "x"); - SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq1, "queue", + scheduler, rmContext, maxResource); Assert.assertEquals(RMNodeLabelsManager.NO_LABEL, resReq1.getNodeLabelExpression()); } catch (InvalidResourceRequestException e) { @@ -571,14 +574,21 @@ public class TestSchedulerUtils { } @Test(timeout = 30000) - public void testValidateResourceRequest() { - YarnScheduler mockScheduler = mock(YarnScheduler.class); + public void testValidateResourceRequest() throws IOException { + ResourceScheduler mockScheduler = mock(ResourceScheduler.class); + + QueueInfo queueInfo = mock(QueueInfo.class); + when(queueInfo.getQueueName()).thenReturn("queue"); Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + when(rmContext.getScheduler()).thenReturn(mockScheduler); + when(mockScheduler.getQueueInfo(Mockito.anyString(), Mockito.anyBoolean(), + Mockito.anyBoolean())).thenReturn(queueInfo); + // zero memory try { Resource resource = @@ -587,8 +597,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { fail("Zero memory should be accepted"); } @@ -601,8 +611,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { fail("Zero vcores should be accepted"); } @@ -616,8 +626,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { fail("Max memory should be accepted"); } @@ -631,8 +641,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); } catch (InvalidResourceRequestException e) { fail("Max vcores should not be accepted"); } @@ -645,8 +655,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); fail("Negative memory should not be accepted"); } catch (InvalidResourceRequestException e) { assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType()); @@ -660,8 +670,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); fail("Negative vcores should not be accepted"); } catch (InvalidResourceRequestException e) { assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType()); @@ -676,8 +686,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); fail("More than max memory should not be accepted"); } catch (InvalidResourceRequestException e) { assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType()); @@ -691,8 +701,8 @@ public class TestSchedulerUtils { ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, - mockScheduler, rmContext); + normalizeAndvalidateRequest(resReq, null, + mockScheduler, rmContext, maxResource); fail("More than max vcores should not be accepted"); } catch (InvalidResourceRequestException e) { assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType()); @@ -805,7 +815,7 @@ public class TestSchedulerUtils { public void testNormalizeNodeLabelExpression() throws IOException { // mock queue and scheduler - YarnScheduler scheduler = mock(YarnScheduler.class); + ResourceScheduler scheduler = mock(ResourceScheduler.class); Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); QueueInfo queueInfo = mock(QueueInfo.class); when(queueInfo.getQueueName()).thenReturn("queue"); @@ -818,6 +828,8 @@ public class TestSchedulerUtils { YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + when(rmContext.getScheduler()).thenReturn(scheduler); + // queue has labels, success cases try { // set queue accessible node labels to [x, y] @@ -831,13 +843,13 @@ public class TestSchedulerUtils { YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); ResourceRequest resReq = BuilderUtils.newResourceRequest( mock(Priority.class), ResourceRequest.ANY, resource, 1); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); Assert.assertEquals("x", resReq.getNodeLabelExpression()); resReq.setNodeLabelExpression(" y "); - SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", - scheduler, rmContext); + normalizeAndvalidateRequest(resReq, "queue", + scheduler, rmContext, maxResource); Assert.assertEquals("y", resReq.getNodeLabelExpression()); } catch (InvalidResourceRequestException e) { e.printStackTrace(); @@ -1033,6 +1045,14 @@ public class TestSchedulerUtils { return rmContext; } + private static void normalizeAndvalidateRequest(ResourceRequest resReq, + String queueName, YarnScheduler scheduler, RMContext rmContext, + Resource maxAllocation) + throws InvalidResourceRequestException { + SchedulerUtils.normalizeAndValidateRequest(resReq, maxAllocation, queueName, + scheduler, rmContext, null); + } + private static class InvalidResourceRequestExceptionMessageGenerator { private StringBuilder sb; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index e77d8e2..f32b467 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -835,6 +837,41 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta); } + @Test + public void testQueueMaximumAllocations() { + CapacityScheduler scheduler = new CapacityScheduler(); + scheduler.setConf(new YarnConfiguration()); + scheduler.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + + setupQueueConfiguration(conf); + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1) + + MAXIMUM_ALLOCATION_MB, "1024"); + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1) + + MAXIMUM_ALLOCATION_VCORES, "1"); + + scheduler.init(conf); + scheduler.start(); + + Resource maxAllocationForQueue = + scheduler.getMaximumResourceCapability("a1"); + Resource maxAllocation1 = scheduler.getMaximumResourceCapability(""); + Resource maxAllocation2 = scheduler.getMaximumResourceCapability(null); + Resource maxAllocation3 = scheduler.getMaximumResourceCapability(); + + Assert.assertEquals(maxAllocation1, maxAllocation2); + Assert.assertEquals(maxAllocation1, maxAllocation3); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + maxAllocation1.getMemorySize()); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + maxAllocation1.getVirtualCores()); + + Assert.assertEquals(1024, maxAllocationForQueue.getMemorySize()); + Assert.assertEquals(1, maxAllocationForQueue.getVirtualCores()); + } + @Test public void testRefreshQueues() throws Exception { @@ -4009,7 +4046,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { private void setMaxAllocMb(CapacitySchedulerConfiguration conf, String queueName, int maxAllocMb) { String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) - + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; + + MAXIMUM_ALLOCATION_MB; conf.setInt(propName, maxAllocMb); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 3ac3849..4f1f20b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -77,6 +77,7 @@ public class FairSchedulerTestBase { public static final float TEST_RESERVATION_THRESHOLD = 0.09f; private static final int SLEEP_DURATION = 10; private static final int SLEEP_RETRIES = 1000; + protected static final int RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE = 10240; final static ContainerUpdates NULL_UPDATE_REQUESTS = new ContainerUpdates(); @@ -93,7 +94,8 @@ public class FairSchedulerTestBase { conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1024); - conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE); conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 50a003e..ac30b23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue; @@ -30,7 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfi import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; import org.apache.hadoop.yarn.util.ControlledClock; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.hadoop.yarn.util.resource.TestResourceUtils; import org.junit.Test; import java.io.File; import java.io.FileOutputStream; @@ -41,6 +44,7 @@ import java.io.PrintWriter; import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -53,6 +57,8 @@ import static org.junit.Assert.fail; public class TestAllocationFileLoaderService { + private static final String A_CUSTOM_RESOURCE = "a-custom-resource"; + final static String TEST_DIR = new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); @@ -203,7 +209,8 @@ public class TestAllocationFileLoaderService { @Test public void testAllocationFileParsing() throws Exception { - Configuration conf = new Configuration(); + Configuration conf = new YarnConfiguration(); + TestResourceUtils.addNewTypesToResources(A_CUSTOM_RESOURCE); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); @@ -246,6 +253,8 @@ public class TestAllocationFileLoaderService { .fairSharePreemptionTimeout(120) .minSharePreemptionTimeout(50) .fairSharePreemptionThreshold(0.6) + .maxContainerAllocation( + "vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10") // Create hierarchical queues G,H, with different min/fair // share preemption timeouts and preemption thresholds. // Also add a child default to make sure it doesn't impact queue H. @@ -253,6 +262,7 @@ public class TestAllocationFileLoaderService { .fairSharePreemptionTimeout(180) .minSharePreemptionTimeout(40) .fairSharePreemptionThreshold(0.7) + .maxContainerAllocation("1024mb,8vcores") .buildSubQueue() .buildQueue() // Set default limit of apps per queue to 15 @@ -286,8 +296,6 @@ public class TestAllocationFileLoaderService { assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size()); assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(Resources.createResource(0), - queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); assertEquals(Resources.createResource(2048, 10), queueConf.getMaxResources("root.queueA").getResource()); @@ -358,6 +366,29 @@ public class TestAllocationFileLoaderService { assertEquals(.4f, queueConf.getQueueMaxAMShare("root.queueD"), 0.01); assertEquals(.5f, queueConf.getQueueMaxAMShare("root.queueE"), 0.01); + Resource expectedResourceWithCustomType = Resources.createResource(512, 16); + expectedResourceWithCustomType.setResourceValue(A_CUSTOM_RESOURCE, 10); + + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation( + "root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation("root.queueA")); + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation("root.queueB")); + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation("root.queueC")); + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation("root.queueD")); + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation("root.queueE")); + assertEquals(Resources.unbounded(), + queueConf.getQueueMaxContainerAllocation("root.queueF")); + assertEquals(expectedResourceWithCustomType, + queueConf.getQueueMaxContainerAllocation("root.queueG")); + assertEquals(Resources.createResource(1024, 8), + queueConf.getQueueMaxContainerAllocation("root.queueG.queueH")); + assertEquals(120000, queueConf.getMinSharePreemptionTimeout("root")); assertEquals(-1, queueConf.getMinSharePreemptionTimeout("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java new file mode 100644 index 0000000..2416d8c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test Application master service using Fair scheduler. + */ +public class TestApplicationMasterServiceWithFS { + + private static final Log LOG = + LogFactory.getLog(TestApplicationMasterServiceWithFS.class); + + private static final int GB = 1024; + private static final int MEMORY_ALLOCATION = 3 * GB; + private static final String TEST_FOLDER = "test-queues"; + private AllocateResponse allocateResponse; + private static YarnConfiguration configuration; + + @BeforeClass + public static void setup() throws IOException { + String allocFile = + GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath(); + + configuration = new YarnConfiguration(); + configuration.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, + ResourceScheduler.class); + configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); + + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println(" <queue name=\"queueA\">"); + out.println( + " <maxContainerAllocation>2048 mb 1 vcores</maxContainerAllocation>"); + out.println(" </queue>"); + out.println(" <queue name=\"queueB\">"); + out.println( + " <maxContainerAllocation>3072 mb 1 vcores</maxContainerAllocation>"); + out.println(" </queue>"); + out.println(" <queue name=\"queueC\">"); + out.println(" </queue>"); + out.println("</allocations>"); + out.close(); + } + + @AfterClass + public static void teardown(){ + File allocFile = GenericTestUtils.getTestDir(TEST_FOLDER); + allocFile.delete(); + } + + @Test(timeout = 3000000) + public void testQueueLevelContainerAllocationFail() throws Exception { + MockRM rm = new MockRM(configuration); + rm.start(); + + // Register node1 + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); + + // Submit an application + RMApp app1 = rm.submitApp(2 * GB, "queueA"); + + // kick the scheduling + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + + am1.addRequests(new String[] { "127.0.0.1" }, MEMORY_ALLOCATION, 1, 1); + try { + allocateResponse = am1.schedule(); // send the request + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof InvalidResourceRequestException); + Assert.assertEquals( + InvalidResourceRequestException.InvalidResourceType.GREATER_THEN_MAX_ALLOCATION, + ((InvalidResourceRequestException) e).getInvalidResourceType()); + + } finally { + rm.stop(); + } + } + + @Test(timeout = 3000000) + public void testQueueLevelContainerAllocationSuccess() throws Exception { + testFairSchedulerContainerAllocationSuccess("queueB"); + } + + @Test(timeout = 3000000) + public void testSchedulerLevelContainerAllocationSuccess() throws Exception { + testFairSchedulerContainerAllocationSuccess("queueC"); + } + + private void testFairSchedulerContainerAllocationSuccess(String queueName) + throws Exception { + MockRM rm = new MockRM(configuration); + rm.start(); + + // Register node1 + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); + + // Submit an application + RMApp app1 = rm.submitApp(2 * GB, queueName); + + // kick the scheduling + nm1.nodeHeartbeat(true); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); + am1.registerAppAttempt(); + + am1.addRequests(new String[] { "127.0.0.1" }, MEMORY_ALLOCATION, 1, 1); + + allocateResponse = am1.schedule(); // send the request + ((FairScheduler) rm.getResourceScheduler()).update(); + + // kick the scheduler + nm1.nodeHeartbeat(true); + GenericTestUtils.waitFor(() -> { + LOG.info("Waiting for containers to be created for app 1"); + try { + allocateResponse = am1.schedule(); + } catch (Exception e) { + Assert.fail("Allocation should be successful"); + } + return allocateResponse.getAllocatedContainers().size() > 0; + }, 1000, 10000); + + Container allocatedContainer = + allocateResponse.getAllocatedContainers().get(0); + Assert.assertEquals(MEMORY_ALLOCATION, + allocatedContainer.getResource().getMemorySize()); + Assert.assertEquals(1, allocatedContainer.getResource().getVirtualCores()); + rm.stop(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 9120d3a..dbf4b2b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -198,8 +199,6 @@ public class TestFairScheduler extends FairSchedulerTestBase { } } - // TESTS - @SuppressWarnings("deprecation") @Test(timeout=2000) public void testLoadConfigurationOnInitialize() throws IOException { @@ -338,6 +337,111 @@ public class TestFairScheduler extends FairSchedulerTestBase { } } + @Test + public void testQueueMaximumCapacityAllocations() throws IOException { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + int tooHighQueueAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE +1; + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println(" <queue name=\"queueA\">"); + out.println( + " <maxContainerAllocation>512 mb 1 vcores</maxContainerAllocation>"); + out.println(" </queue>"); + out.println(" <queue name=\"queueB\">"); + out.println(" </queue>"); + out.println(" <queue name=\"queueC\">"); + out.println( + " <maxContainerAllocation>2048 mb 3 vcores</maxContainerAllocation>"); + out.println(" <queue name=\"queueD\">"); + out.println(" </queue>"); + out.println(" </queue>"); + out.println(" <queue name=\"queueE\">"); + out.println(" <maxContainerAllocation>" + tooHighQueueAllocation + + " mb 1 vcores</maxContainerAllocation>"); + out.println(" </queue>"); + out.println("</allocations>"); + out.close(); + + scheduler.init(conf); + + Assert.assertEquals(1, scheduler.getMaximumResourceCapability("root.queueA") + .getVirtualCores()); + Assert.assertEquals(512, + scheduler.getMaximumResourceCapability("root.queueA").getMemorySize()); + + Assert.assertEquals(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + scheduler.getMaximumResourceCapability("root.queueB") + .getVirtualCores()); + Assert.assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, + scheduler.getMaximumResourceCapability("root.queueB").getMemorySize()); + + Assert.assertEquals(3, scheduler.getMaximumResourceCapability("root.queueC") + .getVirtualCores()); + Assert.assertEquals(2048, + scheduler.getMaximumResourceCapability("root.queueC").getMemorySize()); + + Assert.assertEquals(3, scheduler + .getMaximumResourceCapability("root.queueC.queueD").getVirtualCores()); + Assert.assertEquals(2048, scheduler + .getMaximumResourceCapability("root.queueC.queueD").getMemorySize()); + + Assert.assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, scheduler + .getMaximumResourceCapability("root.queueE").getMemorySize()); + } + + @Test + public void testNormalizationUsingQueueMaximumAllocation() + throws IOException { + + int queueMaxAllocation = 4096; + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println(" <queue name=\"queueA\">"); + out.println(" <maxContainerAllocation>" + queueMaxAllocation + + " mb 1 vcores" + "</maxContainerAllocation>"); + out.println(" </queue>"); + out.println(" <queue name=\"queueB\">"); + out.println(" </queue>"); + out.println("</allocations>"); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + allocateAppAttempt("root.queueA", 1, queueMaxAllocation + 1024); + allocateAppAttempt("root.queueB", 2, + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE + 1024); + + scheduler.update(); + FSQueue queueToCheckA = scheduler.getQueueManager().getQueue("root.queueA"); + FSQueue queueToCheckB = scheduler.getQueueManager().getQueue("root.queueB"); + + assertEquals(queueMaxAllocation, queueToCheckA.getDemand().getMemorySize()); + assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, + queueToCheckB.getDemand().getMemorySize()); + } + + private void allocateAppAttempt(String queueName, int id, int memorySize) { + ApplicationAttemptId id11 = createAppAttemptId(id, id); + createMockRMApp(id11); + scheduler.addApplication(id11.getApplicationId(), queueName, "user1", + false); + scheduler.addApplicationAttempt(id11, false, false); + List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>(); + ResourceRequest request1 = + createResourceRequest(memorySize, ResourceRequest.ANY, 1, 1, true); + ask1.add(request1); + scheduler.allocate(id11, ask1, null, new ArrayList<ContainerId>(), null, + null, NULL_UPDATE_REQUESTS); + } + /** * Test fair shares when max resources are set but are too high to impact * the shares. @@ -1316,8 +1420,9 @@ public class TestFairScheduler extends FairSchedulerTestBase { // New node satisfies resource request scheduler.update(); scheduler.handle(new NodeUpdateSchedulerEvent(node4)); - assertEquals(10240, scheduler.getQueueManager().getQueue("queue1"). - getResourceUsage().getMemorySize()); + assertEquals(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, + scheduler.getQueueManager().getQueue("queue1").getResourceUsage() + .getMemorySize()); scheduler.handle(new NodeUpdateSchedulerEvent(node1)); scheduler.handle(new NodeUpdateSchedulerEvent(node2)); @@ -4099,12 +4204,12 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); - RMNode node1 = - MockNodes.newNodeInfo(1, Resources.createResource(10240, 10), - 1, "127.0.0.1"); - RMNode node2 = - MockNodes.newNodeInfo(1, Resources.createResource(10240, 10), - 2, "127.0.0.2"); + RMNode node1 = MockNodes.newNodeInfo(1, + Resources.createResource(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 10), + 1, "127.0.0.1"); + RMNode node2 = MockNodes.newNodeInfo(1, + Resources.createResource(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 10), + 2, "127.0.0.2"); RMNode node3 = MockNodes.newNodeInfo(1, Resources.createResource(5120, 5), 3, "127.0.0.3"); @@ -4122,10 +4227,12 @@ public class TestFairScheduler extends FairSchedulerTestBase { true); Resource amResource1 = Resource.newInstance(1024, 1); Resource amResource2 = Resource.newInstance(1024, 1); - Resource amResource3 = Resource.newInstance(10240, 1); + Resource amResource3 = + Resource.newInstance(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1); Resource amResource4 = Resource.newInstance(5120, 1); Resource amResource5 = Resource.newInstance(1024, 1); - Resource amResource6 = Resource.newInstance(10240, 1); + Resource amResource6 = + Resource.newInstance(RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1); Resource amResource7 = Resource.newInstance(1024, 1); Resource amResource8 = Resource.newInstance(1024, 1); int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); @@ -4159,7 +4266,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId3 = createAppAttemptId(3, 1); createApplicationWithAMResource(attId3, "queue1", "user1", amResource3); - createSchedulingRequestExistingApplication(10240, 1, amPriority, attId3); + createSchedulingRequestExistingApplication( + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1, amPriority, attId3); FSAppAttempt app3 = scheduler.getSchedulerApp(attId3); scheduler.update(); // app3 reserves a container on node1 because node1's available resource @@ -4233,7 +4341,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId attId6 = createAppAttemptId(6, 1); createApplicationWithAMResource(attId6, "queue1", "user1", amResource6); - createSchedulingRequestExistingApplication(10240, 1, amPriority, attId6); + createSchedulingRequestExistingApplication( + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, 1, amPriority, attId6); FSAppAttempt app6 = scheduler.getSchedulerApp(attId6); scheduler.update(); // app6 can't reserve a container on node1 because @@ -4322,7 +4431,8 @@ public class TestFairScheduler extends FairSchedulerTestBase { // app6 turns the reservation into an allocation on node2. scheduler.handle(updateE2); assertEquals("Application6's AM requests 10240 MB memory", - 10240, app6.getAMResource().getMemorySize()); + RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE, + app6.getAMResource().getMemorySize()); assertEquals("Application6's AM should be running", 1, app6.getLiveContainers().size()); assertEquals("Queue1's AM resource usage should be 11264 MB memory", http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java index f1afe69..db81613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java @@ -60,9 +60,11 @@ class AllocationFileQueue { () -> AllocationFileWriter .createNumberSupplier(properties.getFairSharePreemptionTimeout())); AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold", + () -> AllocationFileWriter.createNumberSupplier( + properties.getFairSharePreemptionThreshold())); + AllocationFileWriter.addIfPresent(pw, "maxContainerAllocation", () -> AllocationFileWriter - .createNumberSupplier( - properties.getFairSharePreemptionThreshold())); + .createNumberSupplier(properties.getMaxContainerAllocation())); printEndTag(pw); pw.close(); return sw.toString(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java index a2faf1d..176024e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java @@ -94,6 +94,12 @@ public abstract class AllocationFileQueueBuilder { return this; } + public AllocationFileQueueBuilder maxContainerAllocation( + String maxContainerAllocation) { + this.queuePropertiesBuilder.maxContainerAllocation(maxContainerAllocation); + return this; + } + public AllocationFileQueueBuilder subQueue(String queueName) { if (this instanceof AllocationFileSimpleQueueBuilder) { return new AllocationFileSubQueueBuilder( http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java index 2c01144..0a0f330 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java @@ -33,6 +33,7 @@ public class AllocationFileQueueProperties { private final String maxChildResources; private final Integer fairSharePreemptionTimeout; private final Double fairSharePreemptionThreshold; + private final String maxContainerAllocation; AllocationFileQueueProperties(Builder builder) { this.queueName = builder.queueName; @@ -48,6 +49,7 @@ public class AllocationFileQueueProperties { this.maxChildResources = builder.maxChildResources; this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout; this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold; + this.maxContainerAllocation = builder.maxContainerAllocation; } public String getQueueName() { @@ -102,6 +104,10 @@ public class AllocationFileQueueProperties { return fairSharePreemptionThreshold; } + public String getMaxContainerAllocation() { + return maxContainerAllocation; + } + /** * Builder class for {@link AllocationFileQueueProperties}. */ @@ -119,6 +125,7 @@ public class AllocationFileQueueProperties { private String maxChildResources; private Integer fairSharePreemptionTimeout; private Double fairSharePreemptionThreshold; + private String maxContainerAllocation; Builder() { } @@ -167,6 +174,11 @@ public class AllocationFileQueueProperties { return this; } + public Builder maxContainerAllocation(String maxContainerAllocation) { + this.maxContainerAllocation = maxContainerAllocation; + return this; + } + public Builder minSharePreemptionTimeout( Integer minSharePreemptionTimeout) { this.minSharePreemptionTimeout = minSharePreemptionTimeout; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd6be589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index b5bcbf5..5f9e779 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -90,6 +90,8 @@ The allocation file must be in XML format. The format contains five types of ele * **maxResources**: maximum resources a queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. A queue will not be assigned a container that would put its aggregate usage over this limit. + * **maxContainerAllocation**: maximum resources a queue can allocate for a single container, expressed in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. If the property is not set it's value is inherited from a parent queue. It's default value is **yarn.scheduler.maximum-allocation-mb**. Cannot be higher than **maxResources**. This property is invalid for root queue. + * **maxChildResources**: maximum resources an ad hoc child queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit. * **maxRunningApps**: limit the number of apps from the queue to run at once --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
