Repository: hadoop Updated Branches: refs/heads/trunk 8e5472b1e -> 06cceba1c
YARN-7738. CapacityScheduler: Support refresh maximum allocation for multiple resource types. Contributed by Wangda Tan. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06cceba1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06cceba1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06cceba1 Branch: refs/heads/trunk Commit: 06cceba1cb07340c412c4467439c16ea6812e685 Parents: 8e5472b Author: Sunil G <sun...@apache.org> Authored: Thu Jan 18 19:05:26 2018 +0530 Committer: Sunil G <sun...@apache.org> Committed: Thu Jan 18 19:05:26 2018 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 3 + .../yarn/util/resource/ResourceUtils.java | 40 ++++-- .../server/resourcemanager/AdminService.java | 3 +- .../scheduler/capacity/CapacityScheduler.java | 10 +- .../CapacitySchedulerConfiguration.java | 22 ++- .../scheduler/capacity/LeafQueue.java | 4 +- .../capacity/TestCapacityScheduler.java | 11 +- ...CapacitySchedulerWithMultiResourceTypes.java | 139 +++++++++++++++++++ 8 files changed, 193 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 304a963..78a691b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -369,6 +369,9 @@ public abstract class Resource implements Comparable<Resource> { return; } if (resource.equals(ResourceInformation.VCORES_URI)) { + if (value > Integer.MAX_VALUE) { + value = (long) Integer.MAX_VALUE; + } this.setVirtualCores((int)value); return; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index b352752..5a660ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -199,9 +199,23 @@ public class ResourceUtils { } } - @VisibleForTesting - static void initializeResourcesMap(Configuration conf) { + /** + * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA* + * @param conf config + * @return maximum allocation + */ + public static Resource fetchMaximumAllocationFromConfig(Configuration conf) { + Map<String, ResourceInformation> resourceInformationMap = + getResourceInformationMapFromConfig(conf); + Resource ret = Resource.newInstance(0, 0); + for (ResourceInformation entry : resourceInformationMap.values()) { + ret.setResourceValue(entry.getName(), entry.getMaximumAllocation()); + } + return ret; + } + private static Map<String, ResourceInformation> getResourceInformationMapFromConfig( + Configuration conf) { Map<String, ResourceInformation> resourceInformationMap = new HashMap<>(); String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); @@ -247,6 +261,13 @@ public class ResourceUtils { setAllocationForMandatoryResources(resourceInformationMap, conf); + return resourceInformationMap; + } + + @VisibleForTesting + static void initializeResourcesMap(Configuration conf) { + Map<String, ResourceInformation> resourceInformationMap = + getResourceInformationMapFromConfig(conf); initializeResourcesFromResourceInformationMap(resourceInformationMap); } @@ -544,19 +565,8 @@ public class ResourceUtils { public static Resource getResourceTypesMaximumAllocation() { Resource ret = Resource.newInstance(0, 0); for (ResourceInformation entry : resourceTypesArray) { - String name = entry.getName(); - if (name.equals(ResourceInformation.MEMORY_MB.getName())) { - ret.setMemorySize(entry.getMaximumAllocation()); - } else if (name.equals(ResourceInformation.VCORES.getName())) { - Long tmp = entry.getMaximumAllocation(); - if (tmp > Integer.MAX_VALUE) { - tmp = (long) Integer.MAX_VALUE; - } - ret.setVirtualCores(tmp.intValue()); - continue; - } else { - ret.setResourceValue(name, entry.getMaximumAllocation()); - } + ret.setResourceValue(entry.getName(), + entry.getMaximumAllocation()); } return ret; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index accf901..3c117bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -404,7 +404,8 @@ public class AdminService extends CompositeService implements throws IOException, YarnException { // Retrieve yarn-site.xml in order to refresh scheduling monitor properties. Configuration conf = getConfiguration(new Configuration(false), - YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE, + YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); // The reason we call Configuration#size() is because when getConfiguration // been called, it invokes Configuration#addResouce, which invokes // Configuration#reloadConfiguration which triggers the reload process in a http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 7e2a623..ba2f85a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -143,6 +143,7 @@ import org.apache.hadoop.yarn.server.utils.Lock; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; @@ -439,12 +440,15 @@ public class CapacityScheduler extends validateConf(this.conf); try { LOG.info("Re-initializing queues..."); - refreshMaximumAllocation(this.conf.getMaximumAllocation()); + refreshMaximumAllocation( + ResourceUtils.fetchMaximumAllocationFromConfig(this.conf)); reinitializeQueues(this.conf); } catch (Throwable t) { this.conf = oldConf; - refreshMaximumAllocation(this.conf.getMaximumAllocation()); - throw new IOException("Failed to re-init queues : "+ t.getMessage(), t); + refreshMaximumAllocation( + ResourceUtils.fetchMaximumAllocationFromConfig(this.conf)); + throw new IOException("Failed to re-init queues : " + t.getMessage(), + t); } // update lazy preemption http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 8aa41ee..e609be9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.Schedulabl import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; @@ -835,16 +836,6 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur return Resources.createResource(minimumMemory, minimumCores); } - public Resource getMaximumAllocation() { - int maximumMemory = getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); - int maximumCores = getInt( - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - return Resources.createResource(maximumMemory, maximumCores); - } - @Private public Priority getQueuePriority(String queue) { String queuePolicyPrefix = getQueuePrefix(queue); @@ -868,6 +859,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur * @return setting specified per queue else falls back to the cluster setting */ public Resource getMaximumAllocationPerQueue(String queue) { + // Only support to specify memory and vcores maximum allocation per queue + // for now. String queuePrefix = getQueuePrefix(queue); long maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED); @@ -879,7 +872,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur LOG.debug("max alloc vcores per queue for " + queue + " is " + maxAllocationVcoresPerQueue); } - Resource clusterMax = getMaximumAllocation(); + Resource clusterMax = ResourceUtils.fetchMaximumAllocationFromConfig(this); if (maxAllocationMbPerQueue == (int)UNDEFINED) { LOG.info("max alloc mb per queue for " + queue + " is undefined"); maxAllocationMbPerQueue = clusterMax.getMemorySize(); @@ -888,8 +881,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur LOG.info("max alloc vcore per queue for " + queue + " is undefined"); maxAllocationVcoresPerQueue = clusterMax.getVirtualCores(); } - Resource result = Resources.createResource(maxAllocationMbPerQueue, - maxAllocationVcoresPerQueue); + // Copy from clusterMax and overwrite per-queue's maximum memory/vcore + // allocation. + Resource result = Resources.clone(clusterMax); + result.setMemorySize(maxAllocationMbPerQueue); + result.setVirtualCores(maxAllocationVcoresPerQueue); if (maxAllocationMbPerQueue > clusterMax.getMemorySize() || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) { throw new IllegalArgumentException( http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 31032b3..8d1428d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -553,8 +553,8 @@ public class LeafQueue extends AbstractCSQueue { // since we have already told running AM's the size Resource oldMax = getMaximumAllocation(); Resource newMax = newlyParsedLeafQueue.getMaximumAllocation(); - if (newMax.getMemorySize() < oldMax.getMemorySize() - || newMax.getVirtualCores() < oldMax.getVirtualCores()) { + + if (!Resources.fitsIn(oldMax, newMax)) { throw new IOException("Trying to reinitialize " + getQueuePath() + " the maximum allocation size can not be decreased!" + " Current setting: " + oldMax + ", trying to set it to: " http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index e91f734..7628312 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -154,6 +154,7 @@ import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; @@ -2942,7 +2943,7 @@ public class TestCapacityScheduler { conf.getMaximumAllocationPerQueue(A1).getMemorySize()); assertEquals("max allocation", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - conf.getMaximumAllocation().getMemorySize()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); CSQueue rootQueue = cs.getRootQueue(); CSQueue queueA = findQueue(rootQueue, A); @@ -3043,10 +3044,10 @@ public class TestCapacityScheduler { conf.getMaximumAllocationPerQueue(A1).getVirtualCores()); assertEquals("cluster max allocation MB", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - conf.getMaximumAllocation().getMemorySize()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); assertEquals("cluster max allocation vcores", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - conf.getMaximumAllocation().getVirtualCores()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); CSQueue rootQueue = cs.getRootQueue(); CSQueue queueA = findQueue(rootQueue, A); @@ -3065,10 +3066,10 @@ public class TestCapacityScheduler { conf.getMaximumAllocationPerQueue(A1).getVirtualCores()); assertEquals("max allocation MB cluster", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - conf.getMaximumAllocation().getMemorySize()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); assertEquals("max allocation vcores cluster", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - conf.getMaximumAllocation().getVirtualCores()); + ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores()); assertEquals("queue max allocation MB", 6144, ((LeafQueue) queueA1).getMaximumAllocation().getMemorySize()); assertEquals("queue max allocation vcores", 3, http://git-wip-us.apache.org/repos/asf/hadoop/blob/06cceba1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 38768e5..2da2cdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -147,4 +148,142 @@ public class TestCapacitySchedulerWithMultiResourceTypes { TestUtils.createResource(2 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)), containerTokenIdentifier.getResource()); } + + @Test + public void testMaximumAllocationRefreshWithMultipleResourceTypes() throws Exception { + + // Initialize resource map + Map<String, ResourceInformation> riMap = new HashMap<>(); + + // Initialize mandatory resources + ResourceInformation memory = ResourceInformation.newInstance( + ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + ResourceInformation vcores = ResourceInformation.newInstance( + ResourceInformation.VCORES.getName(), + ResourceInformation.VCORES.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + riMap.put(ResourceInformation.MEMORY_URI, memory); + riMap.put(ResourceInformation.VCORES_URI, vcores); + riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0, + ResourceTypes.COUNTABLE, 0, 3333L)); + + ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); + + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(); + csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f); + csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); + csconf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", + 100.0f); + csconf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f); + csconf.setResourceComparator(DominantResourceCalculator.class); + csconf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_1); + csconf.setInt(YarnConfiguration.RESOURCE_TYPES + "." + RESOURCE_1 + + ".maximum-allocation", 3333); + + YarnConfiguration conf = new YarnConfiguration(csconf); + // Don't reset resource types since we have already configured resource + // types + conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + MockRM rm = new MockRM(conf); + rm.start(); + + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + Assert.assertEquals(3333L, + cs.getMaximumResourceCapability().getResourceValue(RESOURCE_1)); + Assert.assertEquals(3333L, + cs.getMaximumAllocation().getResourceValue(RESOURCE_1)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability() + .getResourceValue(ResourceInformation.MEMORY_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumAllocation() + .getResourceValue(ResourceInformation.MEMORY_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability() + .getResourceValue(ResourceInformation.VCORES_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumAllocation() + .getResourceValue(ResourceInformation.VCORES_URI)); + + // Set RES_1 to 3332 (less than 3333) and refresh CS, failures expected. + csconf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_1); + csconf.setInt(YarnConfiguration.RESOURCE_TYPES + "." + RESOURCE_1 + + ".maximum-allocation", 3332); + + boolean exception = false; + try { + cs.reinitialize(csconf, rm.getRMContext()); + } catch (IOException e) { + exception = true; + } + + Assert.assertTrue("Should have exception in CS", exception); + + // Maximum allocation won't be updated + Assert.assertEquals(3333L, + cs.getMaximumResourceCapability().getResourceValue(RESOURCE_1)); + Assert.assertEquals(3333L, + cs.getMaximumAllocation().getResourceValue(RESOURCE_1)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability() + .getResourceValue(ResourceInformation.MEMORY_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumAllocation() + .getResourceValue(ResourceInformation.MEMORY_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability() + .getResourceValue(ResourceInformation.VCORES_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumAllocation() + .getResourceValue(ResourceInformation.VCORES_URI)); + + // Set RES_1 to 3334 and refresh CS, should success + csconf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_1); + csconf.setInt(YarnConfiguration.RESOURCE_TYPES + "." + RESOURCE_1 + + ".maximum-allocation", 3334); + cs.reinitialize(csconf, rm.getRMContext()); + + // Maximum allocation will be updated + Assert.assertEquals(3334, + cs.getMaximumResourceCapability().getResourceValue(RESOURCE_1)); + + // Since we haven't updated the real configuration of ResourceUtils, + // cs.getMaximumAllocation won't be updated. + Assert.assertEquals(3333, + cs.getMaximumAllocation().getResourceValue(RESOURCE_1)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumResourceCapability() + .getResourceValue(ResourceInformation.MEMORY_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + cs.getMaximumAllocation() + .getResourceValue(ResourceInformation.MEMORY_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumResourceCapability() + .getResourceValue(ResourceInformation.VCORES_URI)); + Assert.assertEquals( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + cs.getMaximumAllocation() + .getResourceValue(ResourceInformation.VCORES_URI)); + + rm.close(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
