YARN-6933. [YARN-3926] ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan R.
(cherry picked from commit 805095496dfd8e50f71b70cf20845e954d3ba47c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3becd158 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3becd158 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3becd158 Branch: refs/heads/resource-types Commit: 3becd15839426dc129486c5bbdb5379c36bcd4f2 Parents: 6fc855b Author: Sunil G <[email protected]> Authored: Wed Sep 6 18:51:14 2017 +0530 Committer: Daniel Templeton <[email protected]> Committed: Tue Oct 17 09:25:46 2017 -0700 ---------------------------------------------------------------------- .../yarn/util/resource/ResourceUtils.java | 38 ++++++++++---------- .../resource-types/resource-types-error-2.xml | 4 +++ .../resource-types/resource-types-error-3.xml | 2 +- 3 files changed, 23 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3becd158/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 8193a20..df17db6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -41,10 +41,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** @@ -60,13 +58,6 @@ public class ResourceUtils { private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); private static final String VCORES = ResourceInformation.VCORES.getName(); - private static final Set<String> DISALLOWED_NAMES = new HashSet<>(); - static { - DISALLOWED_NAMES.add("memory"); - DISALLOWED_NAMES.add(MEMORY); - DISALLOWED_NAMES.add(VCORES); - } - private static volatile boolean initializedResources = false; private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX = new ConcurrentHashMap<String, Integer>(); @@ -82,9 +73,21 @@ public class ResourceUtils { private ResourceUtils() { } - private static void checkMandatatoryResources( + private static void checkMandatoryResources( Map<String, ResourceInformation> resourceInformationMap) throws YarnRuntimeException { + /* + * Supporting 'memory' also as invalid resource name, in addition to + * 'MEMORY' for historical reasons + */ + String key = "memory"; + if (resourceInformationMap.containsKey(key)) { + LOG.warn("Attempt to define resource '" + key + + "', but it is not allowed."); + throw new YarnRuntimeException("Attempt to re-define mandatory resource '" + + key + "'."); + } + if (resourceInformationMap.containsKey(MEMORY)) { ResourceInformation memInfo = resourceInformationMap.get(MEMORY); String memUnits = ResourceInformation.MEMORY_MB.getUnits(); @@ -110,7 +113,7 @@ public class ResourceUtils { } } - private static void addManadtoryResources( + private static void addMandatoryResources( Map<String, ResourceInformation> res) { ResourceInformation ri; if (!res.containsKey(MEMORY)) { @@ -150,11 +153,6 @@ public class ResourceUtils { "Incomplete configuration for resource type '" + resourceName + "'. One of name, units or type is configured incorrectly."); } - if (DISALLOWED_NAMES.contains(resourceName)) { - throw new YarnRuntimeException( - "Resource type cannot be named '" + resourceName - + "'. That name is disallowed."); - } ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName); LOG.info("Adding resource type - name = " + resourceName + ", units = " + resourceUnits + ", type = " + resourceTypeName); @@ -166,8 +164,8 @@ public class ResourceUtils { .newInstance(resourceName, resourceUnits, 0L, resourceType)); } } - checkMandatatoryResources(resourceInformationMap); - addManadtoryResources(resourceInformationMap); + checkMandatoryResources(resourceInformationMap); + addMandatoryResources(resourceInformationMap); resourceTypes = Collections.unmodifiableMap(resourceInformationMap); updateKnownResources(); updateResourceTypeIndex(); @@ -366,8 +364,8 @@ public class ResourceUtils { if (!initializedNodeResources) { Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation( conf); - addManadtoryResources(nodeResources); - checkMandatatoryResources(nodeResources); + addMandatoryResources(nodeResources); + checkMandatoryResources(nodeResources); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); initializedNodeResources = true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3becd158/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml index ca428eb..fa43b6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml @@ -26,4 +26,8 @@ limitations under the License. See accompanying LICENSE file. <value>G</value> </property> + <property> + <name>yarn.resource-types.vcores.units</name> + <value>Az</value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/3becd158/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml index 08b8a6d..539d657 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml @@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file. <property> <name>yarn.resource-types</name> - <value>vcores,resource1</value> + <value>resource1,resource1</value> </property> <property> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
