Repository: hadoop Updated Branches: refs/heads/branch-3.1 03aba6983 -> f878ddb06
YARN-7328. ResourceUtils allows yarn.nodemanager.resource-types.memory-mb and .vcores to override yarn.nodemanager.resource.memory-mb and .cpu-vcores. (lovekesh bansal via wangda) Change-Id: Ibb1faf0beefec079dae2208986976b2f64650672 (cherry picked from commit ca1043ab9030339d7cdd3275c3f8f4713b8bff59) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f878ddb0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f878ddb0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f878ddb0 Branch: refs/heads/branch-3.1 Commit: f878ddb067c6547f07fcde0c198a37c132a0b214 Parents: ac9e5ee Author: Wangda Tan <wan...@apache.org> Authored: Sat Feb 17 12:28:08 2018 +0800 Committer: Wangda Tan <wan...@apache.org> Committed: Sat Feb 17 13:03:58 2018 +0800 ---------------------------------------------------------------------- .../yarn/util/resource/ResourceUtils.java | 20 ++++++++------ .../yarn/util/resource/TestResourceUtils.java | 28 ++++++++++++++++++- .../resource-types/node-resources-2.xml | 11 -------- .../resource-types/node-resources-error-1.xml | 29 ++++++++++++++++++++ .../resource-types/resource-types-error-2.xml | 4 +-- 5 files changed, 69 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878ddb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 5a660ce..f211f49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -84,20 +84,22 @@ public class ResourceUtils { Map<String, ResourceInformation> resourceInformationMap) throws YarnRuntimeException { /* - * Supporting 'memory' also as invalid resource name, in addition to + * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to * 'MEMORY' for historical reasons */ - String key = "memory"; - if (resourceInformationMap.containsKey(key)) { - LOG.warn( - "Attempt to define resource '" + key + "', but it is not allowed."); - throw new YarnRuntimeException( - "Attempt to re-define mandatory resource '" + key + "'."); + String keys[] = { "memory", ResourceInformation.MEMORY_URI, + ResourceInformation.VCORES_URI }; + for(String key : keys) { + if (resourceInformationMap.containsKey(key)) { + LOG.warn("Attempt to define resource '" + key + "', but it is not allowed."); + throw new YarnRuntimeException( + "Attempt to re-define mandatory resource '" + key + "'."); + } } for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry : ResourceInformation.MANDATORY_RESOURCES.entrySet()) { - key = mandatoryResourceEntry.getKey(); + String key = mandatoryResourceEntry.getKey(); ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue(); ResourceInformation newDefinedRI = resourceInformationMap.get(key); @@ -485,8 +487,8 @@ public class ResourceUtils { if (!initializedNodeResources) { Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation( conf); - addMandatoryResources(nodeResources); checkMandatoryResources(nodeResources); + addMandatoryResources(nodeResources); setAllocationForMandatoryResources(nodeResources, conf); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); initializedNodeResources = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878ddb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 1111ca6..2671de8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -287,7 +287,7 @@ public class TestResourceUtils { Map<String, Resource> testRun = new HashMap<>(); setupResourceTypes(conf, "resource-types-4.xml"); // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); - Resource test3Resources = Resource.newInstance(1024, 1); + Resource test3Resources = Resource.newInstance(0, 0); test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "Gi", 5L)); test3Resources.setResourceInformation("resource2", @@ -316,6 +316,32 @@ public class TestResourceUtils { } @Test + public void testGetNodeResourcesConfigErrors() throws Exception { + Configuration conf = new YarnConfiguration(); + Map<String, Resource> testRun = new HashMap<>(); + setupResourceTypes(conf, "resource-types-4.xml"); + String invalidNodeResFiles[] = { "node-resources-error-1.xml"}; + + for (String resourceFile : invalidNodeResFiles) { + ResourceUtils.resetNodeResources(); + File dest = null; + try { + File source = new File(conf.getClassLoader().getResource(resourceFile).getFile()); + dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map<String, ResourceInformation> actual = ResourceUtils.getNodeResourceInformation(conf); + Assert.fail("Expected error with file " + resourceFile); + } catch (NullPointerException ne) { + throw ne; + } catch (Exception e) { + if (dest != null) { + dest.delete(); + } + } + } + } + + @Test public void testResourceNameFormatValidation() throws Exception { String[] validNames = new String[] { "yarn.io/gpu", http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878ddb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml index 382d5dd..3323294 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml @@ -15,17 +15,6 @@ limitations under the License. See accompanying LICENSE file. --> <configuration> - - <property> - <name>yarn.nodemanager.resource-type.memory-mb</name> - <value>1024Mi</value> - </property> - - <property> - <name>yarn.nodemanager.resource-type.vcores</name> - <value>1</value> - </property> - <property> <name>yarn.nodemanager.resource-type.resource1</name> <value>5Gi</value> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878ddb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml new file mode 100644 index 0000000..4ea3c74 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. See accompanying LICENSE file. +--> + +<configuration> + + <property> + <name>yarn.nodemanager.resource-type.vcores</name> + <value>1024</value> + </property> + + <property> + <name>yarn.nodemanager.resource.resource1</name> + <value>1</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f878ddb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml index fa43b6c..a34db38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml @@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file. <property> <name>yarn.resource-types</name> - <value>vcores,resource1</value> + <value>resource1,resource2</value> </property> <property> @@ -27,7 +27,7 @@ limitations under the License. See accompanying LICENSE file. </property> <property> - <name>yarn.resource-types.vcores.units</name> + <name>yarn.resource-types.resource2.units</name> <value>Az</value> </property> </configuration> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
