[43/50] [abbrv] hadoop git commit: YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev.
YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abc4ea4e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abc4ea4e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abc4ea4e Branch: refs/heads/YARN-3926 Commit: abc4ea43ea1f4d40e96cfe01a7636c7a489f Parents: a948ce4 Author: Varun VasudevAuthored: Sat Jun 11 14:33:46 2016 +0530 Committer: Varun Vasudev Committed: Wed Mar 8 16:16:03 2017 +0530 -- .../hadoop/yarn/api/records/Resource.java | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 19 ++- .../FileSystemBasedConfigurationProvider.java | 3 +- .../hadoop/yarn/LocalConfigurationProvider.java | 3 +- .../api/records/impl/pb/ResourcePBImpl.java | 53 +++--- .../yarn/util/resource/ResourceUtils.java | 168 +++ .../yarn/util/resource/TestResourceUtils.java | 29 +++- .../resource-types/node-resources-1.xml | 29 .../resource-types/node-resources-2.xml | 39 + .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../util/NodeManagerHardwareUtils.java | 52 ++ .../resourcemanager/ResourceTrackerService.java | 9 +- 12 files changed, 342 insertions(+), 72 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/abc4ea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 3efccce..3880bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -272,7 +272,8 @@ public abstract class Resource implements Comparable { continue; } if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits().equals("")) { + && entry.getValue().getUnits() + .equals(ResourceInformation.VCORES.getUnits())) { continue; } sb.append(", ").append(entry.getKey()).append(": ") http://git-wip-us.apache.org/repos/asf/hadoop/blob/abc4ea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 1adf67c..b250c1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -63,6 +63,10 @@ public class YarnConfiguration extends Configuration { "resource-types.xml"; @Private + public static final String NODE_RESOURCES_CONFIGURATION_FILE = + "node-resources.xml"; + + @Private public static final List RM_CONFIGURATION_FILES = Collections.unmodifiableList(Arrays.asList( RESOURCE_TYPES_CONFIGURATION_FILE, @@ -72,6 +76,16 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); + @Private + public static final List NM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + NODE_RESOURCES_CONFIGURATION_FILE, + DR_CONFIGURATION_FILE, + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_CONFIGURATION_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; @@ -100,12 +114,15 @@ public class YarnConfiguration extends Configuration { public static final String YARN_PREFIX = "yarn."; / - // Scheduler resource types configs + // Resource types configs public static final String RESOURCE_TYPES = YarnConfiguration.YARN_PREFIX + "resource-types"; + public static final String NM_RESOURCES_PREFIX = + YarnConfiguration.NM_PREFIX + "resource-type."; + /** Delay before deleting resource to ease debugging of NM issues
[43/50] [abbrv] hadoop git commit: YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev.
YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab252a7a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab252a7a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab252a7a Branch: refs/heads/YARN-3926 Commit: ab252a7a39701fd9536b717d8f3239149c4abc07 Parents: 00d4a56 Author: Varun VasudevAuthored: Sat Jun 11 14:33:46 2016 +0530 Committer: Varun Vasudev Committed: Tue Nov 15 23:26:43 2016 +0530 -- .../hadoop/yarn/api/records/Resource.java | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 19 ++- .../FileSystemBasedConfigurationProvider.java | 3 +- .../hadoop/yarn/LocalConfigurationProvider.java | 3 +- .../api/records/impl/pb/ResourcePBImpl.java | 53 +++--- .../yarn/util/resource/ResourceUtils.java | 168 +++ .../yarn/util/resource/TestResourceUtils.java | 29 +++- .../resource-types/node-resources-1.xml | 29 .../resource-types/node-resources-2.xml | 39 + .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../util/NodeManagerHardwareUtils.java | 52 ++ .../resourcemanager/ResourceTrackerService.java | 9 +- 12 files changed, 342 insertions(+), 72 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab252a7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 3efccce..3880bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -272,7 +272,8 @@ public abstract class Resource implements Comparable { continue; } if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits().equals("")) { + && entry.getValue().getUnits() + .equals(ResourceInformation.VCORES.getUnits())) { continue; } sb.append(", ").append(entry.getKey()).append(": ") http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab252a7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 98c1be9..0bb1b41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -62,6 +62,10 @@ public class YarnConfiguration extends Configuration { "resource-types.xml"; @Private + public static final String NODE_RESOURCES_CONFIGURATION_FILE = + "node-resources.xml"; + + @Private public static final List RM_CONFIGURATION_FILES = Collections.unmodifiableList(Arrays.asList( RESOURCE_TYPES_CONFIGURATION_FILE, @@ -71,6 +75,16 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); + @Private + public static final List NM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + NODE_RESOURCES_CONFIGURATION_FILE, + DR_CONFIGURATION_FILE, + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_CONFIGURATION_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; @@ -99,12 +113,15 @@ public class YarnConfiguration extends Configuration { public static final String YARN_PREFIX = "yarn."; / - // Scheduler resource types configs + // Resource types configs public static final String RESOURCE_TYPES = YarnConfiguration.YARN_PREFIX + "resource-types"; + public static final String NM_RESOURCES_PREFIX = + YarnConfiguration.NM_PREFIX + "resource-type."; + /** Delay before deleting resource to ease debugging of NM issues