[37/50] [abbrv] hadoop git commit: YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha)
YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6278cc71 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6278cc71 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6278cc71 Branch: refs/heads/YARN-1011 Commit: 6278cc716cf0e63d1a86a74b3519d52e0924d7aa Parents: e9b9f48 Author: Karthik Kambatla Authored: Fri Jan 29 14:31:45 2016 -0800 Committer: Haibo Chen Committed: Mon Apr 9 17:07:06 2018 -0700 -- .../hadoop/yarn/conf/YarnConfiguration.java | 13 ++- .../src/main/resources/yarn-default.xml | 21 .../RegisterNodeManagerRequest.java | 14 ++- .../pb/RegisterNodeManagerRequestPBImpl.java| 48 - .../server/api/records/OverAllocationInfo.java | 45 .../server/api/records/ResourceThresholds.java | 45 .../impl/pb/OverAllocationInfoPBImpl.java | 106 +++ .../impl/pb/ResourceThresholdsPBImpl.java | 93 .../yarn_server_common_service_protos.proto | 10 ++ .../hadoop/yarn/server/nodemanager/Context.java | 5 + .../yarn/server/nodemanager/NodeManager.java| 17 +++ .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../monitor/ContainersMonitorImpl.java | 34 ++ .../amrmproxy/BaseAMRMProxyTest.java| 11 ++ 14 files changed, 457 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6278cc71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 2590b6f..2d69fa9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2042,7 +2042,6 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = false; - // Configurations for applicaiton life time monitor feature public static final String RM_APPLICATION_MONITOR_INTERVAL_MS = RM_PREFIX + "application-timeouts.monitor.interval-ms"; @@ -2050,6 +2049,18 @@ public class YarnConfiguration extends Configuration { public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS = 3000; + /** Overallocation (= allocation based on utilization) configs. */ + public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD = + NM_PREFIX + "overallocation.allocation-threshold"; + public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD + = 0f; + @Private + public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f; + public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.preemption-threshold"; + public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD + = 0f; + /** * Interval of time the linux container executor should try cleaning up * cgroups entry when cleaning up a container. This is required due to what http://git-wip-us.apache.org/repos/asf/hadoop/blob/6278cc71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 81b6658..4a7548a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1710,6 +1710,27 @@ +The extent of over-allocation (container-allocation based on + current utilization instead of prior allocation) allowed on this node, + expressed as a float between 0 and 0.95. By default, over-allocation is + turned off (value = 0). When turned on, the node allows running + OPPORTUNISTIC containers when the aggregate utilization is under the + value specified here multiplied by the node's advertised capacity. + +yarn.nodemanager.overallocation.allocation-threshold +0f + + + +When a node is over-allocated to improve utilization by + running OPPORTUNISTIC containers, this conf
[37/50] [abbrv] hadoop git commit: YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha)
YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab91ebd2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab91ebd2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab91ebd2 Branch: refs/heads/YARN-1011 Commit: ab91ebd2cce89ff4e31433a0a179fdd18b4cf5ef Parents: 443523f Author: Karthik Kambatla Authored: Fri Jan 29 14:31:45 2016 -0800 Committer: Haibo Chen Committed: Sun Jan 28 17:43:50 2018 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 13 ++- .../src/main/resources/yarn-default.xml | 21 .../RegisterNodeManagerRequest.java | 14 ++- .../pb/RegisterNodeManagerRequestPBImpl.java| 45 +++- .../server/api/records/OverAllocationInfo.java | 45 .../server/api/records/ResourceThresholds.java | 45 .../impl/pb/OverAllocationInfoPBImpl.java | 106 +++ .../impl/pb/ResourceThresholdsPBImpl.java | 93 .../yarn_server_common_service_protos.proto | 10 ++ .../hadoop/yarn/server/nodemanager/Context.java | 5 + .../yarn/server/nodemanager/NodeManager.java| 17 +++ .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../monitor/ContainersMonitorImpl.java | 34 ++ .../amrmproxy/BaseAMRMProxyTest.java| 11 ++ 14 files changed, 455 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bbbfc52..d8f4d34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1942,7 +1942,6 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = false; - // Configurations for applicaiton life time monitor feature public static final String RM_APPLICATION_MONITOR_INTERVAL_MS = RM_PREFIX + "application-timeouts.monitor.interval-ms"; @@ -1950,6 +1949,18 @@ public class YarnConfiguration extends Configuration { public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS = 3000; + /** Overallocation (= allocation based on utilization) configs. */ + public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD = + NM_PREFIX + "overallocation.allocation-threshold"; + public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD + = 0f; + @Private + public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f; + public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.preemption-threshold"; + public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD + = 0f; + /** * Interval of time the linux container executor should try cleaning up * cgroups entry when cleaning up a container. This is required due to what http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab91ebd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0bb4fca..fc5430a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1668,6 +1668,27 @@ +The extent of over-allocation (container-allocation based on + current utilization instead of prior allocation) allowed on this node, + expressed as a float between 0 and 0.95. By default, over-allocation is + turned off (value = 0). When turned on, the node allows running + OPPORTUNISTIC containers when the aggregate utilization is under the + value specified here multiplied by the node's advertised capacity. + +yarn.nodemanager.overallocation.allocation-threshold +0f + + + +When a node is over-allocated to improve utilization by + running OPPORTUNISTIC containers, this conf