Repository: hadoop Updated Branches: refs/heads/YARN-1011 c9a09d692 -> f7d5d83cf
YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7d5d83c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7d5d83c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7d5d83c Branch: refs/heads/YARN-1011 Commit: f7d5d83cf88752213bf80a9480d92611208997f0 Parents: c9a09d6 Author: Karthik Kambatla <[email protected]> Authored: Fri Jan 29 14:31:45 2016 -0800 Committer: Karthik Kambatla <[email protected]> Committed: Fri Jan 29 14:31:45 2016 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 + .../hadoop/yarn/conf/YarnConfiguration.java | 12 ++- .../src/main/resources/yarn-default.xml | 21 ++++ .../RegisterNodeManagerRequest.java | 19 +++- .../pb/RegisterNodeManagerRequestPBImpl.java | 45 +++++++- .../server/api/records/OverAllocationInfo.java | 45 ++++++++ .../server/api/records/ResourceThresholds.java | 45 ++++++++ .../impl/pb/OverAllocationInfoPBImpl.java | 106 +++++++++++++++++++ .../impl/pb/ResourceThresholdsPBImpl.java | 93 ++++++++++++++++ .../yarn_server_common_service_protos.proto | 10 ++ .../hadoop/yarn/server/nodemanager/Context.java | 5 + .../yarn/server/nodemanager/NodeManager.java | 16 +++ .../nodemanager/NodeStatusUpdaterImpl.java | 6 +- .../monitor/ContainersMonitorImpl.java | 34 ++++++ .../amrmproxy/BaseAMRMProxyTest.java | 11 ++ 15 files changed, 462 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 76cad7f..9f415ba 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -9,6 +9,8 @@ Trunk - Unreleased YARN-2882. Add an OPPORTUNISTIC ExecutionType. (Konstantinos Karanasos and Inigo Goiri via kasha) + YARN-4512 [YARN-1011]. Provide a knob to turn on over-allocation. (kasha) + IMPROVEMENTS YARN-2438. yarn-env.sh cleanup (aw) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d84c155..ea092e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1296,7 +1296,17 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE = false; - + /** Overallocation (= allocation based on utilization) configs. */ + public static final String NM_OVERALLOCATION_ALLOCATION_THRESHOLD = + NM_PREFIX + "overallocation.allocation-threshold"; + public static final float DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD + = 0f; + @Private + public static final float MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD = 0.95f; + public static final String NM_OVERALLOCATION_PREEMPTION_THRESHOLD = + NM_PREFIX + "overallocation.preemption-threshold"; + public static final float DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD + = 0f; /** * Interval of time the linux container executor should try cleaning up http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e33d23e..d7b9784 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1447,6 +1447,27 @@ </property> <property> + <description>The extent of over-allocation (container-allocation based on + current utilization instead of prior allocation) allowed on this node, + expressed as a float between 0 and 0.95. By default, over-allocation is + turned off (value = 0). When turned on, the node allows running + OPPORTUNISTIC containers when the aggregate utilization is under the + value specified here multiplied by the node's advertised capacity. + </description> + <name>yarn.nodemanager.overallocation.allocation-threshold</name> + <value>0f</value> + </property> + + <property> + <description>When a node is over-allocated to improve utilization by + running OPPORTUNISTIC containers, this config captures the utilization + beyond which OPPORTUNISTIC containers should start getting preempted. + </description> + <name>yarn.nodemanager.overallocation.preemption-threshold</name> + <value>1</value> + </property> + + <property> <description>This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not be case-sensitive from a docker perspective, it is best to keep these http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java index 7798ba9..186a80e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; import org.apache.hadoop.yarn.util.Records; public abstract class RegisterNodeManagerRequest { @@ -41,6 +42,15 @@ public abstract class RegisterNodeManagerRequest { int httpPort, Resource resource, String nodeManagerVersionId, List<NMContainerStatus> containerStatuses, List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels) { + return newInstance(nodeId, httpPort, resource, nodeManagerVersionId, + containerStatuses, runningApplications, nodeLabels, null); + } + + public static RegisterNodeManagerRequest newInstance(NodeId nodeId, + int httpPort, Resource resource, String nodeManagerVersionId, + List<NMContainerStatus> containerStatuses, + List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels, + OverAllocationInfo overAllocationInfo) { RegisterNodeManagerRequest request = Records.newRecord(RegisterNodeManagerRequest.class); request.setHttpPort(httpPort); @@ -50,9 +60,10 @@ public abstract class RegisterNodeManagerRequest { request.setContainerStatuses(containerStatuses); request.setRunningApplications(runningApplications); request.setNodeLabels(nodeLabels); + request.setOverAllocationInfo(overAllocationInfo); return request; } - + public abstract NodeId getNodeId(); public abstract int getHttpPort(); public abstract Resource getResource(); @@ -60,7 +71,11 @@ public abstract class RegisterNodeManagerRequest { public abstract List<NMContainerStatus> getNMContainerStatuses(); public abstract Set<NodeLabel> getNodeLabels(); public abstract void setNodeLabels(Set<NodeLabel> nodeLabels); - + + public abstract OverAllocationInfo getOverAllocationInfo(); + public abstract void setOverAllocationInfo( + OverAllocationInfo overAllocationInfo); + /** * We introduce this here because currently YARN RM doesn't persist nodes info * for application running. When RM restart happened, we cannot determinate if http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java index 2a1a268..d59b703 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java @@ -40,11 +40,14 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto.Builder; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; - +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; +import org.apache.hadoop.yarn.server.api.records.impl.pb.OverAllocationInfoPBImpl; + public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest { RegisterNodeManagerRequestProto proto = RegisterNodeManagerRequestProto.getDefaultInstance(); RegisterNodeManagerRequestProto.Builder builder = null; @@ -55,6 +58,7 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest private List<NMContainerStatus> containerStatuses = null; private List<ApplicationId> runningApplications = null; private Set<NodeLabel> labels = null; + private OverAllocationInfo overAllocationInfo = null; public RegisterNodeManagerRequestPBImpl() { builder = RegisterNodeManagerRequestProto.newBuilder(); @@ -93,6 +97,10 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest } builder.setNodeLabels(newBuilder.build()); } + if (this.overAllocationInfo != null) { + builder.setOverAllocationInfo( + convertToProtoFormat(this.overAllocationInfo)); + } } private synchronized void addNMContainerStatusesToProto() { @@ -312,7 +320,30 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest builder.clearNodeLabels(); this.labels = nodeLabels; } - + + @Override + public synchronized OverAllocationInfo getOverAllocationInfo() { + RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.overAllocationInfo != null) { + return this.overAllocationInfo; + } + if (!p.hasOverAllocationInfo()) { + return null; + } + this.overAllocationInfo = convertFromProtoFormat(p.getOverAllocationInfo()); + return this.overAllocationInfo; + } + + @Override + public synchronized void setOverAllocationInfo( + OverAllocationInfo overAllocationInfo) { + maybeInitBuilder(); + if (this.overAllocationInfo == null) { + builder.clearOverAllocationInfo(); + } + this.overAllocationInfo = overAllocationInfo; + } + private synchronized void initNodeLabels() { if (this.labels != null) { return; @@ -371,4 +402,14 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest NMContainerStatus c) { return ((NMContainerStatusPBImpl)c).getProto(); } + + private static OverAllocationInfoProto convertToProtoFormat( + OverAllocationInfo overAllocationInfo) { + return ((OverAllocationInfoPBImpl)overAllocationInfo).getProto(); + } + + private static OverAllocationInfo convertFromProtoFormat( + OverAllocationInfoProto proto) { + return new OverAllocationInfoPBImpl(proto); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java new file mode 100644 index 0000000..77952bf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/OverAllocationInfo.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.api.records.impl.pb + .OverAllocationInfoPBImpl; + +/** + * Captures information on how aggressively the scheduler can over-allocate + * OPPORTUNISTIC containers on a node. This is node-specific, and is sent on + * the wire on each heartbeat. + */ [email protected] [email protected] +public abstract class OverAllocationInfo { + public static OverAllocationInfo newInstance( + ResourceThresholds overAllocationThresholds) { + OverAllocationInfo info = new OverAllocationInfoPBImpl(); + info.setOverAllocationThreshold(overAllocationThresholds); + return info; + } + + public abstract ResourceThresholds getOverAllocationThresholds(); + + public abstract void setOverAllocationThreshold( + ResourceThresholds resourceThresholds); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java new file mode 100644 index 0000000..d57706a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceThresholds.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.server.api.records.impl.pb.ResourceThresholdsPBImpl; + +/** + * Captures resource thresholds to be used for allocation and preemption + * when over-allocation is turned on. + */ [email protected] [email protected] +public abstract class ResourceThresholds { + public static ResourceThresholds newInstance(float threshold) { + ResourceThresholds thresholds = new ResourceThresholdsPBImpl(); + thresholds.setMemoryThreshold(threshold); + thresholds.setCpuThreshold(threshold); + return thresholds; + } + + public abstract float getMemoryThreshold(); + + public abstract float getCpuThreshold(); + + public abstract void setMemoryThreshold(float memoryThreshold); + + public abstract void setCpuThreshold(float cpuThreshold); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java new file mode 100644 index 0000000..758f4fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/OverAllocationInfoPBImpl.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.api.records.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.OverAllocationInfoProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProto; +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; +import org.apache.hadoop.yarn.server.api.records.ResourceThresholds; + +public class OverAllocationInfoPBImpl extends OverAllocationInfo { + private OverAllocationInfoProto proto = + OverAllocationInfoProto.getDefaultInstance(); + private OverAllocationInfoProto.Builder builder = null; + private boolean viaProto = false; + + private ResourceThresholds overAllocationThresholds = null; + + public OverAllocationInfoPBImpl() { + builder = OverAllocationInfoProto.newBuilder(); + } + + public OverAllocationInfoPBImpl(OverAllocationInfoProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized OverAllocationInfoProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private synchronized void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void mergeLocalToBuilder() { + if (overAllocationThresholds != null) { + builder.setOverAllocationThresholds( + convertToProtoFormat(overAllocationThresholds)); + } + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = OverAllocationInfoProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized ResourceThresholds getOverAllocationThresholds() { + OverAllocationInfoProtoOrBuilder p = viaProto ? proto : builder; + if (overAllocationThresholds != null) { + return overAllocationThresholds; + } + if (!p.hasOverAllocationThresholds()) { + return null; + } + overAllocationThresholds = + convertFromProtoFormat(p.getOverAllocationThresholds()); + return overAllocationThresholds; + } + + @Override + public synchronized void setOverAllocationThreshold( + ResourceThresholds resourceThresholds) { + maybeInitBuilder(); + if (this.overAllocationThresholds != null) { + builder.clearOverAllocationThresholds(); + } + this.overAllocationThresholds = resourceThresholds; + } + + private static ResourceThresholdsProto convertToProtoFormat( + ResourceThresholds overAllocationThresholds) { + return ((ResourceThresholdsPBImpl) overAllocationThresholds).getProto(); + } + + private static ResourceThresholds convertFromProtoFormat( + ResourceThresholdsProto overAllocationThresholdsProto) { + return new ResourceThresholdsPBImpl(overAllocationThresholdsProto); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java new file mode 100644 index 0000000..10fb284 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceThresholdsPBImpl.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.records.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ResourceThresholdsProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.records.ResourceThresholds; + +public class ResourceThresholdsPBImpl extends ResourceThresholds{ + private ResourceThresholdsProto proto = + ResourceThresholdsProto.getDefaultInstance(); + private ResourceThresholdsProto.Builder builder = null; + private boolean viaProto = false; + + public ResourceThresholdsPBImpl() { + builder = ResourceThresholdsProto.newBuilder(); + } + + public ResourceThresholdsPBImpl(ResourceThresholdsProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized ResourceThresholdsProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private synchronized void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void mergeLocalToBuilder() { + /* + * Right now, we have only memory and cpu thresholds that are floats. + * This is a no-op until we add other non-static fields to the proto. + */ + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ResourceThresholdsProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized float getMemoryThreshold() { + ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder; + return p.getMemory(); + } + + @Override + public synchronized float getCpuThreshold() { + ResourceThresholdsProtoOrBuilder p = viaProto ? proto : builder; + return p.getCpu(); + } + + @Override + public synchronized void setMemoryThreshold(float memoryThreshold) { + maybeInitBuilder(); + builder.setMemory(memoryThreshold); + } + + @Override + public synchronized void setCpuThreshold(float cpuThreshold) { + maybeInitBuilder(); + builder.setCpu(cpuThreshold); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index a54bbdb..03e1c0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -38,6 +38,7 @@ message RegisterNodeManagerRequestProto { repeated NMContainerStatusProto container_statuses = 6; repeated ApplicationIdProto runningApplications = 7; optional NodeLabelsProto nodeLabels = 8; + optional OverAllocationInfoProto overAllocationInfo = 9; } message RegisterNodeManagerResponseProto { @@ -119,3 +120,12 @@ message SCMUploaderCanUploadRequestProto { message SCMUploaderCanUploadResponseProto { optional bool uploadable = 1; } + +message OverAllocationInfoProto { + optional ResourceThresholdsProto over_allocation_thresholds = 1; +} + +message ResourceThresholdsProto { + optional float memory = 1 [default = 0]; + optional float cpu = 2 [default = 0]; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index 9c2d1fb..39a4c90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; @@ -87,4 +88,8 @@ public interface Context { ConcurrentLinkedQueue<LogAggregationReport> getLogAggregationStatusForApps(); + + boolean isOverAllocationEnabled(); + + OverAllocationInfo getOverAllocationInfo(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index a9a5411..fcd3f17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -458,6 +459,7 @@ public class NodeManager extends CompositeService private boolean isDecommissioned = false; private final ConcurrentLinkedQueue<LogAggregationReport> logAggregationReportForApps; + private OverAllocationInfo overAllocationInfo; public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, @@ -585,6 +587,20 @@ public class NodeManager extends CompositeService getLogAggregationStatusForApps() { return this.logAggregationReportForApps; } + + @Override + public boolean isOverAllocationEnabled() { + return getOverAllocationInfo() != null; + } + + @Override + public OverAllocationInfo getOverAllocationInfo() { + return this.overAllocationInfo; + } + + public void setOverAllocationInfo(OverAllocationInfo overAllocationInfo) { + this.overAllocationInfo = overAllocationInfo; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 5806731..8f80fa7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -326,7 +326,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance(nodeId, httpPort, totalResource, nodeManagerVersionId, containerReports, getRunningApplications(), - nodeLabels); + nodeLabels, context.getOverAllocationInfo()); if (containerReports != null) { LOG.info("Registering with RM using containers :" + containerReports); } @@ -437,8 +437,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements = getIncreasedContainers(); NodeStatus nodeStatus = NodeStatus.newInstance(nodeId, responseId, containersStatuses, - createKeepAliveApplicationList(), nodeHealthStatus, - containersUtilization, nodeUtilization, increasedContainers); + createKeepAliveApplicationList(), nodeHealthStatus, + containersUtilization, nodeUtilization, increasedContainers); return nodeStatus; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 446e7a1..13b30e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -35,8 +35,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; +import org.apache.hadoop.yarn.server.api.records.ResourceThresholds; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; @@ -83,6 +86,8 @@ public class ContainersMonitorImpl extends AbstractService implements private ResourceUtilization containersUtilization; + private ResourceThresholds overAllocationPreemptionThresholds; + private volatile boolean stopped = false; public ContainersMonitorImpl(ContainerExecutor exec, @@ -158,6 +163,13 @@ public class ContainersMonitorImpl extends AbstractService implements LOG.info("Physical memory check enabled: " + pmemCheckEnabled); LOG.info("Virtual memory check enabled: " + vmemCheckEnabled); + initializeOverAllocation(conf); + if (context.isOverAllocationEnabled()) { + pmemCheckEnabled = true; + LOG.info("Force enabling physical memory checks because " + + "overallocation is enabled"); + } + containersMonitorEnabled = isEnabled(); LOG.info("ContainersMonitor enabled: " + containersMonitorEnabled); @@ -191,6 +203,28 @@ public class ContainersMonitorImpl extends AbstractService implements super.serviceInit(conf); } + private void initializeOverAllocation(Configuration conf) { + float overAllocationTreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_ALLOCATION_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_ALLOCATION_THRESHOLD); + overAllocationTreshold = Math.min(overAllocationTreshold, + YarnConfiguration.MAX_NM_OVERALLOCATION_ALLOCATION_THRESHOLD); + overAllocationTreshold = Math.max(0, overAllocationTreshold); + + if (overAllocationTreshold > 0f) { + ((NodeManager.NMContext) context).setOverAllocationInfo( + OverAllocationInfo.newInstance( + ResourceThresholds.newInstance(overAllocationTreshold))); + + float preemptionThreshold = conf.getFloat( + YarnConfiguration.NM_OVERALLOCATION_PREEMPTION_THRESHOLD, + YarnConfiguration.DEFAULT_NM_OVERALLOCATION_PREEMPTION_THRESHOLD); + + this.overAllocationPreemptionThresholds = + ResourceThresholds.newInstance(preemptionThreshold); + } + } + private boolean isEnabled() { if (resourceCalculatorPlugin == null) { LOG.info("ResourceCalculatorPlugin is unavailable on this system. " http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7d5d83c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index 9bc23f6..fc32253 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; +import org.apache.hadoop.yarn.server.api.records.OverAllocationInfo; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor; @@ -674,6 +675,16 @@ public abstract class BaseAMRMProxyTest { } @Override + public boolean isOverAllocationEnabled() { + return false; + } + + @Override + public OverAllocationInfo getOverAllocationInfo() { + return null; + } + + @Override public NodeResourceMonitor getNodeResourceMonitor() { return null; }
