YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2440930 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2440930 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2440930 Branch: refs/heads/YARN-5881 Commit: c244093074d8bf9728ffe31778c3c621ed157c78 Parents: 6bf4a43 Author: Wangda Tan <wan...@apache.org> Authored: Mon Oct 16 11:04:05 2017 -0700 Committer: Sunil G <sun...@apache.org> Committed: Fri Nov 24 15:37:55 2017 +0530 ---------------------------------------------------------------------- .../yarn/api/records/QueueConfigurations.java | 76 +++++++++++ .../yarn/api/records/ResourceInformation.java | 6 + .../src/main/proto/yarn_protos.proto | 4 + .../impl/pb/QueueConfigurationsPBImpl.java | 136 ++++++++++++++++++- .../resource/DominantResourceCalculator.java | 24 +++- .../scheduler/QueueResourceQuotas.java | 38 ------ .../scheduler/capacity/AbstractCSQueue.java | 18 ++- .../scheduler/capacity/CSQueue.java | 4 +- .../scheduler/capacity/LeafQueue.java | 4 +- .../scheduler/capacity/ParentQueue.java | 56 ++++++-- .../scheduler/capacity/UsersManager.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 23 +++- .../webapp/CapacitySchedulerPage.java | 49 +++++-- .../webapp/dao/CapacitySchedulerInfo.java | 3 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 6 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 10 +- .../dao/PartitionQueueCapacitiesInfo.java | 34 ++++- .../webapp/dao/QueueCapacitiesInfo.java | 13 +- ...CapacitySchedulerWithMultiResourceTypes.java | 25 ++-- .../scheduler/capacity/TestLeafQueue.java | 24 +++- 20 files changed, 443 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java index e25c8aa..30096bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java @@ -147,4 +147,80 @@ public abstract class QueueConfigurations { @Private @Unstable public abstract void setMaxAMPercentage(float maxAMPercentage); + + /** + * Get the effective minimum capacity of queue (from absolute resource). + * + * @return minimum resource capability + */ + @Public + @Unstable + public abstract Resource getEffectiveMinCapacity(); + + /** + * Set the effective minimum capacity of queue (from absolute resource). + * + * @param capacity + * minimum resource capability + */ + @Private + @Unstable + public abstract void setEffectiveMinCapacity(Resource capacity); + + /** + * Get the effective maximum capacity of queue (from absolute resource). + * + * @return maximum resource capability + */ + @Public + @Unstable + public abstract Resource getEffectiveMaxCapacity(); + + /** + * Set the effective maximum capacity of queue (from absolute resource). + * + * @param capacity + * maximum resource capability + */ + @Private + @Unstable + public abstract void setEffectiveMaxCapacity(Resource capacity); + + /** + * Get the configured minimum capacity of queue (from absolute resource). + * + * @return minimum resource capability + */ + @Public + @Unstable + public abstract Resource getConfiguredMinCapacity(); + + /** + * Set the configured minimum capacity of queue (from absolute resource). + * + * @param configuredMinResource + * minimum resource capability + */ + @Public + @Unstable + public abstract void setConfiguredMinCapacity(Resource configuredMinResource); + + /** + * Get the configured maximum capacity of queue (from absolute resource). + * + * @return maximum resource capability + */ + @Public + @Unstable + public abstract Resource getConfiguredMaxCapacity(); + + /** + * Set the configured maximum capacity of queue (from absolute resource). + * + * @param configuredMaxResource + * maximum resource capability + */ + @Public + @Unstable + public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 67592cc..a4c1f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -225,6 +225,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> { Long.MAX_VALUE); } + public static ResourceInformation newInstance(String name, String units, + long minRes, long maxRes) { + return ResourceInformation.newInstance(name, units, 0L, + ResourceTypes.COUNTABLE, minRes, maxRes); + } + public static ResourceInformation newInstance(String name, long value) { return ResourceInformation .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L, http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index fdbe2d4..3a9662b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -551,6 +551,10 @@ message QueueConfigurationsProto { optional float maxCapacity = 3; optional float absoluteMaxCapacity = 4; optional float maxAMPercentage = 5; + optional ResourceProto effectiveMinCapacity = 6; + optional ResourceProto effectiveMaxCapacity = 7; + optional ResourceProto configuredMinCapacity = 8; + optional ResourceProto configuredMaxCapacity = 9; } message QueueConfigurationsMapProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java index f308bce..80ef4b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java @@ -19,16 +19,22 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.QueueConfigurations; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; public class QueueConfigurationsPBImpl extends QueueConfigurations { - QueueConfigurationsProto proto = - QueueConfigurationsProto.getDefaultInstance(); + QueueConfigurationsProto proto = QueueConfigurationsProto + .getDefaultInstance(); QueueConfigurationsProto.Builder builder = null; + Resource configuredMinResource = null; + Resource configuredMaxResource = null; + Resource effMinResource = null; + Resource effMaxResource = null; boolean viaProto = false; public QueueConfigurationsPBImpl() { @@ -41,11 +47,40 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations { } public QueueConfigurationsProto getProto() { + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.effMinResource != null) { + builder + .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource)); + } + if (this.effMaxResource != null) { + builder + .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource)); + } + if (this.configuredMinResource != null) { + builder.setEffectiveMinCapacity( + convertToProtoFormat(this.configuredMinResource)); + } + if (this.configuredMaxResource != null) { + builder.setEffectiveMaxCapacity( + convertToProtoFormat(this.configuredMaxResource)); + } + } + @Override public float getCapacity() { QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; @@ -106,6 +141,58 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations { builder.setMaxAMPercentage(maxAMPercentage); } + @Override + public Resource getEffectiveMinCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.effMinResource != null) { + return this.effMinResource; + } + if (!p.hasEffectiveMinCapacity()) { + return null; + } + this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity()); + return this.effMinResource; + } + + @Override + public void setEffectiveMinCapacity(Resource capacity) { + maybeInitBuilder(); + if (capacity == null) { + builder.clearEffectiveMinCapacity(); + } + this.effMinResource = capacity; + } + + @Override + public Resource getEffectiveMaxCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.effMaxResource != null) { + return this.effMaxResource; + } + if (!p.hasEffectiveMaxCapacity()) { + return null; + } + this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity()); + return this.effMaxResource; + } + + @Override + public void setEffectiveMaxCapacity(Resource capacity) { + maybeInitBuilder(); + if (capacity == null) { + builder.clearEffectiveMaxCapacity(); + } + this.effMaxResource = capacity; + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ProtoUtils.convertToProtoFormat(t); + } + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = QueueConfigurationsProto.newBuilder(proto); @@ -134,4 +221,49 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations { return false; } + @Override + public Resource getConfiguredMinCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.configuredMinResource != null) { + return this.configuredMinResource; + } + if (!p.hasConfiguredMinCapacity()) { + return null; + } + this.configuredMinResource = convertFromProtoFormat( + p.getConfiguredMinCapacity()); + return this.configuredMinResource; + } + + @Override + public void setConfiguredMinCapacity(Resource configuredMinResource) { + maybeInitBuilder(); + if (configuredMinResource == null) { + builder.clearConfiguredMinCapacity(); + } + this.configuredMinResource = configuredMinResource; + } + + @Override + public Resource getConfiguredMaxCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.configuredMaxResource != null) { + return this.configuredMaxResource; + } + if (!p.hasConfiguredMaxCapacity()) { + return null; + } + this.configuredMaxResource = convertFromProtoFormat( + p.getConfiguredMaxCapacity()); + return this.configuredMaxResource; + } + + @Override + public void setConfiguredMaxCapacity(Resource configuredMaxResource) { + maybeInitBuilder(); + if (configuredMaxResource == null) { + builder.clearConfiguredMaxCapacity(); + } + this.configuredMaxResource = configuredMaxResource; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 16e4527..cc4acf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -570,8 +570,26 @@ public class DominantResourceCalculator extends ResourceCalculator { @Override public Resource normalizeDown(Resource r, Resource stepFactor) { - return Resources.createResource( - roundDown(r.getMemorySize(), stepFactor.getMemorySize()), - roundDown(r.getVirtualCores(), stepFactor.getVirtualCores())); + Resource ret = Resource.newInstance(r); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation rResourceInformation = r.getResourceInformation(i); + ResourceInformation stepFactorResourceInformation = stepFactor + .getResourceInformation(i); + ResourceInformation tmp = ret.getResourceInformation(i); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil.convert( + stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + + long value = rValue; + if (stepFactorValue != 0) { + value = roundDown(rValue, stepFactorValue); + } + tmp.setValue(value); + } + return ret; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java index 2e653fc..08b4d04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java @@ -112,42 +112,4 @@ public class QueueResourceQuotas extends AbstractResourceUsage { public void setEffectiveMaxResource(String label, Resource res) { _set(label, ResourceType.EFF_MAX_RESOURCE, res); } - - /* - * Effective Minimum Resource - */ - public Resource getEffectiveMinResourceUp() { - return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP); - } - - public Resource getEffectiveMinResourceUp(String label) { - return _get(label, ResourceType.EFF_MIN_RESOURCE_UP); - } - - public void setEffectiveMinResourceUp(String label, Resource res) { - _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res); - } - - public void setEffectiveMinResourceUp(Resource res) { - _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res); - } - - /* - * Effective Maximum Resource - */ - public Resource getEffectiveMaxResourceUp() { - return getEffectiveMaxResourceUp(NL); - } - - public Resource getEffectiveMaxResourceUp(String label) { - return _get(label, ResourceType.EFF_MAX_RESOURCE_UP); - } - - public void setEffectiveMaxResourceUp(Resource res) { - setEffectiveMaxResourceUp(NL, res); - } - - public void setEffectiveMaxResourceUp(String label, Resource res) { - _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index dddac4d..9caf589 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -398,6 +398,10 @@ public abstract class AbstractCSQueue implements CSQueue { Resource maxResource = conf.getMaximumResourceRequirement(label, queuePath, resourceTypes); + if (LOG.isDebugEnabled()) { + LOG.debug("capacityConfigType is '" + capacityConfigType + + "' for queue '" + getQueueName()); + } if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { this.capacityConfigType = (!minResource.equals(Resources.none()) && queueCapacities.getAbsoluteCapacity(label) == 0f) @@ -481,12 +485,6 @@ public abstract class AbstractCSQueue implements CSQueue { } @Override - public Resource getEffectiveCapacityUp(String label) { - return Resources - .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label)); - } - - @Override public Resource getEffectiveCapacityDown(String label, Resource factor) { return Resources.normalizeDown(resourceCalculator, getQueueResourceQuotas().getEffectiveMinResource(label), @@ -621,6 +619,14 @@ public abstract class AbstractCSQueue implements CSQueue { queueConfiguration.setMaxCapacity(maxCapacity); queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity); queueConfiguration.setMaxAMPercentage(maxAMPercentage); + queueConfiguration.setConfiguredMinCapacity( + queueResourceQuotas.getConfiguredMinResource(nodeLabel)); + queueConfiguration.setConfiguredMaxCapacity( + queueResourceQuotas.getConfiguredMaxResource(nodeLabel)); + queueConfiguration.setEffectiveMinCapacity( + queueResourceQuotas.getEffectiveMinResource(nodeLabel)); + queueConfiguration.setEffectiveMaxCapacity( + queueResourceQuotas.getEffectiveMaxResource(nodeLabel)); queueConfigurations.put(nodeLabel, queueConfiguration); } return queueConfigurations; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 2e29a71..624fcc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.security.AccessControlException; @@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet; +import com.google.common.annotations.VisibleForTesting; + /** * <code>CSQueue</code> represents a node in the tree of * hierarchical queues in the {@link CapacityScheduler}. @@ -380,7 +381,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> { * @return effective queue capacity */ Resource getEffectiveCapacity(String label); - Resource getEffectiveCapacityUp(String label); Resource getEffectiveCapacityDown(String label, Resource factor); /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 1460121..41ec4ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -671,7 +671,7 @@ public class LeafQueue extends AbstractCSQueue { 1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1)); effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f); - Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition); + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); Resource userAMLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionResource, @@ -700,7 +700,7 @@ public class LeafQueue extends AbstractCSQueue { * non-labeled), * with per-partition am-resource-percent to get the max am * resource limit for this queue and partition. */ - Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition); + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); Resource queueCurrentLimit = Resources.none(); // For non-labeled partition, we need to consider the current queue http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index c770fac..5ab1494 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -914,9 +914,11 @@ public class ParentQueue extends AbstractCSQueue { queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel); queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel); + queueCapacities.setAbsoluteCapacity(label, 1.0f); } - // Total configured min resources of direct children of queue + // Total configured min resources of direct children of this given parent + // queue. Resource configuredMinResources = Resource.newInstance(0L, 0); for (CSQueue childQueue : getChildQueues()) { Resources.addTo(configuredMinResources, @@ -960,7 +962,7 @@ public class ParentQueue extends AbstractCSQueue { // present could also be taken from effective max resource of parent. Resource parentMaxRes = queueResourceQuotas .getConfiguredMaxResource(label); - if (parentMaxRes.equals(Resources.none())) { + if (parent != null && parentMaxRes.equals(Resources.none())) { parentMaxRes = parent.getQueueResourceQuotas() .getEffectiveMaxResource(label); } @@ -976,6 +978,11 @@ public class ParentQueue extends AbstractCSQueue { parentMaxRes); childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, Resources.clone(effMaxResource)); + + // In cases where we still need to update some units based on + // percentage, we have to calculate percentage and update. + deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc, + childQueue); } else { childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, Resources.multiply(resourceByLabel, @@ -983,16 +990,6 @@ public class ParentQueue extends AbstractCSQueue { childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, Resources.multiply(resourceByLabel, childQueue.getQueueCapacities() .getAbsoluteMaximumCapacity(label))); - - childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label, - Resources.multiplyAndNormalizeUp(rc, resourceByLabel, - childQueue.getQueueCapacities().getAbsoluteCapacity(label), - minimumAllocation)); - childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label, - Resources.multiplyAndNormalizeUp(rc, - resourceByLabel, childQueue.getQueueCapacities() - .getAbsoluteMaximumCapacity(label), - minimumAllocation)); } if (LOG.isDebugEnabled()) { @@ -1006,6 +1003,41 @@ public class ParentQueue extends AbstractCSQueue { } } + private void deriveCapacityFromAbsoluteConfigurations(String label, + Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) { + + /* + * In case when queues are configured with absolute resources, it is better + * to update capacity/max-capacity etc w.r.t absolute resource as well. In + * case of computation, these values wont be used any more. However for + * metrics and UI, its better these values are pre-computed here itself. + */ + + // 1. Update capacity as a float based on parent's minResource + childQueue.getQueueCapacities().setCapacity(label, + rc.divide(clusterResource, + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label), + getQueueResourceQuotas().getEffectiveMinResource(label))); + + // 2. Update max-capacity as a float based on parent's maxResource + childQueue.getQueueCapacities().setMaximumCapacity(label, + rc.divide(clusterResource, + childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label), + getQueueResourceQuotas().getEffectiveMaxResource(label))); + + // 3. Update absolute capacity as a float based on parent's minResource and + // cluster resource. + childQueue.getQueueCapacities().setAbsoluteCapacity(label, + (float) childQueue.getQueueCapacities().getCapacity() + / getQueueCapacities().getAbsoluteCapacity(label)); + + // 4. Update absolute max-capacity as a float based on parent's maxResource + // and cluster resource. + childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label, + (float) childQueue.getQueueCapacities().getMaximumCapacity(label) + / getQueueCapacities().getAbsoluteMaximumCapacity(label)); + } + @Override public List<CSQueue> getChildQueues() { try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index efc20e9..7287c5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -686,7 +686,7 @@ public class UsersManager implements AbstractUsersManager { * * If we're running over capacity, then its (usedResources + required) * (which extra resources we are allocating) */ - Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition); + Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition); /* * Assume we have required resource equals to minimumAllocation, this can http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index e9bee14..e3e47b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; @@ -905,6 +906,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { StringBuilder diagnosticMessage) { LeafQueue queue = getCSLeafQueue(); QueueCapacities queueCapacities = queue.getQueueCapacities(); + QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas(); diagnosticMessage.append(" Details : AM Partition = "); diagnosticMessage.append(appAMNodePartitionName.isEmpty() ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName); @@ -926,6 +928,18 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName) * 100); diagnosticMessage.append(" % ; "); + diagnosticMessage.append("Queue's capacity (absolute resource) = "); + diagnosticMessage.append( + queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); + diagnosticMessage.append("Queue's used capacity (absolute resource) = "); + diagnosticMessage + .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); + diagnosticMessage.append("Queue's max capacity (absolute resource) = "); + diagnosticMessage.append( + queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); } /** @@ -989,13 +1003,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); if (!calc.isInvalidDivisor(totalPartitionRes)) { - float queueAbsMaxCapPerPartition = - ((AbstractCSQueue) getQueue()).getQueueCapacities() - .getAbsoluteCapacity(getAppAMNodePartitionName()); + Resource effCap = ((AbstractCSQueue) getQueue()) + .getEffectiveCapacity(getAppAMNodePartitionName()); float queueUsagePerc = calc.divide(totalPartitionRes, - report.getUsedResources(), - Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition)) - * 100; + report.getUsedResources(), effCap) * 100; report.setQueueUsagePercentage(queueUsagePerc); } return report; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 74b4e79..3f68e07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -155,21 +155,36 @@ class CapacitySchedulerPage extends RmView { ? new ResourceInfo(Resources.none()) : resourceUsages.getAmUsed(); ri. - __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)). - __("Configured Capacity:", percent(capacities.getCapacity() / 100)). - __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)). - __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)). - __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)). - __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)). - __("Used Resources:", resourceUsages.getUsed().toString()). - __("Configured Max Application Master Limit:", StringUtils.format("%.1f", - capacities.getMaxAMLimitPercentage())). - __("Max Application Master Resources:", - resourceUsages.getAMLimit().toString()). - __("Used Application Master Resources:", - amUsed.toString()). - __("Max Application Master Resources Per User:", - userAMResourceLimit.toString()); + __("Used Capacity:", + appendPercent(resourceUsages.getUsed().toString(), + capacities.getUsedCapacity() / 100)) + .__("Configured Capacity:", + capacities.getConfiguredMinResource().toString()) + .__("Configured Max Capacity:", + capacities.getConfiguredMaxResource().getResource() + .equals(Resources.none()) + ? "unlimited" + : capacities.getConfiguredMaxResource().toString()) + .__("Effective Capacity:", + appendPercent(capacities.getEffectiveMinResource().toString(), + capacities.getCapacity() / 100)) + .__("Effective Max Capacity:", + appendPercent(capacities.getEffectiveMaxResource().toString(), + capacities.getMaxCapacity() / 100)) + .__("Absolute Used Capacity:", + percent(capacities.getAbsoluteUsedCapacity() / 100)) + .__("Absolute Configured Capacity:", + percent(capacities.getAbsoluteCapacity() / 100)) + .__("Absolute Configured Max Capacity:", + percent(capacities.getAbsoluteMaxCapacity() / 100)) + .__("Used Resources:", resourceUsages.getUsed().toString()) + .__("Configured Max Application Master Limit:", + StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage())) + .__("Max Application Master Resources:", + resourceUsages.getAMLimit().toString()) + .__("Used Application Master Resources:", amUsed.toString()) + .__("Max Application Master Resources Per User:", + userAMResourceLimit.toString()); } private void renderCommonLeafQueueInfo(ResponseInfo ri) { @@ -615,6 +630,10 @@ class CapacitySchedulerPage extends RmView { return QueuesBlock.class; } + static String appendPercent(String message, float f) { + return message + " (" + StringUtils.formatPercent(f, 1) + ")"; + } + static String percent(float f) { return StringUtils.formatPercent(f, 1); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 32e4ac5..ca42041 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -59,7 +59,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo { max = 1f; this.maxCapacity = max * 100; - capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false); + capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), + parent.getQueueResourceQuotas(), false); queues = getQueues(parent); health = new CapacitySchedulerHealthInfo(cs); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 7dcdf58..343c78e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -89,8 +90,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { } @Override - protected void populateQueueCapacities(QueueCapacities qCapacities) { - capacities = new QueueCapacitiesInfo(qCapacities); + protected void populateQueueCapacities(QueueCapacities qCapacities, + QueueResourceQuotas qResQuotas) { + capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas); } public int getNumActiveApplications() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 86b2fea..d4de9ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; @@ -103,7 +104,8 @@ public class CapacitySchedulerQueueInfo { Collections.sort(nodeLabels); } QueueCapacities qCapacities = q.getQueueCapacities(); - populateQueueCapacities(qCapacities); + QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas(); + populateQueueCapacities(qCapacities, qResQuotas); ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); populateQueueResourceUsage(queueResourceUsage); @@ -118,8 +120,10 @@ public class CapacitySchedulerQueueInfo { resources = new ResourcesInfo(queueResourceUsage, false); } - protected void populateQueueCapacities(QueueCapacities qCapacities) { - capacities = new QueueCapacitiesInfo(qCapacities, false); + protected void populateQueueCapacities(QueueCapacities qCapacities, + QueueResourceQuotas qResQuotas) { + capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas, + false); } public float getCapacity() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java index 5e298f9..2a15502 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java @@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; + /** * This class represents queue capacities for a given partition */ @@ -36,13 +39,19 @@ public class PartitionQueueCapacitiesInfo { private float absoluteUsedCapacity; private float absoluteMaxCapacity = 100; private float maxAMLimitPercentage; + private ResourceInfo configuredMinResource; + private ResourceInfo configuredMaxResource; + private ResourceInfo effectiveMinResource; + private ResourceInfo effectiveMaxResource; public PartitionQueueCapacitiesInfo() { } public PartitionQueueCapacitiesInfo(String partitionName, float capacity, float usedCapacity, float maxCapacity, float absCapacity, - float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) { + float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage, + Resource confMinRes, Resource confMaxRes, Resource effMinRes, + Resource effMaxRes) { super(); this.partitionName = partitionName; this.capacity = capacity; @@ -52,6 +61,10 @@ public class PartitionQueueCapacitiesInfo { this.absoluteUsedCapacity = absUsedCapacity; this.absoluteMaxCapacity = absMaxCapacity; this.maxAMLimitPercentage = maxAMLimitPercentage; + this.configuredMinResource = new ResourceInfo(confMinRes); + this.configuredMaxResource = new ResourceInfo(confMaxRes); + this.effectiveMinResource = new ResourceInfo(effMinRes); + this.effectiveMaxResource = new ResourceInfo(effMaxRes); } public float getCapacity() { @@ -117,4 +130,23 @@ public class PartitionQueueCapacitiesInfo { public void setMaxAMLimitPercentage(float maxAMLimitPercentage) { this.maxAMLimitPercentage = maxAMLimitPercentage; } + + public ResourceInfo getConfiguredMinResource() { + return configuredMinResource; + } + + public ResourceInfo getConfiguredMaxResource() { + if (configuredMaxResource.getResource().equals(Resources.none())) { + return null; + } + return configuredMaxResource; + } + + public ResourceInfo getEffectiveMinResource() { + return effectiveMinResource; + } + + public ResourceInfo getEffectiveMaxResource() { + return effectiveMaxResource; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java index 9a3e439..35c80d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; /** @@ -39,6 +40,7 @@ public class QueueCapacitiesInfo { } public QueueCapacitiesInfo(QueueCapacities capacities, + QueueResourceQuotas resourceQuotas, boolean considerAMUsage) { if (capacities == null) { return; @@ -68,12 +70,17 @@ public class QueueCapacitiesInfo { queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo( partitionName, capacity, usedCapacity, maxCapacity, absCapacity, absUsedCapacity, absMaxCapacity, - considerAMUsage ? maxAMLimitPercentage : 0f)); + considerAMUsage ? maxAMLimitPercentage : 0f, + resourceQuotas.getConfiguredMinResource(partitionName), + resourceQuotas.getConfiguredMaxResource(partitionName), + resourceQuotas.getEffectiveMinResource(partitionName), + resourceQuotas.getEffectiveMaxResource(partitionName))); } } - public QueueCapacitiesInfo(QueueCapacities capacities) { - this(capacities, true); + public QueueCapacitiesInfo(QueueCapacities capacities, + QueueResourceQuotas resourceQuotas) { + this(capacities, resourceQuotas, true); } public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index b386c18..38768e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -61,11 +61,20 @@ public class TestCapacitySchedulerWithMultiResourceTypes { Map<String, ResourceInformation> riMap = new HashMap<>(); // Initialize mandatory resources - riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB); - riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES); - riMap.put(RESOURCE_1, ResourceInformation - .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0, - Integer.MAX_VALUE)); + ResourceInformation memory = ResourceInformation.newInstance( + ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + ResourceInformation vcores = ResourceInformation.newInstance( + ResourceInformation.VCORES.getName(), + ResourceInformation.VCORES.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + riMap.put(ResourceInformation.MEMORY_URI, memory); + riMap.put(ResourceInformation.VCORES_URI, vcores); + riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0, + ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE)); ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); @@ -107,7 +116,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes { RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - Assert.assertEquals(Resource.newInstance(1 * GB, 0), + Assert.assertEquals(Resource.newInstance(1 * GB, 1), leafQueue.getUsedResources()); RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); @@ -123,9 +132,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes { // Do node heartbeats 1 time and check container allocated. cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); - // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2> + // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2> Assert.assertEquals( - TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)), + TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)), leafQueue.getUsedResources()); // Acquire container http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 96f8970..30bff78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; @@ -1355,7 +1356,7 @@ public class TestLeafQueue { // TODO, fix headroom in the future patch assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // User limit = 2G, 2 in use - assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); + assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // the application is not yet active // Again one to user_0 since he hasn't exceeded user limit yet @@ -1366,8 +1367,8 @@ public class TestLeafQueue { assertEquals(3*GB, a.getUsedResources().getMemorySize()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G - assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G + assertEquals(0*GB, app_0.getHeadroom().getMemorySize()); + assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); @@ -3874,7 +3875,7 @@ public class TestLeafQueue { // Queue "test" consumes 100% of the cluster, so its capacity and absolute // capacity are both 1.0f. - Queue queue = createQueue("test", null, 1.0f, 1.0f); + Queue queue = createQueue("test", null, 1.0f, 1.0f, res); final String user = "user1"; FiCaSchedulerApp app = new FiCaSchedulerApp(appAttId, user, queue, @@ -3891,7 +3892,8 @@ public class TestLeafQueue { // Queue "test2" is a child of root and its capacity is 50% of root. As a // child of root, its absolute capaicty is also 50%. - queue = createQueue("test2", null, 0.5f, 0.5f); + queue = createQueue("test2", null, 0.5f, 0.5f, + Resources.divideAndCeil(dominantResourceCalculator, res, 2)); app = new FiCaSchedulerApp(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext); app.getAppAttemptResourceUsage().incUsed(requestedResource); @@ -3903,7 +3905,8 @@ public class TestLeafQueue { // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster. // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%. - AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f); + AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f, + Resources.divideAndCeil(dominantResourceCalculator, res, 4)); app = new FiCaSchedulerApp(appAttId, user, qChild, qChild.getAbstractUsersManager(), rmContext); app.getAppAttemptResourceUsage().incUsed(requestedResource); @@ -3922,7 +3925,7 @@ public class TestLeafQueue { } private AbstractCSQueue createQueue(String name, Queue parent, float capacity, - float absCap) { + float absCap, Resource res) { CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf()); QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, null, QueueState.RUNNING, null, "", null, false); @@ -3934,6 +3937,13 @@ public class TestLeafQueue { QueueCapacities qCaps = mock(QueueCapacities.class); when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap); when(queue.getQueueCapacities()).thenReturn(qCaps); + QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class); + when(qQuotas.getConfiguredMinResource(any())).thenReturn(res); + when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res); + when(qQuotas.getEffectiveMinResource(any())).thenReturn(res); + when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res); + when(queue.getQueueResourceQuotas()).thenReturn(qQuotas); + when(queue.getEffectiveCapacity(any())).thenReturn(res); return queue; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
