This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 91ce09e YARN-9360. Do not expose innards of QueueMetrics object into
FSLeafQueue#computeMaxAMResource. Contributed by Peter Bacsko
91ce09e is described below
commit 91ce09e7065bacd7b4f09696fff35b789c52bcd7
Author: Szilard Nemeth <[email protected]>
AuthorDate: Mon Jul 15 10:47:10 2019 +0200
YARN-9360. Do not expose innards of QueueMetrics object into
FSLeafQueue#computeMaxAMResource. Contributed by Peter Bacsko
---
.../resourcemanager/scheduler/QueueMetrics.java | 34 ++++++++++++++++++++--
.../scheduler/fair/FSLeafQueue.java | 33 ++++-----------------
2 files changed, 38 insertions(+), 29 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index d126f09..c126338 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -833,7 +833,37 @@ public class QueueMetrics implements MetricsSource {
return aggregateContainersPreempted.value();
}
- public QueueMetricsForCustomResources getQueueMetricsForCustomResources() {
- return queueMetricsForCustomResources;
+ /**
+ * Fills in Resource values from available metrics values of custom resources
+ * to @code{targetResource}, only if the corresponding
+ * value of @code{targetResource} is zero.
+ * If @code{fromResource} has a value less than the available metrics value
+ * for a particular resource, it will be set to the @code{targetResource}
+ * instead.
+ *
+ * @param fromResource The resource to compare available resource values
with.
+ * @param targetResource The resource to save the values into.
+ */
+ public void fillInValuesFromAvailableResources(Resource fromResource,
+ Resource targetResource) {
+ if (queueMetricsForCustomResources != null) {
+ QueueMetricsCustomResource availableResources =
+ queueMetricsForCustomResources.getAvailable();
+
+ // We expect all custom resources contained in availableResources,
+ // so we will loop through all of them.
+ for (Map.Entry<String, Long> availableEntry : availableResources
+ .getValues().entrySet()) {
+ String resourceName = availableEntry.getKey();
+
+ // We only update the value if fairshare is 0 for that resource.
+ if (targetResource.getResourceValue(resourceName) == 0) {
+ Long availableValue = availableEntry.getValue();
+ long value = Math.min(availableValue,
+ fromResource.getResourceValue(resourceName));
+ targetResource.setResourceValue(resourceName, value);
+ }
+ }
+ }
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 361355b..afea3d5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
@@ -43,8 +42,6 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetricsCustomResource;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetricsForCustomResources;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -507,40 +504,22 @@ public class FSLeafQueue extends FSQueue {
*/
private Resource computeMaxAMResource() {
Resource maxResource = Resources.clone(getFairShare());
+ Resource maxShare = getMaxShare();
+
if (maxResource.getMemorySize() == 0) {
maxResource.setMemorySize(
Math.min(scheduler.getRootQueueMetrics().getAvailableMB(),
- getMaxShare().getMemorySize()));
+ maxShare.getMemorySize()));
}
if (maxResource.getVirtualCores() == 0) {
maxResource.setVirtualCores(Math.min(
scheduler.getRootQueueMetrics().getAvailableVirtualCores(),
- getMaxShare().getVirtualCores()));
+ maxShare.getVirtualCores()));
}
- QueueMetricsForCustomResources metricsForCustomResources =
- scheduler.getRootQueueMetrics().getQueueMetricsForCustomResources();
-
- if (metricsForCustomResources != null) {
- QueueMetricsCustomResource availableResources =
- metricsForCustomResources.getAvailable();
-
- // We expect all custom resources contained in availableResources,
- // so we will loop through all of them.
- for (Map.Entry<String, Long> availableEntry : availableResources
- .getValues().entrySet()) {
- String resourceName = availableEntry.getKey();
-
- // We only update the value if fairshare is 0 for that resource.
- if (maxResource.getResourceValue(resourceName) == 0) {
- Long availableValue = availableEntry.getValue();
- long value = Math.min(availableValue,
- getMaxShare().getResourceValue(resourceName));
- maxResource.setResourceValue(resourceName, value);
- }
- }
- }
+ scheduler.getRootQueueMetrics()
+ .fillInValuesFromAvailableResources(maxShare, maxResource);
// Round up to allow AM to run when there is only one vcore on the cluster
return Resources.multiplyAndRoundUp(maxResource, maxAMShare);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
