This is an automated email from the ASF dual-hosted git repository.
bibinchundatt pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8bc2dfb YARN-10519. Refactor QueueMetricsForCustomResources class to
move to yarn-common package. Contributed by Minni Mittal
8bc2dfb is described below
commit 8bc2dfbf36bddd57de3496bece88ddb86529426b
Author: bibinchundatt <[email protected]>
AuthorDate: Wed Jan 20 11:00:04 2021 +0530
YARN-10519. Refactor QueueMetricsForCustomResources class to move to
yarn-common package. Contributed by Minni Mittal
---
.../yarn/metrics/CustomResourceMetricValue.java} | 17 ++-
.../hadoop/yarn/metrics/CustomResourceMetrics.java | 133 +++++++++++++++++++++
.../apache/hadoop/yarn/metrics/package-info.java | 23 ++++
.../CSQueueMetricsForCustomResources.java | 9 +-
.../FSQueueMetricsForCustomResources.java | 39 +++---
.../resourcemanager/scheduler/QueueMetrics.java | 106 ++++++----------
.../scheduler/QueueMetricsForCustomResources.java | 76 +++---------
.../scheduler/capacity/CSQueueMetrics.java | 39 +++---
.../TestQueueMetricsForCustomResources.java | 3 +-
.../scheduler/fair/TestFSLeafQueue.java | 4 +-
10 files changed, 271 insertions(+), 178 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsCustomResource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/CustomResourceMetricValue.java
similarity index 83%
rename from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsCustomResource.java
rename to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/CustomResourceMetricValue.java
index f47b023..e548292 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsCustomResource.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/CustomResourceMetricValue.java
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+package org.apache.hadoop.yarn.metrics;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -29,26 +29,26 @@ import java.util.function.BiFunction;
* the name of the custom resource.
* There are different kinds of values like allocated, available and others.
*/
-public class QueueMetricsCustomResource {
+public class CustomResourceMetricValue {
private final Map<String, Long> values = Maps.newHashMap();
- protected void increase(Resource res) {
+ public void increase(Resource res) {
update(res, Long::sum);
}
- void increaseWithMultiplier(Resource res, long multiplier) {
+ public void increaseWithMultiplier(Resource res, long multiplier) {
update(res, (v1, v2) -> v1 + v2 * multiplier);
}
- protected void decrease(Resource res) {
+ public void decrease(Resource res) {
update(res, (v1, v2) -> v1 - v2);
}
- void decreaseWithMultiplier(Resource res, int containers) {
+ public void decreaseWithMultiplier(Resource res, int containers) {
update(res, (v1, v2) -> v1 - v2 * containers);
}
- protected void set(Resource res) {
+ public void set(Resource res) {
update(res, (v1, v2) -> v2);
}
@@ -64,8 +64,7 @@ public class QueueMetricsCustomResource {
if (!values.containsKey(resource.getName())) {
values.put(resource.getName(), 0L);
}
- values.merge(resource.getName(),
- resource.getValue(), operation);
+ values.merge(resource.getName(), resource.getValue(), operation);
}
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/CustomResourceMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/CustomResourceMetrics.java
new file mode 100644
index 0000000..926f852
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/CustomResourceMetrics.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.metrics;
+
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This is base class for allocated and available metrics for
+ * custom resources.
+ */
+public class CustomResourceMetrics {
+ private static final String ALLOCATED_RESOURCE_METRIC_PREFIX =
+ "AllocatedResource.";
+ private static final String ALLOCATED_RESOURCE_METRIC_DESC = "Allocated
NAME";
+
+ private static final String AVAILABLE_RESOURCE_METRIC_PREFIX =
+ "AvailableResource.";
+ private static final String AVAILABLE_RESOURCE_METRIC_DESC = "Available
NAME";
+
+ private final CustomResourceMetricValue allocated =
+ new CustomResourceMetricValue();
+ private final CustomResourceMetricValue available =
+ new CustomResourceMetricValue();
+
+ /**
+ * Register all custom resources metrics as part of initialization.
+ * @param customResources Map containing all custom resource types
+ * @param registry of the metric type
+ */
+ public void registerCustomResources(Map<String, Long> customResources,
+ MetricsRegistry registry) {
+ registerCustomResources(customResources, registry,
+ ALLOCATED_RESOURCE_METRIC_PREFIX, ALLOCATED_RESOURCE_METRIC_DESC);
+ registerCustomResources(customResources, registry,
+ AVAILABLE_RESOURCE_METRIC_PREFIX, AVAILABLE_RESOURCE_METRIC_DESC);
+ }
+
+ /**
+ * Get a map of all custom resource metric.
+ * @return map of custom resource
+ */
+ public Map<String, Long> initAndGetCustomResources() {
+ Map<String, Long> customResources = new HashMap<String, Long>();
+ ResourceInformation[] resources = ResourceUtils.getResourceTypesArray();
+
+ for (int i = 2; i < resources.length; i++) {
+ ResourceInformation resource = resources[i];
+ customResources.put(resource.getName(), Long.valueOf(0));
+ }
+ return customResources;
+ }
+
+ /**
+ * As and when this metric object construction happens for any queue, all
+ * custom resource metrics value would be initialized with '0' like any other
+ * mandatory resources metrics.
+ * @param customResources Map containing all custom resource types
+ * @param registry of the metric type
+ * @param metricPrefix prefix in metric name
+ * @param metricDesc suffix for metric name
+ */
+ public void registerCustomResources(Map<String, Long> customResources,
+ MetricsRegistry registry, String metricPrefix, String metricDesc) {
+ for (Map.Entry<String, Long> entry : customResources.entrySet()) {
+ String resourceName = entry.getKey();
+ Long resourceValue = entry.getValue();
+
+ MutableGaugeLong resourceMetric =
+ (MutableGaugeLong) registry.get(metricPrefix + resourceName);
+
+ if (resourceMetric == null) {
+ resourceMetric = registry.newGauge(metricPrefix + resourceName,
+ metricDesc.replace("NAME", resourceName), 0L);
+ }
+ resourceMetric.set(resourceValue);
+ }
+ }
+
+ public void setAvailable(Resource res) {
+ available.set(res);
+ }
+
+ public void increaseAllocated(Resource res) {
+ allocated.increase(res);
+ }
+
+ public void increaseAllocated(Resource res, int containers) {
+ allocated.increaseWithMultiplier(res, containers);
+ }
+
+ public void decreaseAllocated(Resource res) {
+ allocated.decrease(res);
+ }
+
+ public void decreaseAllocated(Resource res, int containers) {
+ allocated.decreaseWithMultiplier(res, containers);
+ }
+
+ public Map<String, Long> getAllocatedValues() {
+ return allocated.getValues();
+ }
+
+ public Map<String, Long> getAvailableValues() {
+ return available.getValues();
+ }
+
+ public CustomResourceMetricValue getAvailable() {
+ return available;
+ }
+}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java
new file mode 100644
index 0000000..5df20b1
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Provides common metrics (available, allocated) for custom resources.
+ */
[email protected]
+package org.apache.hadoop.yarn.metrics;
+import org.apache.hadoop.classification.InterfaceAudience;
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
index 77fe42b..39fc975 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
import java.util.Map;
@@ -27,10 +28,10 @@ import java.util.Map;
*/
public class CSQueueMetricsForCustomResources
extends QueueMetricsForCustomResources {
- private final QueueMetricsCustomResource guaranteedCapacity =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource maxCapacity =
- new QueueMetricsCustomResource();
+ private final CustomResourceMetricValue guaranteedCapacity =
+ new CustomResourceMetricValue();
+ private final CustomResourceMetricValue maxCapacity =
+ new CustomResourceMetricValue();
public void setGuaranteedCapacity(Resource res) {
guaranteedCapacity.set(res);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/FSQueueMetricsForCustomResources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/FSQueueMetricsForCustomResources.java
index 2f73d6b..0139e3c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/FSQueueMetricsForCustomResources.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/FSQueueMetricsForCustomResources.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
import java.util.Map;
@@ -26,20 +27,20 @@ import java.util.Map;
* It provides increase and decrease methods for all types of metrics.
*/
public class FSQueueMetricsForCustomResources {
- private final QueueMetricsCustomResource fairShare =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource steadyFairShare =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource minShare =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource maxShare =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource maxAMShare =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource amResourceUsage =
- new QueueMetricsCustomResource();
-
- public QueueMetricsCustomResource getFairShare() {
+ private final CustomResourceMetricValue
+ fairShare = new CustomResourceMetricValue();
+ private final CustomResourceMetricValue steadyFairShare =
+ new CustomResourceMetricValue();
+ private final CustomResourceMetricValue
+ minShare = new CustomResourceMetricValue();
+ private final CustomResourceMetricValue
+ maxShare = new CustomResourceMetricValue();
+ private final CustomResourceMetricValue
+ maxAMShare = new CustomResourceMetricValue();
+ private final CustomResourceMetricValue amResourceUsage =
+ new CustomResourceMetricValue();
+
+ public CustomResourceMetricValue getFairShare() {
return fairShare;
}
@@ -51,7 +52,7 @@ public class FSQueueMetricsForCustomResources {
return fairShare.getValues();
}
- public QueueMetricsCustomResource getSteadyFairShare() {
+ public CustomResourceMetricValue getSteadyFairShare() {
return steadyFairShare;
}
@@ -63,7 +64,7 @@ public class FSQueueMetricsForCustomResources {
return steadyFairShare.getValues();
}
- public QueueMetricsCustomResource getMinShare() {
+ public CustomResourceMetricValue getMinShare() {
return minShare;
}
@@ -75,7 +76,7 @@ public class FSQueueMetricsForCustomResources {
return minShare.getValues();
}
- public QueueMetricsCustomResource getMaxShare() {
+ public CustomResourceMetricValue getMaxShare() {
return maxShare;
}
@@ -87,7 +88,7 @@ public class FSQueueMetricsForCustomResources {
return maxShare.getValues();
}
- public QueueMetricsCustomResource getMaxAMShare() {
+ public CustomResourceMetricValue getMaxAMShare() {
return maxAMShare;
}
@@ -99,7 +100,7 @@ public class FSQueueMetricsForCustomResources {
return maxAMShare.getValues();
}
- public QueueMetricsCustomResource getAMResourceUsage() {
+ public CustomResourceMetricValue getAMResourceUsage() {
return amResourceUsage;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index c3d466a..1e42e7a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.metrics2.lib.Interns.info;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -43,8 +42,8 @@ import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -533,8 +532,8 @@ public class QueueMetrics implements MetricsSource {
availableVCores.set(limit.getVirtualCores());
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.setAvailable(limit);
- registerCustomResources(
- queueMetricsForCustomResources.getAvailableValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getAvailableValues(), registry,
AVAILABLE_RESOURCE_METRIC_PREFIX, AVAILABLE_RESOURCE_METRIC_DESC);
}
}
@@ -616,16 +615,6 @@ public class QueueMetrics implements MetricsSource {
}
}
- protected Map<String, Long> initAndGetCustomResources() {
- Map<String, Long> customResources = new HashMap<String, Long>();
- ResourceInformation[] resources = ResourceUtils.getResourceTypesArray();
-
- for (int i = 2; i < resources.length; i++) {
- ResourceInformation resource = resources[i];
- customResources.put(resource.getName(), Long.valueOf(0));
- }
- return customResources;
- }
protected void createQueueMetricsForCustomResources() {
if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
@@ -635,43 +624,21 @@ public class QueueMetrics implements MetricsSource {
}
}
- /**
- * Register all custom resources metrics as part of initialization. As and
- * when this metric object construction happens for any queue, all custom
- * resource metrics value would be initialized with '0' like any other
- * mandatory resources metrics
- */
protected void registerCustomResources() {
- Map<String, Long> customResources = initAndGetCustomResources();
- registerCustomResources(customResources, ALLOCATED_RESOURCE_METRIC_PREFIX,
- ALLOCATED_RESOURCE_METRIC_DESC);
- registerCustomResources(customResources, AVAILABLE_RESOURCE_METRIC_PREFIX,
- AVAILABLE_RESOURCE_METRIC_DESC);
- registerCustomResources(customResources, PENDING_RESOURCE_METRIC_PREFIX,
- PENDING_RESOURCE_METRIC_DESC);
- registerCustomResources(customResources, RESERVED_RESOURCE_METRIC_PREFIX,
- RESERVED_RESOURCE_METRIC_DESC);
- registerCustomResources(customResources,
- AGGREGATE_PREEMPTED_SECONDS_METRIC_PREFIX,
- AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC);
- }
-
- protected void registerCustomResources(Map<String, Long> customResources,
- String metricPrefix, String metricDesc) {
- for (Entry<String, Long> entry : customResources.entrySet()) {
- String resourceName = entry.getKey();
- Long resourceValue = entry.getValue();
-
- MutableGaugeLong resourceMetric =
- (MutableGaugeLong) this.registry.get(metricPrefix + resourceName);
-
- if (resourceMetric == null) {
- resourceMetric =
- this.registry.newGauge(metricPrefix + resourceName,
- metricDesc.replace("NAME", resourceName), 0L);
- }
- resourceMetric.set(resourceValue);
- }
+ Map<String, Long> customResources =
+ queueMetricsForCustomResources.initAndGetCustomResources();
+ queueMetricsForCustomResources
+ .registerCustomResources(customResources, this.registry);
+ queueMetricsForCustomResources
+ .registerCustomResources(customResources, this.registry,
+ PENDING_RESOURCE_METRIC_PREFIX, PENDING_RESOURCE_METRIC_DESC);
+ queueMetricsForCustomResources
+ .registerCustomResources(customResources, this.registry,
+ RESERVED_RESOURCE_METRIC_PREFIX, RESERVED_RESOURCE_METRIC_DESC);
+ queueMetricsForCustomResources
+ .registerCustomResources(customResources, this.registry,
+ AGGREGATE_PREEMPTED_SECONDS_METRIC_PREFIX,
+ AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC);
}
private void incrementPendingResources(int containers, Resource res) {
@@ -680,7 +647,8 @@ public class QueueMetrics implements MetricsSource {
pendingVCores.incr(res.getVirtualCores() * containers);
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.increasePending(res, containers);
-
registerCustomResources(queueMetricsForCustomResources.getPendingValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getPendingValues(), this.registry,
PENDING_RESOURCE_METRIC_PREFIX, PENDING_RESOURCE_METRIC_DESC);
}
}
@@ -722,7 +690,8 @@ public class QueueMetrics implements MetricsSource {
pendingVCores.decr(res.getVirtualCores() * containers);
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.decreasePending(res, containers);
-
registerCustomResources(queueMetricsForCustomResources.getPendingValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getPendingValues(), this.registry,
PENDING_RESOURCE_METRIC_PREFIX, PENDING_RESOURCE_METRIC_DESC);
}
}
@@ -793,8 +762,8 @@ public class QueueMetrics implements MetricsSource {
allocatedVCores.incr(res.getVirtualCores() * containers);
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.increaseAllocated(res, containers);
- registerCustomResources(
- queueMetricsForCustomResources.getAllocatedValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getAllocatedValues(), this.registry,
ALLOCATED_RESOURCE_METRIC_PREFIX, ALLOCATED_RESOURCE_METRIC_DESC);
}
if (decrPending) {
@@ -813,8 +782,8 @@ public class QueueMetrics implements MetricsSource {
allocatedVCores.incr(res.getVirtualCores());
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.increaseAllocated(res);
- registerCustomResources(
- queueMetricsForCustomResources.getAllocatedValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getAllocatedValues(), this.registry,
ALLOCATED_RESOURCE_METRIC_PREFIX, ALLOCATED_RESOURCE_METRIC_DESC);
}
@@ -822,7 +791,8 @@ public class QueueMetrics implements MetricsSource {
pendingVCores.decr(res.getVirtualCores());
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.decreasePending(res);
-
registerCustomResources(queueMetricsForCustomResources.getPendingValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getPendingValues(), this.registry,
PENDING_RESOURCE_METRIC_PREFIX, PENDING_RESOURCE_METRIC_DESC);
}
@@ -879,8 +849,8 @@ public class QueueMetrics implements MetricsSource {
allocatedVCores.decr(res.getVirtualCores() * containers);
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.decreaseAllocated(res, containers);
- registerCustomResources(
- queueMetricsForCustomResources.getAllocatedValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getAllocatedValues(), this.registry,
ALLOCATED_RESOURCE_METRIC_PREFIX, ALLOCATED_RESOURCE_METRIC_DESC);
}
}
@@ -928,9 +898,9 @@ public class QueueMetrics implements MetricsSource {
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources
.increaseAggregatedPreemptedSeconds(res, seconds);
- registerCustomResources(
+ queueMetricsForCustomResources.registerCustomResources(
queueMetricsForCustomResources.getAggregatePreemptedSeconds()
- .getValues(),
+ .getValues(), this.registry,
AGGREGATE_PREEMPTED_SECONDS_METRIC_PREFIX,
AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC);
}
@@ -971,8 +941,8 @@ public class QueueMetrics implements MetricsSource {
reservedVCores.incr(res.getVirtualCores());
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.increaseReserved(res);
- registerCustomResources(
- queueMetricsForCustomResources.getReservedValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getReservedValues(), this.registry,
RESERVED_RESOURCE_METRIC_PREFIX, RESERVED_RESOURCE_METRIC_DESC);
}
}
@@ -1010,8 +980,8 @@ public class QueueMetrics implements MetricsSource {
reservedVCores.decr(res.getVirtualCores());
if (queueMetricsForCustomResources != null) {
queueMetricsForCustomResources.decreaseReserved(res);
- registerCustomResources(
- queueMetricsForCustomResources.getReservedValues(),
+ queueMetricsForCustomResources.registerCustomResources(
+ queueMetricsForCustomResources.getReservedValues(), this.registry,
RESERVED_RESOURCE_METRIC_PREFIX, RESERVED_RESOURCE_METRIC_DESC);
}
}
@@ -1114,7 +1084,7 @@ public class QueueMetrics implements MetricsSource {
* @return QueueMetricsCustomResource
*/
@VisibleForTesting
- public QueueMetricsCustomResource getAggregatedPreemptedSecondsResources() {
+ public CustomResourceMetricValue getAggregatedPreemptedSecondsResources() {
return queueMetricsForCustomResources.getAggregatePreemptedSeconds();
}
@@ -1232,7 +1202,7 @@ public class QueueMetrics implements MetricsSource {
public void fillInValuesFromAvailableResources(Resource fromResource,
Resource targetResource) {
if (queueMetricsForCustomResources != null) {
- QueueMetricsCustomResource availableResources =
+ CustomResourceMetricValue availableResources =
queueMetricsForCustomResources.getAvailable();
// We expect all custom resources contained in availableResources,
@@ -1257,7 +1227,7 @@ public class QueueMetrics implements MetricsSource {
return this.queueMetricsForCustomResources;
}
- public void setQueueMetricsForCustomResources(
+ protected void setQueueMetricsForCustomResources(
QueueMetricsForCustomResources metrics) {
this.queueMetricsForCustomResources = metrics;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
index ddfffc8..cd970a7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetricsForCustomResources.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,7 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,28 +19,20 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetrics;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
import java.util.Map;
-/**
- * This class is a main entry-point for any kind of metrics for
- * custom resources.
- * It provides increase and decrease methods for all types of metrics.
- */
-public class QueueMetricsForCustomResources {
- private final QueueMetricsCustomResource aggregatePreemptedSeconds =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource aggregatePreempted =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource allocated =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource available =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource pending =
- new QueueMetricsCustomResource();
-
- private final QueueMetricsCustomResource reserved =
- new QueueMetricsCustomResource();
+public class QueueMetricsForCustomResources extends CustomResourceMetrics {
+ private final CustomResourceMetricValue aggregatePreemptedSeconds =
+ new CustomResourceMetricValue();
+ private final CustomResourceMetricValue aggregatePreempted =
+ new CustomResourceMetricValue();
+ private final CustomResourceMetricValue pending =
+ new CustomResourceMetricValue();
+ private final CustomResourceMetricValue reserved =
+ new CustomResourceMetricValue();
public void increaseReserved(Resource res) {
reserved.increase(res);
@@ -48,10 +42,6 @@ public class QueueMetricsForCustomResources {
reserved.decrease(res);
}
- public void setAvailable(Resource res) {
- available.set(res);
- }
-
public void increasePending(Resource res, int containers) {
pending.increaseWithMultiplier(res, containers);
}
@@ -64,20 +54,12 @@ public class QueueMetricsForCustomResources {
pending.decreaseWithMultiplier(res, containers);
}
- public void increaseAllocated(Resource res) {
- allocated.increase(res);
- }
-
- public void increaseAllocated(Resource res, int containers) {
- allocated.increaseWithMultiplier(res, containers);
- }
-
- public void decreaseAllocated(Resource res) {
- allocated.decrease(res);
+ public Map<String, Long> getPendingValues() {
+ return pending.getValues();
}
- public void decreaseAllocated(Resource res, int containers) {
- allocated.decreaseWithMultiplier(res, containers);
+ public Map<String, Long> getReservedValues() {
+ return reserved.getValues();
}
public void increaseAggregatedPreemptedSeconds(Resource res, long seconds) {
@@ -88,27 +70,7 @@ public class QueueMetricsForCustomResources {
aggregatePreempted.increase(res);
}
- Map<String, Long> getAllocatedValues() {
- return allocated.getValues();
- }
-
- Map<String, Long> getAvailableValues() {
- return available.getValues();
- }
-
- Map<String, Long> getPendingValues() {
- return pending.getValues();
- }
-
- Map<String, Long> getReservedValues() {
- return reserved.getValues();
- }
-
- QueueMetricsCustomResource getAggregatePreemptedSeconds() {
+ CustomResourceMetricValue getAggregatePreemptedSeconds() {
return aggregatePreemptedSeconds;
}
-
- public QueueMetricsCustomResource getAvailable() {
- return available;
- }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 106f565..586f837 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -78,6 +78,8 @@ public class CSQueueMetrics extends QueueMetrics {
private static final String MAX_CAPACITY_METRIC_DESC =
"MaxCapacity of NAME";
+ private CSQueueMetricsForCustomResources csQueueMetricsForCustomResources;
+
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
super(ms, queueName, parent, enableUserMetrics, conf);
@@ -90,11 +92,14 @@ public class CSQueueMetrics extends QueueMetrics {
* mandatory resources metrics
*/
protected void registerCustomResources() {
- Map<String, Long> customResources = initAndGetCustomResources();
- registerCustomResources(customResources, GUARANTEED_CAPACITY_METRIC_PREFIX,
- GUARANTEED_CAPACITY_METRIC_DESC);
- registerCustomResources(customResources, MAX_CAPACITY_METRIC_PREFIX,
- MAX_CAPACITY_METRIC_DESC);
+ Map<String, Long> customResources =
+ csQueueMetricsForCustomResources.initAndGetCustomResources();
+ csQueueMetricsForCustomResources
+ .registerCustomResources(customResources, this.registry,
+ GUARANTEED_CAPACITY_METRIC_PREFIX,
GUARANTEED_CAPACITY_METRIC_DESC);
+ csQueueMetricsForCustomResources
+ .registerCustomResources(customResources, this.registry,
+ MAX_CAPACITY_METRIC_PREFIX, MAX_CAPACITY_METRIC_DESC);
super.registerCustomResources();
}
@@ -184,12 +189,10 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
guaranteedMB.set(res.getMemorySize());
guaranteedVCores.set(res.getVirtualCores());
- if (getQueueMetricsForCustomResources() != null) {
- ((CSQueueMetricsForCustomResources)
getQueueMetricsForCustomResources())
- .setGuaranteedCapacity(res);
- registerCustomResources(
- ((CSQueueMetricsForCustomResources)
- getQueueMetricsForCustomResources()).getGuaranteedCapacity(),
+ if (csQueueMetricsForCustomResources != null) {
+ csQueueMetricsForCustomResources.setGuaranteedCapacity(res);
+ csQueueMetricsForCustomResources.registerCustomResources(
+ csQueueMetricsForCustomResources.getGuaranteedCapacity(), registry,
GUARANTEED_CAPACITY_METRIC_PREFIX,
GUARANTEED_CAPACITY_METRIC_DESC);
}
}
@@ -207,12 +210,10 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
maxCapacityMB.set(res.getMemorySize());
maxCapacityVCores.set(res.getVirtualCores());
- if (getQueueMetricsForCustomResources() != null) {
- ((CSQueueMetricsForCustomResources)
getQueueMetricsForCustomResources())
- .setMaxCapacity(res);
- registerCustomResources(
- ((CSQueueMetricsForCustomResources)
- getQueueMetricsForCustomResources()).getMaxCapacity(),
+ if (csQueueMetricsForCustomResources != null) {
+ csQueueMetricsForCustomResources.setMaxCapacity(res);
+ csQueueMetricsForCustomResources.registerCustomResources(
+ csQueueMetricsForCustomResources.getMaxCapacity(), registry,
MAX_CAPACITY_METRIC_PREFIX, MAX_CAPACITY_METRIC_DESC);
}
}
@@ -221,7 +222,9 @@ public class CSQueueMetrics extends QueueMetrics {
@Override
protected void createQueueMetricsForCustomResources() {
if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- setQueueMetricsForCustomResources(new
CSQueueMetricsForCustomResources());
+ this.csQueueMetricsForCustomResources =
+ new CSQueueMetricsForCustomResources();
+ setQueueMetricsForCustomResources(csQueueMetricsForCustomResources);
registerCustomResources();
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
index 0d84999..7acfe5d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -293,7 +294,7 @@ public class TestQueueMetricsForCustomResources {
}
private Resource convertPreemptedSecondsToResource(QueueMetrics qm) {
- QueueMetricsCustomResource customValues = qm
+ CustomResourceMetricValue customValues = qm
.getAggregatedPreemptedSecondsResources();
MutableCounterLong vcoreSeconds = qm
.getAggregateVcoreSecondsPreempted();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 8e29414..5af9bc3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -38,11 +38,11 @@ import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetricsCustomResource;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -344,7 +344,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
private Map<String, Long> verifyQueueMetricsForCustomResources(
FSLeafQueue schedulable) {
- QueueMetricsCustomResource maxAMShareCustomResources =
+ CustomResourceMetricValue maxAMShareCustomResources =
schedulable.getMetrics().getCustomResources().getMaxAMShare();
Map<String, Long> customResourceValues = maxAMShareCustomResources
.getValues();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
