YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)
Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0cfb0a3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0cfb0a3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0cfb0a3 Branch: refs/heads/YARN-1011 Commit: e0cfb0a31a1c6ad0f9dcf0705b44229593a0915f Parents: 2e58656 Author: Wangda Tan <[email protected]> Authored: Fri Jan 26 15:43:27 2018 +0800 Committer: Wangda Tan <[email protected]> Committed: Fri Jan 26 15:43:27 2018 +0800 ---------------------------------------------------------------------- .../webapp/JAXBContextResolver.java | 2 +- .../webapp/dao/ClusterMetricsInfo.java | 24 ++++++++++ .../resourcemanager/webapp/dao/NodeInfo.java | 20 ++++++++ .../webapp/dao/ResourceInfo.java | 18 ++++++-- .../webapp/dao/ResourceInformationsInfo.java | 48 ++++++++++++++++++++ .../main/webapp/app/models/cluster-metric.js | 4 +- .../src/main/webapp/app/models/yarn-rm-node.js | 4 +- .../main/webapp/app/serializers/yarn-rm-node.js | 4 +- 8 files changed, 114 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 2f50a24..2e4204e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> { UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class }; + AppTimeoutInfo.class, ResourceInformationsInfo.class }; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 3214cb9..84f70d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -61,6 +61,12 @@ public class ClusterMetricsInfo { private int activeNodes; private int shutdownNodes; + // Total used resource of the cluster, including all partitions + private ResourceInfo totalUsedResourcesAcrossPartition; + + // Total registered resources of the cluster, including all partitions + private ResourceInfo totalClusterResourcesAcrossPartition; + public ClusterMetricsInfo() { } // JAXB needs this @@ -92,9 +98,20 @@ public class ClusterMetricsInfo { this.containersReserved = metrics.getReservedContainers(); if (rs instanceof CapacityScheduler) { + CapacityScheduler cs = (CapacityScheduler) rs; this.totalMB = availableMB + allocatedMB + reservedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores + containersReserved; + // TODO, add support of other schedulers to get total used resources + // across partition. + if (cs.getRootQueue() != null + && cs.getRootQueue().getQueueResourceUsage() != null + && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) { + totalUsedResourcesAcrossPartition = new ResourceInfo( + cs.getRootQueue().getQueueResourceUsage().getAllUsed()); + totalClusterResourcesAcrossPartition = new ResourceInfo( + cs.getClusterResource()); + } } else { this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; @@ -310,4 +327,11 @@ public class ClusterMetricsInfo { this.shutdownNodes = shutdownNodes; } + public ResourceInfo getTotalUsedResourcesAcrossPartition() { + return totalUsedResourcesAcrossPartition; + } + + public ResourceInfo getTotalClusterResourcesAcrossPartition() { + return totalClusterResourcesAcrossPartition; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 2530c8e..3cec215 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -58,6 +58,8 @@ public class NodeInfo { private int numQueuedContainers; protected ArrayList<String> nodeLabels = new ArrayList<String>(); protected ResourceUtilizationInfo resourceUtilization; + protected ResourceInfo usedResource; + protected ResourceInfo availableResource; public NodeInfo() { } // JAXB needs this @@ -75,6 +77,8 @@ public class NodeInfo { this.usedVirtualCores = report.getUsedResource().getVirtualCores(); this.availableVirtualCores = report.getAvailableResource().getVirtualCores(); + this.usedResource = new ResourceInfo(report.getUsedResource()); + this.availableResource = new ResourceInfo(report.getAvailableResource()); } this.id = id.toString(); this.rack = ni.getRackName(); @@ -183,6 +187,22 @@ public class NodeInfo { return this.nodeLabels; } + public ResourceInfo getUsedResource() { + return usedResource; + } + + public void setUsedResource(ResourceInfo used) { + this.usedResource = used; + } + + public ResourceInfo getAvailableResource() { + return availableResource; + } + + public void setAvailableResource(ResourceInfo avail) { + this.availableResource = avail; + } + public ResourceUtilizationInfo getResourceUtilization() { return this.resourceUtilization; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index e13980a..5bed936 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.Resources; + @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) public class ResourceInfo { @@ -34,6 +35,9 @@ public class ResourceInfo { long memory; @XmlElement int vCores; + @XmlElement + ResourceInformationsInfo resourceInformations = + new ResourceInformationsInfo(); private Resource resources; @@ -41,9 +45,13 @@ public class ResourceInfo { } public ResourceInfo(Resource res) { - memory = res.getMemorySize(); - vCores = res.getVirtualCores(); - resources = Resources.clone(res); + // Make sure no NPE. + if (res != null) { + memory = res.getMemorySize(); + vCores = res.getVirtualCores(); + resources = Resources.clone(res); + resourceInformations.addAll(res.getAllResourcesListCopy()); + } } public long getMemorySize() { @@ -84,4 +92,8 @@ public class ResourceInfo { public Resource getResource() { return Resource.newInstance(resources); } + + public ResourceInformationsInfo getResourcesInformations() { + return resourceInformations; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java new file mode 100644 index 0000000..651d7df --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +@XmlRootElement(name = "resourceInformations") +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceInformationsInfo { + + @XmlElement(name = "resourceInformation") + protected ArrayList<ResourceInformation> resourceInformation = + new ArrayList<ResourceInformation>(); + + public ResourceInformationsInfo() { + } // JAXB needs this + + public ArrayList<ResourceInformation> getApps() { + return resourceInformation; + } + + public void addAll(List<ResourceInformation> resourcesInformationsInfo) { + resourceInformation.addAll(resourcesInformationsInfo); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js index bbc6f08..764723a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js @@ -169,8 +169,8 @@ export default DS.Model.extend({ && this.get("totalUsedResourcesAcrossPartition")) { var usages = []; - var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations; - var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations; + var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation; + var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation; clusterResourceInformations.forEach(function(cluster) { var perResourceTypeUsage = { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js index 3b2385f..64fe7b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js @@ -99,7 +99,7 @@ export default DS.Model.extend({ const usedResource = this.get("usedResource"); const availableResource = this.get("availableResource"); - var resourceInformations = usedResource ? usedResource.resourcesInformations : []; + var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : []; for (var i = 0; i < resourceInformations.length; i++) { ri = resourceInformations[i]; if (ri.name === "yarn.io/gpu") { @@ -108,7 +108,7 @@ export default DS.Model.extend({ } var available = 0; - resourceInformations = availableResource ? availableResource.resourcesInformations : []; + resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : []; for (i = 0; i < resourceInformations.length; i++) { ri = resourceInformations[i]; if (ri.name === "yarn.io/gpu") { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js index a3a1d59..c27fd92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js @@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({ availableVirtualCores: payload.availableVirtualCores, version: payload.version, nodeLabels: payload.nodeLabels, - usedResource: payload.used, - availableResource: payload.avail + usedResource: payload.usedResource, + availableResource: payload.availableResource } }; return fixedPayload; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
