YARN-4292. ResourceUtilization should be a part of NodeInfo REST API. (Sunil G
via wangda)
(cherry picked from commit a2c3bfc8c1349102a7f2bc4ea96b80b429ac227b)
(cherry picked from commit 51e448d295f6d2214c1d6455446a8e05e8a967b1)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15b3ff19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15b3ff19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15b3ff19
Branch: refs/heads/branch-2.8
Commit: 15b3ff19715cd671ed782ce2e193eaf139f768fd
Parents: 9c22910
Author: Wangda Tan <[email protected]>
Authored: Thu Dec 3 14:28:00 2015 -0800
Committer: Wangda Tan <[email protected]>
Committed: Fri Dec 18 16:33:53 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../resourcemanager/webapp/dao/NodeInfo.java | 8 ++
.../webapp/dao/ResourceUtilizationInfo.java | 89 ++++++++++++++++++++
.../webapp/TestRMWebServicesNodes.java | 79 +++++++++++++++--
4 files changed, 174 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 02dbf3d..165fa47 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -536,6 +536,9 @@ Release 2.8.0 - UNRELEASED
YARN-4248. REST API for submit/update/delete Reservations. (curino)
+ YARN-4292. ResourceUtilization should be a part of NodeInfo REST API.
+ (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 3104117..0f877f8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -50,6 +50,7 @@ public class NodeInfo {
protected long usedVirtualCores;
protected long availableVirtualCores;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
+ protected ResourceUtilizationInfo resourceUtilization;
public NodeInfo() {
} // JAXB needs this
@@ -82,6 +83,9 @@ public class NodeInfo {
nodeLabels.addAll(labelSet);
Collections.sort(nodeLabels);
}
+
+ // update node and containers resource utilization
+ this.resourceUtilization = new ResourceUtilizationInfo(ni);
}
public String getRack() {
@@ -139,4 +143,8 @@ public class NodeInfo {
public ArrayList<String> getNodeLabels() {
return this.nodeLabels;
}
+
+ public ResourceUtilizationInfo getResourceUtilization() {
+ return this.resourceUtilization;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
new file mode 100644
index 0000000..67d0d64
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+
+/**
+ * DAO object represents resource utilization of node and containers.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourceUtilizationInfo {
+
+ protected int nodePhysicalMemoryMB;
+ protected int nodeVirtualMemoryMB;
+ protected double nodeCPUUsage;
+ protected int aggregatedContainersPhysicalMemoryMB;
+ protected int aggregatedContainersVirtualMemoryMB;
+ protected double containersCPUUsage;
+
+ public ResourceUtilizationInfo() {
+ } // JAXB needs this
+
+ public ResourceUtilizationInfo(RMNode ni) {
+
+ // update node and containers resource utilization
+ ResourceUtilization nodeUtilization = ni.getNodeUtilization();
+ if (nodeUtilization != null) {
+ this.nodePhysicalMemoryMB = nodeUtilization.getPhysicalMemory();
+ this.nodeVirtualMemoryMB = nodeUtilization.getVirtualMemory();
+ this.nodeCPUUsage = nodeUtilization.getCPU();
+ }
+
+ ResourceUtilization containerAggrUtilization = ni
+ .getAggregatedContainersUtilization();
+ if (containerAggrUtilization != null) {
+ this.aggregatedContainersPhysicalMemoryMB = containerAggrUtilization
+ .getPhysicalMemory();
+ this.aggregatedContainersVirtualMemoryMB = containerAggrUtilization
+ .getVirtualMemory();
+ this.containersCPUUsage = containerAggrUtilization.getCPU();
+ }
+ }
+
+ public int getNodePhysicalMemoryMB() {
+ return nodePhysicalMemoryMB;
+ }
+
+ public int getNodeVirtualMemoryMB() {
+ return nodeVirtualMemoryMB;
+ }
+
+ public int getAggregatedContainersPhysicalMemoryMB() {
+ return aggregatedContainersPhysicalMemoryMB;
+ }
+
+ public int getAggregatedContainersVirtualMemoryMB() {
+ return aggregatedContainersVirtualMemoryMB;
+ }
+
+ public double getNodeCPUUsage() {
+ return nodeCPUUsage;
+ }
+
+ public double getContainersCPUUsage() {
+ return containersCPUUsage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 206edb1..462ce50 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -639,6 +640,44 @@ public class TestRMWebServicesNodes extends JerseyTestBase
{
assertEquals("incorrect number of elements", 3, nodeArray.length());
}
+ @Test
+ public void testNodesResourceUtilization() throws JSONException, Exception {
+ WebResource r = resource();
+ MockNM nm1 = rm.registerNode("h1:1234", 5120);
+ rm.sendNodeStarted(nm1);
+ rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+
+ RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
+ .get(nm1.getNodeId());
+ NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
+ "test health report", System.currentTimeMillis());
+ ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0,
+ (float) 10.5);
+ ResourceUtilization containerResource = ResourceUtilization.newInstance(
+ 2048, 0, (float) 5.05);
+ NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
+ new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource,
+ nodeResource, null);
+ node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, null));
+ rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("nodes").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject nodes = json.getJSONObject("nodes");
+ assertEquals("incorrect number of elements", 1, nodes.length());
+ JSONArray nodeArray = nodes.getJSONArray("node");
+ assertEquals("incorrect number of elements", 1, nodeArray.length());
+ JSONObject info = nodeArray.getJSONObject(0);
+
+ // verify the resource utilization
+ verifyNodeInfo(info, nm1);
+ }
+
public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
Exception {
for (int i = 0; i < nodes.getLength(); i++) {
@@ -656,14 +695,23 @@ public class TestRMWebServicesNodes extends
JerseyTestBase {
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
- WebServicesTestUtils.getXmlString(element, "version"));
+ WebServicesTestUtils.getXmlString(element, "version"),
+ WebServicesTestUtils.getXmlInt(element, "nodePhysicalMemoryMB"),
+ WebServicesTestUtils.getXmlInt(element, "nodeVirtualMemoryMB"),
+ WebServicesTestUtils.getXmlFloat(element, "nodeCPUUsage"),
+ WebServicesTestUtils.getXmlInt(element,
+ "aggregatedContainersPhysicalMemoryMB"),
+ WebServicesTestUtils.getXmlInt(element,
+ "aggregatedContainersVirtualMemoryMB"),
+ WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 13, nodeInfo.length());
+ assertEquals("incorrect number of elements", 14, nodeInfo.length());
+ JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization");
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
nodeInfo.getString("id"), nodeInfo.getString("nodeHostName"),
@@ -672,15 +720,23 @@ public class TestRMWebServicesNodes extends
JerseyTestBase {
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
nodeInfo.getLong("usedVirtualCores"),
nodeInfo.getLong("availableVirtualCores"),
- nodeInfo.getString("version"));
-
+ nodeInfo.getString("version"),
+ resourceInfo.getInt("nodePhysicalMemoryMB"),
+ resourceInfo.getInt("nodeVirtualMemoryMB"),
+ resourceInfo.getDouble("nodeCPUUsage"),
+ resourceInfo.getInt("aggregatedContainersPhysicalMemoryMB"),
+ resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"),
+ resourceInfo.getDouble("containersCPUUsage"));
}
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB, long
usedVirtualCores,
- long availVirtualCores, String version)
+ long availVirtualCores, String version, int nodePhysicalMemoryMB,
+ int nodeVirtualMemoryMB, double nodeCPUUsage,
+ int containersPhysicalMemoryMB, int containersVirtualMemoryMB,
+ double containersCPUUsage)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@@ -701,6 +757,19 @@ public class TestRMWebServicesNodes extends JerseyTestBase
{
expectedHttpAddress, nodeHTTPAddress);
WebServicesTestUtils.checkStringMatch("version",
node.getNodeManagerVersion(), version);
+ if (node.getNodeUtilization() != null) {
+ ResourceUtilization nodeResource = ResourceUtilization.newInstance(
+ nodePhysicalMemoryMB, nodeVirtualMemoryMB, (float) nodeCPUUsage);
+ assertEquals("nodeResourceUtilization doesn't match",
+ node.getNodeUtilization(), nodeResource);
+ }
+ if (node.getAggregatedContainersUtilization() != null) {
+ ResourceUtilization containerResource = ResourceUtilization.newInstance(
+ containersPhysicalMemoryMB, containersVirtualMemoryMB,
+ (float) containersCPUUsage);
+ assertEquals("containerResourceUtilization doesn't match",
+ node.getAggregatedContainersUtilization(), containerResource);
+ }
long expectedHealthUpdate = node.getLastHealthReportTime();
assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
