Repository: hadoop Updated Branches: refs/heads/branch-2 23fe1abc0 -> 51e448d29
YARN-4292. ResourceUtilization should be a part of NodeInfo REST API. (Sunil G via wangda) (cherry picked from commit a2c3bfc8c1349102a7f2bc4ea96b80b429ac227b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51e448d2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51e448d2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51e448d2 Branch: refs/heads/branch-2 Commit: 51e448d295f6d2214c1d6455446a8e05e8a967b1 Parents: 23fe1ab Author: Wangda Tan <[email protected]> Authored: Thu Dec 3 14:28:00 2015 -0800 Committer: Wangda Tan <[email protected]> Committed: Thu Dec 3 14:30:25 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/webapp/dao/NodeInfo.java | 8 ++ .../webapp/dao/ResourceUtilizationInfo.java | 89 ++++++++++++++++++++ .../webapp/TestRMWebServicesNodes.java | 79 +++++++++++++++-- 4 files changed, 174 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/51e448d2/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5c67fea..bbfebec 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -534,6 +534,9 @@ Release 2.8.0 - UNRELEASED YARN-4132. Separate configs for nodemanager to resourcemanager connection timeout and retries (Chang Li via jlowe) + YARN-4292. ResourceUtilization should be a part of NodeInfo REST API. + (Sunil G via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/51e448d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 3104117..0f877f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -50,6 +50,7 @@ public class NodeInfo { protected long usedVirtualCores; protected long availableVirtualCores; protected ArrayList<String> nodeLabels = new ArrayList<String>(); + protected ResourceUtilizationInfo resourceUtilization; public NodeInfo() { } // JAXB needs this @@ -82,6 +83,9 @@ public class NodeInfo { nodeLabels.addAll(labelSet); Collections.sort(nodeLabels); } + + // update node and containers resource utilization + this.resourceUtilization = new ResourceUtilizationInfo(ni); } public String getRack() { @@ -139,4 +143,8 @@ public class NodeInfo { public ArrayList<String> getNodeLabels() { return this.nodeLabels; } + + public ResourceUtilizationInfo getResourceUtilization() { + return this.resourceUtilization; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/51e448d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java new file mode 100644 index 0000000..67d0d64 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; + +/** + * DAO object represents resource utilization of node and containers. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceUtilizationInfo { + + protected int nodePhysicalMemoryMB; + protected int nodeVirtualMemoryMB; + protected double nodeCPUUsage; + protected int aggregatedContainersPhysicalMemoryMB; + protected int aggregatedContainersVirtualMemoryMB; + protected double containersCPUUsage; + + public ResourceUtilizationInfo() { + } // JAXB needs this + + public ResourceUtilizationInfo(RMNode ni) { + + // update node and containers resource utilization + ResourceUtilization nodeUtilization = ni.getNodeUtilization(); + if (nodeUtilization != null) { + this.nodePhysicalMemoryMB = nodeUtilization.getPhysicalMemory(); + this.nodeVirtualMemoryMB = nodeUtilization.getVirtualMemory(); + this.nodeCPUUsage = nodeUtilization.getCPU(); + } + + ResourceUtilization containerAggrUtilization = ni + .getAggregatedContainersUtilization(); + if (containerAggrUtilization != null) { + this.aggregatedContainersPhysicalMemoryMB = containerAggrUtilization + .getPhysicalMemory(); + this.aggregatedContainersVirtualMemoryMB = containerAggrUtilization + .getVirtualMemory(); + this.containersCPUUsage = containerAggrUtilization.getCPU(); + } + } + + public int getNodePhysicalMemoryMB() { + return nodePhysicalMemoryMB; + } + + public int getNodeVirtualMemoryMB() { + return nodeVirtualMemoryMB; + } + + public int getAggregatedContainersPhysicalMemoryMB() { + return aggregatedContainersPhysicalMemoryMB; + } + + public int getAggregatedContainersVirtualMemoryMB() { + return aggregatedContainersVirtualMemoryMB; + } + + public double getNodeCPUUsage() { + return nodeCPUUsage; + } + + public double getContainersCPUUsage() { + return containersCPUUsage; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/51e448d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index 206edb1..462ce50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -639,6 +640,44 @@ public class TestRMWebServicesNodes extends JerseyTestBase { assertEquals("incorrect number of elements", 3, nodeArray.length()); } + @Test + public void testNodesResourceUtilization() throws JSONException, Exception { + WebResource r = resource(); + MockNM nm1 = rm.registerNode("h1:1234", 5120); + rm.sendNodeStarted(nm1); + rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); + + RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes() + .get(nm1.getNodeId()); + NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, + "test health report", System.currentTimeMillis()); + ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0, + (float) 10.5); + ResourceUtilization containerResource = ResourceUtilization.newInstance( + 2048, 0, (float) 5.05); + NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, + new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource, + nodeResource, null); + node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, null)); + rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject nodes = json.getJSONObject("nodes"); + assertEquals("incorrect number of elements", 1, nodes.length()); + JSONArray nodeArray = nodes.getJSONArray("node"); + assertEquals("incorrect number of elements", 1, nodeArray.length()); + JSONObject info = nodeArray.getJSONObject(0); + + // verify the resource utilization + verifyNodeInfo(info, nm1); + } + public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException, Exception { for (int i = 0; i < nodes.getLength(); i++) { @@ -656,14 +695,23 @@ public class TestRMWebServicesNodes extends JerseyTestBase { WebServicesTestUtils.getXmlLong(element, "availMemoryMB"), WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"), WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"), - WebServicesTestUtils.getXmlString(element, "version")); + WebServicesTestUtils.getXmlString(element, "version"), + WebServicesTestUtils.getXmlInt(element, "nodePhysicalMemoryMB"), + WebServicesTestUtils.getXmlInt(element, "nodeVirtualMemoryMB"), + WebServicesTestUtils.getXmlFloat(element, "nodeCPUUsage"), + WebServicesTestUtils.getXmlInt(element, + "aggregatedContainersPhysicalMemoryMB"), + WebServicesTestUtils.getXmlInt(element, + "aggregatedContainersVirtualMemoryMB"), + WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage")); } } public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm) throws JSONException, Exception { - assertEquals("incorrect number of elements", 13, nodeInfo.length()); + assertEquals("incorrect number of elements", 14, nodeInfo.length()); + JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization"); verifyNodeInfoGeneric(nm, nodeInfo.getString("state"), nodeInfo.getString("rack"), nodeInfo.getString("id"), nodeInfo.getString("nodeHostName"), @@ -672,15 +720,23 @@ public class TestRMWebServicesNodes extends JerseyTestBase { nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"), nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"), nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"), - nodeInfo.getString("version")); - + nodeInfo.getString("version"), + resourceInfo.getInt("nodePhysicalMemoryMB"), + resourceInfo.getInt("nodeVirtualMemoryMB"), + resourceInfo.getDouble("nodeCPUUsage"), + resourceInfo.getInt("aggregatedContainersPhysicalMemoryMB"), + resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"), + resourceInfo.getDouble("containersCPUUsage")); } public void verifyNodeInfoGeneric(MockNM nm, String state, String rack, String id, String nodeHostName, String nodeHTTPAddress, long lastHealthUpdate, String healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores, - long availVirtualCores, String version) + long availVirtualCores, String version, int nodePhysicalMemoryMB, + int nodeVirtualMemoryMB, double nodeCPUUsage, + int containersPhysicalMemoryMB, int containersVirtualMemoryMB, + double containersCPUUsage) throws JSONException, Exception { RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId()); @@ -701,6 +757,19 @@ public class TestRMWebServicesNodes extends JerseyTestBase { expectedHttpAddress, nodeHTTPAddress); WebServicesTestUtils.checkStringMatch("version", node.getNodeManagerVersion(), version); + if (node.getNodeUtilization() != null) { + ResourceUtilization nodeResource = ResourceUtilization.newInstance( + nodePhysicalMemoryMB, nodeVirtualMemoryMB, (float) nodeCPUUsage); + assertEquals("nodeResourceUtilization doesn't match", + node.getNodeUtilization(), nodeResource); + } + if (node.getAggregatedContainersUtilization() != null) { + ResourceUtilization containerResource = ResourceUtilization.newInstance( + containersPhysicalMemoryMB, containersVirtualMemoryMB, + (float) containersCPUUsage); + assertEquals("containerResourceUtilization doesn't match", + node.getAggregatedContainersUtilization(), containerResource); + } long expectedHealthUpdate = node.getLastHealthReportTime(); assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
