YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00aed24b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00aed24b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00aed24b Branch: refs/heads/YARN-1011 Commit: 00aed24b1baa888d4ed4f45a8f2dd02bcaa8e58a Parents: 1ac1486 Author: Miklos Szegedi <szege...@apache.org> Authored: Wed Nov 22 08:39:03 2017 -0800 Committer: Haibo Chen <haiboc...@apache.org> Committed: Mon Apr 9 17:13:04 2018 -0700 ---------------------------------------------------------------------- .../hadoop/mapreduce/v2/TestRMNMInfo.java | 14 ++- .../hadoop/yarn/api/records/NodeReport.java | 98 +++++++++++++++----- .../src/main/proto/yarn_protos.proto | 7 +- .../applications/distributedshell/Client.java | 2 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 49 +++++++--- .../hadoop/yarn/client/cli/TestYarnCLI.java | 43 ++++++--- .../api/records/impl/pb/NodeReportPBImpl.java | 94 +++++++++++++++---- .../hadoop/yarn/server/utils/BuilderUtils.java | 37 ++++---- .../server/resourcemanager/ClientRMService.java | 21 +++-- .../resourcemanager/DefaultAMSProcessor.java | 22 +++-- .../yarn/server/resourcemanager/RMNMInfo.java | 45 +++++---- .../resourcemanager/rmnode/RMNodeImpl.java | 2 +- .../scheduler/SchedulerNodeReport.java | 50 +++++++--- .../webapp/dao/FifoSchedulerInfo.java | 9 +- .../resourcemanager/webapp/dao/NodeInfo.java | 17 ++-- .../server/resourcemanager/NodeManager.java | 4 +- .../capacity/TestApplicationPriority.java | 66 ++++++++----- .../capacity/TestCapacityScheduler.java | 32 ++++--- .../TestNodeLabelContainerAllocation.java | 40 ++++---- .../scheduler/fifo/TestFifoScheduler.java | 50 ++++++---- .../webapp/TestRMWebServicesNodes.java | 13 +-- 21 files changed, 491 insertions(+), 224 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java index efea709..76b5493 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java @@ -124,11 +124,17 @@ public class TestRMNMInfo { Assert.assertNotNull(n.get("NodeManagerVersion")); Assert.assertNotNull(n.get("NumContainers")); Assert.assertEquals( - n.get("NodeId") + ": Unexpected number of used containers", - 0, n.get("NumContainers").asInt()); + n.get("NodeId") + ": Unexpected number of guaranteed containers used", + 0, n.get("NumContainers").asInt()); + Assert.assertEquals(n.get("NodeId") + + ": Unexpected number of opportunistic containers used", + 0, n.get("NumOpportunisticContainers").asInt()); Assert.assertEquals( - n.get("NodeId") + ": Unexpected amount of used memory", + n.get("NodeId") + ": Unexpected amount of guaranteed memory used", 0, n.get("UsedMemoryMB").asInt()); + Assert.assertEquals( + n.get("NodeId") + ": Unexpected amount of used opportunistic memory", + 0, n.get("UsedOpportunisticMemoryMB").asInt()); Assert.assertNotNull(n.get("AvailableMemoryMB")); } } @@ -161,6 +167,8 @@ public class TestRMNMInfo { Assert.assertNotNull(n.get("NodeManagerVersion")); Assert.assertNull(n.get("NumContainers")); Assert.assertNull(n.get("UsedMemoryMB")); + Assert.assertNull(n.get("NumOpportunisticContainers")); + Assert.assertNull(n.get("UsedOpportunisticMemoryMB")); Assert.assertNull(n.get("AvailableMemoryMB")); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java index 3a80641..53e9f92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java @@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records; * <li>{@link NodeId} of the node.</li> * <li>HTTP Tracking URL of the node.</li> * <li>Rack name for the node.</li> - * <li>Used {@link Resource} on the node.</li> + * <li>Used guaranteed {@link Resource} on the node.</li> + * <li>Used opportunistic {@link Resource} on the node.</li> * <li>Total available {@link Resource} of the node.</li> - * <li>Number of running containers on the node.</li> + * <li>Number of total running containers on the node.</li> + * <li>Number of running guaranteed containers on the node.</li> + * <li>Number of running opportunistic containers on the node.</li> * </ul> * * @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest) @@ -50,33 +53,37 @@ public abstract class NodeReport { @Private @Unstable public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime) { - return newInstance(nodeId, nodeState, httpAddress, rackName, used, - capability, numContainers, healthReport, lastHealthReportTime, - null, null, null); + String httpAddress, String rackName, Resource guaranteedUsed, + Resource capability, int numGuaranteedContainers, String healthReport, + long lastHealthReportTime) { + return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed, + capability, numGuaranteedContainers, healthReport, lastHealthReportTime, + null, null, null, null, 0); } @Private @Unstable public static NodeReport newInstance(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime, - Set<String> nodeLabels, Integer decommissioningTimeout, - NodeUpdateType nodeUpdateType) { + String httpAddress, String rackName, Resource guaranteedUsed, + Resource capability, int numGuaranteedContainers, String healthReport, + long lastHealthReportTime, Set<String> nodeLabels, + Integer decommissioningTimeout, NodeUpdateType nodeUpdateType, + Resource opportunisticUsed, int numOpportunisticContainers) { NodeReport nodeReport = Records.newRecord(NodeReport.class); nodeReport.setNodeId(nodeId); nodeReport.setNodeState(nodeState); nodeReport.setHttpAddress(httpAddress); nodeReport.setRackName(rackName); - nodeReport.setUsed(used); nodeReport.setCapability(capability); - nodeReport.setNumContainers(numContainers); + nodeReport.setGuaranteedResourceUsed(guaranteedUsed); + nodeReport.setNumGuaranteedContainers(numGuaranteedContainers); nodeReport.setHealthReport(healthReport); nodeReport.setLastHealthReportTime(lastHealthReportTime); nodeReport.setNodeLabels(nodeLabels); nodeReport.setDecommissioningTimeout(decommissioningTimeout); nodeReport.setNodeUpdateType(nodeUpdateType); + nodeReport.setOpportunisticResourceUsed(opportunisticUsed); + nodeReport.setNumOpportunisticContainers(numOpportunisticContainers); return nodeReport; } @@ -129,17 +136,43 @@ public abstract class NodeReport { public abstract void setRackName(String rackName); /** - * Get <em>used</em> <code>Resource</code> on the node. - * @return <em>used</em> <code>Resource</code> on the node + * Get <em>guaranteed</em> <code>Resource</code> used on the node. + * @return <em>guaranteed</em> <code>Resource</code> used on the node */ @Public @Stable + @Deprecated public abstract Resource getUsed(); @Private @Unstable + @Deprecated public abstract void setUsed(Resource used); - + + /** + * Get <em>guaranteed</em> <code>Resource</code> used on the node. + * @return <em>guaranteed</em> <code>Resource</code> used on the node + */ + @Public + @Unstable + public abstract Resource getGuaranteedResourceUsed(); + + @Private + @Unstable + public abstract void setGuaranteedResourceUsed(Resource guaranteed); + + /** + * Get <em>opportunistic</em> <code>Resource</code> used on the node. + * @return <em>opportunistic</em> <code>Resource</code> used on the node + */ + @Public + @Unstable + public abstract Resource getOpportunisticResourceUsed(); + + @Private + @Unstable + public abstract void setOpportunisticResourceUsed(Resource opportunistic); + /** * Get the <em>total</em> <code>Resource</code> on the node. * @return <em>total</em> <code>Resource</code> on the node @@ -153,19 +186,40 @@ public abstract class NodeReport { public abstract void setCapability(Resource capability); /** - * Get the <em>number of allocated containers</em> on the node. - * @return <em>number of allocated containers</em> on the node + * Get the <em>number of guaranteed containers</em> allocated on the node. + * @return <em>number of guaranteed containers</em> allocated on the node */ @Private @Unstable - public abstract int getNumContainers(); + public abstract int getNumGuaranteedContainers(); @Private @Unstable - public abstract void setNumContainers(int numContainers); - + public abstract void setNumGuaranteedContainers(int numContainers); - /** + /** + * Get the <em>number of opportunistic containers</em> allocated on the node. + * @return <em>number of opportunistic containers</em> allocated on the node + */ + @Private + @Unstable + public abstract int getNumOpportunisticContainers(); + + @Private + @Unstable + public abstract void setNumOpportunisticContainers(int numContainers); + + /** + * Get the <em>number of containers</em> allocated on the node. + * @return <em>number of containers</em> allocated on the node + */ + @Private + @Unstable + public int getNumTotalContainers() { + return getNumGuaranteedContainers() + getNumOpportunisticContainers(); + } + + /** * Get the <em>diagnostic health report</em> of the node. * @return <em>diagnostic health report</em> of the node */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5e200dc..4b3302b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -343,9 +343,9 @@ message NodeReportProto { optional NodeIdProto nodeId = 1; optional string httpAddress = 2; optional string rackName = 3; - optional ResourceProto used = 4; + optional ResourceProto guaranteedResourceUsed = 4; optional ResourceProto capability = 5; - optional int32 numContainers = 6; + optional int32 numGuaranteedContainers = 6; optional NodeStateProto node_state = 7; optional string health_report = 8; optional int64 last_health_report_time = 9; @@ -354,6 +354,9 @@ message NodeReportProto { optional ResourceUtilizationProto node_utilization = 12; optional uint32 decommissioning_timeout = 13; optional NodeUpdateTypeProto node_update_type = 14; + optional int32 numOpportunisticContainers = 15; + optional ResourceProto opportunisticResourceUsed = 16; + optional int32 numTotalContainers = 17; } message NodeIdToLabelsProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 61879d0..00a2ae7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -642,7 +642,7 @@ public class Client { + ", nodeId=" + node.getNodeId() + ", nodeAddress=" + node.getHttpAddress() + ", nodeRackName=" + node.getRackName() - + ", nodeNumContainers=" + node.getNumContainers()); + + ", nodeNumContainers=" + node.getNumGuaranteedContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java index 288a5d2..be466fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java @@ -43,8 +43,8 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ConverterUtils; @Private @Unstable @@ -185,7 +185,7 @@ public class NodeCLI extends YarnCLI { for (NodeReport nodeReport : nodesReport) { writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport .getNodeState(), nodeReport.getHttpAddress(), nodeReport - .getNumContainers()); + .getNumGuaranteedContainers()); } writer.flush(); } @@ -210,13 +210,18 @@ public class NodeCLI extends YarnCLI { for (NodeReport nodeReport : nodesReport) { writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport.getNodeState(), nodeReport.getHttpAddress(), - nodeReport.getNumContainers()); + nodeReport.getNumGuaranteedContainers()); writer.println("Detailed Node Information :"); writer.print("\tConfigured Resources : "); writer.println(nodeReport.getCapability()); - writer.print("\tAllocated Resources : "); - if (nodeReport.getUsed() != null) { - writer.print(nodeReport.getUsed()); + writer.print("\tAllocated Guaranteed Resources : "); + if (nodeReport.getGuaranteedResourceUsed() != null) { + writer.print(nodeReport.getGuaranteedResourceUsed()); + } + writer.println(); + writer.print("\tAllocated Opportunistic Resources : "); + if (nodeReport.getOpportunisticResourceUsed() != null) { + writer.print(nodeReport.getOpportunisticResourceUsed()); } writer.println(); @@ -287,16 +292,32 @@ public class NodeCLI extends YarnCLI { nodeReportStr.print("\tHealth-Report : "); nodeReportStr .println(nodeReport.getHealthReport()); - nodeReportStr.print("\tContainers : "); - nodeReportStr.println(nodeReport.getNumContainers()); - nodeReportStr.print("\tMemory-Used : "); - nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB" - : (nodeReport.getUsed().getMemorySize() + "MB")); + nodeReportStr.print("\tGuaranteed Containers : "); + nodeReportStr.println(nodeReport.getNumGuaranteedContainers()); + nodeReportStr.print("\tOpportunistic Containers : "); + nodeReportStr.println(nodeReport.getNumOpportunisticContainers()); + Resource guaranteedResourceUsed = + nodeReport.getGuaranteedResourceUsed(); + Resource opportunisticResourceUsed = + nodeReport.getOpportunisticResourceUsed(); + nodeReportStr.print("\tGuaranteed-Memory-Used : "); + nodeReportStr.println( + (guaranteedResourceUsed == null) ? "0MB" : + (guaranteedResourceUsed.getMemorySize() + "MB")); + nodeReportStr.print("\tOpportunistic-Memory-Used : "); + nodeReportStr.println( + (opportunisticResourceUsed == null) ? "0MB" : + (opportunisticResourceUsed.getMemorySize() + "MB")); nodeReportStr.print("\tMemory-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB"); - nodeReportStr.print("\tCPU-Used : "); - nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores" - : (nodeReport.getUsed().getVirtualCores() + " vcores")); + nodeReportStr.print("\tGuaranteed-CPU-Used : "); + nodeReportStr.println( + (guaranteedResourceUsed == null) ? "0 vcores" : + (guaranteedResourceUsed.getVirtualCores() + " vcores")); + nodeReportStr.print("\tOpportunistic-CPU-Used : "); + nodeReportStr.println( + (opportunisticResourceUsed == null) ? "0 vcores" : + (opportunisticResourceUsed.getVirtualCores() + " vcores")); nodeReportStr.print("\tCPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores"); nodeReportStr.print("\tNode-Labels : "); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1f6488d..fe00fa9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -1352,7 +1352,8 @@ public class TestYarnCLI { pw.println(" 0"); pw.println("Detailed Node Information :"); pw.println("\tConfigured Resources : <memory:0, vCores:0>"); - pw.println("\tAllocated Resources : <memory:0, vCores:0>"); + pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>"); + pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>"); pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0"); pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0"); pw.println("\tNode-Labels : "); @@ -1360,7 +1361,8 @@ public class TestYarnCLI { pw.println(" 0"); pw.println("Detailed Node Information :"); pw.println("\tConfigured Resources : <memory:0, vCores:0>"); - pw.println("\tAllocated Resources : <memory:0, vCores:0>"); + pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>"); + pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>"); pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0"); pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0"); pw.println("\tNode-Labels : "); @@ -1561,10 +1563,13 @@ public class TestYarnCLI { pw.println("\tLast-Health-Update : " + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz")); pw.println("\tHealth-Report : "); - pw.println("\tContainers : 0"); - pw.println("\tMemory-Used : 0MB"); + pw.println("\tGuaranteed Containers : 0"); + pw.println("\tOpportunistic Containers : 0"); + pw.println("\tGuaranteed-Memory-Used : 0MB"); + pw.println("\tOpportunistic-Memory-Used : 0MB"); pw.println("\tMemory-Capacity : 0MB"); - pw.println("\tCPU-Used : 0 vcores"); + pw.println("\tGuaranteed-CPU-Used : 0 vcores"); + pw.println("\tOpportunistic-CPU-Used : 0 vcores"); pw.println("\tCPU-Capacity : 0 vcores"); pw.println("\tNode-Labels : a,b,c,x,y,z"); pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0"); @@ -1597,10 +1602,13 @@ public class TestYarnCLI { pw.println("\tLast-Health-Update : " + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz")); pw.println("\tHealth-Report : "); - pw.println("\tContainers : 0"); - pw.println("\tMemory-Used : 0MB"); + pw.println("\tGuaranteed Containers : 0"); + pw.println("\tOpportunistic Containers : 0"); + pw.println("\tGuaranteed-Memory-Used : 0MB"); + pw.println("\tOpportunistic-Memory-Used : 0MB"); pw.println("\tMemory-Capacity : 0MB"); - pw.println("\tCPU-Used : 0 vcores"); + pw.println("\tGuaranteed-CPU-Used : 0 vcores"); + pw.println("\tOpportunistic-CPU-Used : 0 vcores"); pw.println("\tCPU-Capacity : 0 vcores"); pw.println("\tNode-Labels : "); pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0"); @@ -1633,10 +1641,13 @@ public class TestYarnCLI { pw.println("\tLast-Health-Update : " + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz")); pw.println("\tHealth-Report : "); - pw.println("\tContainers : 0"); - pw.println("\tMemory-Used : 0MB"); + pw.println("\tGuaranteed Containers : 0"); + pw.println("\tOpportunistic Containers : 0"); + pw.println("\tGuaranteed-Memory-Used : 0MB"); + pw.println("\tOpportunistic-Memory-Used : 0MB"); pw.println("\tMemory-Capacity : 0MB"); - pw.println("\tCPU-Used : 0 vcores"); + pw.println("\tGuaranteed-CPU-Used : 0 vcores"); + pw.println("\tOpportunistic-CPU-Used : 0 vcores"); pw.println("\tCPU-Capacity : 0 vcores"); pw.println("\tNode-Labels : a,b,c,x,y,z"); pw.println("\tResource Utilization by Node : "); @@ -2069,10 +2080,12 @@ public class TestYarnCLI { // ordered nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y"); } - NodeReport nodeReport = NodeReport.newInstance(NodeId - .newInstance("host" + i, 0), state, "host" + 1 + ":8888", - "rack1", Records.newRecord(Resource.class), Records - .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null); + NodeReport nodeReport = NodeReport.newInstance( + NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888", + "rack1", Records.newRecord(Resource.class), + Records.newRecord(Resource.class), 0, "", + 0, nodeLabels, null, null, + Records.newRecord(Resource.class), 0); if (!emptyResourceUtilization) { ResourceUtilization containersUtilization = ResourceUtilization .newInstance(1024, 2048, 4); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java index ced588d..0960eb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java @@ -45,7 +45,8 @@ public class NodeReportPBImpl extends NodeReport { private NodeReportProto.Builder builder = null; private boolean viaProto = false; private NodeId nodeId; - private Resource used; + private Resource guaranteedResourceUsed; + private Resource opportunisticResourceUsed; private Resource capability; private ResourceUtilization containersUtilization = null; private ResourceUtilization nodeUtilization = null; @@ -109,9 +110,17 @@ public class NodeReportPBImpl extends NodeReport { } @Override - public int getNumContainers() { + public int getNumGuaranteedContainers() { NodeReportProtoOrBuilder p = viaProto ? proto : builder; - return (p.hasNumContainers()) ? p.getNumContainers() : 0; + return (p.hasNumGuaranteedContainers()) ? + p.getNumGuaranteedContainers() : 0; + } + + @Override + public int getNumOpportunisticContainers() { + NodeReportProtoOrBuilder p = viaProto ? proto : builder; + return (p.hasNumOpportunisticContainers()) ? + p.getNumOpportunisticContainers() : 0; } @Override @@ -120,18 +129,25 @@ public class NodeReportPBImpl extends NodeReport { return (p.hasRackName()) ? p.getRackName() : null; } + @Deprecated @Override public Resource getUsed() { - if (this.used != null) { - return this.used; + return getGuaranteedResourceUsed(); + } + + @Override + public Resource getOpportunisticResourceUsed() { + if (this.opportunisticResourceUsed != null) { + return this.opportunisticResourceUsed; } NodeReportProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasUsed()) { + if (!p.hasOpportunisticResourceUsed()) { return null; } - this.used = convertFromProtoFormat(p.getUsed()); - return this.used; + this.opportunisticResourceUsed = + convertFromProtoFormat(p.getOpportunisticResourceUsed()); + return this.opportunisticResourceUsed; } @Override @@ -196,13 +212,24 @@ public class NodeReportPBImpl extends NodeReport { } @Override - public void setNumContainers(int numContainers) { + public void setNumGuaranteedContainers(int numContainers) { + maybeInitBuilder(); + if (numContainers == 0) { + builder.clearNumGuaranteedContainers(); + return; + } + builder.setNumGuaranteedContainers(numContainers); + } + + @Override + public void setNumOpportunisticContainers(int numContainers) { maybeInitBuilder(); if (numContainers == 0) { - builder.clearNumContainers(); + builder.clearNumOpportunisticContainers(); return; } - builder.setNumContainers(numContainers); + builder.setNumOpportunisticContainers(numContainers); + } @Override @@ -215,13 +242,43 @@ public class NodeReportPBImpl extends NodeReport { builder.setRackName(rackName); } + @Deprecated @Override public void setUsed(Resource used) { + setGuaranteedResourceUsed(used); + } + + @Override + public Resource getGuaranteedResourceUsed() { + if (this.guaranteedResourceUsed != null) { + return this.guaranteedResourceUsed; + } + + NodeReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasGuaranteedResourceUsed()) { + return null; + } + this.guaranteedResourceUsed = + convertFromProtoFormat(p.getGuaranteedResourceUsed()); + return this.guaranteedResourceUsed; + } + + @Override + public void setGuaranteedResourceUsed(Resource guaranteed) { + maybeInitBuilder(); + if (guaranteedResourceUsed == null) { + builder.clearGuaranteedResourceUsed(); + } + this.guaranteedResourceUsed = guaranteed; + } + + @Override + public void setOpportunisticResourceUsed(Resource opportunisticUsed) { maybeInitBuilder(); - if (used == null) { - builder.clearUsed(); + if (opportunisticUsed == null) { + builder.clearOpportunisticResourceUsed(); } - this.used = used; + this.opportunisticResourceUsed = opportunisticUsed; } public NodeReportProto getProto() { @@ -258,8 +315,13 @@ public class NodeReportPBImpl extends NodeReport { builder.getNodeId())) { builder.setNodeId(convertToProtoFormat(this.nodeId)); } - if (this.used != null) { - builder.setUsed(convertToProtoFormat(this.used)); + if (this.guaranteedResourceUsed != null) { + builder.setGuaranteedResourceUsed( + convertToProtoFormat(this.guaranteedResourceUsed)); + } + if (this.opportunisticResourceUsed != null) { + builder.setOpportunisticResourceUsed( + convertToProtoFormat(this.opportunisticResourceUsed)); } if (this.capability != null) { builder.setCapability(convertToProtoFormat(this.capability)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 4ba1bdf..c405506 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -185,26 +185,23 @@ public class BuilderUtils { } public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime) { - return newNodeReport(nodeId, nodeState, httpAddress, rackName, used, - capability, numContainers, healthReport, lastHealthReportTime, - null, null, null); + String httpAddress, String rackName, Resource guaranteedResourceUsed, + Resource capability, int numGuaranteedContainers, + Resource opportunisticResourceUsed, int numOpportunisticContainer, + String healthReport, long lastHealthReportTime, Set<String> nodeLabels, + Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) { + return newNodeReport(nodeId, nodeState, httpAddress, rackName, + guaranteedResourceUsed, capability, numGuaranteedContainers, + opportunisticResourceUsed, numOpportunisticContainer, healthReport, + lastHealthReportTime, nodeLabels, null, null, + decommissioningTimeout, nodeUpdateType); } public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime, - Set<String> nodeLabels, Integer decommissioningTimeout, - NodeUpdateType nodeUpdateType) { - return newNodeReport(nodeId, nodeState, httpAddress, rackName, used, - capability, numContainers, healthReport, lastHealthReportTime, - nodeLabels, null, null, decommissioningTimeout, nodeUpdateType); - } - - public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, - String httpAddress, String rackName, Resource used, Resource capability, - int numContainers, String healthReport, long lastHealthReportTime, + String httpAddress, String rackName, Resource guaranteedResourceUsed, + Resource capability, int numGuaranteedContainers, + Resource opportunisticResourceUsed, int numOpportunisticContainers, + String healthReport, long lastHealthReportTime, Set<String> nodeLabels, ResourceUtilization containersUtilization, ResourceUtilization nodeUtilization, Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) { @@ -213,9 +210,11 @@ public class BuilderUtils { nodeReport.setNodeState(nodeState); nodeReport.setHttpAddress(httpAddress); nodeReport.setRackName(rackName); - nodeReport.setUsed(used); + nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed); nodeReport.setCapability(capability); - nodeReport.setNumContainers(numContainers); + nodeReport.setNumGuaranteedContainers(numGuaranteedContainers); + nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed); + nodeReport.setNumOpportunisticContainers(numOpportunisticContainers); nodeReport.setHealthReport(healthReport); nodeReport.setLastHealthReportTime(lastHealthReportTime); nodeReport.setNodeLabels(nodeLabels); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 55a3f0b..7b560fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1027,17 +1027,26 @@ public class ClientRMService extends AbstractService implements private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); - Resource used = BuilderUtils.newResource(0, 0); - int numContainers = 0; + Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0); + int numGuaranteedContainers = 0; + Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0); + int numOpportunisticContainers = 0; if (schedulerNodeReport != null) { - used = schedulerNodeReport.getUsedResource(); - numContainers = schedulerNodeReport.getNumContainers(); + guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed(); + opportunisticResourceUsed = + schedulerNodeReport.getOpportunisticResourceUsed(); + numGuaranteedContainers = + schedulerNodeReport.getNumGuaranteedContainers(); + numOpportunisticContainers = + schedulerNodeReport.getNumOpportunisticContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), - rmNode.getHttpAddress(), rmNode.getRackName(), used, - rmNode.getTotalCapability(), numContainers, + rmNode.getHttpAddress(), rmNode.getRackName(), + guaranteedResourceUsed, + rmNode.getTotalCapability(), numGuaranteedContainers, + opportunisticResourceUsed, numOpportunisticContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(), rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index 71558a7..1fa1d00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -345,17 +345,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { RMNode rmNode = rmNodeEntry.getKey(); SchedulerNodeReport schedulerNodeReport = getScheduler().getNodeReport(rmNode.getNodeID()); - Resource used = BuilderUtils.newResource(0, 0); - int numContainers = 0; + Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0); + int numGuaranteedContainers = 0; + Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0); + int numOpportunisticContainers = 0; if (schedulerNodeReport != null) { - used = schedulerNodeReport.getUsedResource(); - numContainers = schedulerNodeReport.getNumContainers(); + opportunisticResourceUsed = + schedulerNodeReport.getOpportunisticResourceUsed(); + guaranteedResourceUsed = + schedulerNodeReport.getGuaranteedResourceUsed(); + numGuaranteedContainers = + schedulerNodeReport.getNumGuaranteedContainers(); + numOpportunisticContainers = + schedulerNodeReport.getNumOpportunisticContainers(); } NodeId nodeId = rmNode.getNodeID(); NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), - rmNode.getHttpAddress(), rmNode.getRackName(), used, - rmNode.getTotalCapability(), numContainers, + rmNode.getHttpAddress(), rmNode.getRackName(), + guaranteedResourceUsed, rmNode.getTotalCapability(), + numGuaranteedContainers, opportunisticResourceUsed, + numOpportunisticContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(), rmNodeEntry.getValue()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java index 1b7ddd3..849db5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java @@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans { List<InfoMap> nodesInfo = new ArrayList<InfoMap>(); for (final RMNode ni : nodes) { - SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID()); - InfoMap info = new InfoMap(); - info.put("HostName", ni.getHostName()); - info.put("Rack", ni.getRackName()); - info.put("State", ni.getState().toString()); - info.put("NodeId", ni.getNodeID()); - info.put("NodeHTTPAddress", ni.getHttpAddress()); - info.put("LastHealthUpdate", - ni.getLastHealthReportTime()); - info.put("HealthReport", - ni.getHealthReport()); - info.put("NodeManagerVersion", - ni.getNodeManagerVersion()); - if(report != null) { - info.put("NumContainers", report.getNumContainers()); - info.put("UsedMemoryMB", report.getUsedResource().getMemorySize()); - info.put("AvailableMemoryMB", - report.getAvailableResource().getMemorySize()); - } + SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID()); + InfoMap info = new InfoMap(); + info.put("HostName", ni.getHostName()); + info.put("Rack", ni.getRackName()); + info.put("State", ni.getState().toString()); + info.put("NodeId", ni.getNodeID()); + info.put("NodeHTTPAddress", ni.getHttpAddress()); + info.put("LastHealthUpdate", + ni.getLastHealthReportTime()); + info.put("HealthReport", + ni.getHealthReport()); + info.put("NodeManagerVersion", + ni.getNodeManagerVersion()); + if(report != null) { + info.put("NumContainers", report.getNumGuaranteedContainers()); + info.put("NumOpportunisticContainers", + report.getNumOpportunisticContainers()); + info.put("UsedMemoryMB", + report.getGuaranteedResourceUsed().getMemorySize()); + info.put("UsedOpportunisticMemoryMB", + report.getOpportunisticResourceUsed().getMemorySize()); + info.put("AvailableMemoryMB", + report.getAvailableGuaranteedResource().getMemorySize()); + } - nodesInfo.add(info); + nodesInfo.add(info); } return JSON.toString(nodesInfo); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 5a7c008..01d0452 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -139,7 +139,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> { /** Physical resources in the node. */ private volatile Resource physicalResource; - /* Container Queue Information for the node.. Used by Distributed Scheduler */ + /* Container Queue Information for the node..*/ private OpportunisticContainersStatus opportunisticContainersStatus; private final ContainerAllocationExpirer containerAllocationExpirer; http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java index ea30d78..25621a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java @@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource; @Private @Stable public class SchedulerNodeReport { - private final Resource used; - private final Resource avail; - private final int num; + private final Resource guaranteedResourceUsage; + private final Resource opportunisticResourceUsage; + private final Resource guaranteedResourceAvail; + private final int numOfGuaranteedContainers; + private final int numOfOpportunisticContainers; public SchedulerNodeReport(SchedulerNode node) { - this.used = node.getAllocatedResource(); - this.avail = node.getUnallocatedResource(); - this.num = node.getNumGuaranteedContainers(); + this.guaranteedResourceUsage = node.getAllocatedResource(); + this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated(); + this.guaranteedResourceAvail = node.getUnallocatedResource(); + this.numOfGuaranteedContainers = node.getNumGuaranteedContainers(); + this.numOfOpportunisticContainers = node.getNumOpportunisticContainers(); } /** - * @return the amount of resources currently used by the node. + * @return the amount of guaranteed resources currently used by the node. */ - public Resource getUsedResource() { - return used; + public Resource getGuaranteedResourceUsed() { + return guaranteedResourceUsage; } /** - * @return the amount of resources currently available on the node + * @return the amount of opportunistic resources currently used by the node. */ - public Resource getAvailableResource() { - return avail; + public Resource getOpportunisticResourceUsed() { + return opportunisticResourceUsage; } /** - * @return the number of containers currently running on this node. + * @return the amount of guaranteed resources currently available on the node */ - public int getNumContainers() { - return num; + public Resource getAvailableGuaranteedResource() { + return guaranteedResourceAvail; + } + + /** + * @return the number of guaranteed containers currently running on + * this node. + */ + public int getNumGuaranteedContainers() { + return numOfGuaranteedContainers; + } + + /** + * @return the number of opportunistic containers currently running on + * this node. + */ + public int getNumOpportunisticContainers() { + return numOfOpportunisticContainers; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java index 1752546..f5d8d07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java @@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo { for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); - this.usedNodeCapacity += report.getUsedResource().getMemorySize(); - this.availNodeCapacity += report.getAvailableResource().getMemorySize(); + this.usedNodeCapacity += + report.getGuaranteedResourceUsed().getMemorySize(); + this.availNodeCapacity += + report.getAvailableGuaranteedResource().getMemorySize(); this.totalNodeCapacity += ni.getTotalCapability().getMemorySize(); - this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); + this.numContainers += + fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 46a6e60..6f4a608 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -73,14 +73,17 @@ public class NodeInfo { this.usedMemoryMB = 0; this.availMemoryMB = 0; if (report != null) { - this.numContainers = report.getNumContainers(); - this.usedMemoryMB = report.getUsedResource().getMemorySize(); - this.availMemoryMB = report.getAvailableResource().getMemorySize(); - this.usedVirtualCores = report.getUsedResource().getVirtualCores(); + this.numContainers = report.getNumGuaranteedContainers(); + this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize(); + this.availMemoryMB = + report.getAvailableGuaranteedResource().getMemorySize(); + this.usedVirtualCores = + report.getGuaranteedResourceUsed().getVirtualCores(); this.availableVirtualCores = - report.getAvailableResource().getVirtualCores(); - this.usedResource = new ResourceInfo(report.getUsedResource()); - this.availableResource = new ResourceInfo(report.getAvailableResource()); + report.getAvailableGuaranteedResource().getVirtualCores(); + this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed()); + this.availableResource = + new ResourceInfo(report.getAvailableGuaranteedResource()); } this.id = id.toString(); this.rack = ni.getRackName(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index ee974e3..96613fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol { LOG.info("Checking resource usage for " + containerManagerAddress); Assert.assertEquals(available.getMemorySize(), resourceManager.getResourceScheduler().getNodeReport( - this.nodeId).getAvailableResource().getMemorySize()); + this.nodeId).getAvailableGuaranteedResource().getMemorySize()); Assert.assertEquals(used.getMemorySize(), resourceManager.getResourceScheduler().getNodeReport( - this.nodeId).getUsedResource().getMemorySize()); + this.nodeId).getGuaranteedResourceUsed().getMemorySize()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index cad0151..9c317e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -176,8 +176,10 @@ public class TestApplicationPriority { // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); - Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(15 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); @@ -189,8 +191,10 @@ public class TestApplicationPriority { // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // get scheduler CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); @@ -210,8 +214,10 @@ public class TestApplicationPriority { // check node report, 12 GB used and 4 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(12 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // send updated request for App1 am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>()); @@ -226,8 +232,10 @@ public class TestApplicationPriority { // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); rm.stop(); } @@ -261,8 +269,10 @@ public class TestApplicationPriority { // check node report, 8 GB used (1 AM and 7 containers) and 0 GB available SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); - Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(8 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Submit the second app App2 with priority 7 Priority appPriority2 = Priority.newInstance(7); @@ -288,8 +298,10 @@ public class TestApplicationPriority { // check node report, 1 GB used and 7 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(7 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); rm.stop(); } @@ -486,8 +498,10 @@ public class TestApplicationPriority { // check node report, 15 GB used (1 AM and 7 containers) and 1 GB available SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); - Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(15 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Submit the second app App2 with priority 8 (Higher than App1) Priority appPriority2 = Priority.newInstance(8); @@ -499,8 +513,10 @@ public class TestApplicationPriority { // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // get scheduler CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); @@ -522,8 +538,10 @@ public class TestApplicationPriority { // check node report, 12 GB used and 4 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(12 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers App1 am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>()); @@ -535,8 +553,10 @@ public class TestApplicationPriority { Assert.assertEquals(2, allocated2.size()); // check node report, 16 GB used and 0 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(16 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // kill 1 more counter = 0; @@ -552,8 +572,10 @@ public class TestApplicationPriority { // check node report, 14 GB used and 2 GB available report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(14 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Change the priority of App1 to 3 (lowest) Priority appPriority3 = Priority.newInstance(3); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index b7f7237..89a92fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -1249,8 +1249,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); // check node report, 2 GB used and 2 GB available - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); @@ -1271,8 +1273,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available - Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(0, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); @@ -1291,7 +1295,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { waitCount = 0; while (waitCount++ != 20) { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - if (report_nm1.getAvailableResource().getMemorySize() != 0) { + if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) { break; } LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried " @@ -1300,8 +1304,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { } // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(-2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.newContainerStatus( @@ -1317,9 +1323,11 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); // As container return 2 GB back, the available resource becomes 0 again. - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Verify no NPE is trigger in schedule after resource is updated. am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1); @@ -2819,8 +2827,10 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report - Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(1 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(9 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 02c0cc5..8023457 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation { SchedulerNodeReport reportNm1 = rm1.getResourceScheduler() .getNodeReport(nm1.getNodeId()); - Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize()); Assert.assertEquals(5 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(5 * GB, + reportNm1.getAvailableGuaranteedResource().getMemorySize()); SchedulerNodeReport reportNm2 = rm1.getResourceScheduler() .getNodeReport(nm2.getNodeId()); - Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize()); + Assert.assertEquals(0 * GB, + reportNm2.getGuaranteedResourceUsed().getMemorySize()); Assert.assertEquals(10 * GB, - reportNm2.getAvailableResource().getMemorySize()); + reportNm2.getAvailableGuaranteedResource().getMemorySize()); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a"); assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB()); @@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation { SchedulerNodeReport reportNm1 = rm1.getResourceScheduler() .getNodeReport(nm1.getNodeId()); - Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize()); + Assert.assertEquals(3 * GB, + reportNm1.getGuaranteedResourceUsed().getMemorySize()); Assert.assertEquals(7 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getAvailableGuaranteedResource().getMemorySize()); SchedulerNodeReport reportNm2 = rm1.getResourceScheduler() .getNodeReport(nm2.getNodeId()); - Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize()); + Assert.assertEquals(1 * GB, + reportNm2.getGuaranteedResourceUsed().getMemorySize()); Assert.assertEquals(9 * GB, - reportNm2.getAvailableResource().getMemorySize()); + reportNm2.getAvailableGuaranteedResource().getMemorySize()); LeafQueue leafQueue = (LeafQueue) cs.getQueue("a"); double delta = 0.0001; @@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation { SchedulerNodeReport reportNm1 = rm1.getResourceScheduler() .getNodeReport(nm1.getNodeId()); - Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize()); + Assert.assertEquals(6 * GB, + reportNm1.getGuaranteedResourceUsed().getMemorySize()); Assert.assertEquals(14 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getAvailableGuaranteedResource().getMemorySize()); // Try to launch app2 in a2, asked 2GB, should success // app2 -> a2 @@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation { reportNm1 = rm1.getResourceScheduler() .getNodeReport(nm1.getNodeId()); - Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize()); Assert.assertEquals(10 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(10 * GB, + reportNm1.getAvailableGuaranteedResource().getMemorySize()); // Kill all apps in queue a2 cs.killAllAppsInQueue("a2"); @@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation { checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(), cs.getApplicationAttempt(am3.getApplicationAttemptId())); - Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize()); Assert.assertEquals(10 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(10 * GB, + reportNm1.getAvailableGuaranteedResource().getMemorySize()); // Kill all apps in queue a1 cs.killAllAppsInQueue("a1"); @@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation { checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(), cs.getApplicationAttempt(am4.getApplicationAttemptId())); - Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize()); Assert.assertEquals(10 * GB, - reportNm1.getAvailableResource().getMemorySize()); + reportNm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(10 * GB, + reportNm1.getAvailableGuaranteedResource().getMemorySize()); rm1.close(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 11a28a1..b7bd4bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -702,7 +702,8 @@ public class TestFifoScheduler { am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); RMApp app2 = rm.submitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 @@ -712,7 +713,8 @@ public class TestFifoScheduler { am2.registerAppAttempt(); SchedulerNodeReport report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); - Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm2.getGuaranteedResourceUsed().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1); @@ -748,11 +750,15 @@ public class TestFifoScheduler { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); - Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize()); + Assert.assertEquals(0, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm2.getAvailableGuaranteedResource().getMemorySize()); - Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize()); + Assert.assertEquals(6 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm2.getGuaranteedResourceUsed().getMemorySize()); Container c1 = allocated1.get(0); Assert.assertEquals(GB, c1.getResource().getMemorySize()); @@ -770,7 +776,8 @@ public class TestFifoScheduler { Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses() .size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals( + 5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize()); rm.stop(); } @@ -827,7 +834,8 @@ public class TestFifoScheduler { int checkAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); - Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals( + checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize()); rm.stop(); } @@ -1118,8 +1126,10 @@ public class TestFifoScheduler { SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 2 GB used and 2 GB available - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); @@ -1140,8 +1150,10 @@ public class TestFifoScheduler { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available - Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(0, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); @@ -1160,7 +1172,7 @@ public class TestFifoScheduler { while (waitCount++ != 20) { report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); if (null != report_nm1 && - report_nm1.getAvailableResource().getMemorySize() != 0) { + report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) { break; } LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried " @@ -1170,8 +1182,10 @@ public class TestFifoScheduler { // Now, the used resource is still 4 GB, and available resource is minus // value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize()); - Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(4 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); + Assert.assertEquals(-2 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); // Check container can complete successfully in case of resource // over-commitment. @@ -1189,9 +1203,11 @@ public class TestFifoScheduler { Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses() .size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); - Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize()); + Assert.assertEquals(2 * GB, + report_nm1.getGuaranteedResourceUsed().getMemorySize()); // As container return 2 GB back, the available resource becomes 0 again. - Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize()); + Assert.assertEquals(0 * GB, + report_nm1.getAvailableGuaranteedResource().getMemorySize()); rm.stop(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/00aed24b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index 7ea7e81..a4ae69a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -817,15 +817,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase { if (report != null) { assertEquals("numContainers doesn't match: " + numContainers, - report.getNumContainers(), numContainers); + report.getNumGuaranteedContainers(), numContainers); assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report - .getUsedResource().getMemorySize(), usedMemoryMB); + .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB); assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report - .getAvailableResource().getMemorySize(), availMemoryMB); + .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB); assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report - .getUsedResource().getVirtualCores(), usedVirtualCores); - assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report - .getAvailableResource().getVirtualCores(), availVirtualCores); + .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores); + assertEquals("availVirtualCores doesn't match: " + availVirtualCores, + report.getAvailableGuaranteedResource().getVirtualCores(), + availVirtualCores); } if (opportunisticStatus != null) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
