YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources. (Bibin A Chundatt via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ec26de Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ec26de Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ec26de Branch: refs/heads/YARN-1197 Commit: 76ec26de8099dc48ce3812c595b7ab857a600442 Parents: 1b3bceb Author: Wangda Tan <wan...@apache.org> Authored: Wed Jul 22 11:54:02 2015 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Wed Jul 22 11:54:02 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/SchedulerApplicationAttempt.java | 2 +- .../scheduler/capacity/LeafQueue.java | 8 ++- .../TestCapacitySchedulerNodeLabelUpdate.java | 64 ++++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5100cdf..f751862 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -653,6 +653,9 @@ Release 2.8.0 - UNRELEASED YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue is more than 2 level. (Ajith S via wangda) + YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo + should based on total-used-resources. (Bibin A Chundatt via wangda) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index cf543bd..317e61c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -598,7 +598,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage(); Resource usedResourceClone = - Resources.clone(attemptResourceUsage.getUsed()); + Resources.clone(attemptResourceUsage.getAllUsed()); Resource reservedResourceClone = Resources.clone(attemptResourceUsage.getReserved()); return ApplicationResourceUsageReport.newInstance(liveContainers.size(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 0ce4d68..5c283f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -439,7 +439,7 @@ public class LeafQueue extends AbstractCSQueue { for (Map.Entry<String, User> entry : users.entrySet()) { User user = entry.getValue(); usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user - .getUsed()), user.getActiveApplications(), user + .getAllUsed()), user.getActiveApplications(), user .getPendingApplications(), Resources.clone(user .getConsumedAMResources()), Resources.clone(user .getUserResourceLimit()))); @@ -1894,7 +1894,11 @@ public class LeafQueue extends AbstractCSQueue { public Resource getUsed() { return userResourceUsage.getUsed(); } - + + public Resource getAllUsed() { + return userResourceUsage.getAllUsed(); + } + public Resource getUsed(String label) { return userResourceUsage.getUsed(label); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index e60e496..0a701d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -95,6 +97,68 @@ public class TestCapacitySchedulerNodeLabelUpdate { .getMemory()); } + @Test(timeout = 60000) + public void testResourceUsage() throws Exception { + // set node -> label + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", + "z")); + + // set mapping: + // h1 -> x + // h2 -> y + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); + mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y"))); + + // inject node label manager + MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return mgr; + } + }; + rm.getRMContext().setNodeLabelManager(mgr); + rm.start(); + MockNM nm1 = rm.registerNode("h1:1234", 2048); + MockNM nm2 = rm.registerNode("h2:1234", 2048); + MockNM nm3 = rm.registerNode("h3:1234", 2048); + + ContainerId containerId; + // launch an app to queue a1 (label = x), and check all container will + // be allocated in h1 + RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); + ApplicationResourceUsageReport appResourceUsageReport = + rm.getResourceScheduler().getAppResourceUsageReport( + am1.getApplicationAttemptId()); + Assert.assertEquals(1024, appResourceUsageReport.getUsedResources() + .getMemory()); + Assert.assertEquals(1, appResourceUsageReport.getUsedResources() + .getVirtualCores()); + // request a container. + am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x"); + containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); + rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000); + appResourceUsageReport = + rm.getResourceScheduler().getAppResourceUsageReport( + am1.getApplicationAttemptId()); + Assert.assertEquals(2048, appResourceUsageReport.getUsedResources() + .getMemory()); + Assert.assertEquals(2, appResourceUsageReport.getUsedResources() + .getVirtualCores()); + LeafQueue queue = + (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler()) + .getQueue("a"); + ArrayList<UserInfo> users = queue.getUsers(); + for (UserInfo userInfo : users) { + if (userInfo.getUsername().equals("user")) { + ResourceInfo resourcesUsed = userInfo.getResourcesUsed(); + Assert.assertEquals(2048, resourcesUsed.getMemory()); + Assert.assertEquals(2, resourcesUsed.getvCores()); + } + } + rm.stop(); + } + @Test (timeout = 60000) public void testNodeUpdate() throws Exception { // set node -> label
