Repository: hadoop Updated Branches: refs/heads/branch-2 a6370dde9 -> 01b2e8b9c
YARN-4090. Make Collections.sort() more efficient by caching resource usage. (Contributed by Yufei Gu, Shilong Zhang and Xianyin Xin) (cherry picked from commit 1f4cdf10681b6903207a63fb5c306c9665ed9464) (cherry picked from commit 96106b8f5fe50e2d5c0c4df5dbddea4f89f278d9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01b2e8b9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01b2e8b9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01b2e8b9 Branch: refs/heads/branch-2 Commit: 01b2e8b9caaad50f25b7900428e7c3ee8cd31e0e Parents: a6370dd Author: Yufei Gu <yu...@apache.org> Authored: Fri Oct 20 01:32:20 2017 -0700 Committer: Yufei Gu <yu...@apache.org> Committed: Fri Oct 20 01:38:07 2017 -0700 ---------------------------------------------------------------------- .../scheduler/fair/FSAppAttempt.java | 19 ++++++++++ .../scheduler/fair/FSLeafQueue.java | 19 +--------- .../scheduler/fair/FSParentQueue.java | 14 ------- .../resourcemanager/scheduler/fair/FSQueue.java | 34 +++++++++++++++++ .../scheduler/fair/TestFairScheduler.java | 39 +++++++++++++++++++- 5 files changed, 93 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/01b2e8b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 06ba4e3..006acea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -174,6 +175,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt rmContainer.getNodeLabelExpression(), getUser(), 1, containerResource); this.attemptResourceUsage.decUsed(containerResource); + getQueue().decUsedResource(containerResource); // Clear resource utilization metrics cache. lastMemoryAggregateAllocationUpdateTime = -1; @@ -468,6 +470,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate( type, node, schedulerKey, container); this.attemptResourceUsage.incUsed(container.getResource()); + getQueue().incUsedResource(container.getResource()); // Update resource requests related to "request" and store in RMContainer ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList); @@ -651,6 +654,22 @@ public class FSAppAttempt extends SchedulerApplicationAttempt schedulerKey.getAllocationRequestId()); } + @Override + public synchronized void recoverContainer(SchedulerNode node, + RMContainer rmContainer) { + try { + writeLock.lock(); + + super.recoverContainer(node, rmContainer); + + if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) { + getQueue().incUsedResource(rmContainer.getContainer().getResource()); + } + } finally { + writeLock.unlock(); + } + } + /** * Reserve a spot for {@code container} on this {@code node}. If * the container is {@code alreadyReserved} on the node, simply http://git-wip-us.apache.org/repos/asf/hadoop/blob/01b2e8b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 3c51829..b3f5035 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -90,6 +90,7 @@ public class FSLeafQueue extends FSQueue { } else { nonRunnableApps.add(app); } + incUsedResource(app.getResourceUsage()); } finally { writeLock.unlock(); } @@ -124,6 +125,7 @@ public class FSLeafQueue extends FSQueue { getMetrics().setAMResourceUsage(amResourceUsage); } + decUsedResource(app.getResourceUsage()); return runnable; } @@ -293,23 +295,6 @@ public class FSLeafQueue extends FSQueue { return demand; } - @Override - public Resource getResourceUsage() { - Resource usage = Resources.createResource(0); - readLock.lock(); - try { - for (FSAppAttempt app : runnableApps) { - Resources.addTo(usage, app.getResourceUsage()); - } - for (FSAppAttempt app : nonRunnableApps) { - Resources.addTo(usage, app.getResourceUsage()); - } - } finally { - readLock.unlock(); - } - return usage; - } - Resource getAmResourceUsage() { return amResourceUsage; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/01b2e8b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index e42c9f5..a8e53fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -119,20 +119,6 @@ public class FSParentQueue extends FSQueue { } @Override - public Resource getResourceUsage() { - Resource usage = Resources.createResource(0); - readLock.lock(); - try { - for (FSQueue child : childQueues) { - Resources.addTo(usage, child.getResourceUsage()); - } - } finally { - readLock.unlock(); - } - return usage; - } - - @Override public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources http://git-wip-us.apache.org/repos/asf/hadoop/blob/01b2e8b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 9d07c79..9be56a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -58,6 +58,7 @@ public abstract class FSQueue implements Queue, Schedulable { private Resource fairShare = Resources.createResource(0, 0); private Resource steadyFairShare = Resources.createResource(0, 0); private Resource reservedResource = Resources.createResource(0, 0); + private final Resource resourceUsage = Resource.newInstance(0, 0); private final String name; protected final FairScheduler scheduler; private final YarnAuthorizationProvider authorizer; @@ -480,6 +481,39 @@ public abstract class FSQueue implements Queue, Schedulable { } @Override + public Resource getResourceUsage() { + return resourceUsage; + } + + /** + * Increase resource usage for this queue and all parent queues. + * + * @param res the resource to increase + */ + protected void incUsedResource(Resource res) { + synchronized (resourceUsage) { + Resources.addTo(resourceUsage, res); + if (parent != null) { + parent.incUsedResource(res); + } + } + } + + /** + * Decrease resource usage for this queue and all parent queues. + * + * @param res the resource to decrease + */ + protected void decUsedResource(Resource res) { + synchronized (resourceUsage) { + Resources.subtractFrom(resourceUsage, res); + if (parent != null) { + parent.decUsedResource(res); + } + } + } + + @Override public Priority getDefaultApplicationPriority() { // TODO add implementation for FSParentQueue return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/01b2e8b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 28bfa0b..f1092bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4584,6 +4584,43 @@ public class TestFairScheduler extends FairSchedulerTestBase { TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler( scheduler.getSchedulerApplications(), scheduler, "default"); } + + @Test + public void testResourceUsageByMoveApp() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node1 = MockNodes.newNodeInfo( + 1, Resources.createResource(1 * GB, 4), 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + ApplicationAttemptId appAttId = + createSchedulingRequest(1 * GB, 2, "parent1.queue1", "user1", 2); + scheduler.update(); + + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(updateEvent); + + QueueManager queueMgr = scheduler.getQueueManager(); + FSQueue parent1 = queueMgr.getParentQueue("parent1", true); + FSQueue parent2 = queueMgr.getParentQueue("parent2", true); + FSQueue queue2 = queueMgr.getLeafQueue("parent2.queue2", true); + FSQueue queue1 = queueMgr.getLeafQueue("parent1.queue1", true); + + Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 0); + Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 0); + Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 1 * GB); + Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 1 * GB); + + scheduler.moveApplication(appAttId.getApplicationId(), "parent2.queue2"); + + Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 1 * GB); + Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 1 * GB); + Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 0); + Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 0); + } @Test (expected = YarnException.class) public void testMoveWouldViolateMaxAppsConstraints() throws Exception { @@ -4597,7 +4634,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); - + scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
