Repository: hadoop Updated Branches: refs/heads/trunk c8bc7a847 -> 86ac1ad9f
YARN-5453. FairScheduler#update may skip update demand resource of child queue/app if current demand reached maxResource. (sandflee via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86ac1ad9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86ac1ad9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86ac1ad9 Branch: refs/heads/trunk Commit: 86ac1ad9fd65c7dd12278372b369de38dc4616db Parents: c8bc7a8 Author: Karthik Kambatla <ka...@apache.org> Authored: Wed Nov 9 23:44:02 2016 -0800 Committer: Karthik Kambatla <ka...@apache.org> Committed: Wed Nov 9 23:44:02 2016 -0800 ---------------------------------------------------------------------- .../scheduler/fair/FSLeafQueue.java | 15 +++---- .../scheduler/fair/FSParentQueue.java | 6 +-- .../scheduler/fair/TestFairScheduler.java | 41 ++++++++++++++++++++ 3 files changed, 48 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 9d5bbe5..c393759 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -268,20 +268,16 @@ public class FSLeafQueue extends FSQueue { readLock.lock(); try { for (FSAppAttempt sched : runnableApps) { - if (Resources.equals(demand, maxShare)) { - break; - } - updateDemandForApp(sched, maxShare); + updateDemandForApp(sched); } for (FSAppAttempt sched : nonRunnableApps) { - if (Resources.equals(demand, maxShare)) { - break; - } - updateDemandForApp(sched, maxShare); + updateDemandForApp(sched); } } finally { readLock.unlock(); } + // Cap demand to maxShare to limit allocation to maxShare + demand = Resources.componentwiseMin(demand, maxShare); if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxShare); @@ -290,7 +286,7 @@ public class FSLeafQueue extends FSQueue { } } - private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) { + private void updateDemandForApp(FSAppAttempt sched) { sched.updateDemand(); Resource toAdd = sched.getDemand(); if (LOG.isDebugEnabled()) { @@ -299,7 +295,6 @@ public class FSLeafQueue extends FSQueue { + demand); } demand = Resources.add(demand, toAdd); - demand = Resources.componentwiseMin(demand, maxRes); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index d05390b..53ac8c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -159,16 +159,14 @@ public class FSParentQueue extends FSQueue { childQueue.updateDemand(); Resource toAdd = childQueue.getDemand(); demand = Resources.add(demand, toAdd); - demand = Resources.componentwiseMin(demand, maxShare); if (LOG.isDebugEnabled()) { LOG.debug("Counting resource from " + childQueue.getName() + " " + toAdd + "; Total resource demand for " + getName() + " now " + demand); } - if (Resources.equals(demand, maxShare)) { - break; - } } + // Cap demand to maxShare to limit allocation to maxShare + demand = Resources.componentwiseMin(demand, maxShare); } finally { writeLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/86ac1ad9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index f17726c..21d22c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4813,4 +4813,45 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(0, metrics.getReservedMB()); assertEquals(0, metrics.getReservedVirtualCores()); } + + + @Test + public void testUpdateDemand() throws IOException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + Resource maxResource = Resources.createResource(1024 * 8); + + FSAppAttempt app1 = mock(FSAppAttempt.class); + Mockito.when(app1.getDemand()).thenReturn(maxResource); + FSAppAttempt app2 = mock(FSAppAttempt.class); + Mockito.when(app2.getDemand()).thenReturn(maxResource); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); + + FSLeafQueue aQueue = + new FSLeafQueue("root.queue1.a", scheduler, queue1); + aQueue.setMaxShare(maxResource); + aQueue.addAppSchedulable(app1); + + FSLeafQueue bQueue = + new FSLeafQueue("root.queue1.b", scheduler, queue1); + bQueue.setMaxShare(maxResource); + bQueue.addAppSchedulable(app2); + + queue1.setMaxShare(maxResource); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + queue1.updateDemand(); + + assertTrue("Demand is greater than max allowed ", + Resources.equals(queue1.getDemand(), maxResource)); + assertTrue("Demand of child queue not updated ", + Resources.equals(aQueue.getDemand(), maxResource) && + Resources.equals(bQueue.getDemand(), maxResource)); + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
