Repository: hadoop Updated Branches: refs/heads/branch-2.7.1 ac0538aac -> 5a1d236ba
YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any allocations. (subru via curino) (cherry picked from commit 0fcb4a8cf2add3f112907ff4e833e2f04947b53e) (cherry picked from commit 6a71a0a230b337c8d62cd116a9742c54f00728e9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a1d236b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a1d236b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a1d236b Branch: refs/heads/branch-2.7.1 Commit: 5a1d236ba0f9292bff2a27e3bc8d0e73d7755898 Parents: ac0538a Author: carlo curino <Carlo Curino> Authored: Thu Jul 23 19:33:59 2015 -0700 Committer: carlo curino <Carlo Curino> Committed: Thu Jul 23 19:45:04 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/capacity/ReservationQueue.java | 4 --- .../capacity/TestReservationQueue.java | 26 +++++++++++--------- 3 files changed, 17 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a1d236b/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 450ec67..079ca5c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -184,6 +184,9 @@ Release 2.7.1 - 2015-07-06 YARN-3850. NM fails to read files from full disks which can lead to container logs being lost and other issues (Varun Saxena via jlowe) + YARN-3969. Allow jobs to be submitted to reservation that is active + but does not have any allocations. (subru via curino) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a1d236b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java index a8d17cf..19ce9f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java @@ -39,12 +39,9 @@ public class ReservationQueue extends LeafQueue { private PlanQueue parent; - private int maxSystemApps; - public ReservationQueue(CapacitySchedulerContext cs, String queueName, PlanQueue parent) throws IOException { super(cs, queueName, parent, null); - maxSystemApps = cs.getConfiguration().getMaximumSystemApplications(); // the following parameters are common to all reservation in the plan updateQuotas(parent.getUserLimitForReservation(), parent.getUserLimitFactor(), @@ -90,7 +87,6 @@ public class ReservationQueue extends LeafQueue { } setCapacity(capacity); setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity()); - setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity())); // note: we currently set maxCapacity to capacity // this might be revised later setMaxCapacity(entitlement.getMaxCapacity()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a1d236b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java index 4e6c73d..e23e93c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -39,6 +40,7 @@ public class TestReservationQueue { CapacitySchedulerConfiguration csConf; CapacitySchedulerContext csContext; + final static int DEF_MAX_APPS = 10000; final static int GB = 1024; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -66,7 +68,13 @@ public class TestReservationQueue { // create a queue PlanQueue pq = new PlanQueue(csContext, "root", null, null); reservationQueue = new ReservationQueue(csContext, "a", pq); + } + private void validateReservationQueue(double capacity) { + assertTrue(" actual capacity: " + reservationQueue.getCapacity(), + reservationQueue.getCapacity() - capacity < CSQueueUtils.EPSILON); + assertEquals(reservationQueue.maxApplications, DEF_MAX_APPS); + assertEquals(reservationQueue.maxApplicationsPerUser, DEF_MAX_APPS); } @Test @@ -74,25 +82,20 @@ public class TestReservationQueue { // verify that setting, adding, subtracting capacity works reservationQueue.setCapacity(1.0F); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); reservationQueue.setEntitlement(new QueueEntitlement(0.9f, 1f)); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 0.9 < CSQueueUtils.EPSILON); + validateReservationQueue(0.9); reservationQueue.setEntitlement(new QueueEntitlement(1f, 1f)); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); reservationQueue.setEntitlement(new QueueEntitlement(0f, 1f)); - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() < CSQueueUtils.EPSILON); + validateReservationQueue(0); try { reservationQueue.setEntitlement(new QueueEntitlement(1.1f, 1f)); fail(); } catch (SchedulerDynamicEditException iae) { // expected - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); } try { @@ -100,8 +103,7 @@ public class TestReservationQueue { fail(); } catch (SchedulerDynamicEditException iae) { // expected - assertTrue(" actual capacity: " + reservationQueue.getCapacity(), - reservationQueue.getCapacity() - 1 < CSQueueUtils.EPSILON); + validateReservationQueue(1); } }
