Repository: hadoop Updated Branches: refs/heads/trunk c5ee7fded -> a96afae12
YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a96afae1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a96afae1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a96afae1 Branch: refs/heads/trunk Commit: a96afae125ba02fb4480542d3fb0891623ee4c37 Parents: c5ee7fde Author: Karthik Kambatla <ka...@cloudera.com> Authored: Thu Mar 9 23:11:54 2017 -0800 Committer: Karthik Kambatla <ka...@cloudera.com> Committed: Thu Mar 9 23:11:54 2017 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/util/resource/Resources.java | 7 +++++ .../yarn/util/resource/TestResources.java | 24 ++++++++++++++++- .../scheduler/fair/FSLeafQueue.java | 3 ++- .../scheduler/fair/TestFairScheduler.java | 28 ++++++++++---------- 4 files changed, 46 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 57b3a46..7020300 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -242,6 +242,13 @@ public class Resources { out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } + + public static Resource multiplyAndRoundUp(Resource lhs, double by) { + Resource out = clone(lhs); + out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by)); + out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by)); + return out; + } public static Resource normalize( ResourceCalculator calculator, Resource lhs, Resource min, http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 057214b..f8570a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.yarn.api.records.Resource; import org.junit.Test; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestResources { @@ -46,5 +48,25 @@ public class TestResources { assertTrue(Resources.none().compareTo( createResource(0, 1)) < 0); } - + + @Test + public void testMultipleRoundUp() { + final double by = 0.5; + final String memoryErrorMsg = "Invalid memory size."; + final String vcoreErrorMsg = "Invalid virtual core number."; + Resource resource = Resources.createResource(1, 1); + Resource result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(2, 2); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(0, 0); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 0); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index aad2916..5be4b36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -515,7 +515,8 @@ public class FSLeafQueue extends FSQueue { getMaxShare().getVirtualCores())); } - return Resources.multiply(maxResource, maxAMShare); + // Round up to allow AM to run when there is only one vcore on the cluster + return Resources.multiplyAndRoundUp(maxResource, maxAMShare); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 028eea6..cdf0d81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -659,15 +659,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // case, we use maxShare, since it is smaller than available resource. assertEquals("QueueFSZeroWithMax's fair share should be zero", 0, queueFSZeroWithMax.getFairShare().getMemorySize()); + Resource expectedAMResource = Resources.multiplyAndRoundUp( + queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getMemorySize() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getMemorySize(), queueFSZeroWithMax.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getVirtualCores(), queueFSZeroWithMax.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -689,17 +687,19 @@ public class TestFairScheduler extends FairSchedulerTestBase { // the min(maxShare, available resource) to compute maxAMShare, in this // case, we use available resource since it is smaller than the // default maxShare. + expectedAMResource = Resources.multiplyAndRoundUp( + Resources.createResource(memCapacity - amResource.getMemorySize(), + cpuCapacity - amResource.getVirtualCores()), + queueFSZeroWithAVL.getMaxAMShare()); assertEquals("QueueFSZeroWithAVL's fair share should be zero", 0, queueFSZeroWithAVL.getFairShare().getMemorySize()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((memCapacity - amResource.getMemorySize()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getMemorySize(), queueFSZeroWithAVL.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((cpuCapacity - amResource.getVirtualCores()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getVirtualCores(), queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -721,13 +721,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // fair share to compute maxAMShare assertNotEquals("QueueFSNonZero's fair share shouldn't be zero", 0, queueFSNonZero.getFairShare().getMemorySize()); + expectedAMResource = Resources.multiplyAndRoundUp( + queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(memCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getMemorySize(), queueFSNonZero.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(cpuCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getVirtualCores(), queueFSNonZero.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSNonZero's AM resource usage should be the same to " + "AM resource request", --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
