Repository: hadoop Updated Branches: refs/heads/branch-2.7 23994e824 -> fb57a1aac
YARN-3686. CapacityScheduler should trim default_node_label_expression. (Sunil G via wangda) (cherry picked from commit cdbd66be111c93c85a409d47284e588c453ecae9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb57a1aa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb57a1aa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb57a1aa Branch: refs/heads/branch-2.7 Commit: fb57a1aac81139a4a1032dd91da83e14fdeea43d Parents: 23994e8 Author: Wangda Tan <[email protected]> Authored: Tue May 26 15:58:47 2015 -0700 Committer: Wangda Tan <[email protected]> Committed: Tue May 26 16:29:27 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../api/records/impl/pb/QueueInfoPBImpl.java | 2 +- .../records/impl/pb/ResourceRequestPBImpl.java | 2 +- .../CapacitySchedulerConfiguration.java | 7 ++- .../scheduler/TestSchedulerUtils.java | 46 ++++++++++++++++++++ .../capacity/TestCapacityScheduler.java | 21 +++++++++ 6 files changed, 78 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57a1aa/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2b3b11d..f017496 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -128,6 +128,9 @@ Release 2.7.1 - UNRELEASED YARN-2238. filtering on UI sticks even if I move away from the page. (Jian He via xgong) + YARN-3686. CapacityScheduler should trim default_node_label_expression. + (Sunil G via wangda) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57a1aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java index 4b83500..14a31be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java @@ -356,7 +356,7 @@ public class QueueInfoPBImpl extends QueueInfo { public String getDefaultNodeLabelExpression() { QueueInfoProtoOrBuilder p = viaProto ? proto : builder; return (p.hasDefaultNodeLabelExpression()) ? p - .getDefaultNodeLabelExpression() : null; + .getDefaultNodeLabelExpression().trim() : null; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57a1aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index 27fb5ae..791e6a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -193,7 +193,7 @@ public class ResourceRequestPBImpl extends ResourceRequest { if (!p.hasNodeLabelExpression()) { return null; } - return (p.getNodeLabelExpression()); + return (p.getNodeLabelExpression().trim()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57a1aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 1062be0..58a902e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -473,7 +473,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur } public String getDefaultNodeLabelExpression(String queue) { - return get(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION); + String defaultLabelExpression = get(getQueuePrefix(queue) + + DEFAULT_NODE_LABEL_EXPRESSION); + if (defaultLabelExpression == null) { + return null; + } + return defaultLabelExpression.trim(); } public void setDefaultNodeLabelExpression(String queue, String exp) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57a1aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 4410573..479f981 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -689,6 +689,52 @@ public class TestSchedulerUtils { ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x"); Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus()); } + + @Test (timeout = 30000) + public void testNormalizeNodeLabelExpression() + throws IOException { + // mock queue and scheduler + YarnScheduler scheduler = mock(YarnScheduler.class); + Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); + QueueInfo queueInfo = mock(QueueInfo.class); + when(queueInfo.getQueueName()).thenReturn("queue"); + when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels); + when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x "); + when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) + .thenReturn(queueInfo); + + Resource maxResource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + + // queue has labels, success cases + try { + // set queue accessible node labels to [x, y] + queueAccessibleNodeLabels.clear(); + queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of("x", "y")); + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + Assert.assertTrue(resReq.getNodeLabelExpression().equals("x")); + + resReq.setNodeLabelExpression(" y "); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + Assert.assertTrue(resReq.getNodeLabelExpression().equals("y")); + } catch (InvalidResourceRequestException e) { + e.printStackTrace(); + fail("Should be valid when request labels is a subset of queue labels"); + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); + } + } public static SchedulerApplication<SchedulerApplicationAttempt> verifyAppAddedAndRemovedFromScheduler( http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb57a1aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 0735e45..d27cc47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2618,6 +2618,27 @@ public class TestCapacityScheduler { rm.stop(); } + + @Test + public void testDefaultNodeLabelExpressionQueueConfig() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setDefaultNodeLabelExpression("root.a", " x"); + conf.setDefaultNodeLabelExpression("root.b", " y "); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + + QueueInfo queueInfoA = cs.getQueueInfo("a", true, false); + Assert.assertEquals(queueInfoA.getQueueName(), "a"); + Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x"); + + QueueInfo queueInfoB = cs.getQueueInfo("b", true, false); + Assert.assertEquals(queueInfoB.getQueueName(), "b"); + Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y"); + } private void setMaxAllocMb(Configuration conf, int maxAllocMb) { conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
