Repository: hadoop Updated Branches: refs/heads/branch-2 3dec58dd7 -> 0728a86b3
YARN-3686. CapacityScheduler should trim default_node_label_expression. (Sunil G via wangda) (cherry picked from commit cdbd66be111c93c85a409d47284e588c453ecae9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0728a86b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0728a86b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0728a86b Branch: refs/heads/branch-2 Commit: 0728a86b3affd65a7f60edc2a461a98c0458415b Parents: 3dec58d Author: Wangda Tan <wan...@apache.org> Authored: Tue May 26 15:58:47 2015 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Tue May 26 16:00:17 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../api/records/impl/pb/QueueInfoPBImpl.java | 2 +- .../records/impl/pb/ResourceRequestPBImpl.java | 2 +- .../CapacitySchedulerConfiguration.java | 7 ++- .../scheduler/TestSchedulerUtils.java | 47 ++++++++++++++++++++ .../capacity/TestCapacityScheduler.java | 21 +++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0728a86b/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0f4a4d7..9d78a6c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -526,6 +526,9 @@ Release 2.7.1 - UNRELEASED YARN-2238. filtering on UI sticks even if I move away from the page. (Jian He via xgong) + YARN-3686. CapacityScheduler should trim default_node_label_expression. + (Sunil G via wangda) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0728a86b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java index 4f12da2..90aecf4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java @@ -358,7 +358,7 @@ public class QueueInfoPBImpl extends QueueInfo { public String getDefaultNodeLabelExpression() { QueueInfoProtoOrBuilder p = viaProto ? proto : builder; return (p.hasDefaultNodeLabelExpression()) ? p - .getDefaultNodeLabelExpression() : null; + .getDefaultNodeLabelExpression().trim() : null; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/0728a86b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index 27fb5ae..791e6a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -193,7 +193,7 @@ public class ResourceRequestPBImpl extends ResourceRequest { if (!p.hasNodeLabelExpression()) { return null; } - return (p.getNodeLabelExpression()); + return (p.getNodeLabelExpression().trim()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/0728a86b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 031ca5d..563643c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -524,7 +524,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur } public String getDefaultNodeLabelExpression(String queue) { - return get(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION); + String defaultLabelExpression = get(getQueuePrefix(queue) + + DEFAULT_NODE_LABEL_EXPRESSION); + if (defaultLabelExpression == null) { + return null; + } + return defaultLabelExpression.trim(); } public void setDefaultNodeLabelExpression(String queue, String exp) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0728a86b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index 4befe0c..ab053d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -694,6 +694,53 @@ public class TestSchedulerUtils { ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x"); Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus()); } + + @Test (timeout = 30000) + public void testNormalizeNodeLabelExpression() + throws IOException { + // mock queue and scheduler + YarnScheduler scheduler = mock(YarnScheduler.class); + Set<String> queueAccessibleNodeLabels = Sets.newHashSet(); + QueueInfo queueInfo = mock(QueueInfo.class); + when(queueInfo.getQueueName()).thenReturn("queue"); + when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels); + when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x "); + when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())) + .thenReturn(queueInfo); + + Resource maxResource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + + // queue has labels, success cases + try { + // set queue accessible node labels to [x, y] + queueAccessibleNodeLabels.clear(); + queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y")); + rmContext.getNodeLabelManager().addToCluserNodeLabels( + ImmutableSet.of(NodeLabel.newInstance("x"), + NodeLabel.newInstance("y"))); + Resource resource = Resources.createResource( + 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + Assert.assertTrue(resReq.getNodeLabelExpression().equals("x")); + + resReq.setNodeLabelExpression(" y "); + SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", + scheduler, rmContext); + Assert.assertTrue(resReq.getNodeLabelExpression().equals("y")); + } catch (InvalidResourceRequestException e) { + e.printStackTrace(); + fail("Should be valid when request labels is a subset of queue labels"); + } finally { + rmContext.getNodeLabelManager().removeFromClusterNodeLabels( + Arrays.asList("x", "y")); + } + } public static SchedulerApplication<SchedulerApplicationAttempt> verifyAppAddedAndRemovedFromScheduler( http://git-wip-us.apache.org/repos/asf/hadoop/blob/0728a86b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 7b665e4..0361424 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2930,6 +2930,27 @@ public class TestCapacityScheduler { rm.stop(); } + + @Test + public void testDefaultNodeLabelExpressionQueueConfig() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + conf.setDefaultNodeLabelExpression("root.a", " x"); + conf.setDefaultNodeLabelExpression("root.b", " y "); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + cs.init(conf); + cs.start(); + + QueueInfo queueInfoA = cs.getQueueInfo("a", true, false); + Assert.assertEquals(queueInfoA.getQueueName(), "a"); + Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x"); + + QueueInfo queueInfoB = cs.getQueueInfo("b", true, false); + Assert.assertEquals(queueInfoB.getQueueName(), "b"); + Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y"); + } private void setMaxAllocMb(Configuration conf, int maxAllocMb) { conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
