Repository: hadoop Updated Branches: refs/heads/trunk f2ea555ac -> b3056c266
YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per label when a host runs multiple node-managers. Contributed by Wangda Tan. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3056c26 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3056c26 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3056c26 Branch: refs/heads/trunk Commit: b3056c266a628a65cf7ceb61b55ab3bd0a09baf2 Parents: f2ea555 Author: Vinod Kumar Vavilapalli <vino...@apache.org> Authored: Wed Oct 15 18:47:26 2014 -0700 Committer: Vinod Kumar Vavilapalli <vino...@apache.org> Committed: Wed Oct 15 18:47:26 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 4 +++ .../nodelabels/CommonNodeLabelsManager.java | 1 + .../nodelabels/TestRMNodeLabelsManager.java | 29 ++++++++++++++++++++ 3 files changed, 34 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3056c26/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 97fea49..b0e307c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -583,6 +583,10 @@ Release 2.6.0 - UNRELEASED YARN-2628. Capacity scheduler with DominantResourceCalculator carries out reservation even though slots are free. (Varun Vasudev via jianhe) + YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource + tracking per label when a host runs multiple node-managers. (Wangda Tan via + vinodkv) + BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS YARN-1707. Introduce APIs to add/remove/resize queues in the http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3056c26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 89fbf09..511b5ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -129,6 +129,7 @@ public class CommonNodeLabelsManager extends AbstractService { if (labels != null) { c.labels = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); + c.labels.addAll(labels); } else { c.labels = null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3056c26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 26284e2..81eead9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -25,6 +26,7 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -364,4 +366,31 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { Assert.assertEquals(clusterResource, mgr.getQueueResource("Q5", q5Label, clusterResource)); } + + @Test(timeout=5000) + public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException { + // active two NM to n1, one large and one small + mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE); + + // check resource of no label, it should be small * 4 + Assert.assertEquals( + mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), + Resources.multiply(SMALL_RESOURCE, 4)); + + // change two of these nodes to p1, check resource of no_label and P1 + mgr.addToCluserNodeLabels(toSet("p1")); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), + toNodeId("n1:2"), toSet("p1"))); + + // check resource + Assert.assertEquals( + mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), + Resources.multiply(SMALL_RESOURCE, 2)); + Assert.assertEquals( + mgr.getResourceByLabel("p1", null), + Resources.multiply(SMALL_RESOURCE, 2)); + } }
