Repository: hadoop Updated Branches: refs/heads/trunk c46d4bafe -> ec0a852a3
YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager to get NodeLabel object. (Sunil G via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec0a852a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec0a852a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec0a852a Branch: refs/heads/trunk Commit: ec0a852a37d5c91a62d3d0ff3ddbd9d58235b312 Parents: c46d4ba Author: Wangda Tan <[email protected]> Authored: Wed May 27 12:51:20 2015 -0700 Committer: Wangda Tan <[email protected]> Committed: Wed May 27 12:51:20 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodelabels/CommonNodeLabelsManager.java | 17 ++++--- .../nodelabels/RMNodeLabelsManager.java | 1 + .../resourcemanager/webapp/RMWebServices.java | 27 +++++------ .../webapp/TestRMWebServicesNodeLabels.java | 51 +++++++++++++------- 5 files changed, 62 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7d122ff..4c75a39 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -265,6 +265,9 @@ Release 2.8.0 - UNRELEASED YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams. (Lars Francke via junping_du) + YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager + to get NodeLabel object. (Sunil G via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 2d57ad6..f741f68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -993,13 +993,18 @@ public class CommonNodeLabelsManager extends AbstractService { } } - private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) { - Set<String> labels = getLabelsByNode(nodeId, nodeCollections); - if (labels.isEmpty()) { - return EMPTY_NODELABEL_SET; + public Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) { + try { + readLock.lock(); + Set<String> labels = getLabelsByNode(nodeId, nodeCollections); + if (labels.isEmpty()) { + return EMPTY_NODELABEL_SET; + } + Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels); + return nodeLabels; + } finally { + readLock.unlock(); } - Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels); - return nodeLabels; } private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 696b99b..10092ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -34,6 +34,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index c39f2b9..b1fa80a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -810,12 +810,12 @@ public class RMWebServices { NodeToLabelsInfo ntl = new NodeToLabelsInfo(); HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels(); - Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext() - .getNodeLabelManager().getNodeLabels(); + Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext() + .getNodeLabelManager().getNodeLabelsInfo(); - for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) { - ntlMap.put(nitle.getKey().toString(), - new NodeLabelsInfo(nitle.getValue())); + for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) { + List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue()); + ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels)); } return ntl; @@ -830,16 +830,16 @@ public class RMWebServices { LabelsToNodesInfo lts = new LabelsToNodesInfo(); Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes(); - Map<String, Set<NodeId>> labelsToNodeId = null; + Map<NodeLabel, Set<NodeId>> labelsToNodeId = null; if (labels == null || labels.size() == 0) { labelsToNodeId = - rm.getRMContext().getNodeLabelManager().getLabelsToNodes(); + rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(); } else { labelsToNodeId = - rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels); + rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels); } - for (Entry<String, Set<NodeId>> entry : labelsToNodeId.entrySet()) { + for (Entry<NodeLabel, Set<NodeId>> entry : labelsToNodeId.entrySet()) { List<String> nodeIdStrList = new ArrayList<String>(); for (NodeId nodeId : entry.getValue()) { nodeIdStrList.add(nodeId.toString()); @@ -985,14 +985,13 @@ public class RMWebServices { @Path("/nodes/{nodeId}/get-labels") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr, - @PathParam("nodeId") String nodeId) - throws IOException { + @PathParam("nodeId") String nodeId) throws IOException { init(); NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId); - return new NodeLabelsInfo( - rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid)); - + List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext() + .getNodeLabelManager().getLabelsInfoByNode(nid)); + return new NodeLabelsInfo(labels); } protected Response killApp(RMApp app, UserGroupInformation callerUGI, http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java index 54fe00d..53a9902 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java @@ -133,12 +133,15 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName()); assertEquals(1, nlsifo.getNodeLabels().size()); + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + assertEquals("a", nl.getName()); + assertTrue(nl.getExclusivity()); + } // Add another nlsifo = new NodeLabelsInfo(); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b")); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false)); response = r.path("ws").path("v1").path("cluster") .path("add-node-labels").queryParam("user.name", userName) @@ -154,6 +157,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); assertEquals(2, nlsifo.getNodeLabels().size()); + // Verify exclusivity for 'y' as false + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + if (nl.getName().equals("b")) { + assertFalse(nl.getExclusivity()); + } + } // Add labels to a node MultivaluedMapImpl params = new MultivaluedMapImpl(); @@ -202,7 +211,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class); assertEquals(2, ltni.getLabelsToNodes().size()); - NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b")); + NodeIDsInfo nodes = ltni.getLabelsToNodes().get( + new NodeLabelInfo("b", false)); assertTrue(nodes.getNodeIDs().contains("nid2:0")); assertTrue(nodes.getNodeIDs().contains("nid1:0")); nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a")); @@ -231,7 +241,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("a")); + assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Replace @@ -255,7 +265,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("b")); + assertTrue(nlsifo.getNodeLabelsInfo().contains( + new NodeLabelInfo("b", false))); // Replace labels using node-to-labels NodeToLabelsEntryList ntli = new NodeToLabelsEntryList(); @@ -281,7 +292,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class); NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); assertEquals(1, nlinfo.getNodeLabels().size()); - assertTrue(nlinfo.getNodeLabelsName().contains("a")); + assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Remove all params = new MultivaluedMapImpl(); @@ -303,7 +314,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("")); + assertTrue(nlsifo.getNodeLabelsInfo().isEmpty()); // Add a label back for auth tests params = new MultivaluedMapImpl(); @@ -326,7 +337,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("a")); + assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Auth fail replace labels on node params = new MultivaluedMapImpl(); @@ -347,7 +358,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertTrue(nlsifo.getNodeLabelsName().contains("a")); + assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a"))); // Fail to add a label with post response = @@ -383,8 +394,11 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); - assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName()); assertEquals(1, nlsifo.getNodeLabels().size()); + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + assertEquals("a", nl.getName()); + assertTrue(nl.getExclusivity()); + } // Remove cluster label with post params = new MultivaluedMapImpl(); @@ -409,8 +423,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { // configuration is on // Reset for testing : add cluster labels nlsifo = new NodeLabelsInfo(); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x")); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y")); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false)); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false)); response = r.path("ws") .path("v1") @@ -421,8 +435,6 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON).post(ClientResponse.class); // Reset for testing : Add labels to a node - nlsifo = new NodeLabelsInfo(); - nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y")); params = new MultivaluedMapImpl(); params.add("labels", "y"); response = @@ -461,7 +473,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { ntlinfo = response.getEntity(NodeToLabelsInfo.class); nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); assertEquals(1, nlinfo.getNodeLabels().size()); - assertFalse(nlinfo.getNodeLabels().contains("x")); + assertFalse(nlinfo.getNodeLabelsInfo().contains( + new NodeLabelInfo("x", false))); // Case2 : failure to Replace labels using replace-labels response = @@ -481,7 +494,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { ntlinfo = response.getEntity(NodeToLabelsInfo.class); nlinfo = ntlinfo.getNodeToLabels().get("nid:0"); assertEquals(1, nlinfo.getNodeLabels().size()); - assertFalse(nlinfo.getNodeLabels().contains("x")); + assertFalse(nlinfo.getNodeLabelsInfo().contains( + new NodeLabelInfo("x", false))); // Case3 : Remove cluster label should be successful params = new MultivaluedMapImpl(); @@ -500,7 +514,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); + assertEquals(new NodeLabelInfo("y", false), + nlsifo.getNodeLabelsInfo().get(0)); assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName()); + assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); // Remove y params = new MultivaluedMapImpl(); @@ -543,7 +560,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); nlsifo = response.getEntity(NodeLabelsInfo.class); assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName()); - assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); + assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity()); assertEquals(1, nlsifo.getNodeLabels().size()); }
