Repository: hadoop Updated Branches: refs/heads/trunk 0f9592144 -> d4f53fc96
YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4f53fc9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4f53fc9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4f53fc9 Branch: refs/heads/trunk Commit: d4f53fc9631d682cd79ba440aefa6750dcc898be Parents: 0f95921 Author: Wangda Tan <[email protected]> Authored: Wed May 13 13:29:09 2015 -0700 Committer: Wangda Tan <[email protected]> Committed: Wed May 13 13:29:09 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../nodelabels/CommonNodeLabelsManager.java | 149 ++++++++++++++++--- .../hadoop/yarn/nodelabels/RMNodeLabel.java | 7 +- .../yarn/nodelabels/NodeLabelTestBase.java | 30 ++++ .../nodelabels/TestCommonNodeLabelsManager.java | 22 +++ 5 files changed, 186 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e5d5ecf..16c2dd9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -420,6 +420,9 @@ Release 2.7.1 - UNRELEASED YARN-3539. Updated timeline server documentation and marked REST APIs evolving. (Steve Loughran via zjshen) + YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string + label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index f2ff0f6..bf34837 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -64,6 +64,8 @@ public class CommonNodeLabelsManager extends AbstractService { private static final int MAX_LABEL_LENGTH = 255; public static final Set<String> EMPTY_STRING_SET = Collections .unmodifiableSet(new HashSet<String>(0)); + public static final Set<NodeLabel> EMPTY_NODELABEL_SET = Collections + .unmodifiableSet(new HashSet<NodeLabel>(0)); public static final String ANY = "*"; public static final Set<String> ACCESS_ANY_LABEL_SET = ImmutableSet.of(ANY); private static final Pattern LABEL_PATTERN = Pattern @@ -716,23 +718,53 @@ public class CommonNodeLabelsManager extends AbstractService { * @return nodes to labels map */ public Map<NodeId, Set<String>> getNodeLabels() { + Map<NodeId, Set<String>> nodeToLabels = + generateNodeLabelsInfoPerNode(String.class); + return nodeToLabels; + } + + /** + * Get mapping of nodes to label info + * + * @return nodes to labels map + */ + public Map<NodeId, Set<NodeLabel>> getNodeLabelsInfo() { + Map<NodeId, Set<NodeLabel>> nodeToLabels = + generateNodeLabelsInfoPerNode(NodeLabel.class); + return nodeToLabels; + } + + @SuppressWarnings("unchecked") + private <T> Map<NodeId, Set<T>> generateNodeLabelsInfoPerNode(Class<T> type) { try { readLock.lock(); - Map<NodeId, Set<String>> nodeToLabels = - new HashMap<NodeId, Set<String>>(); + Map<NodeId, Set<T>> nodeToLabels = new HashMap<>(); for (Entry<String, Host> entry : nodeCollections.entrySet()) { String hostName = entry.getKey(); Host host = entry.getValue(); for (NodeId nodeId : host.nms.keySet()) { - Set<String> nodeLabels = getLabelsByNode(nodeId); - if (nodeLabels == null || nodeLabels.isEmpty()) { - continue; + if (type.isAssignableFrom(String.class)) { + Set<String> nodeLabels = getLabelsByNode(nodeId); + if (nodeLabels == null || nodeLabels.isEmpty()) { + continue; + } + nodeToLabels.put(nodeId, (Set<T>) nodeLabels); + } else { + Set<NodeLabel> nodeLabels = getLabelsInfoByNode(nodeId); + if (nodeLabels == null || nodeLabels.isEmpty()) { + continue; + } + nodeToLabels.put(nodeId, (Set<T>) nodeLabels); } - nodeToLabels.put(nodeId, nodeLabels); } if (!host.labels.isEmpty()) { - nodeToLabels - .put(NodeId.newInstance(hostName, WILDCARD_PORT), host.labels); + if (type.isAssignableFrom(String.class)) { + nodeToLabels.put(NodeId.newInstance(hostName, WILDCARD_PORT), + (Set<T>) host.labels); + } else { + nodeToLabels.put(NodeId.newInstance(hostName, WILDCARD_PORT), + (Set<T>) createNodeLabelFromLabelNames(host.labels)); + } } } return Collections.unmodifiableMap(nodeToLabels); @@ -741,6 +773,7 @@ public class CommonNodeLabelsManager extends AbstractService { } } + /** * Get mapping of labels to nodes for all the labels. * @@ -765,28 +798,72 @@ public class CommonNodeLabelsManager extends AbstractService { public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels) { try { readLock.lock(); - Map<String, Set<NodeId>> labelsToNodes = - new HashMap<String, Set<NodeId>>(); - for (String label : labels) { - if(label.equals(NO_LABEL)) { - continue; - } - RMNodeLabel nodeLabelInfo = labelCollections.get(label); - if(nodeLabelInfo != null) { - Set<NodeId> nodeIds = nodeLabelInfo.getAssociatedNodeIds(); - if (!nodeIds.isEmpty()) { - labelsToNodes.put(label, nodeIds); - } - } else { - LOG.warn("getLabelsToNodes : Label [" + label + "] cannot be found"); - } - } + Map<String, Set<NodeId>> labelsToNodes = getLabelsToNodesMapping(labels, + String.class); + return Collections.unmodifiableMap(labelsToNodes); + } finally { + readLock.unlock(); + } + } + + + /** + * Get mapping of labels to nodes for all the labels. + * + * @return labels to nodes map + */ + public Map<NodeLabel, Set<NodeId>> getLabelsInfoToNodes() { + try { + readLock.lock(); + return getLabelsInfoToNodes(labelCollections.keySet()); + } finally { + readLock.unlock(); + } + } + + /** + * Get mapping of labels info to nodes for specified set of labels. + * + * @param nodelabels + * set of nodelabels for which labels to nodes mapping will be + * returned. + * @return labels to nodes map + */ + public Map<NodeLabel, Set<NodeId>> getLabelsInfoToNodes(Set<String> labels) { + try { + readLock.lock(); + Map<NodeLabel, Set<NodeId>> labelsToNodes = getLabelsToNodesMapping( + labels, NodeLabel.class); return Collections.unmodifiableMap(labelsToNodes); } finally { readLock.unlock(); } } + private <T> Map<T, Set<NodeId>> getLabelsToNodesMapping(Set<String> labels, + Class<T> type) { + Map<T, Set<NodeId>> labelsToNodes = new HashMap<T, Set<NodeId>>(); + for (String label : labels) { + if (label.equals(NO_LABEL)) { + continue; + } + RMNodeLabel nodeLabelInfo = labelCollections.get(label); + if (nodeLabelInfo != null) { + Set<NodeId> nodeIds = nodeLabelInfo.getAssociatedNodeIds(); + if (!nodeIds.isEmpty()) { + if (type.isAssignableFrom(String.class)) { + labelsToNodes.put(type.cast(label), nodeIds); + } else { + labelsToNodes.put(type.cast(nodeLabelInfo.getNodeLabel()), nodeIds); + } + } + } else { + LOG.warn("getLabelsToNodes : Label [" + label + "] cannot be found"); + } + } + return labelsToNodes; + } + /** * Get existing valid labels in repository * @@ -914,6 +991,30 @@ public class CommonNodeLabelsManager extends AbstractService { } } + private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) { + Set<String> labels = getLabelsByNode(nodeId, nodeCollections); + if (labels.isEmpty()) { + return EMPTY_NODELABEL_SET; + } + Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels); + return nodeLabels; + } + + private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) { + Set<NodeLabel> nodeLabels = new HashSet<NodeLabel>(); + for (String label : labels) { + if (label.equals(NO_LABEL)) { + continue; + } + RMNodeLabel rmLabel = labelCollections.get(label); + if (rmLabel == null) { + continue; + } + nodeLabels.add(rmLabel.getNodeLabel()); + } + return nodeLabels; + } + protected void createNodeIfNonExisted(NodeId nodeId) throws IOException { Host host = nodeCollections.get(nodeId.getHost()); if (null == host) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java index 1c4fd17..feeeaf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java @@ -25,7 +25,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.resource.Resources; public class RMNodeLabel implements Comparable<RMNodeLabel> { @@ -34,6 +33,7 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> { private String labelName; private Set<NodeId> nodeIds; private boolean exclusive; + private NodeLabel nodeLabel; public RMNodeLabel(NodeLabel nodeLabel) { this(nodeLabel.getName(), Resource.newInstance(0, 0), 0, @@ -52,6 +52,7 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> { this.numActiveNMs = activeNMs; this.nodeIds = new HashSet<NodeId>(); this.exclusive = exclusive; + this.nodeLabel = NodeLabel.newInstance(labelName, exclusive); } public void addNodeId(NodeId node) { @@ -100,6 +101,10 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> { return new RMNodeLabel(labelName, resource, numActiveNMs, exclusive); } + public NodeLabel getNodeLabel() { + return this.nodeLabel; + } + @Override public int compareTo(RMNodeLabel o) { // We should always put empty label entry first after sorting http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java index 4d406fd..8301d96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.Map.Entry; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.junit.Assert; import com.google.common.collect.ImmutableMap; @@ -41,6 +42,15 @@ public class NodeLabelTestBase { } } + public static void assertLabelInfoMapEquals(Map<NodeId, Set<NodeLabel>> m1, + ImmutableMap<NodeId, Set<NodeLabel>> m2) { + Assert.assertEquals(m1.size(), m2.size()); + for (NodeId k : m1.keySet()) { + Assert.assertTrue(m2.containsKey(k)); + assertNLCollectionEquals(m1.get(k), m2.get(k)); + } + } + public static void assertLabelsToNodesEquals(Map<String, Set<NodeId>> m1, ImmutableMap<String, Set<NodeId>> m2) { Assert.assertEquals(m1.size(), m2.size()); @@ -88,6 +98,14 @@ public class NodeLabelTestBase { Assert.assertTrue(s1.containsAll(s2)); } + public static void assertNLCollectionEquals(Collection<NodeLabel> c1, + Collection<NodeLabel> c2) { + Set<NodeLabel> s1 = new HashSet<NodeLabel>(c1); + Set<NodeLabel> s2 = new HashSet<NodeLabel>(c2); + Assert.assertEquals(s1, s2); + Assert.assertTrue(s1.containsAll(s2)); + } + @SuppressWarnings("unchecked") public static <E> Set<E> toSet(E... elements) { Set<E> set = Sets.newHashSet(elements); @@ -105,4 +123,16 @@ public class NodeLabelTestBase { return NodeId.newInstance(str, CommonNodeLabelsManager.WILDCARD_PORT); } } + + public static void assertLabelsInfoToNodesEquals( + Map<NodeLabel, Set<NodeId>> m1, ImmutableMap<NodeLabel, Set<NodeId>> m2) { + Assert.assertEquals(m1.size(), m2.size()); + for (NodeLabel k : m1.keySet()) { + Assert.assertTrue(m2.containsKey(k)); + Set<NodeId> s1 = new HashSet<NodeId>(m1.get(k)); + Set<NodeId> s2 = new HashSet<NodeId>(m2.get(k)); + Assert.assertEquals(s1, s2); + Assert.assertTrue(s1.containsAll(s2)); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index 09838b4..c25b0bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -579,4 +579,26 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { labelsByNode); Assert.assertTrue(labelsByNode.contains("p1")); } + + @Test(timeout = 5000) + public void testLabelsInfoToNodes() throws IOException { + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", false), + NodeLabel.newInstance("p2", true), NodeLabel.newInstance("p3", true))); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); + Map<NodeLabel, Set<NodeId>> labelsToNodes = mgr.getLabelsInfoToNodes(); + assertLabelsInfoToNodesEquals(labelsToNodes, ImmutableMap.of( + NodeLabel.newInstance("p1", false), toSet(toNodeId("n1")))); + } + + @Test(timeout = 5000) + public void testGetNodeLabelsInfo() throws IOException { + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", false), + NodeLabel.newInstance("p2", true), NodeLabel.newInstance("p3", false))); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p2"))); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n2"), toSet("p3"))); + + assertLabelInfoMapEquals(mgr.getNodeLabelsInfo(), ImmutableMap.of( + toNodeId("n1"), toSet(NodeLabel.newInstance("p2", true)), + toNodeId("n2"), toSet(NodeLabel.newInstance("p3", false)))); + } }
