YARN-6680. Avoid locking overhead for NO_LABEL lookups. Contributed by Daryn Sharp
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee89ac84 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee89ac84 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee89ac84 Branch: refs/heads/HADOOP-13345 Commit: ee89ac84e68d3e181b75c63f74a0444f9d28146f Parents: e9c2aa1 Author: Naganarasimha <[email protected]> Authored: Mon Jun 19 22:42:45 2017 +0530 Committer: Naganarasimha <[email protected]> Committed: Mon Jun 19 22:42:45 2017 +0530 ---------------------------------------------------------------------- .../yarn/nodelabels/CommonNodeLabelsManager.java | 8 +++++++- .../nodelabels/RMNodeLabelsManager.java | 8 ++++++-- .../resourcemanager/scheduler/ResourceUsage.java | 14 +++++++------- .../server/resourcemanager/webapp/TestRMWebApp.java | 8 ++++++-- 4 files changed, 26 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 60ade2d..66e945f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -99,6 +99,8 @@ public class CommonNodeLabelsManager extends AbstractService { protected ConcurrentMap<String, Host> nodeCollections = new ConcurrentHashMap<String, Host>(); + protected RMNodeLabel noNodeLabel; + protected final ReadLock readLock; protected final WriteLock writeLock; @@ -225,7 +227,8 @@ public class CommonNodeLabelsManager extends AbstractService { isCentralizedNodeLabelConfiguration = YarnConfiguration.isCentralizedNodeLabelConfiguration(conf); - labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL)); + noNodeLabel = new RMNodeLabel(NO_LABEL); + labelCollections.put(NO_LABEL, noNodeLabel); } /** @@ -947,6 +950,9 @@ public class CommonNodeLabelsManager extends AbstractService { } public boolean isExclusiveNodeLabel(String nodeLabel) throws IOException { + if (nodeLabel.equals(NO_LABEL)) { + return noNodeLabel.getIsExclusive(); + } try { readLock.lock(); RMNodeLabel label = labelCollections.get(nodeLabel); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 79b25ed..507f696 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -519,12 +519,16 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { public Resource getResourceByLabel(String label, Resource clusterResource) { label = normalizeLabel(label); + if (label.equals(NO_LABEL)) { + return noNodeLabel.getResource(); + } try { readLock.lock(); - if (null == labelCollections.get(label)) { + RMNodeLabel nodeLabel = labelCollections.get(label); + if (nodeLabel == null) { return Resources.none(); } - return labelCollections.get(label).getResource(); + return nodeLabel.getResource(); } finally { readLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index 2857379..6f0c7d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -28,7 +28,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.util.resource.Resources; /** @@ -46,6 +45,7 @@ public class ResourceUsage { private Map<String, UsageByLabel> usages; // short for no-label :) private static final String NL = CommonNodeLabelsManager.NO_LABEL; + private final UsageByLabel usageNoLabel; public ResourceUsage() { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @@ -53,7 +53,8 @@ public class ResourceUsage { writeLock = lock.writeLock(); usages = new HashMap<String, UsageByLabel>(); - usages.put(NL, new UsageByLabel(NL)); + usageNoLabel = new UsageByLabel(NL); + usages.put(NL, usageNoLabel); } // Usage enum here to make implement cleaner @@ -323,10 +324,9 @@ public class ResourceUsage { } private Resource _get(String label, ResourceType type) { - if (label == null) { - label = RMNodeLabelsManager.NO_LABEL; + if (label == null || label.equals(NL)) { + return normalize(usageNoLabel.resArr[type.idx]); } - try { readLock.lock(); UsageByLabel usage = usages.get(label); @@ -362,8 +362,8 @@ public class ResourceUsage { } private UsageByLabel getAndAddIfMissing(String label) { - if (label == null) { - label = RMNodeLabelsManager.NO_LABEL; + if (label == null || label.equals(NL)) { + return usageNoLabel; } if (!usages.containsKey(label)) { UsageByLabel u = new UsageByLabel(label); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 03fc206..8a65040 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -229,12 +230,15 @@ public class TestRMWebApp { setupQueueConfiguration(conf); CapacityScheduler cs = new CapacityScheduler(); - cs.setConf(new YarnConfiguration()); + YarnConfiguration yarnConf = new YarnConfiguration(); + cs.setConf(yarnConf); RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); - rmContext.setNodeLabelManager(new NullRMNodeLabelsManager()); + RMNodeLabelsManager labelManager = new NullRMNodeLabelsManager(); + labelManager.init(yarnConf); + rmContext.setNodeLabelManager(labelManager); cs.setRMContext(rmContext); cs.init(conf); return cs; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
