Repository: hadoop Updated Branches: refs/heads/branch-2.9 61a3bf465 -> df9a01810
YARN-9085. Add Guaranteed and MaxCapacity to CSQueueMetrics (cherry picked from commit cc51607ccdf60f4ebecf7afa3e2056297ca0ac5f) (cherry picked from commit eb8b1eaa9a8aa1424864c94a7f796f33dd425f07) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df9a0181 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df9a0181 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df9a0181 Branch: refs/heads/branch-2.9 Commit: df9a018109b841744128369cadffda5ac87f373e Parents: 61a3bf4 Author: Jonathan Hung <[email protected]> Authored: Fri Dec 7 10:32:53 2018 -0800 Committer: Jonathan Hung <[email protected]> Committed: Fri Dec 7 10:50:06 2018 -0800 ---------------------------------------------------------------------- .../scheduler/capacity/CSQueueMetrics.java | 39 +++++++++++++++++ .../scheduler/capacity/CSQueueUtils.java | 17 ++++++++ .../scheduler/capacity/LeafQueue.java | 4 ++ .../scheduler/capacity/ParentQueue.java | 4 ++ .../capacity/TestCapacityScheduler.java | 44 ++++++++++++++++++++ 5 files changed, 108 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/df9a0181/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java index 87fc234..8382665 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java @@ -24,6 +24,7 @@ import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MutableGaugeFloat; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -46,6 +47,14 @@ public class CSQueueMetrics extends QueueMetrics { MutableGaugeFloat usedCapacity; @Metric("Percent of Absolute Capacity Used") MutableGaugeFloat absoluteUsedCapacity; + @Metric("Guaranteed memory in MB") + MutableGaugeLong guaranteedMB; + @Metric("Guaranteed CPU in virtual cores") + MutableGaugeInt guaranteedVCores; + @Metric("Maximum memory in MB") + MutableGaugeLong maxCapacityMB; + @Metric("Maximum CPU in virtual cores") + MutableGaugeInt maxCapacityVCores; CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { @@ -126,6 +135,36 @@ public class CSQueueMetrics extends QueueMetrics { } } + public long getGuaranteedMB() { + return guaranteedMB.value(); + } + + public int getGuaranteedVCores() { + return guaranteedVCores.value(); + } + + public void setGuaranteedResources(String partition, Resource res) { + if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { + guaranteedMB.set(res.getMemorySize()); + guaranteedVCores.set(res.getVirtualCores()); + } + } + + public long getMaxCapacityMB() { + return maxCapacityMB.value(); + } + + public int getMaxCapacityVCores() { + return maxCapacityVCores.value(); + } + + public void setMaxCapacityResources(String partition, Resource res) { + if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { + maxCapacityMB.set(res.getMemorySize()); + maxCapacityVCores.set(res.getVirtualCores()); + } + } + public synchronized static CSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { MetricsSystem ms = DefaultMetricsSystem.instance(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/df9a0181/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 9554086..da1d304 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -295,4 +295,21 @@ class CSQueueUtils { childQueue.getMetrics().setAvailableResourcesToQueue(nodePartition, getMaxAvailableResourceToQueue(rc, nlm, childQueue, cluster)); } + + /** + * Updated configured capacity/max-capacity for queue. + * @param rc resource calculator + * @param partitionResource total cluster resources for this partition + * @param partition partition being updated + * @param queue queue + */ + public static void updateConfiguredCapacityMetrics(ResourceCalculator rc, + Resource partitionResource, String partition, AbstractCSQueue queue) { + queue.getMetrics().setGuaranteedResources(partition, rc.multiplyAndNormalizeDown( + partitionResource, queue.getQueueCapacities().getAbsoluteCapacity(partition), + queue.getMinimumAllocation())); + queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown( + partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition), + queue.getMinimumAllocation())); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/df9a0181/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 41c6f09..c352ba4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1794,6 +1794,10 @@ public class LeafQueue extends AbstractCSQueue { // Update metrics CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, null); + // Update configured capacity/max-capacity for default partition only + CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator, + labelManager.getResourceByLabel(null, clusterResource), + RMNodeLabelsManager.NO_LABEL, this); // queue metrics are updated, more resource may be available // activate the pending applications if possible http://git-wip-us.apache.org/repos/asf/hadoop/blob/df9a0181/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 213bf07..2de13cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -841,6 +841,10 @@ public class ParentQueue extends AbstractCSQueue { CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, null); + // Update configured capacity/max-capacity for default partition only + CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator, + labelManager.getResourceByLabel(null, clusterResource), + RMNodeLabelsManager.NO_LABEL, this); } finally { writeLock.unlock(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/df9a0181/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index bbac2bd..0f97793 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -5157,4 +5157,48 @@ public class TestCapacityScheduler { assertEquals(4, appsInB1.size()); rm.close(); } + + @Test + public void testCSQueueMetrics() throws Exception { + CapacityScheduler cs = new CapacityScheduler(); + cs.setConf(new YarnConfiguration()); + cs.setRMContext(resourceManager.getRMContext()); + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupQueueConfiguration(conf); + cs.init(conf); + cs.start(); + + RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 1, "n1"); + RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 2, "n2"); + cs.handle(new NodeAddedSchedulerEvent(n1)); + cs.handle(new NodeAddedSchedulerEvent(n2)); + + assertEquals(10240, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB()); + assertEquals(71680, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB()); + assertEquals(102400, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB()); + assertEquals(102400, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB()); + + // Remove a node, metrics should be updated + cs.handle(new NodeRemovedSchedulerEvent(n2)); + assertEquals(5120, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB()); + assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB()); + assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB()); + assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB()); + + // Add child queue to a, and reinitialize. Metrics should be updated + conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} ); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 30.0f); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.0f); + conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 50.0f); + + cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, + null, new RMContainerTokenSecretManager(conf), + new NMTokenSecretManagerInRM(conf), + new ClientToAMTokenSecretManagerInRM(), null)); + + assertEquals(1024, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getGuaranteedMB()); + assertEquals(2048, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getGuaranteedMB()); + assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getMaxCapacityMB()); + assertEquals(25600, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getMaxCapacityMB()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
