This is an automated email from the ASF dual-hosted git repository.
taoyang pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 1f6f4a2 YARN-9795. ClusterMetrics to include AM allocation delay.
Contributed by Fengnan Li.
1f6f4a2 is described below
commit 1f6f4a2457441ff2c1fddeb1714d8c47b195d427
Author: Tao Yang <[email protected]>
AuthorDate: Sat Sep 7 07:52:39 2019 +0800
YARN-9795. ClusterMetrics to include AM allocation delay. Contributed by
Fengnan Li.
---
.../hadoop/yarn/server/resourcemanager/ClusterMetrics.java | 9 +++++++++
.../server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java | 8 ++++++++
.../hadoop/yarn/server/resourcemanager/TestClusterMetrics.java | 8 +++++++-
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
index 5917b99..ba5fc40 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
@@ -48,6 +48,8 @@ public class ClusterMetrics {
@Metric("# of Shutdown NMs") MutableGaugeInt numShutdownNMs;
@Metric("AM container launch delay") MutableRate aMLaunchDelay;
@Metric("AM register delay") MutableRate aMRegisterDelay;
+ @Metric("AM container allocation delay")
+ private MutableRate aMContainerAllocationDelay;
private static final MetricsInfo RECORD_INFO = info("ClusterMetrics",
"Metrics for the Yarn Cluster");
@@ -190,4 +192,11 @@ public class ClusterMetrics {
aMRegisterDelay.add(delay);
}
+ public void addAMContainerAllocationDelay(long delay) {
+ aMContainerAllocationDelay.add(delay);
+ }
+
+ public MutableRate getAMContainerAllocationDelay() {
+ return aMContainerAllocationDelay;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index ef85408..862c43a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -176,6 +176,8 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
private long finishTime = 0;
private long launchAMStartTime = 0;
private long launchAMEndTime = 0;
+ private long scheduledTime = 0;
+ private long containerAllocatedTime = 0;
// Set to null initially. Will eventually get set
// if an RMAppAttemptUnregistrationEvent occurs
@@ -1170,6 +1172,7 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
&& amContainerAllocation.getContainers() != null) {
assert (amContainerAllocation.getContainers().size() == 0);
}
+ appAttempt.scheduledTime = System.currentTimeMillis();
return RMAppAttemptState.SCHEDULED;
} else {
// save state and then go to LAUNCHED state
@@ -1226,6 +1229,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
.clearNodeSetForAttempt(appAttempt.applicationAttemptId);
appAttempt.getSubmissionContext().setResource(
appAttempt.getMasterContainer().getResource());
+ appAttempt.containerAllocatedTime = System.currentTimeMillis();
+ long allocationDelay =
+ appAttempt.containerAllocatedTime - appAttempt.scheduledTime;
+ ClusterMetrics.getMetrics().addAMContainerAllocationDelay(
+ allocationDelay);
appAttempt.storeAttempt();
return RMAppAttemptState.ALLOCATED_SAVING;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java
index 5b9105f..d81e278 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java
@@ -29,17 +29,23 @@ public class TestClusterMetrics {
private ClusterMetrics metrics;
/**
- * Test aMLaunchDelay and aMRegisterDelay Metrics
+ * Test below metrics
+ * - aMLaunchDelay
+ * - aMRegisterDelay
+ * - aMContainerAllocationDelay
*/
@Test
public void testAmMetrics() throws Exception {
assert(metrics != null);
Assert.assertTrue(!metrics.aMLaunchDelay.changed());
Assert.assertTrue(!metrics.aMRegisterDelay.changed());
+ Assert.assertTrue(!metrics.getAMContainerAllocationDelay().changed());
metrics.addAMLaunchDelay(1);
metrics.addAMRegisterDelay(1);
+ metrics.addAMContainerAllocationDelay(1);
Assert.assertTrue(metrics.aMLaunchDelay.changed());
Assert.assertTrue(metrics.aMRegisterDelay.changed());
+ Assert.assertTrue(metrics.getAMContainerAllocationDelay().changed());
}
@Before
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
