This is an automated email from the ASF dual-hosted git repository. rohithsharmaks pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7fa62e150ce86f7cb0e88d593b8f07e3384a9a60 Author: Rohith Sharma K S <[email protected]> AuthorDate: Wed Feb 6 09:31:20 2019 +0530 YARN-8219. Add application launch time to ATSV2. Contributed by Abhishek Modi. --- .../yarn/server/metrics/ApplicationMetricsConstants.java | 3 +++ .../yarn/util/timeline/TimelineEntityV2Converter.java | 9 +++++++-- .../metrics/AbstractSystemMetricsPublisher.java | 5 +++++ .../metrics/CombinedSystemMetricsPublisher.java | 6 ++++++ .../resourcemanager/metrics/NoOpSystemMetricPublisher.java | 4 ++++ .../resourcemanager/metrics/SystemMetricsPublisher.java | 2 ++ .../metrics/TimelineServiceV2Publisher.java | 14 ++++++++++++++ .../yarn/server/resourcemanager/rmapp/RMAppImpl.java | 4 +++- .../metrics/TestSystemMetricsPublisherForV2.java | 4 +++- 9 files changed, 47 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 93da565..6428fa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -34,6 +34,9 @@ public class ApplicationMetricsConstants { public static final String FINISHED_EVENT_TYPE = "YARN_APPLICATION_FINISHED"; + public static final String LAUNCHED_EVENT_TYPE = + "YARN_APPLICATION_LAUNCHED"; + public static final String ACLS_UPDATED_EVENT_TYPE = "YARN_APPLICATION_ACLS_UPDATED"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java index 98e1e7a..9fefc09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java @@ -258,6 +258,7 @@ public final class TimelineEntityV2Converter { String type = null; boolean unmanagedApplication = false; long createdTime = 0; + long launchTime = 0; long finishedTime = 0; float progress = 0.0f; int applicationPriority = 0; @@ -417,6 +418,9 @@ public final class TimelineEntityV2Converter { ApplicationMetricsConstants.CREATED_EVENT_TYPE)) { createdTime = event.getTimestamp(); } else if (event.getId().equals( + ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE)) { + launchTime = event.getTimestamp(); + } else if (event.getId().equals( ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) { // This type of events are parsed in time-stamp descending order // which means the previous event could override the information @@ -449,8 +453,9 @@ public final class TimelineEntityV2Converter { return ApplicationReport.newInstance( ApplicationId.fromString(entity.getId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, - appResources, null, progress, type, null, appTags, unmanagedApplication, + diagnosticsInfo, null, createdTime, launchTime, + finishedTime, finalStatus, appResources, null, + progress, type, null, appTags, unmanagedApplication, Priority.newInstance(applicationPriority), appNodeLabelExpression, amNodeLabelExpression); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java index 96b153f..09f44d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; /** * Abstract implementation of SystemMetricsPublisher which is then extended by @@ -122,6 +123,10 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService PUBLISH_ENTITY, PUBLISH_APPLICATION_FINISHED_ENTITY } + @Override + public void appLaunched(RMApp app, long launchTime) { + } + /** * TimelinePublishEvent's hash code should be based on application's id this * will ensure all the events related to a particular app goes to particular diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java index 9646747..8e04a1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java @@ -45,6 +45,12 @@ public class CombinedSystemMetricsPublisher implements SystemMetricsPublisher { } } + @Override public void appLaunched(RMApp app, long launchTime) { + for (SystemMetricsPublisher publisher : this.publishers) { + publisher.appLaunched(app, launchTime); + } + } + @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { for (SystemMetricsPublisher publisher : this.publishers) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java index e0bf3c7..537c88c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java @@ -67,4 +67,8 @@ public class NoOpSystemMetricPublisher implements SystemMetricsPublisher{ public void appStateUpdated(RMApp app, YarnApplicationState appState, long updatedTime) { } + + @Override + public void appLaunched(RMApp app, long launchTime) { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index eeea434..137d790 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -32,6 +32,8 @@ public interface SystemMetricsPublisher { void appCreated(RMApp app, long createdTime); + void appLaunched(RMApp app, long launchTime); + void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime); void appUpdated(RMApp app, long updatedTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 95d75b2..e93c26d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -155,6 +155,20 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { @SuppressWarnings("unchecked") @Override + public void appLaunched(RMApp app, long launchTime) { + ApplicationEntity entity = + createApplicationEntity(app.getApplicationId()); + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setId(ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE); + tEvent.setTimestamp(launchTime); + entity.addEvent(tEvent); + + getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( + SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); + } + + @SuppressWarnings("unchecked") + @Override public void appFinished(RMApp app, RMAppState state, long finishedTime) { ApplicationEntity entity = createApplicationEntity(app.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index bd645d7..696b39d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -293,7 +293,7 @@ public class RMAppImpl implements RMApp, Recoverable { .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) - // Handle AppAttemptLaunch to upate the launchTime and publish to ATS + // Handle AppAttemptLaunch to update the launchTime and publish to ATS .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.ATTEMPT_LAUNCHED, new AttemptLaunchedTransition()) @@ -1067,6 +1067,8 @@ public class RMAppImpl implements RMApp, Recoverable { app.getCurrentAppAttempt().getAppAttemptId()+ "launchTime: "+event.getTimestamp()); app.launchTime = event.getTimestamp(); + app.rmContext.getSystemMetricsPublisher().appLaunched( + app, app.launchTime); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 3b503e5..ca3f47d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -203,6 +203,7 @@ public class TestSystemMetricsPublisherForV2 { RMApp app = createAppAndRegister(appId); metricsPublisher.appCreated(app, app.getStartTime()); + metricsPublisher.appLaunched(app, app.getLaunchTime()); metricsPublisher.appACLsUpdated(app, "user1,user2", 4L); metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime()); dispatcher.await(); @@ -221,7 +222,7 @@ public class TestSystemMetricsPublisherForV2 { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity( - appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); + appFile, 4, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); } @Test(timeout = 10000) @@ -356,6 +357,7 @@ public class TestSystemMetricsPublisherForV2 { when(app.getQueue()).thenReturn("test queue"); when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); + when(app.getLaunchTime()).thenReturn(Integer.MAX_VALUE + 2L); when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); when(app.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
