This is an automated email from the ASF dual-hosted git repository. eyang pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new 68a98be YARN-6695. Fixed NPE in publishing appFinished events to ATSv2. Contributed by Prabhu Joseph 68a98be is described below commit 68a98be8a24b2ab875cb1a1f2157dde24cfc9a24 Author: Eric Yang <ey...@apache.org> AuthorDate: Thu Apr 18 12:29:37 2019 -0400 YARN-6695. Fixed NPE in publishing appFinished events to ATSv2. Contributed by Prabhu Joseph (cherry picked from commit df76cdc8959c51b71704ab5c38335f745a6f35d8) --- .../metrics/TimelineServiceV2Publisher.java | 13 ++++-- .../metrics/TestSystemMetricsPublisherForV2.java | 47 ++++++++++++++++++++++ .../src/site/markdown/TimelineServiceV2.md | 8 ---- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 89905e5..11db6b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -454,10 +454,15 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { } TimelineCollector timelineCollector = rmTimelineCollectorManager.get(appId); - TimelineEntities entities = new TimelineEntities(); - entities.addEntity(entity); - timelineCollector.putEntities(entities, - UserGroupInformation.getCurrentUser()); + if (timelineCollector != null) { + TimelineEntities entities = new TimelineEntities(); + entities.addEntity(entity); + timelineCollector.putEntities(entities, + UserGroupInformation.getCurrentUser()); + } else { + LOG.debug("Cannot find active collector while publishing entity " + + entity); + } } catch (IOException e) { LOG.error("Error when publishing entity " + entity); if (LOG.isDebugEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 3b503e5..ca26f58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -28,12 +28,17 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; @@ -291,6 +296,48 @@ public class TestSystemMetricsPublisherForV2 { ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, 0); } + @Test(timeout = 10000) + public void testPutEntityWhenNoCollector() throws Exception { + // Validating the logs as DrainDispatcher won't throw exception + class TestAppender extends AppenderSkeleton { + private final List<LoggingEvent> log = new ArrayList<>(); + + @Override + public boolean requiresLayout() { + return false; + } + + @Override + protected void append(final LoggingEvent loggingEvent) { + log.add(loggingEvent); + } + + @Override + public void close() { + } + + public List<LoggingEvent> getLog() { + return new ArrayList<>(log); + } + } + + TestAppender appender = new TestAppender(); + final Logger logger = Logger.getRootLogger(); + logger.addAppender(appender); + + try { + RMApp app = createRMApp(ApplicationId.newInstance(0, 1)); + metricsPublisher.appCreated(app, app.getStartTime()); + dispatcher.await(); + for (LoggingEvent event : appender.getLog()) { + assertFalse("Dispatcher Crashed", + event.getRenderedMessage().contains("Error in dispatcher thread")); + } + } finally { + logger.removeAppender(appender); + } + } + private RMApp createAppAndRegister(ApplicationId appId) { RMApp app = createRMApp(appId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md index 9aa0a3e..dd7c506 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md @@ -295,14 +295,6 @@ Following are the basic configurations to start Timeline service v.2: <name>yarn.system-metrics-publisher.enabled</name> <value>true</value> </property> - -<property> - <description>The setting that controls whether yarn container events are - published to the timeline service or not by RM. This configuration setting - is for ATS V2.</description> - <name>yarn.rm.system-metrics-publisher.emit-container-events</name> - <value>true</value> -</property> ``` In addition, you may want to set the YARN cluster name to a reasonably unique value in case you --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org