This is an automated email from the ASF dual-hosted git repository. pbacsko pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new a967ab0 YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery to take too much time. Contributed by Qi Zhu. a967ab0 is described below commit a967ab06f2e802f30ed34eb4ceb2742e4df8be52 Author: Peter Bacsko <pbac...@cloudera.com> AuthorDate: Tue Apr 27 14:01:53 2021 +0200 YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery to take too much time. Contributed by Qi Zhu. --- .../org/apache/hadoop/yarn/event/AsyncDispatcher.java | 18 +++++++++++++++++- .../apache/hadoop/yarn/event/TestAsyncDispatcher.java | 2 -- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java index 667515d..ba6bb43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java @@ -25,7 +25,11 @@ import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.yarn.metrics.EventTypeMetrics; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -93,6 +97,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher { private Clock clock = new MonotonicClock(); + private ThreadPoolExecutor printEventDetailsExecutor; + /** * The thread name for dispatcher. */ @@ -179,6 +185,15 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher { YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD, YarnConfiguration. DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD); + + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat("PrintEventDetailsThread #%d") + .build(); + // Thread pool for async print event details, + // to prevent wasting too much time for RM. + printEventDetailsExecutor = new ThreadPoolExecutor( + 1, 5, 10, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), threadFactory); } @Override @@ -222,6 +237,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher { LOG.warn("Interrupted Exception while stopping", ie); } } + printEventDetailsExecutor.shutdownNow(); // stop all the components super.serviceStop(); @@ -319,7 +335,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher { if (qSize != 0 && qSize % detailsInterval == 0 && lastEventDetailsQueueSizeLogged != qSize) { lastEventDetailsQueueSizeLogged = qSize; - printEventQueueDetails(); + printEventDetailsExecutor.submit(this::printEventQueueDetails); printTrigger = true; } int remCapacity = eventQueue.remainingCapacity(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java index 7d2572a..8b2dfa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java @@ -187,8 +187,6 @@ public class TestAsyncDispatcher { when(event.getType()).thenReturn(TestEnum.TestEventType); dispatcher.getEventHandler().handle(event); } - verify(log, atLeastOnce()).info("Event type: TestEventType, " + - "Event record counter: 5000"); Thread.sleep(2000); //Make sure more than one event to take verify(log, atLeastOnce()). --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org