YARN-6931. Make the aggregation interval in AppLevelTimelineCollector configurable. (Abhishek Modi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24a89825 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24a89825 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24a89825 Branch: refs/heads/HADOOP-15461 Commit: 24a89825f0cbc92b0a462152bc765e2195edd5a1 Parents: e9ea902 Author: Haibo Chen <haiboc...@apache.org> Authored: Tue Jun 12 10:03:07 2018 -0700 Committer: Haibo Chen <haiboc...@apache.org> Committed: Tue Jun 12 10:03:34 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++++ .../src/main/resources/yarn-default.xml | 9 +++++++++ .../collector/AppLevelTimelineCollectorWithAgg.java | 14 +++++++++----- 3 files changed, 27 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a89825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f7f82f8..5292a25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2665,6 +2665,15 @@ public class YarnConfiguration extends Configuration { TIMELINE_SERVICE_PREFIX + "read.authentication.enabled"; /** + * The name for setting that controls how often in-memory app level + * aggregation is kicked off in timeline collector. + */ + public static final String TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS = + TIMELINE_SERVICE_PREFIX + "app-aggregation-interval-secs"; + + public static final int + DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS = 15; + /** * The default setting for authentication checks for reading timeline * service v2 data. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a89825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index b0ffc48..2cc842f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2547,6 +2547,15 @@ <property> <description> + The setting that controls how often in-memory app level + aggregation is kicked off in timeline collector. + </description> + <name>yarn.timeline-service.app-aggregation-interval-secs</name> + <value>15</value> + </property> + + <property> + <description> The default hdfs location for flowrun coprocessor jar. </description> <name>yarn.timeline-service.hbase.coprocessor.jar.hdfs.location http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a89825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java index d7f47c8..aa041a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -50,7 +51,7 @@ public class AppLevelTimelineCollectorWithAgg LoggerFactory.getLogger(TimelineCollector.class); private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1; - private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15; + private int aggregationExecutorIntervalSecs; private static Set<String> entityTypesSkipAggregation = initializeSkipSet(); @@ -71,6 +72,11 @@ public class AppLevelTimelineCollectorWithAgg @Override protected void serviceInit(Configuration conf) throws Exception { + aggregationExecutorIntervalSecs = conf.getInt( + YarnConfiguration.TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS, + YarnConfiguration. + DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS + ); super.serviceInit(conf); } @@ -84,10 +90,8 @@ public class AppLevelTimelineCollectorWithAgg .build()); appAggregator = new AppLevelAggregator(); appAggregationExecutor.scheduleAtFixedRate(appAggregator, - AppLevelTimelineCollectorWithAgg. - AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS, - AppLevelTimelineCollectorWithAgg. - AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS, + aggregationExecutorIntervalSecs, + aggregationExecutorIntervalSecs, TimeUnit.SECONDS); super.serviceStart(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
