AMBARI-17843 : App data aggregated for hosted apps is being calcualted for all apps, not just configured ones (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e8751bba Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e8751bba Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e8751bba Branch: refs/heads/branch-2.4 Commit: e8751bba7043c954bd21ccacfcf470e61c7a0add Parents: 30b8ddf Author: Aravindan Vijayan <[email protected]> Authored: Fri Jul 22 13:38:46 2016 -0700 Committer: Aravindan Vijayan <[email protected]> Committed: Fri Jul 22 14:02:19 2016 -0700 ---------------------------------------------------------------------- .../TimelineMetricAppAggregator.java | 61 ++++++++++++++------ .../discovery/TimelineMetricMetadataKey.java | 4 ++ .../TimelineMetricMetadataManager.java | 6 ++ 3 files changed, 52 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e8751bba/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java index 05beb76..0d6a09e 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java @@ -21,6 +21,9 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; import java.util.ArrayList; @@ -47,11 +50,13 @@ public class TimelineMetricAppAggregator { private final List<String> appIdsToAggregate; private final Map<String, Set<String>> hostedAppsMap; Map<TimelineClusterMetric, MetricClusterAggregate> aggregateClusterMetrics; + TimelineMetricMetadataManager metadataManagerInstance; public TimelineMetricAppAggregator(TimelineMetricMetadataManager metadataManager, Configuration metricsConf) { appIdsToAggregate = getAppIdsForHostAggregation(metricsConf); hostedAppsMap = metadataManager.getHostedAppsCache(); + metadataManagerInstance = metadataManager; LOG.info("AppIds configured for aggregation: " + appIdsToAggregate); } @@ -122,27 +127,45 @@ public class TimelineMetricAppAggregator { return; } + TimelineMetricMetadataKey appKey = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), HOST_APP_ID); Set<String> apps = hostedAppsMap.get(hostname); for (String appId : apps) { - // Add a new cluster aggregate metric if none exists - TimelineClusterMetric appTimelineClusterMetric = - new TimelineClusterMetric(clusterMetric.getMetricName(), - appId, - clusterMetric.getInstanceId(), - clusterMetric.getTimestamp(), - clusterMetric.getType() - ); - - MetricClusterAggregate clusterAggregate = aggregateClusterMetrics.get(appTimelineClusterMetric); - - if (clusterAggregate == null) { - clusterAggregate = new MetricClusterAggregate(metricValue, 1, null, metricValue, metricValue); - aggregateClusterMetrics.put(appTimelineClusterMetric, clusterAggregate); - } else { - clusterAggregate.updateSum(metricValue); - clusterAggregate.updateNumberOfHosts(1); - clusterAggregate.updateMax(metricValue); - clusterAggregate.updateMin(metricValue); + if (appIdsToAggregate.contains(appId)) { + + appKey.setAppId(appId); + TimelineMetricMetadata appMetadata = metadataManagerInstance.getMetadataCacheValue(appKey); + if (appMetadata == null) { + TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(clusterMetric.getMetricName(), HOST_APP_ID); + TimelineMetricMetadata hostMetricMetadata = metadataManagerInstance.getMetadataCacheValue(key); + + if (hostMetricMetadata != null) { + TimelineMetricMetadata timelineMetricMetadata = new TimelineMetricMetadata(clusterMetric.getMetricName(), + appId, hostMetricMetadata.getUnits(), hostMetricMetadata.getType(), hostMetricMetadata.getSeriesStartTime(), + hostMetricMetadata.isSupportsAggregates()); + metadataManagerInstance.putIfModifiedTimelineMetricMetadata(timelineMetricMetadata); + } + } + + // Add a new cluster aggregate metric if none exists + TimelineClusterMetric appTimelineClusterMetric = + new TimelineClusterMetric(clusterMetric.getMetricName(), + appId, + clusterMetric.getInstanceId(), + clusterMetric.getTimestamp(), + clusterMetric.getType() + ); + + MetricClusterAggregate clusterAggregate = aggregateClusterMetrics.get(appTimelineClusterMetric); + + if (clusterAggregate == null) { + clusterAggregate = new MetricClusterAggregate(metricValue, 1, null, metricValue, metricValue); + aggregateClusterMetrics.put(appTimelineClusterMetric, clusterAggregate); + } else { + clusterAggregate.updateSum(metricValue); + clusterAggregate.updateNumberOfHosts(1); + clusterAggregate.updateMax(metricValue); + clusterAggregate.updateMin(metricValue); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/e8751bba/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java index ec97ee5..504b502 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataKey.java @@ -34,6 +34,10 @@ public class TimelineMetricMetadataKey { return appId; } + public void setAppId(String appId) { + this.appId = appId; + } + @Override public boolean equals(Object o) { if (this == o) return true; http://git-wip-us.apache.org/repos/asf/ambari/blob/e8751bba/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java index 9fa2066..a79f9f6 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java @@ -24,6 +24,8 @@ import org.apache.hadoop.metrics2.sink.timeline.MetadataException; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineClusterMetric; + import java.sql.SQLException; import java.util.Collection; import java.util.HashSet; @@ -96,6 +98,10 @@ public class TimelineMetricMetadataManager { return METADATA_CACHE; } + public TimelineMetricMetadata getMetadataCacheValue(TimelineMetricMetadataKey key) { + return METADATA_CACHE.get(key); + } + public Map<String, Set<String>> getHostedAppsCache() { return HOSTED_APPS_MAP; }
