Repository: ambari Updated Branches: refs/heads/trunk 4999e6cb9 -> 1be5dfd92
AMBARI-17027: Metrics Collector API: Introduce basic series aggregation functions (Jungtaek Lim via avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1be5dfd9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1be5dfd9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1be5dfd9 Branch: refs/heads/trunk Commit: 1be5dfd92a0b3aa085abb777d309312c324bb3e7 Parents: 4999e6c Author: Aravindan Vijayan <[email protected]> Authored: Thu Jun 30 11:04:18 2016 -0700 Committer: Aravindan Vijayan <[email protected]> Committed: Thu Jun 30 11:04:18 2016 -0700 ---------------------------------------------------------------------- .../ambari-metrics/datasource.js | 22 ++- .../ambari-metrics/partials/query.editor.html | 12 ++ .../ambari-metrics/queryCtrl.js | 8 +- .../timeline/HBaseTimelineMetricStore.java | 31 ++- .../metrics/timeline/TimelineMetricStore.java | 4 +- .../timeline/TimelineMetricStoreWatcher.java | 2 +- ...tTimelineMetricsSeriesAggregateFunction.java | 97 ++++++++++ .../function/SeriesAggregateFunction.java | 42 +++++ .../TimelineMetricsSeriesAggregateFunction.java | 25 +++ ...neMetricsSeriesAggregateFunctionFactory.java | 41 ++++ ...melineMetricsSeriesAvgAggregateFunction.java | 39 ++++ ...melineMetricsSeriesMaxAggregateFunction.java | 41 ++++ ...melineMetricsSeriesMinAggregateFunction.java | 41 ++++ ...melineMetricsSeriesSumAggregateFunction.java | 39 ++++ .../webapp/TimelineWebServices.java | 8 +- .../timeline/TestTimelineMetricStore.java | 3 +- .../TimelineMetricStoreWatcherTest.java | 5 +- ...elineMetricsSeriesAggregateFunctionTest.java | 188 +++++++++++++++++++ 18 files changed, 629 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js index 6a4f01b..66043c5 100644 --- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js +++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js @@ -165,9 +165,10 @@ define([ + target.precision; var metricAggregator = target.aggregator === "none" ? '' : '._' + target.aggregator; var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform + metricAggregator + "&hostname=" + target.hosts + '&appId=' + target.app + '&startTime=' + from + - '&endTime=' + to + precision).then( + '&endTime=' + to + precision + seriesAggregator).then( getMetricsData(target) ); }; @@ -186,9 +187,10 @@ define([ + target.precision; var metricAggregator = target.aggregator === "none" ? '' : '._' + target.aggregator; var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform + metricAggregator + '&hostname=' + tHost + '&appId=' + target.app + '&startTime=' + from + - '&endTime=' + to + precision).then( + '&endTime=' + to + precision + seriesAggregator).then( getMetricsData(target) ); }; @@ -206,10 +208,11 @@ define([ topN = '&topN=' + metricTopN[0].current.value +'&topNFunction=' + metricTopAgg[0].current.value + '&isBottomN='+ isBottomN; } var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; var templatedComponent = (_.isEmpty(tComponent)) ? target.app : tComponent; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + metricTransform + metricAggregator + '&hostname=' + target.templatedHost + '&appId=' + templatedComponent + '&startTime=' + from + - '&endTime=' + to + precision + topN).then( + '&endTime=' + to + precision + topN + seriesAggregator).then( allHostMetricsData(target) ); }; @@ -218,17 +221,19 @@ define([ + target.precision; var metricAggregator = target.aggregator === "none" ? '' : '._' + target.aggregator; var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.queue + metricTransform + metricAggregator + '&appId=resourcemanager&startTime=' + from + - '&endTime=' + to + precision).then( + '&endTime=' + to + precision + seriesAggregator).then( getMetricsData(target) ); }; var getHbaseAppIdData = function(target) { var precision = target.precision === 'default' || typeof target.precision == 'undefined' ? '' : '&precision=' + target.precision; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.hbMetric + '&appId=hbase&startTime=' - + from + '&endTime=' + to + precision).then( + + from + '&endTime=' + to + precision + seriesAggregator).then( allHostMetricsData(target) ); }; @@ -238,9 +243,10 @@ define([ + target.precision; var metricAggregator = target.aggregator === "none" ? '' : '._' + target.aggregator; var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.kbMetric + metricTransform + metricAggregator + '&appId=kafka_broker&startTime=' + from + - '&endTime=' + to + precision).then( + '&endTime=' + to + precision + seriesAggregator).then( getMetricsData(target) ); }; @@ -249,8 +255,9 @@ define([ + target.precision; var metricAggregator = target.aggregator === "none" ? '' : '._' + target.aggregator; var metricTransform = !target.transform || target.transform === "none" ? '' : '._' + target.transform; + var seriesAggregator = !target.seriesAggregator || target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' + target.seriesAggregator; return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.nnMetric + metricTransform - + metricAggregator + '&appId=namenode&startTime=' + from + '&endTime=' + to + precision).then( + + metricAggregator + '&appId=namenode&startTime=' + from + '&endTime=' + to + precision + seriesAggregator).then( allHostMetricsData(target) ); }; @@ -672,6 +679,7 @@ define([ ]); return aggregatorsPromise; }; + return AmbariMetricsDatasource; }); } http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html index b034c03..3f322c1 100644 --- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html +++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html @@ -109,6 +109,7 @@ <i class="fa fa-warning"></i> </a> </li> + </ul> <div class="clearfix"></div> @@ -150,5 +151,16 @@ ng-change="targetBlur()"> </select> </li> + + <li class="tight-form-item"> + Series Aggregator + </li> + <li> + <select ng-model="target.seriesAggregator" class="tight-form-input input-small" + ng-options="seriesAggregator for seriesAggregator in seriesAggregators" + ng-init="seriesAggregator()" + ng-change="targetBlur()"> + </select> + </li> <div class="clearfix"></div> </div> http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-grafana/ambari-metrics/queryCtrl.js ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/queryCtrl.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/queryCtrl.js index 2eb3613..a26e7d0 100644 --- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/queryCtrl.js +++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/queryCtrl.js @@ -31,6 +31,7 @@ define([ $scope.aggregators = ['none','avg', 'sum', 'min', 'max']; $scope.precisions = ['default','seconds', 'minutes', 'hours', 'days']; $scope.transforms = ['none','diff','rate']; + $scope.seriesAggregators = ['none', 'avg', 'sum', 'min', 'max']; if (!$scope.target.aggregator) { $scope.target.aggregator = 'avg'; @@ -45,6 +46,11 @@ define([ $scope.target.transform = "none"; } }; + $scope.seriesAggregator = function () { + if (typeof $scope.target.seriesAggregator == 'undefined') { + $scope.target.seriesAggregator = "none"; + } + }; $scope.$watch('target.app', function (newValue) { if (newValue === '') { $scope.target.metric = ''; @@ -144,4 +150,4 @@ define([ $scope.init(); }); - }); \ No newline at end of file + }); http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index 1b2d02f..d8c1ddb 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -41,6 +40,9 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline. import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.Condition; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.ConditionBuilder; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.TopNCondition; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function.SeriesAggregateFunction; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function.TimelineMetricsSeriesAggregateFunction; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function.TimelineMetricsSeriesAggregateFunctionFactory; import java.io.IOException; import java.net.UnknownHostException; @@ -191,7 +193,7 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin public TimelineMetrics getTimelineMetrics(List<String> metricNames, List<String> hostnames, String applicationId, String instanceId, Long startTime, Long endTime, Precision precision, Integer limit, - boolean groupedByHosts, TopNConfig topNConfig) throws SQLException, IOException { + boolean groupedByHosts, TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException, IOException { if (metricNames == null || metricNames.isEmpty()) { throw new IllegalArgumentException("No metric name filter specified."); @@ -203,6 +205,13 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin if (limit != null && limit > PhoenixHBaseAccessor.RESULTSET_LIMIT){ throw new IllegalArgumentException("Limit too big"); } + + TimelineMetricsSeriesAggregateFunction seriesAggrFunctionInstance = null; + if (!StringUtils.isEmpty(seriesAggregateFunction)) { + SeriesAggregateFunction func = SeriesAggregateFunction.getFunction(seriesAggregateFunction); + seriesAggrFunctionInstance = TimelineMetricsSeriesAggregateFunctionFactory.newInstance(func); + } + Map<String, List<Function>> metricFunctions = parseMetricNamesToAggregationFunctions(metricNames); @@ -242,7 +251,14 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin } else { metrics = hBaseAccessor.getMetricRecords(condition, metricFunctions); } - return postProcessMetrics(metrics); + + metrics = postProcessMetrics(metrics); + + if (metrics.getMetrics().size() == 0) { + return metrics; + } + + return seriesAggregateMetrics(seriesAggrFunctionInstance, metrics); } private TimelineMetrics postProcessMetrics(TimelineMetrics metrics) { @@ -260,6 +276,15 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin return metrics; } + private TimelineMetrics seriesAggregateMetrics(TimelineMetricsSeriesAggregateFunction seriesAggrFuncInstance, + TimelineMetrics metrics) { + if (seriesAggrFuncInstance != null) { + TimelineMetric appliedMetric = seriesAggrFuncInstance.apply(metrics); + metrics.setMetrics(Collections.singletonList(appliedMetric)); + } + return metrics; + } + static Map<Long, Double> updateValuesAsRate(Map<Long, Double> metricValues, boolean isDiff) { Long prevTime = null; Double prevVal = null; http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java index e37bc4d..15644ed 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java @@ -44,6 +44,8 @@ public interface TimelineMetricStore { * @param limit Override default result limit * @param groupedByHosts Group {@link TimelineMetric} by metric name, hostname, * app id and instance id + * @param seriesAggregateFunction Specify this when caller want to aggregate multiple metrics + * series into one. [ SUM, AVG, MIN, MAX ] * * @return {@link TimelineMetric} * @throws java.sql.SQLException @@ -51,7 +53,7 @@ public interface TimelineMetricStore { TimelineMetrics getTimelineMetrics(List<String> metricNames, List<String> hostnames, String applicationId, String instanceId, Long startTime, Long endTime, Precision precision, Integer limit, boolean groupedByHosts, - TopNConfig topNConfig) + TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException, IOException; /** http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java index 7d49070..aa53430 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java @@ -105,7 +105,7 @@ public class TimelineMetricStoreWatcher implements Runnable { TimelineMetrics timelineMetrics = timelineMetricStore.getTimelineMetrics( Collections.singletonList(FAKE_METRIC_NAME), Collections.singletonList(FAKE_HOSTNAME), FAKE_APP_ID, null, startTime - delay * 2 * 1000, - startTime + delay * 2 * 1000, Precision.SECONDS, 1, true, null); + startTime + delay * 2 * 1000, Precision.SECONDS, 1, true, null, null); return timelineMetrics.getMetrics().get(0); } }; http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java new file mode 100644 index 0000000..634e51d --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/AbstractTimelineMetricsSeriesAggregateFunction.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import com.google.common.base.Joiner; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +public abstract class AbstractTimelineMetricsSeriesAggregateFunction + implements TimelineMetricsSeriesAggregateFunction { + + @Override + public TimelineMetric apply(TimelineMetrics timelineMetrics) { + Set<String> metricNameSet = new TreeSet<>(); + Set<String> hostNameSet = new TreeSet<>(); + Set<String> appIdSet = new TreeSet<>(); + Set<String> instanceIdSet = new TreeSet<>(); + TreeMap<Long, List<Double>> metricValues = new TreeMap<>(); + + for (TimelineMetric timelineMetric : timelineMetrics.getMetrics()) { + metricNameSet.add(timelineMetric.getMetricName()); + addToSetOnlyNotNull(hostNameSet, timelineMetric.getHostName()); + addToSetOnlyNotNull(appIdSet, timelineMetric.getAppId()); + addToSetOnlyNotNull(instanceIdSet, timelineMetric.getInstanceId()); + + for (Map.Entry<Long, Double> metricValue : timelineMetric.getMetricValues().entrySet()) { + Long timestamp = metricValue.getKey(); + Double value = metricValue.getValue(); + if (!metricValues.containsKey(timestamp)) { + metricValues.put(timestamp, new LinkedList<Double>()); + } + metricValues.get(timestamp).add(value); + } + } + + TreeMap<Long, Double> aggregatedMetricValues = new TreeMap<>(); + for (Map.Entry<Long, List<Double>> metricValue : metricValues.entrySet()) { + List<Double> values = metricValue.getValue(); + if (values.size() == 0) { + throw new IllegalArgumentException("count of values should be more than 0"); + } + aggregatedMetricValues.put(metricValue.getKey(), applyFunction(values)); + } + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName(getMetricName(metricNameSet.iterator())); + timelineMetric.setHostName(joinStringsWithComma(hostNameSet.iterator())); + timelineMetric.setAppId(joinStringsWithComma(appIdSet.iterator())); + timelineMetric.setInstanceId(joinStringsWithComma(instanceIdSet.iterator())); + if (aggregatedMetricValues.size() > 0) { + timelineMetric.setStartTime(aggregatedMetricValues.firstKey()); + } + timelineMetric.setMetricValues(aggregatedMetricValues); + return timelineMetric; + } + + protected String getMetricName(Iterator<String> metricNames) { + return getFunctionName() + "(" + Joiner.on(",").join(metricNames) + ")"; + } + + protected String joinStringsWithComma(Iterator<String> hostNames) { + return Joiner.on(",").join(hostNames); + } + + protected abstract Double applyFunction(List<Double> values); + protected abstract String getFunctionName(); + + private void addToSetOnlyNotNull(Set<String> set, String value) { + if (value != null) { + set.add(value); + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java new file mode 100644 index 0000000..ef5e441 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/SeriesAggregateFunction.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function; + +public enum SeriesAggregateFunction { + AVG, MIN, MAX, SUM; + + public static boolean isPresent(String functionName) { + try { + SeriesAggregateFunction.valueOf(functionName.toUpperCase()); + } catch (IllegalArgumentException e) { + return false; + } + return true; + } + + public static SeriesAggregateFunction getFunction(String functionName) throws Function.FunctionFormatException { + try { + return SeriesAggregateFunction.valueOf(functionName.toUpperCase()); + } catch (NullPointerException | IllegalArgumentException e) { + throw new Function.FunctionFormatException( + "Function should be sum, avg, min, max. Got " + functionName, e); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java new file mode 100644 index 0000000..bdb5fe5 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunction.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; + +public interface TimelineMetricsSeriesAggregateFunction { + TimelineMetric apply(TimelineMetrics timelineMetrics); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java new file mode 100644 index 0000000..63a0fdc --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionFactory.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function; + +public class TimelineMetricsSeriesAggregateFunctionFactory { + private TimelineMetricsSeriesAggregateFunctionFactory() { + } + + public static TimelineMetricsSeriesAggregateFunction newInstance(SeriesAggregateFunction func) { + switch (func) { + case AVG: + return new TimelineMetricsSeriesAvgAggregateFunction(); + case MIN: + return new TimelineMetricsSeriesMinAggregateFunction(); + case MAX: + return new TimelineMetricsSeriesMaxAggregateFunction(); + case SUM: + return new TimelineMetricsSeriesSumAggregateFunction(); + default: + throw new Function.FunctionFormatException("Function should be sum, avg, min, max. Got " + + func.name()); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java new file mode 100644 index 0000000..f7c66ed --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAvgAggregateFunction.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import java.util.List; + +public class TimelineMetricsSeriesAvgAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction { + private static final String FUNCTION_NAME = "AVG"; + + @Override + protected Double applyFunction(List<Double> values) { + double sum = 0.0d; + for (Double value : values) { + sum += value; + } + + return sum / values.size(); + } + + @Override + protected String getFunctionName() { + return FUNCTION_NAME; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java new file mode 100644 index 0000000..0b79f78 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMaxAggregateFunction.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import java.util.List; + +public class TimelineMetricsSeriesMaxAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction { + private static final String FUNCTION_NAME = "MAX"; + + @Override + protected Double applyFunction(List<Double> values) { + double max = Double.MIN_VALUE; + for (Double value : values) { + if (value > max) { + max = value; + } + } + + return max; + } + + @Override + protected String getFunctionName() { + return FUNCTION_NAME; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java new file mode 100644 index 0000000..7146aa2 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesMinAggregateFunction.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import java.util.List; + +public class TimelineMetricsSeriesMinAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction { + private static final String FUNCTION_NAME = "MIN"; + + @Override + protected Double applyFunction(List<Double> values) { + double min = Double.MAX_VALUE; + for (Double value : values) { + if (value < min) { + min = value; + } + } + + return min; + } + + @Override + protected String getFunctionName() { + return FUNCTION_NAME; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java new file mode 100644 index 0000000..2a15c95 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesSumAggregateFunction.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import java.util.List; + +public class TimelineMetricsSeriesSumAggregateFunction extends AbstractTimelineMetricsSeriesAggregateFunction { + private static final String FUNCTION_NAME = "SUM"; + + @Override + protected Double applyFunction(List<Double> values) { + double sum = 0.0d; + for (Double value : values) { + sum += value; + } + + return sum; + } + + @Override + protected String getFunctionName() { + return FUNCTION_NAME; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index ee3a097..6b15e29 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -343,7 +343,8 @@ public class TimelineWebServices { @QueryParam("grouped") String grouped, @QueryParam("topN") String topN, @QueryParam("topNFunction") String topNFunction, - @QueryParam("isBottomN") String isBottomN + @QueryParam("isBottomN") String isBottomN, + @QueryParam("seriesAggregateFunction") String seriesAggregateFunction ) { init(res); try { @@ -352,14 +353,15 @@ public class TimelineWebServices { "appId: " + appId + ", instanceId: " + instanceId + ", " + "hostname: " + hostname + ", startTime: " + startTime + ", " + "endTime: " + endTime + ", " + - "precision: " + precision); + "precision: " + precision + "seriesAggregateFunction: " + seriesAggregateFunction); } return timelineMetricStore.getTimelineMetrics( parseListStr(metricNames, ","), parseListStr(hostname, ","), appId, instanceId, parseLongStr(startTime), parseLongStr(endTime), Precision.getPrecision(precision), parseIntStr(limit), - parseBoolean(grouped), parseTopNConfig(topN, topNFunction, isBottomN)); + parseBoolean(grouped), parseTopNConfig(topN, topNFunction, isBottomN), + seriesAggregateFunction); } catch (NumberFormatException ne) { throw new BadRequestException("startTime and limit should be numeric " + http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java index cfd1f58..2e2d3a8 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java @@ -38,7 +38,8 @@ public class TestTimelineMetricStore implements TimelineMetricStore { @Override public TimelineMetrics getTimelineMetrics(List<String> metricNames, List<String> hostnames, String applicationId, String instanceId, Long startTime, - Long endTime, Precision precision, Integer limit, boolean groupedByHost, TopNConfig topNConfig) throws SQLException, + Long endTime, Precision precision, Integer limit, boolean groupedByHost, + TopNConfig topNConfig, String seriesAggregateFunction) throws SQLException, IOException { TimelineMetrics timelineMetrics = new TimelineMetrics(); List<TimelineMetric> metricList = new ArrayList<TimelineMetric>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java index a94f4c5..54b8442 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcherTest.java @@ -32,6 +32,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import java.util.List; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; @@ -57,7 +58,7 @@ public class TimelineMetricStoreWatcherTest { expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(), EasyMock.<List<String>>anyObject(), anyObject(String.class), anyObject(String.class), anyObject(Long.class), anyObject(Long.class), - eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class))) + eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class), anyString())) .andReturn(null).anyTimes(); mockStatic(ExitUtil.class); @@ -84,7 +85,7 @@ public class TimelineMetricStoreWatcherTest { expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(), EasyMock.<List<String>>anyObject(), anyObject(String.class), anyObject(String.class), anyObject(Long.class), anyObject(Long.class), - eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class))) + eq(Precision.SECONDS), eq(1), eq(true), anyObject(TopNConfig.class), anyString())) .andReturn(null).anyTimes(); String msg = "Error getting metrics from TimelineMetricStore. " + http://git-wip-us.apache.org/repos/asf/ambari/blob/1be5dfd9/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java new file mode 100644 index 0000000..bd54bad --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/function/TimelineMetricsSeriesAggregateFunctionTest.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.function; + +import com.google.common.collect.Lists; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; +import org.junit.Test; + +import java.util.Map; +import java.util.TreeMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TimelineMetricsSeriesAggregateFunctionTest { + + public static final double DELTA = 0.00000000001; + + @Test public void testSeriesAggregateBySummation() throws Exception { + TimelineMetrics testMetrics = getTestObject(); + // all TimelineMetric are having same values + TreeMap<Long, Double> metricValues = testMetrics.getMetrics().get(0).getMetricValues(); + + TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory + .newInstance(SeriesAggregateFunction.SUM); + TimelineMetric aggregatedMetric = function.apply(testMetrics); + + String aggregatedMetricName = aggregatedMetric.getMetricName(); + String aggregatedHostName = aggregatedMetric.getHostName(); + String aggregatedAppId = aggregatedMetric.getAppId(); + String aggregatedInstanceId = aggregatedMetric.getInstanceId(); + + for (TimelineMetric testMetric : testMetrics.getMetrics()) { + assertTrue(aggregatedMetricName.contains(testMetric.getMetricName())); + assertTrue(aggregatedHostName.contains(testMetric.getHostName())); + if (!testMetric.getMetricName().equals("byte_in.3")) { + assertTrue(aggregatedAppId.contains(testMetric.getAppId())); + assertTrue(aggregatedInstanceId.contains(testMetric.getInstanceId())); + } + } + + TreeMap<Long, Double> summationMetricValues = aggregatedMetric.getMetricValues(); + assertEquals(3, summationMetricValues.size()); + for (Map.Entry<Long, Double> tsAndValue : summationMetricValues.entrySet()) { + assertEquals(metricValues.get(tsAndValue.getKey()) * 3, tsAndValue.getValue(), DELTA); + } + } + + @Test public void testSeriesAggregateByAverage() throws Exception { + TimelineMetrics testMetrics = getTestObject(); + // all TimelineMetric are having same values + TreeMap<Long, Double> metricValues = testMetrics.getMetrics().get(0).getMetricValues(); + + TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory + .newInstance(SeriesAggregateFunction.AVG); + TimelineMetric aggregatedMetric = function.apply(testMetrics); + + // checks only values, others are covered by testSeriesAggregateBySummation + TreeMap<Long, Double> averageMetricValues = aggregatedMetric.getMetricValues(); + assertEquals(3, averageMetricValues.size()); + for (Map.Entry<Long, Double> tsAndValue : averageMetricValues.entrySet()) { + assertEquals(metricValues.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA); + } + } + + @Test public void testSeriesAggregateByMax() throws Exception { + TimelineMetrics testMetrics = getTestObject(); + + // override metric values + TreeMap<Long, Double> metricValues = new TreeMap<>(); + metricValues.put(1L, 1.0); + metricValues.put(2L, 2.0); + metricValues.put(3L, 3.0); + + testMetrics.getMetrics().get(0).setMetricValues(metricValues); + + TreeMap<Long, Double> metricValues2 = new TreeMap<>(); + metricValues2.put(1L, 2.0); + metricValues2.put(2L, 4.0); + metricValues2.put(3L, 6.0); + + testMetrics.getMetrics().get(1).setMetricValues(metricValues2); + + TreeMap<Long, Double> metricValues3 = new TreeMap<>(); + metricValues3.put(1L, 3.0); + metricValues3.put(2L, 6.0); + metricValues3.put(3L, 9.0); + + testMetrics.getMetrics().get(2).setMetricValues(metricValues3); + + TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory + .newInstance(SeriesAggregateFunction.MAX); + TimelineMetric aggregatedMetric = function.apply(testMetrics); + + // checks only values, others are covered by testSeriesAggregateBySummation + TreeMap<Long, Double> maxMetricValues = aggregatedMetric.getMetricValues(); + assertEquals(3, maxMetricValues.size()); + for (Map.Entry<Long, Double> tsAndValue : maxMetricValues.entrySet()) { + assertEquals(metricValues3.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA); + } + } + + @Test public void testSeriesAggregateByMin() throws Exception { + TimelineMetrics testMetrics = getTestObject(); + + // override metric values + TreeMap<Long, Double> metricValues = new TreeMap<>(); + metricValues.put(1L, 1.0); + metricValues.put(2L, 2.0); + metricValues.put(3L, 3.0); + + testMetrics.getMetrics().get(0).setMetricValues(metricValues); + + TreeMap<Long, Double> metricValues2 = new TreeMap<>(); + metricValues2.put(1L, 2.0); + metricValues2.put(2L, 4.0); + metricValues2.put(3L, 6.0); + + testMetrics.getMetrics().get(1).setMetricValues(metricValues2); + + TreeMap<Long, Double> metricValues3 = new TreeMap<>(); + metricValues3.put(1L, 3.0); + metricValues3.put(2L, 6.0); + metricValues3.put(3L, 9.0); + + testMetrics.getMetrics().get(2).setMetricValues(metricValues3); + + TimelineMetricsSeriesAggregateFunction function = TimelineMetricsSeriesAggregateFunctionFactory + .newInstance(SeriesAggregateFunction.MIN); + TimelineMetric aggregatedMetric = function.apply(testMetrics); + + // checks only values, others are covered by testSeriesAggregateBySummation + TreeMap<Long, Double> minMetricValues = aggregatedMetric.getMetricValues(); + assertEquals(3, minMetricValues.size()); + for (Map.Entry<Long, Double> tsAndValue : minMetricValues.entrySet()) { + assertEquals(metricValues.get(tsAndValue.getKey()), tsAndValue.getValue(), DELTA); + } + } + + private TimelineMetrics getTestObject() { + TimelineMetric metric = new TimelineMetric(); + metric.setMetricName("byte_in.1"); + metric.setHostName("host1"); + metric.setAppId("app1"); + metric.setInstanceId("instance1"); + + TimelineMetric metric2 = new TimelineMetric(); + metric2.setMetricName("byte_in.2"); + metric2.setHostName("host2"); + metric2.setAppId("app2"); + metric2.setInstanceId("instance2"); + + TimelineMetric metric3 = new TimelineMetric(); + metric3.setMetricName("byte_in.3"); + metric3.setHostName("host3"); + // appId and instanceId for metric3 are null + + TreeMap<Long, Double> metricValues = new TreeMap<>(); + metricValues.put(1L, 3.0); + metricValues.put(2L, 2 * 3.0); + metricValues.put(3L, 3 * 3.0); + + metric.setMetricValues(metricValues); + metric2.setMetricValues(metricValues); + metric3.setMetricValues(metricValues); + + TimelineMetrics metrics = new TimelineMetrics(); + metrics.setMetrics(Lists.newArrayList(metric, metric2, metric3)); + + return metrics; + } +} \ No newline at end of file
