Repository: ambari Updated Branches: refs/heads/trunk 128f26ce9 -> 45bac47f8
AMBARI-15749 : Add AMS post processing function to show the metrics 'diff' over time (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/45bac47f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/45bac47f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/45bac47f Branch: refs/heads/trunk Commit: 45bac47f8980cef1871155b5b3dd201c3055bbcd Parents: 602fb5d Author: Aravindan Vijayan <[email protected]> Authored: Mon Apr 11 16:23:21 2016 -0700 Committer: Aravindan Vijayan <[email protected]> Committed: Mon Apr 11 16:37:19 2016 -0700 ---------------------------------------------------------------------- .../timeline/HBaseTimelineMetricStore.java | 14 ++++++++----- .../metrics/timeline/aggregators/Function.java | 3 ++- .../metrics/timeline/FunctionTest.java | 10 ++++++++++ .../timeline/HBaseTimelineMetricStoreTest.java | 21 +++++++++++++++++++- 4 files changed, 41 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/45bac47f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index a5204e1..ab11333 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -42,6 +42,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -211,30 +212,33 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin for (TimelineMetric metric : metricsList){ String name = metric.getMetricName(); if (name.contains("._rate")){ - updateValuesAsRate(metric.getMetricValues()); + updateValuesAsRate(metric.getMetricValues(), false); + } else if (name.contains("._diff")) { + updateValuesAsRate(metric.getMetricValues(), true); } } return metrics; } - static Map<Long, Double> updateValuesAsRate(Map<Long, Double> metricValues) { + static Map<Long, Double> updateValuesAsRate(Map<Long, Double> metricValues, boolean isDiff) { Long prevTime = null; Double prevVal = null; long step; Double diff; - for (Map.Entry<Long, Double> timeValueEntry : metricValues.entrySet()) { + for(Iterator<Map.Entry<Long, Double>> it = metricValues.entrySet().iterator(); it.hasNext(); ) { + Map.Entry<Long, Double> timeValueEntry = it.next(); Long currTime = timeValueEntry.getKey(); Double currVal = timeValueEntry.getValue(); if (prevTime != null) { step = currTime - prevTime; diff = currVal - prevVal; - Double rate = diff / TimeUnit.MILLISECONDS.toSeconds(step); + Double rate = isDiff ? diff : (diff / TimeUnit.MILLISECONDS.toSeconds(step)); timeValueEntry.setValue(rate); } else { - timeValueEntry.setValue(0.0); + it.remove(); } prevTime = currTime; http://git-wip-us.apache.org/repos/asf/ambari/blob/45bac47f/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java index 6f408a5..ab9d2e9 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/Function.java @@ -125,7 +125,8 @@ public class Function { public enum PostProcessingFunction { NONE(""), - RATE("._rate"); + RATE("._rate"), + DIFF("._diff"); PostProcessingFunction(String suffix){ this.suffix = suffix; http://git-wip-us.apache.org/repos/asf/ambari/blob/45bac47f/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java index 46bc6f8..188f634 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/FunctionTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.fromMetricName; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.ReadFunction.AVG; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.RATE; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.DIFF; import static org.assertj.core.api.Assertions.assertThat; public class FunctionTest { @@ -42,6 +43,15 @@ public class FunctionTest { // Rate support without aggregates f = fromMetricName("Metric._rate"); assertThat(f).isEqualTo(new Function(null, RATE)); + + // Diff support + f = fromMetricName("Metric._diff._avg"); + assertThat(f).isEqualTo(new Function(AVG, DIFF)); + + // Diff support without aggregates + f = fromMetricName("Metric._diff"); + assertThat(f).isEqualTo(new Function(null, DIFF)); + } @Ignore // If unknown function: behavior is best effort query without function http://git-wip-us.apache.org/repos/asf/ambari/blob/45bac47f/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStoreTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStoreTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStoreTest.java index 512a7db..29e2664 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStoreTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStoreTest.java @@ -23,9 +23,11 @@ import org.junit.Test; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.TimeUnit; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.ReadFunction.AVG; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function.PostProcessingFunction.RATE; @@ -78,7 +80,7 @@ public class HBaseTimelineMetricStoreTest { metricValues.put(1454016728371L, 1011.25); // Calculate rate - Map<Long, Double> rates = HBaseTimelineMetricStore.updateValuesAsRate(new TreeMap<>(metricValues)); + Map<Long, Double> rates = HBaseTimelineMetricStore.updateValuesAsRate(new TreeMap<>(metricValues), false); // Make sure rate is zero for (Map.Entry<Long, Double> rateEntry : rates.entrySet()) { @@ -86,4 +88,21 @@ public class HBaseTimelineMetricStoreTest { + ", value = " + rateEntry.getValue(), 0.0, rateEntry.getValue()); } } + + @Test + public void testDiffCalculation() throws Exception { + Map<Long, Double> metricValues = new TreeMap<>(); + metricValues.put(1454016368371L, 1011.25); + metricValues.put(1454016428371L, 1010.25); + metricValues.put(1454016488371L, 1012.25); + metricValues.put(1454016548371L, 1010.25); + metricValues.put(1454016608371L, 1010.25); + + Map<Long, Double> rates = HBaseTimelineMetricStore.updateValuesAsRate(new TreeMap<>(metricValues), true); + + Assert.assertTrue(rates.size()==4); + Assert.assertTrue(rates.containsValue(-1.0)); + Assert.assertTrue(rates.containsValue(2.0)); + Assert.assertTrue(rates.containsValue(0.0)); + } }
