AMBARI-21686 : Implement a test driver that provides a set of metric series with different kinds of metric behavior. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/63e74355 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/63e74355 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/63e74355 Branch: refs/heads/branch-3.0-ams Commit: 63e74355776b454762ab423e3516dfc3c12355e5 Parents: 8cad9eb Author: Aravindan Vijayan <[email protected]> Authored: Thu Sep 21 15:40:35 2017 -0700 Committer: Aravindan Vijayan <[email protected]> Committed: Thu Sep 21 15:40:35 2017 -0700 ---------------------------------------------------------------------- .../ambari-metrics-alertservice/pom.xml | 17 +- .../ambari/metrics/alertservice/R/AmsRTest.java | 147 ------- .../alertservice/R/RFunctionInvoker.java | 192 ---------- .../metrics/alertservice/common/DataSet.java | 38 -- .../alertservice/common/MethodResult.java | 27 -- .../alertservice/common/MetricAnomaly.java | 69 ---- .../metrics/alertservice/common/ResultSet.java | 43 --- .../common/SingleValuedTimelineMetric.java | 103 ----- .../alertservice/common/StatisticUtils.java | 77 ---- .../alertservice/common/TimelineMetric.java | 238 ------------ .../alertservice/common/TimelineMetrics.java | 129 ------- .../methods/MetricAnomalyModel.java | 29 -- .../metrics/alertservice/methods/ema/EmaDS.java | 70 ---- .../alertservice/methods/ema/EmaModel.java | 129 ------- .../methods/ema/EmaModelLoader.java | 46 --- .../alertservice/methods/ema/EmaResult.java | 36 -- .../alertservice/methods/ema/TestEmaModel.java | 68 ---- .../prototype/AmbariServerInterface.java | 122 ++++++ .../MetricAnomalyDetectorTestInput.java | 126 ++++++ .../prototype/MetricAnomalyTester.java | 163 ++++++++ .../prototype/MetricKafkaProducer.java | 56 +++ .../prototype/MetricSparkConsumer.java | 178 +++++++++ .../prototype/MetricsCollectorInterface.java | 237 ++++++++++++ .../prototype/PointInTimeADSystem.java | 256 +++++++++++++ .../prototype/RFunctionInvoker.java | 222 +++++++++++ .../prototype/TestSeriesInputRequest.java | 88 +++++ .../alertservice/prototype/TrendADSystem.java | 331 ++++++++++++++++ .../alertservice/prototype/TrendMetric.java | 33 ++ .../prototype/common/DataSeries.java | 38 ++ .../prototype/common/ResultSet.java | 43 +++ .../prototype/common/StatisticUtils.java | 62 +++ .../methods/AnomalyDetectionTechnique.java | 32 ++ .../prototype/methods/MetricAnomaly.java | 86 +++++ .../prototype/methods/ema/EmaModel.java | 124 ++++++ .../prototype/methods/ema/EmaModelLoader.java | 46 +++ .../prototype/methods/ema/EmaTechnique.java | 142 +++++++ .../prototype/methods/hsdev/HsdevTechnique.java | 77 ++++ .../prototype/methods/kstest/KSTechnique.java | 101 +++++ .../seriesgenerator/AbstractMetricSeries.java | 25 ++ .../seriesgenerator/DualBandMetricSeries.java | 88 +++++ .../MetricSeriesGeneratorFactory.java | 379 +++++++++++++++++++ .../seriesgenerator/MonotonicMetricSeries.java | 101 +++++ .../seriesgenerator/NormalMetricSeries.java | 81 ++++ .../SteadyWithTurbulenceMetricSeries.java | 115 ++++++ .../StepFunctionMetricSeries.java | 107 ++++++ .../seriesgenerator/UniformMetricSeries.java | 95 +++++ .../alertservice/spark/AmsKafkaProducer.java | 92 ----- .../spark/AnomalyMetricPublisher.java | 196 ---------- .../spark/MetricAnomalyDetector.java | 147 ------- .../src/main/resources/R-scripts/hsdev.r | 12 +- .../src/main/resources/R-scripts/kstest.r | 2 +- .../src/main/resources/R-scripts/tukeys.r | 9 +- .../src/main/resources/R-scripts/util.R | 36 -- .../prototype/TestEmaTechnique.java | 86 +++++ .../prototype/TestRFunctionInvoker.java | 161 ++++++++ .../alertservice/prototype/TestTukeys.java | 100 +++++ .../MetricSeriesGeneratorTest.java | 108 ++++++ .../metrics2/sink/timeline/TimelineMetric.java | 5 +- .../metrics2/sink/timeline/TimelineMetrics.java | 3 +- .../metrics/spark/MetricAnomalyDetector.scala | 18 +- .../metrics/spark/SparkPhoenixReader.scala | 18 +- .../ambari-metrics-timelineservice/pom.xml | 2 +- .../timeline/HBaseTimelineMetricsService.java | 24 -- .../timeline/MetricsPaddingMethodTest.java | 7 - 64 files changed, 4058 insertions(+), 1980 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/pom.xml b/ambari-metrics/ambari-metrics-alertservice/pom.xml index 4afc80f..4db8a6a 100644 --- a/ambari-metrics/ambari-metrics-alertservice/pom.xml +++ b/ambari-metrics/ambari-metrics-alertservice/pom.xml @@ -31,7 +31,6 @@ <build> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> <source>1.8</source> @@ -130,5 +129,21 @@ <artifactId>spark-mllib_2.10</artifactId> <version>1.3.0</version> </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + <version>4.10</version> + </dependency> + <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-metrics-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>4.2.5</version> + </dependency> </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/AmsRTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/AmsRTest.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/AmsRTest.java deleted file mode 100644 index 2bbc250..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/AmsRTest.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.R; - -import org.apache.ambari.metrics.alertservice.common.ResultSet; -import org.apache.ambari.metrics.alertservice.common.DataSet; -import org.apache.commons.lang.ArrayUtils; -import org.rosuda.JRI.REXP; -import org.rosuda.JRI.RVector; -import org.rosuda.JRI.Rengine; - -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -public class AmsRTest { - - public static void main(String[] args) { - - String metricName = "TestMetric"; - double[] ts = getTS(1000); - - double[] train_ts = ArrayUtils.subarray(ts, 0,750); - double[] train_x = getData(750); - DataSet trainData = new DataSet(metricName, train_ts, train_x); - - double[] test_ts = ArrayUtils.subarray(ts, 750,1000); - double[] test_x = getData(250); - test_x[50] = 5.5; //Anomaly - DataSet testData = new DataSet(metricName, test_ts, test_x); - ResultSet rs; - - Map<String, String> configs = new HashMap(); - - System.out.println("TUKEYS"); - configs.put("tukeys.n", "3"); - rs = RFunctionInvoker.tukeys(trainData, testData, configs); - rs.print(); - System.out.println("--------------"); - - System.out.println("EMA Global"); - configs.put("ema.n", "3"); - configs.put("ema.w", "0.8"); - rs = RFunctionInvoker.ema_global(trainData, testData, configs); - rs.print(); - System.out.println("--------------"); - - System.out.println("EMA Daily"); - rs = RFunctionInvoker.ema_daily(trainData, testData, configs); - rs.print(); - System.out.println("--------------"); - - configs.put("ks.p_value", "0.05"); - System.out.println("KS Test"); - rs = RFunctionInvoker.ksTest(trainData, testData, configs); - rs.print(); - System.out.println("--------------"); - - ts = getTS(5000); - train_ts = ArrayUtils.subarray(ts, 30,4800); - train_x = getData(4800); - trainData = new DataSet(metricName, train_ts, train_x); - test_ts = ArrayUtils.subarray(ts, 4800,5000); - test_x = getData(200); - for (int i =0; i<200;i++) { - test_x[i] = test_x[i]*5; - } - testData = new DataSet(metricName, test_ts, test_x); - configs.put("hsdev.n", "3"); - configs.put("hsdev.nhp", "3"); - configs.put("hsdev.interval", "86400000"); - configs.put("hsdev.period", "604800000"); - System.out.println("HSdev"); - rs = RFunctionInvoker.hsdev(trainData, testData, configs); - rs.print(); - System.out.println("--------------"); - - } - - static double[] getTS(int n) { - long currentTime = System.currentTimeMillis(); - double[] ts = new double[n]; - currentTime = currentTime - (currentTime % (5*60*1000)); - - for (int i = 0,j=n-1; i<n; i++,j--) { - ts[j] = currentTime; - currentTime = currentTime - (5*60*1000); - } - return ts; - } - - static void testBasic() { - Rengine r = new Rengine(new String[]{"--no-save"}, false, null); - try { - r.eval("library(ambarimetricsAD)"); - r.eval("source('~/dev/AMS/AD/ambarimetricsAD/org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.alerting.R/test.org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.alerting.R', echo=TRUE)"); - r.eval("source('~/dev/AMS/AD/ambarimetricsAD/org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.alerting.R/util.org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.alerting.R', echo=TRUE)"); - r.eval("source('~/dev/AMS/AD/ambarimetricsAD/org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.alerting.R/tukeys.org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.alerting.R', echo=TRUE)"); - double[] ts = getTS(5000); - double[] x = getData(5000); - r.assign("ts", ts); - r.assign("x", x); - r.eval("x[1000] <- 4.5"); - r.eval("x[2000] <- 4.75"); - r.eval("x[3000] <- 3.5"); - r.eval("x[4000] <- 5.5"); - r.eval("x[5000] <- 5.0"); - r.eval("data <- data.frame(ts,x)"); - r.eval("names(data) <- c(\"TS\", \"Metric\")"); - System.out.println(r.eval("data")); - REXP exp = r.eval("t_an <- test_methods(data)"); - exp = r.eval("t_an"); - String strExp = exp.asString(); - System.out.println("result:" + exp); - RVector cont = (RVector) exp.getContent(); - double[] an_ts = cont.at(0).asDoubleArray(); - double[] an_x = cont.at(1).asDoubleArray(); - System.out.println("result:" + strExp); - } - finally { - r.end(); - } - } - static double[] getData(int n) { - double[] metrics = new double[n]; - Random random = new Random(); - for (int i = 0; i<n; i++) { - metrics[i] = random.nextDouble(); - } - return metrics; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/RFunctionInvoker.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/RFunctionInvoker.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/RFunctionInvoker.java deleted file mode 100644 index 2713b71..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/R/RFunctionInvoker.java +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.R; - - -import org.apache.ambari.metrics.alertservice.common.ResultSet; -import org.apache.ambari.metrics.alertservice.common.DataSet; -import org.rosuda.JRI.REXP; -import org.rosuda.JRI.RVector; -import org.rosuda.JRI.Rengine; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class RFunctionInvoker { - - public static Rengine r = new Rengine(new String[]{"--no-save"}, false, null); - - - private static void loadDataSets(Rengine r, DataSet trainData, DataSet testData) { - r.assign("train_ts", trainData.ts); - r.assign("train_x", trainData.values); - r.eval("train_data <- data.frame(train_ts,train_x)"); - r.eval("names(train_data) <- c(\"TS\", " + trainData.metricName + ")"); - - r.assign("test_ts", testData.ts); - r.assign("test_x", testData.values); - r.eval("test_data <- data.frame(test_ts,test_x)"); - r.eval("names(test_data) <- c(\"TS\", " + testData.metricName + ")"); - } - - - public static ResultSet tukeys(DataSet trainData, DataSet testData, Map<String, String> configs) { - try { - r.eval("source('tukeys.r', echo=TRUE)"); - - int n = Integer.parseInt(configs.get("tukeys.n")); - r.eval("n <- " + n); - - loadDataSets(r, trainData, testData); - - r.eval("an <- ams_tukeys(train_data, test_data, n)"); - REXP exp = r.eval("an"); - RVector cont = (RVector) exp.getContent(); - List<double[]> result = new ArrayList(); - for (int i = 0; i< cont.size(); i++) { - result.add(cont.at(i).asDoubleArray()); - } - return new ResultSet(result); - } catch(Exception e) { - e.printStackTrace(); - } finally { - r.end(); - } - return null; - } - - public static ResultSet ema_global(DataSet trainData, DataSet testData, Map<String, String> configs) { - try { - r.eval("source('ema.R', echo=TRUE)"); - - int n = Integer.parseInt(configs.get("ema.n")); - r.eval("n <- " + n); - - double w = Double.parseDouble(configs.get("ema.w")); - r.eval("w <- " + w); - - loadDataSets(r, trainData, testData); - - r.eval("an <- ema_global(train_data, test_data, w, n)"); - REXP exp = r.eval("an"); - RVector cont = (RVector) exp.getContent(); - List<double[]> result = new ArrayList(); - for (int i = 0; i< cont.size(); i++) { - result.add(cont.at(i).asDoubleArray()); - } - return new ResultSet(result); - - } catch(Exception e) { - e.printStackTrace(); - } finally { - r.end(); - } - return null; - } - - public static ResultSet ema_daily(DataSet trainData, DataSet testData, Map<String, String> configs) { - try { - r.eval("source('ema.R', echo=TRUE)"); - - int n = Integer.parseInt(configs.get("ema.n")); - r.eval("n <- " + n); - - double w = Double.parseDouble(configs.get("ema.w")); - r.eval("w <- " + w); - - loadDataSets(r, trainData, testData); - - r.eval("an <- ema_daily(train_data, test_data, w, n)"); - REXP exp = r.eval("an"); - RVector cont = (RVector) exp.getContent(); - List<double[]> result = new ArrayList(); - for (int i = 0; i< cont.size(); i++) { - result.add(cont.at(i).asDoubleArray()); - } - return new ResultSet(result); - - } catch(Exception e) { - e.printStackTrace(); - } finally { - r.end(); - } - return null; - } - - public static ResultSet ksTest(DataSet trainData, DataSet testData, Map<String, String> configs) { - try { - r.eval("source('kstest.r', echo=TRUE)"); - - double p_value = Double.parseDouble(configs.get("ks.p_value")); - r.eval("p_value <- " + p_value); - - loadDataSets(r, trainData, testData); - - r.eval("an <- ams_ks(train_data, test_data, p_value)"); - REXP exp = r.eval("an"); - RVector cont = (RVector) exp.getContent(); - List<double[]> result = new ArrayList(); - for (int i = 0; i< cont.size(); i++) { - result.add(cont.at(i).asDoubleArray()); - } - return new ResultSet(result); - - } catch(Exception e) { - e.printStackTrace(); - } finally { - r.end(); - } - return null; - } - - public static ResultSet hsdev(DataSet trainData, DataSet testData, Map<String, String> configs) { - try { - r.eval("source('hsdev.r', echo=TRUE)"); - - int n = Integer.parseInt(configs.get("hsdev.n")); - r.eval("n <- " + n); - - int nhp = Integer.parseInt(configs.get("hsdev.nhp")); - r.eval("nhp <- " + nhp); - - long interval = Long.parseLong(configs.get("hsdev.interval")); - r.eval("interval <- " + interval); - - long period = Long.parseLong(configs.get("hsdev.period")); - r.eval("period <- " + period); - - loadDataSets(r, trainData, testData); - - r.eval("an2 <- hsdev_daily(train_data, test_data, n, nhp, interval, period)"); - REXP exp = r.eval("an2"); - RVector cont = (RVector) exp.getContent(); - - List<double[]> result = new ArrayList(); - for (int i = 0; i< cont.size(); i++) { - result.add(cont.at(i).asDoubleArray()); - } - return new ResultSet(result); - } catch(Exception e) { - e.printStackTrace(); - } finally { - r.end(); - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/DataSet.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/DataSet.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/DataSet.java deleted file mode 100644 index a709c73..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/DataSet.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - -import java.util.Arrays; - -public class DataSet { - - public String metricName; - public double[] ts; - public double[] values; - - public DataSet(String metricName, double[] ts, double[] values) { - this.metricName = metricName; - this.ts = ts; - this.values = values; - } - - @Override - public String toString() { - return metricName + Arrays.toString(ts) + Arrays.toString(values); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MethodResult.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MethodResult.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MethodResult.java deleted file mode 100644 index 6bf58df..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MethodResult.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - -public abstract class MethodResult { - protected String methodType; - public abstract String prettyPrint(); - - public String getMethodType() { - return methodType; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MetricAnomaly.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MetricAnomaly.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MetricAnomaly.java deleted file mode 100644 index 4dbb425..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/MetricAnomaly.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - -public class MetricAnomaly { - - private String metricKey; - private long timestamp; - private double metricValue; - private MethodResult methodResult; - - public MetricAnomaly(String metricKey, long timestamp, double metricValue, MethodResult methodResult) { - this.metricKey = metricKey; - this.timestamp = timestamp; - this.metricValue = metricValue; - this.methodResult = methodResult; - } - - public String getMetricKey() { - return metricKey; - } - - public void setMetricName(String metricName) { - this.metricKey = metricName; - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public double getMetricValue() { - return metricValue; - } - - public void setMetricValue(double metricValue) { - this.metricValue = metricValue; - } - - public MethodResult getMethodResult() { - return methodResult; - } - - public void setMethodResult(MethodResult methodResult) { - this.methodResult = methodResult; - } - - public String getAnomalyAsString() { - return metricKey + ":" + timestamp + ":" + metricValue + ":" + methodResult.prettyPrint(); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/ResultSet.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/ResultSet.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/ResultSet.java deleted file mode 100644 index 9415c1b..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/ResultSet.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - - -import java.util.ArrayList; -import java.util.List; - -public class ResultSet { - - List<double[]> resultset = new ArrayList<>(); - - public ResultSet(List<double[]> resultset) { - this.resultset = resultset; - } - - public void print() { - System.out.println("Result : "); - if (!resultset.isEmpty()) { - for (int i = 0; i<resultset.get(0).length;i++) { - for (double[] entity : resultset) { - System.out.print(entity[i] + " "); - } - System.out.println(); - } - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/SingleValuedTimelineMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/SingleValuedTimelineMetric.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/SingleValuedTimelineMetric.java deleted file mode 100644 index acd4452..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/SingleValuedTimelineMetric.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - - -public class SingleValuedTimelineMetric { - private Long timestamp; - private Double value; - private String metricName; - private String appId; - private String instanceId; - private String hostName; - private Long startTime; - private String type; - - public void setSingleTimeseriesValue(Long timestamp, Double value) { - this.timestamp = timestamp; - this.value = value; - } - - public SingleValuedTimelineMetric(String metricName, String appId, - String instanceId, String hostName, - long timestamp, long startTime, String type) { - this.metricName = metricName; - this.appId = appId; - this.instanceId = instanceId; - this.hostName = hostName; - this.timestamp = timestamp; - this.startTime = startTime; - this.type = type; - } - - public Long getTimestamp() { - return timestamp; - } - - public long getStartTime() { - return startTime; - } - - public String getType() { - return type; - } - - public Double getValue() { - return value; - } - - public String getMetricName() { - return metricName; - } - - public String getAppId() { - return appId; - } - - public String getInstanceId() { - return instanceId; - } - - public String getHostName() { - return hostName; - } - - public boolean equalsExceptTime(TimelineMetric metric) { - if (!metricName.equals(metric.getMetricName())) return false; - if (hostName != null ? !hostName.equals(metric.getHostName()) : metric.getHostName() != null) - return false; - if (appId != null ? !appId.equals(metric.getAppId()) : metric.getAppId() != null) - return false; - if (instanceId != null ? !instanceId.equals(metric.getInstanceId()) : metric.getInstanceId() != null) return false; - - return true; - } - - public TimelineMetric getTimelineMetric() { - TimelineMetric metric = new TimelineMetric(); - metric.setMetricName(this.metricName); - metric.setAppId(this.appId); - metric.setHostName(this.hostName); - metric.setType(this.type); - metric.setInstanceId(this.instanceId); - metric.setStartTime(this.startTime); - metric.setTimestamp(this.timestamp); - metric.getMetricValues().put(timestamp, value); - return metric; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/StatisticUtils.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/StatisticUtils.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/StatisticUtils.java deleted file mode 100644 index 81bd77b..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/StatisticUtils.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; - -public class StatisticUtils { - - public static double mean(Collection<Double> values) { - double sum = 0; - for (double d : values) { - sum += d; - } - return sum / values.size(); - } - - public static double variance(Collection<Double> values) { - double avg = mean(values); - double variance = 0; - for (double d : values) { - variance += Math.pow(d - avg, 2.0); - } - return variance; - } - - public static double sdev(Collection<Double> values, boolean useBesselsCorrection) { - double variance = variance(values); - int n = (useBesselsCorrection) ? values.size() - 1 : values.size(); - return Math.sqrt(variance / n); - } - - public static double median(Collection<Double> values) { - ArrayList<Double> clonedValues = new ArrayList<Double>(values); - Collections.sort(clonedValues); - int n = values.size(); - - if (n % 2 != 0) { - return clonedValues.get((n-1)/2); - } else { - return ( clonedValues.get((n-1)/2) + clonedValues.get(n/2) ) / 2; - } - } - - - -// public static void main(String[] args) { -// -// Collection<Double> values = new ArrayList<>(); -// values.add(1.0); -// values.add(2.0); -// values.add(3.0); -// values.add(4.0); -// values.add(5.0); -// -// System.out.println(mean(values)); -// System.out.println(sdev(values, false)); -// System.out.println(median(values)); -// } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetric.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetric.java deleted file mode 100644 index 88ad834..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetric.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -@XmlRootElement(name = "metric") -@XmlAccessorType(XmlAccessType.NONE) [email protected] [email protected] -public class TimelineMetric implements Comparable<TimelineMetric>, Serializable { - - private String metricName; - private String appId; - private String instanceId; - private String hostName; - private long timestamp; - private long startTime; - private String type; - private String units; - private TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>(); - private Map<String, String> metadata = new HashMap<>(); - - // default - public TimelineMetric() { - - } - - public TimelineMetric(String metricName, String appId, String hostName, TreeMap<Long,Double> metricValues) { - this.metricName = metricName; - this.appId = appId; - this.hostName = hostName; - this.metricValues.putAll(metricValues); - } - - // copy constructor - public TimelineMetric(TimelineMetric metric) { - setMetricName(metric.getMetricName()); - setType(metric.getType()); - setUnits(metric.getUnits()); - setTimestamp(metric.getTimestamp()); - setAppId(metric.getAppId()); - setInstanceId(metric.getInstanceId()); - setHostName(metric.getHostName()); - setStartTime(metric.getStartTime()); - setMetricValues(new TreeMap<Long, Double>(metric.getMetricValues())); - } - - @XmlElement(name = "metricname") - public String getMetricName() { - return metricName; - } - - public void setMetricName(String metricName) { - this.metricName = metricName; - } - - @XmlElement(name = "appid") - public String getAppId() { - return appId; - } - - public void setAppId(String appId) { - this.appId = appId; - } - - @XmlElement(name = "instanceid") - public String getInstanceId() { - return instanceId; - } - - public void setInstanceId(String instanceId) { - this.instanceId = instanceId; - } - - @XmlElement(name = "hostname") - public String getHostName() { - return hostName; - } - - public void setHostName(String hostName) { - this.hostName = hostName; - } - - @XmlElement(name = "timestamp") - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - @XmlElement(name = "starttime") - public long getStartTime() { - return startTime; - } - - public void setStartTime(long startTime) { - this.startTime = startTime; - } - - @XmlElement(name = "type", defaultValue = "UNDEFINED") - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - @XmlElement(name = "units") - public String getUnits() { - return units; - } - - public void setUnits(String units) { - this.units = units; - } - - @XmlElement(name = "metrics") - public TreeMap<Long, Double> getMetricValues() { - return metricValues; - } - - public void setMetricValues(TreeMap<Long, Double> metricValues) { - this.metricValues = metricValues; - } - - public void addMetricValues(Map<Long, Double> metricValues) { - this.metricValues.putAll(metricValues); - } - - @XmlElement(name = "metadata") - public Map<String,String> getMetadata () { - return metadata; - } - - public void setMetadata (Map<String,String> metadata) { - this.metadata = metadata; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TimelineMetric metric = (TimelineMetric) o; - - if (!metricName.equals(metric.metricName)) return false; - if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null) - return false; - if (appId != null ? !appId.equals(metric.appId) : metric.appId != null) - return false; - if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) - return false; - if (timestamp != metric.timestamp) return false; - if (startTime != metric.startTime) return false; - - return true; - } - - public boolean equalsExceptTime(TimelineMetric metric) { - if (!metricName.equals(metric.metricName)) return false; - if (hostName != null ? !hostName.equals(metric.hostName) : metric.hostName != null) - return false; - if (appId != null ? !appId.equals(metric.appId) : metric.appId != null) - return false; - if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) - return false; - - return true; - } - - @Override - public int hashCode() { - int result = metricName.hashCode(); - result = 31 * result + (appId != null ? appId.hashCode() : 0); - result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0); - result = 31 * result + (hostName != null ? hostName.hashCode() : 0); - result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); - return result; - } - - @Override - public int compareTo(TimelineMetric other) { - if (timestamp > other.timestamp) { - return -1; - } else if (timestamp < other.timestamp) { - return 1; - } else { - return metricName.compareTo(other.metricName); - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetrics.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetrics.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetrics.java deleted file mode 100644 index 7df6a9c..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/common/TimelineMetrics.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.common; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * The class that hosts a list of timeline entities. - */ -@XmlRootElement(name = "metrics") -@XmlAccessorType(XmlAccessType.NONE) [email protected] [email protected] -public class TimelineMetrics implements Serializable { - - private List<TimelineMetric> allMetrics = new ArrayList<TimelineMetric>(); - - public TimelineMetrics() {} - - @XmlElement(name = "metrics") - public List<TimelineMetric> getMetrics() { - return allMetrics; - } - - public void setMetrics(List<TimelineMetric> allMetrics) { - this.allMetrics = allMetrics; - } - - private boolean isEqualTimelineMetrics(TimelineMetric metric1, - TimelineMetric metric2) { - - boolean isEqual = true; - - if (!metric1.getMetricName().equals(metric2.getMetricName())) { - return false; - } - - if (metric1.getHostName() != null) { - isEqual = metric1.getHostName().equals(metric2.getHostName()); - } - - if (metric1.getAppId() != null) { - isEqual = metric1.getAppId().equals(metric2.getAppId()); - } - - return isEqual; - } - - /** - * Merge with existing TimelineMetric if everything except startTime is - * the same. - * @param metric {@link TimelineMetric} - */ - public void addOrMergeTimelineMetric(TimelineMetric metric) { - TimelineMetric metricToMerge = null; - - if (!allMetrics.isEmpty()) { - for (TimelineMetric timelineMetric : allMetrics) { - if (timelineMetric.equalsExceptTime(metric)) { - metricToMerge = timelineMetric; - break; - } - } - } - - if (metricToMerge != null) { - metricToMerge.addMetricValues(metric.getMetricValues()); - if (metricToMerge.getTimestamp() > metric.getTimestamp()) { - metricToMerge.setTimestamp(metric.getTimestamp()); - } - if (metricToMerge.getStartTime() > metric.getStartTime()) { - metricToMerge.setStartTime(metric.getStartTime()); - } - } else { - allMetrics.add(metric); - } - } - - // Optimization that addresses too many TreeMaps from getting created. - public void addOrMergeTimelineMetric(SingleValuedTimelineMetric metric) { - TimelineMetric metricToMerge = null; - - if (!allMetrics.isEmpty()) { - for (TimelineMetric timelineMetric : allMetrics) { - if (metric.equalsExceptTime(timelineMetric)) { - metricToMerge = timelineMetric; - break; - } - } - } - - if (metricToMerge != null) { - metricToMerge.getMetricValues().put(metric.getTimestamp(), metric.getValue()); - if (metricToMerge.getTimestamp() > metric.getTimestamp()) { - metricToMerge.setTimestamp(metric.getTimestamp()); - } - if (metricToMerge.getStartTime() > metric.getStartTime()) { - metricToMerge.setStartTime(metric.getStartTime()); - } - } else { - allMetrics.add(metric.getTimelineMetric()); - } - } -} - http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/MetricAnomalyModel.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/MetricAnomalyModel.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/MetricAnomalyModel.java deleted file mode 100644 index af33d26..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/MetricAnomalyModel.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.methods; - -import org.apache.ambari.metrics.alertservice.common.MetricAnomaly; -import org.apache.ambari.metrics.alertservice.common.TimelineMetric; - -import java.util.List; - -public interface MetricAnomalyModel { - - public List<MetricAnomaly> onNewMetric(TimelineMetric metric); - public List<MetricAnomaly> test(TimelineMetric metric); -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaDS.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaDS.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaDS.java deleted file mode 100644 index 32cd96b..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaDS.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.methods.ema; - -import com.sun.org.apache.commons.logging.Log; -import com.sun.org.apache.commons.logging.LogFactory; - -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; - -@XmlRootElement -public class EmaDS implements Serializable { - - String metricName; - String appId; - String hostname; - double ema; - double ems; - double weight; - int timessdev; - private static final Log LOG = LogFactory.getLog(EmaDS.class); - - public EmaDS(String metricName, String appId, String hostname, double weight, int timessdev) { - this.metricName = metricName; - this.appId = appId; - this.hostname = hostname; - this.weight = weight; - this.timessdev = timessdev; - this.ema = 0.0; - this.ems = 0.0; - } - - - public EmaResult testAndUpdate(double metricValue) { - - double diff = Math.abs(ema - metricValue) - (timessdev * ems); - - ema = weight * ema + (1 - weight) * metricValue; - ems = Math.sqrt(weight * Math.pow(ems, 2.0) + (1 - weight) * Math.pow(metricValue - ema, 2.0)); - LOG.info(ema + ", " + ems); - return diff > 0 ? new EmaResult(diff) : null; - } - - public void update(double metricValue) { - ema = weight * ema + (1 - weight) * metricValue; - ems = Math.sqrt(weight * Math.pow(ems, 2.0) + (1 - weight) * Math.pow(metricValue - ema, 2.0)); - LOG.info(ema + ", " + ems); - } - - public EmaResult test(double metricValue) { - double diff = Math.abs(ema - metricValue) - (timessdev * ems); - return diff > 0 ? new EmaResult(diff) : null; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModel.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModel.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModel.java deleted file mode 100644 index 13a0f55..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModel.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.methods.ema; - -import com.google.gson.Gson; -import com.sun.org.apache.commons.logging.Log; -import com.sun.org.apache.commons.logging.LogFactory; -import org.apache.ambari.metrics.alertservice.common.MethodResult; -import org.apache.ambari.metrics.alertservice.common.MetricAnomaly; -import org.apache.ambari.metrics.alertservice.common.TimelineMetric; -import org.apache.ambari.metrics.alertservice.methods.MetricAnomalyModel; -import org.apache.spark.SparkContext; -import org.apache.spark.mllib.util.Saveable; - -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@XmlRootElement -public class EmaModel implements MetricAnomalyModel, Saveable, Serializable { - - @XmlElement(name = "trackedEmas") - private Map<String, EmaDS> trackedEmas = new HashMap<>(); - private static final Log LOG = LogFactory.getLog(EmaModel.class); - - public List<MetricAnomaly> onNewMetric(TimelineMetric metric) { - - String metricName = metric.getMetricName(); - String appId = metric.getAppId(); - String hostname = metric.getHostName(); - String key = metricName + "_" + appId + "_" + hostname; - List<MetricAnomaly> anomalies = new ArrayList<>(); - - if (!trackedEmas.containsKey(metricName)) { - trackedEmas.put(key, new EmaDS(metricName, appId, hostname, 0.8, 3)); - } - - EmaDS emaDS = trackedEmas.get(key); - for (Long timestamp : metric.getMetricValues().keySet()) { - double metricValue = metric.getMetricValues().get(timestamp); - MethodResult result = emaDS.testAndUpdate(metricValue); - if (result != null) { - MetricAnomaly metricAnomaly = new MetricAnomaly(key,timestamp, metricValue, result); - anomalies.add(metricAnomaly); - } - } - return anomalies; - } - - public EmaDS train(TimelineMetric metric, double weight, int timessdev) { - - String metricName = metric.getMetricName(); - String appId = metric.getAppId(); - String hostname = metric.getHostName(); - String key = metricName + "_" + appId + "_" + hostname; - - EmaDS emaDS = new EmaDS(metric.getMetricName(), metric.getAppId(), metric.getHostName(), weight, timessdev); - LOG.info("In EMA Train step"); - for (Long timestamp : metric.getMetricValues().keySet()) { - emaDS.update(metric.getMetricValues().get(timestamp)); - } - trackedEmas.put(key, emaDS); - return emaDS; - } - - public List<MetricAnomaly> test(TimelineMetric metric) { - String metricName = metric.getMetricName(); - String appId = metric.getAppId(); - String hostname = metric.getHostName(); - String key = metricName + "_" + appId + "_" + hostname; - - EmaDS emaDS = trackedEmas.get(key); - - if (emaDS == null) { - return new ArrayList<>(); - } - - List<MetricAnomaly> anomalies = new ArrayList<>(); - - for (Long timestamp : metric.getMetricValues().keySet()) { - double metricValue = metric.getMetricValues().get(timestamp); - MethodResult result = emaDS.testAndUpdate(metricValue); - if (result != null) { - MetricAnomaly metricAnomaly = new MetricAnomaly(key,timestamp, metricValue, result); - anomalies.add(metricAnomaly); - } - } - return anomalies; - } - - @Override - public void save(SparkContext sc, String path) { - Gson gson = new Gson(); - try { - String json = gson.toJson(this); - try (Writer writer = new BufferedWriter(new OutputStreamWriter( - new FileOutputStream(path), "utf-8"))) { - writer.write(json); - } } catch (IOException e) { - LOG.error(e); - } - } - - @Override - public String formatVersion() { - return "1.0"; - } - -} - http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModelLoader.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModelLoader.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModelLoader.java deleted file mode 100644 index 0205844..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaModelLoader.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.methods.ema; - -import com.google.gson.Gson; -import com.sun.org.apache.commons.logging.Log; -import com.sun.org.apache.commons.logging.LogFactory; -import org.apache.spark.SparkContext; -import org.apache.spark.mllib.util.Loader; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; - -public class EmaModelLoader implements Loader<EmaModel> { - private static final Log LOG = LogFactory.getLog(EmaModelLoader.class); - - @Override - public EmaModel load(SparkContext sc, String path) { - Gson gson = new Gson(); - try { - String fileString = new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8); - return gson.fromJson(fileString, EmaModel.class); - } catch (IOException e) { - LOG.error(e); - } - return null; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaResult.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaResult.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaResult.java deleted file mode 100644 index 2d24a9c..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/EmaResult.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.methods.ema; - -import org.apache.ambari.metrics.alertservice.common.MethodResult; - -public class EmaResult extends MethodResult{ - - double diff; - - public EmaResult(double diff) { - this.methodType = "EMA"; - this.diff = diff; - } - - - @Override - public String prettyPrint() { - return methodType + "(` = " + diff + ")"; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/TestEmaModel.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/TestEmaModel.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/TestEmaModel.java deleted file mode 100644 index b851dab..0000000 --- a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/methods/ema/TestEmaModel.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.methods.ema; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import org.apache.ambari.metrics.alertservice.common.TimelineMetric; - -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -public class TestEmaModel { - - public static void main(String[] args) throws IOException { - - long now = System.currentTimeMillis(); - TimelineMetric metric1 = new TimelineMetric(); - metric1.setMetricName("dummy_metric"); - metric1.setHostName("dummy_host"); - metric1.setTimestamp(now); - metric1.setStartTime(now - 1000); - metric1.setAppId("HOST"); - metric1.setType("Integer"); - - TreeMap<Long, Double> metricValues = new TreeMap<Long, Double>(); - - for (int i = 0; i<20;i++) { - double metric = 9 + Math.random(); - metricValues.put(now - i*100, metric); - } - metric1.setMetricValues(metricValues); - - EmaModel emaModel = new EmaModel(); - - emaModel.train(metric1, 0.8, 3); - } - - /* - {{ - put(now - 100, 1.20); - put(now - 200, 1.25); - put(now - 300, 1.30); - put(now - 400, 4.50); - put(now - 500, 1.35); - put(now - 400, 5.50); - }} - */ -} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/AmbariServerInterface.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/AmbariServerInterface.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/AmbariServerInterface.java new file mode 100644 index 0000000..0c1c6fc --- /dev/null +++ b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/AmbariServerInterface.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.metrics.alertservice.prototype; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Serializable; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class AmbariServerInterface implements Serializable{ + + private static final Log LOG = LogFactory.getLog(AmbariServerInterface.class); + + private String ambariServerHost; + private String clusterName; + + public AmbariServerInterface(String ambariServerHost, String clusterName) { + this.ambariServerHost = ambariServerHost; + this.clusterName = clusterName; + } + + public int getPointInTimeSensitivity() { + + String url = constructUri("http", ambariServerHost, "8080", "/api/v1/clusters/" + clusterName + "/alert_definitions?fields=*"); + + URL obj = null; + BufferedReader in = null; + + try { + obj = new URL(url); + HttpURLConnection con = (HttpURLConnection) obj.openConnection(); + con.setRequestMethod("GET"); + + String encoded = Base64.getEncoder().encodeToString(("admin:admin").getBytes(StandardCharsets.UTF_8)); + con.setRequestProperty("Authorization", "Basic "+encoded); + + int responseCode = con.getResponseCode(); + LOG.info("Sending 'GET' request to URL : " + url); + LOG.info("Response Code : " + responseCode); + + in = new BufferedReader( + new InputStreamReader(con.getInputStream())); + + StringBuilder responseJsonSb = new StringBuilder(); + String line; + while ((line = in.readLine()) != null) { + responseJsonSb.append(line); + } + + JSONObject jsonObject = new JSONObject(responseJsonSb.toString()); + JSONArray array = jsonObject.getJSONArray("items"); + for(int i = 0 ; i < array.length() ; i++){ + JSONObject alertDefn = array.getJSONObject(i).getJSONObject("AlertDefinition"); + LOG.info("alertDefn : " + alertDefn.get("name")); + if (alertDefn.get("name") != null && alertDefn.get("name").equals("point_in_time_metrics_anomalies")) { + JSONObject sourceNode = alertDefn.getJSONObject("source"); + JSONArray params = sourceNode.getJSONArray("parameters"); + for(int j = 0 ; j < params.length() ; j++){ + JSONObject param = params.getJSONObject(j); + if (param.get("name").equals("sensitivity")) { + return param.getInt("value"); + } + } + break; + } + } + + } catch (Exception e) { + LOG.error(e); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + LOG.warn(e); + } + } + } + + return -1; + } + + private String constructUri(String protocol, String host, String port, String path) { + StringBuilder sb = new StringBuilder(protocol); + sb.append("://"); + sb.append(host); + sb.append(":"); + sb.append(port); + sb.append(path); + return sb.toString(); + } + +// public static void main(String[] args) { +// AmbariServerInterface ambariServerInterface = new AmbariServerInterface(); +// ambariServerInterface.getPointInTimeSensitivity("avijayan-ams-1.openstacklocal","c1"); +// } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyDetectorTestInput.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyDetectorTestInput.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyDetectorTestInput.java new file mode 100644 index 0000000..490328a --- /dev/null +++ b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyDetectorTestInput.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.metrics.alertservice.prototype; + +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; +import java.util.Map; + +@XmlRootElement +public class MetricAnomalyDetectorTestInput { + + public MetricAnomalyDetectorTestInput() { + } + + //Train data + private String trainDataName; + private String trainDataType; + private Map<String, String> trainDataConfigs; + private int trainDataSize; + + //Test data + private String testDataName; + private String testDataType; + private Map<String, String> testDataConfigs; + private int testDataSize; + + //Algorithm data + private List<String> methods; + private Map<String, String> methodConfigs; + + public String getTrainDataName() { + return trainDataName; + } + + public void setTrainDataName(String trainDataName) { + this.trainDataName = trainDataName; + } + + public String getTrainDataType() { + return trainDataType; + } + + public void setTrainDataType(String trainDataType) { + this.trainDataType = trainDataType; + } + + public Map<String, String> getTrainDataConfigs() { + return trainDataConfigs; + } + + public void setTrainDataConfigs(Map<String, String> trainDataConfigs) { + this.trainDataConfigs = trainDataConfigs; + } + + public String getTestDataName() { + return testDataName; + } + + public void setTestDataName(String testDataName) { + this.testDataName = testDataName; + } + + public String getTestDataType() { + return testDataType; + } + + public void setTestDataType(String testDataType) { + this.testDataType = testDataType; + } + + public Map<String, String> getTestDataConfigs() { + return testDataConfigs; + } + + public void setTestDataConfigs(Map<String, String> testDataConfigs) { + this.testDataConfigs = testDataConfigs; + } + + public Map<String, String> getMethodConfigs() { + return methodConfigs; + } + + public void setMethodConfigs(Map<String, String> methodConfigs) { + this.methodConfigs = methodConfigs; + } + + public int getTrainDataSize() { + return trainDataSize; + } + + public void setTrainDataSize(int trainDataSize) { + this.trainDataSize = trainDataSize; + } + + public int getTestDataSize() { + return testDataSize; + } + + public void setTestDataSize(int testDataSize) { + this.testDataSize = testDataSize; + } + + public List<String> getMethods() { + return methods; + } + + public void setMethods(List<String> methods) { + this.methods = methods; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyTester.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyTester.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyTester.java new file mode 100644 index 0000000..bff8120 --- /dev/null +++ b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricAnomalyTester.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.metrics.alertservice.prototype; + +import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries; +import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet; +import org.apache.ambari.metrics.alertservice.seriesgenerator.MetricSeriesGeneratorFactory; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +public class MetricAnomalyTester { + + public static String appId = MetricsCollectorInterface.serviceName; + static final Log LOG = LogFactory.getLog(MetricAnomalyTester.class); + static Map<String, TimelineMetric> timelineMetricMap = new HashMap<>(); + + public static TimelineMetrics runTestAnomalyRequest(MetricAnomalyDetectorTestInput input) throws UnknownHostException { + + long currentTime = System.currentTimeMillis(); + TimelineMetrics timelineMetrics = new TimelineMetrics(); + String hostname = InetAddress.getLocalHost().getHostName(); + + //Train data + TimelineMetric metric1 = new TimelineMetric(); + if (StringUtils.isNotEmpty(input.getTrainDataName())) { + metric1 = timelineMetricMap.get(input.getTrainDataName()); + if (metric1 == null) { + metric1 = new TimelineMetric(); + double[] trainSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTrainDataType(), input.getTrainDataSize(), input.getTrainDataConfigs()); + metric1.setMetricName(input.getTrainDataName()); + metric1.setAppId(appId); + metric1.setHostName(hostname); + metric1.setStartTime(currentTime); + metric1.setInstanceId(null); + metric1.setMetricValues(getAsTimeSeries(currentTime, trainSeries)); + timelineMetricMap.put(input.getTrainDataName(), metric1); + } + timelineMetrics.getMetrics().add(metric1); + } else { + LOG.error("No train data name specified"); + } + + //Test data + TimelineMetric metric2 = new TimelineMetric(); + if (StringUtils.isNotEmpty(input.getTestDataName())) { + metric2 = timelineMetricMap.get(input.getTestDataName()); + if (metric2 == null) { + metric2 = new TimelineMetric(); + double[] testSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTestDataType(), input.getTestDataSize(), input.getTestDataConfigs()); + metric2.setMetricName(input.getTestDataName()); + metric2.setAppId(appId); + metric2.setHostName(hostname); + metric2.setStartTime(currentTime); + metric2.setInstanceId(null); + metric2.setMetricValues(getAsTimeSeries(currentTime, testSeries)); + timelineMetricMap.put(input.getTestDataName(), metric2); + } + timelineMetrics.getMetrics().add(metric2); + } else { + LOG.warn("No test data name specified"); + } + + //Invoke method + if (CollectionUtils.isNotEmpty(input.getMethods())) { + RFunctionInvoker.setScriptsDir("/etc/ambari-metrics-collector/conf/R-scripts"); + for (String methodType : input.getMethods()) { + ResultSet result = RFunctionInvoker.executeMethod(methodType, getAsDataSeries(metric1), getAsDataSeries(metric2), input.getMethodConfigs()); + TimelineMetric timelineMetric = getAsTimelineMetric(result, methodType, input, currentTime, hostname); + if (timelineMetric != null) { + timelineMetrics.getMetrics().add(timelineMetric); + } + } + } else { + LOG.warn("No anomaly method requested"); + } + + return timelineMetrics; + } + + + private static TimelineMetric getAsTimelineMetric(ResultSet result, String methodType, MetricAnomalyDetectorTestInput input, long currentTime, String hostname) { + + if (result == null) { + return null; + } + + TimelineMetric timelineMetric = new TimelineMetric(); + if (methodType.equals("tukeys") || methodType.equals("ema")) { + timelineMetric.setMetricName(input.getTrainDataName() + "_" + input.getTestDataName() + "_" + methodType + "_" + currentTime); + timelineMetric.setHostName(hostname); + timelineMetric.setAppId(appId); + timelineMetric.setInstanceId(null); + timelineMetric.setStartTime(currentTime); + + TreeMap<Long, Double> metricValues = new TreeMap<>(); + if (result.resultset.size() > 0) { + double[] ts = result.resultset.get(0); + double[] metrics = result.resultset.get(1); + for (int i = 0; i < ts.length; i++) { + if (i == 0) { + timelineMetric.setStartTime((long) ts[i]); + } + metricValues.put((long) ts[i], metrics[i]); + } + } + timelineMetric.setMetricValues(metricValues); + return timelineMetric; + } + return null; + } + + + private static TreeMap<Long, Double> getAsTimeSeries(long currentTime, double[] values) { + + long startTime = currentTime - (values.length - 1) * 60 * 1000; + TreeMap<Long, Double> metricValues = new TreeMap<>(); + + for (int i = 0; i < values.length; i++) { + metricValues.put(startTime, values[i]); + startTime += (60 * 1000); + } + return metricValues; + } + + private static DataSeries getAsDataSeries(TimelineMetric timelineMetric) { + + TreeMap<Long, Double> metricValues = timelineMetric.getMetricValues(); + double[] timestamps = new double[metricValues.size()]; + double[] values = new double[metricValues.size()]; + int i = 0; + + for (Long timestamp : metricValues.keySet()) { + timestamps[i] = timestamp; + values[i++] = metricValues.get(timestamp); + } + return new DataSeries(timelineMetric.getMetricName() + "_" + timelineMetric.getAppId() + "_" + timelineMetric.getHostName(), timestamps, values); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/63e74355/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricKafkaProducer.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricKafkaProducer.java b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricKafkaProducer.java new file mode 100644 index 0000000..8023d15 --- /dev/null +++ b/ambari-metrics/ambari-metrics-alertservice/src/main/java/org/apache/ambari/metrics/alertservice/prototype/MetricKafkaProducer.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.metrics.alertservice.prototype; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; + +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class MetricKafkaProducer { + + Producer producer; + private static String topicName = "ambari-metrics-topic"; + + public MetricKafkaProducer(String kafkaServers) { + Properties configProperties = new Properties(); + configProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaServers); //"avijayan-ams-2.openstacklocal:6667" + configProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.ByteArraySerializer"); + configProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.connect.json.JsonSerializer"); + producer = new KafkaProducer(configProperties); + } + + public void sendMetrics(TimelineMetrics timelineMetrics) throws InterruptedException, ExecutionException { + + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.valueToTree(timelineMetrics); + ProducerRecord<String, JsonNode> rec = new ProducerRecord<String, JsonNode>(topicName,jsonNode); + Future<RecordMetadata> kafkaFuture = producer.send(rec); + + System.out.println(kafkaFuture.isDone()); + System.out.println(kafkaFuture.get().topic()); + } +}
