http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java deleted file mode 100644 index 6485ebb..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/MetricAnomalyTester.java +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.metrics.alertservice.prototype.testing.utilities; - -import org.apache.ambari.metrics.alertservice.prototype.core.MetricsCollectorInterface; -import org.apache.ambari.metrics.alertservice.prototype.core.RFunctionInvoker; -import org.apache.ambari.metrics.alertservice.prototype.common.DataSeries; -import org.apache.ambari.metrics.alertservice.prototype.common.ResultSet; -import org.apache.ambari.metrics.alertservice.seriesgenerator.MetricSeriesGeneratorFactory; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -public class MetricAnomalyTester { - - public static String appId = MetricsCollectorInterface.serviceName; - static final Log LOG = LogFactory.getLog(MetricAnomalyTester.class); - static Map<String, TimelineMetric> timelineMetricMap = new HashMap<>(); - - public static TimelineMetrics runTestAnomalyRequest(MetricAnomalyDetectorTestInput input) throws UnknownHostException { - - long currentTime = System.currentTimeMillis(); - TimelineMetrics timelineMetrics = new TimelineMetrics(); - String hostname = InetAddress.getLocalHost().getHostName(); - - //Train data - TimelineMetric metric1 = new TimelineMetric(); - if (StringUtils.isNotEmpty(input.getTrainDataName())) { - metric1 = timelineMetricMap.get(input.getTrainDataName()); - if (metric1 == null) { - metric1 = new TimelineMetric(); - double[] trainSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTrainDataType(), input.getTrainDataSize(), input.getTrainDataConfigs()); - metric1.setMetricName(input.getTrainDataName()); - metric1.setAppId(appId); - metric1.setHostName(hostname); - metric1.setStartTime(currentTime); - metric1.setInstanceId(null); - metric1.setMetricValues(getAsTimeSeries(currentTime, trainSeries)); - timelineMetricMap.put(input.getTrainDataName(), metric1); - } - timelineMetrics.getMetrics().add(metric1); - } else { - LOG.error("No train data name specified"); - } - - //Test data - TimelineMetric metric2 = new TimelineMetric(); - if (StringUtils.isNotEmpty(input.getTestDataName())) { - metric2 = timelineMetricMap.get(input.getTestDataName()); - if (metric2 == null) { - metric2 = new TimelineMetric(); - double[] testSeries = MetricSeriesGeneratorFactory.generateSeries(input.getTestDataType(), input.getTestDataSize(), input.getTestDataConfigs()); - metric2.setMetricName(input.getTestDataName()); - metric2.setAppId(appId); - metric2.setHostName(hostname); - metric2.setStartTime(currentTime); - metric2.setInstanceId(null); - metric2.setMetricValues(getAsTimeSeries(currentTime, testSeries)); - timelineMetricMap.put(input.getTestDataName(), metric2); - } - timelineMetrics.getMetrics().add(metric2); - } else { - LOG.warn("No test data name specified"); - } - - //Invoke method - if (CollectionUtils.isNotEmpty(input.getMethods())) { - RFunctionInvoker.setScriptsDir("/etc/ambari-metrics-collector/conf/R-scripts"); - for (String methodType : input.getMethods()) { - ResultSet result = RFunctionInvoker.executeMethod(methodType, getAsDataSeries(metric1), getAsDataSeries(metric2), input.getMethodConfigs()); - TimelineMetric timelineMetric = getAsTimelineMetric(result, methodType, input, currentTime, hostname); - if (timelineMetric != null) { - timelineMetrics.getMetrics().add(timelineMetric); - } - } - } else { - LOG.warn("No anomaly method requested"); - } - - return timelineMetrics; - } - - - private static TimelineMetric getAsTimelineMetric(ResultSet result, String methodType, MetricAnomalyDetectorTestInput input, long currentTime, String hostname) { - - if (result == null) { - return null; - } - - TimelineMetric timelineMetric = new TimelineMetric(); - if (methodType.equals("tukeys") || methodType.equals("ema")) { - timelineMetric.setMetricName(input.getTrainDataName() + "_" + input.getTestDataName() + "_" + methodType + "_" + currentTime); - timelineMetric.setHostName(hostname); - timelineMetric.setAppId(appId); - timelineMetric.setInstanceId(null); - timelineMetric.setStartTime(currentTime); - - TreeMap<Long, Double> metricValues = new TreeMap<>(); - if (result.resultset.size() > 0) { - double[] ts = result.resultset.get(0); - double[] metrics = result.resultset.get(1); - for (int i = 0; i < ts.length; i++) { - if (i == 0) { - timelineMetric.setStartTime((long) ts[i]); - } - metricValues.put((long) ts[i], metrics[i]); - } - } - timelineMetric.setMetricValues(metricValues); - return timelineMetric; - } - return null; - } - - - private static TreeMap<Long, Double> getAsTimeSeries(long currentTime, double[] values) { - - long startTime = currentTime - (values.length - 1) * 60 * 1000; - TreeMap<Long, Double> metricValues = new TreeMap<>(); - - for (int i = 0; i < values.length; i++) { - metricValues.put(startTime, values[i]); - startTime += (60 * 1000); - } - return metricValues; - } - - private static DataSeries getAsDataSeries(TimelineMetric timelineMetric) { - - TreeMap<Long, Double> metricValues = timelineMetric.getMetricValues(); - double[] timestamps = new double[metricValues.size()]; - double[] values = new double[metricValues.size()]; - int i = 0; - - for (Long timestamp : metricValues.keySet()) { - timestamps[i] = timestamp; - values[i++] = metricValues.get(timestamp); - } - return new DataSeries(timelineMetric.getMetricName() + "_" + timelineMetric.getAppId() + "_" + timelineMetric.getHostName(), timestamps, values); - } -}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java deleted file mode 100644 index b817f3e..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestMetricSeriesGenerator.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.prototype.testing.utilities; - -/** - * Class which was originally used to send test series from AMS to Spark through Kafka. - */ - -public class TestMetricSeriesGenerator { - //implements Runnable { - -// private Map<TestSeriesInputRequest, AbstractMetricSeries> configuredSeries = new HashMap<>(); -// private static final Log LOG = LogFactory.getLog(TestMetricSeriesGenerator.class); -// private TimelineMetricStore metricStore; -// private String hostname; -// -// public TestMetricSeriesGenerator(TimelineMetricStore metricStore) { -// this.metricStore = metricStore; -// try { -// this.hostname = InetAddress.getLocalHost().getHostName(); -// } catch (UnknownHostException e) { -// e.printStackTrace(); -// } -// } -// -// public void addSeries(TestSeriesInputRequest inputRequest) { -// if (!configuredSeries.containsKey(inputRequest)) { -// AbstractMetricSeries metricSeries = MetricSeriesGeneratorFactory.generateSeries(inputRequest.getSeriesType(), inputRequest.getConfigs()); -// configuredSeries.put(inputRequest, metricSeries); -// LOG.info("Added series " + inputRequest.getSeriesName()); -// } -// } -// -// public void removeSeries(String seriesName) { -// boolean isPresent = false; -// TestSeriesInputRequest tbd = null; -// for (TestSeriesInputRequest inputRequest : configuredSeries.keySet()) { -// if (inputRequest.getSeriesName().equals(seriesName)) { -// isPresent = true; -// tbd = inputRequest; -// } -// } -// if (isPresent) { -// LOG.info("Removing series " + seriesName); -// configuredSeries.remove(tbd); -// } else { -// LOG.info("Series not found : " + seriesName); -// } -// } -// -// @Override -// public void run() { -// long currentTime = System.currentTimeMillis(); -// TimelineMetrics timelineMetrics = new TimelineMetrics(); -// -// for (TestSeriesInputRequest input : configuredSeries.keySet()) { -// AbstractMetricSeries metricSeries = configuredSeries.get(input); -// TimelineMetric timelineMetric = new TimelineMetric(); -// timelineMetric.setMetricName(input.getSeriesName()); -// timelineMetric.setAppId("anomaly-engine-test-metric"); -// timelineMetric.setInstanceId(null); -// timelineMetric.setStartTime(currentTime); -// timelineMetric.setHostName(hostname); -// TreeMap<Long, Double> metricValues = new TreeMap(); -// metricValues.put(currentTime, metricSeries.nextValue()); -// timelineMetric.setMetricValues(metricValues); -// timelineMetrics.addOrMergeTimelineMetric(timelineMetric); -// LOG.info("Emitting metric with appId = " + timelineMetric.getAppId()); -// } -// try { -// LOG.info("Publishing test metrics for " + timelineMetrics.getMetrics().size() + " series."); -// metricStore.putMetrics(timelineMetrics); -// } catch (Exception e) { -// LOG.error(e); -// } -// } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java deleted file mode 100644 index a424f8e..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/prototype/testing/utilities/TestSeriesInputRequest.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.prototype.testing.utilities; - -import org.apache.htrace.fasterxml.jackson.core.JsonProcessingException; -import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper; - -import javax.xml.bind.annotation.XmlRootElement; -import java.util.Collections; -import java.util.Map; - -@XmlRootElement -public class TestSeriesInputRequest { - - private String seriesName; - private String seriesType; - private Map<String, String> configs; - - public TestSeriesInputRequest() { - } - - public TestSeriesInputRequest(String seriesName, String seriesType, Map<String, String> configs) { - this.seriesName = seriesName; - this.seriesType = seriesType; - this.configs = configs; - } - - public String getSeriesName() { - return seriesName; - } - - public void setSeriesName(String seriesName) { - this.seriesName = seriesName; - } - - public String getSeriesType() { - return seriesType; - } - - public void setSeriesType(String seriesType) { - this.seriesType = seriesType; - } - - public Map<String, String> getConfigs() { - return configs; - } - - public void setConfigs(Map<String, String> configs) { - this.configs = configs; - } - - @Override - public boolean equals(Object o) { - TestSeriesInputRequest anotherInput = (TestSeriesInputRequest)o; - return anotherInput.getSeriesName().equals(this.getSeriesName()); - } - - @Override - public int hashCode() { - return seriesName.hashCode(); - } - - public static void main(String[] args) { - - ObjectMapper objectMapper = new ObjectMapper(); - TestSeriesInputRequest testSeriesInputRequest = new TestSeriesInputRequest("test", "ema", Collections.singletonMap("key","value")); - try { - System.out.print(objectMapper.writeValueAsString(testSeriesInputRequest)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java deleted file mode 100644 index a8e31bf..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/AbstractMetricSeries.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -public interface AbstractMetricSeries { - - public double nextValue(); - public double[] getSeries(int n); - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java deleted file mode 100644 index 4158ff4..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/DualBandMetricSeries.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Random; - -public class DualBandMetricSeries implements AbstractMetricSeries { - - double lowBandValue = 0.0; - double lowBandDeviationPercentage = 0.0; - int lowBandPeriodSize = 10; - double highBandValue = 1.0; - double highBandDeviationPercentage = 0.0; - int highBandPeriodSize = 10; - - Random random = new Random(); - double lowBandValueLowerLimit, lowBandValueHigherLimit; - double highBandLowerLimit, highBandUpperLimit; - int l = 0, h = 0; - - public DualBandMetricSeries(double lowBandValue, - double lowBandDeviationPercentage, - int lowBandPeriodSize, - double highBandValue, - double highBandDeviationPercentage, - int highBandPeriodSize) { - this.lowBandValue = lowBandValue; - this.lowBandDeviationPercentage = lowBandDeviationPercentage; - this.lowBandPeriodSize = lowBandPeriodSize; - this.highBandValue = highBandValue; - this.highBandDeviationPercentage = highBandDeviationPercentage; - this.highBandPeriodSize = highBandPeriodSize; - init(); - } - - private void init() { - lowBandValueLowerLimit = lowBandValue - lowBandDeviationPercentage * lowBandValue; - lowBandValueHigherLimit = lowBandValue + lowBandDeviationPercentage * lowBandValue; - highBandLowerLimit = highBandValue - highBandDeviationPercentage * highBandValue; - highBandUpperLimit = highBandValue + highBandDeviationPercentage * highBandValue; - } - - @Override - public double nextValue() { - - double value = 0.0; - - if (l < lowBandPeriodSize) { - value = lowBandValueLowerLimit + (lowBandValueHigherLimit - lowBandValueLowerLimit) * random.nextDouble(); - l++; - } else if (h < highBandPeriodSize) { - value = highBandLowerLimit + (highBandUpperLimit - highBandLowerLimit) * random.nextDouble(); - h++; - } - - if (l == lowBandPeriodSize && h == highBandPeriodSize) { - l = 0; - h = 0; - } - - return value; - } - - @Override - public double[] getSeries(int n) { - double[] series = new double[n]; - for (int i = 0; i < n; i++) { - series[i] = nextValue(); - } - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java deleted file mode 100644 index 1e37ff3..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MetricSeriesGeneratorFactory.java +++ /dev/null @@ -1,379 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Arrays; -import java.util.Map; -import java.util.Random; - -public class MetricSeriesGeneratorFactory { - - /** - * Return a normally distributed data series with some deviation % and outliers. - * - * @param n size of the data series - * @param value The value around which the uniform data series is centered on. - * @param deviationPercentage The allowed deviation % on either side of the uniform value. For example, if value = 10, and deviation % is 0.1, the series values lie between 0.9 to 1.1. - * @param outlierProbability The probability of finding an outlier in the series. - * @param outlierDeviationLowerPercentage min percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13. - * @param outlierDeviationHigherPercentage max percentage outlier should be away from the uniform value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16. - * @param outliersAboveValue Outlier should be greater or smaller than the value. - * @return uniform series - */ - public static double[] createUniformSeries(int n, - double value, - double deviationPercentage, - double outlierProbability, - double outlierDeviationLowerPercentage, - double outlierDeviationHigherPercentage, - boolean outliersAboveValue) { - - UniformMetricSeries metricSeries = new UniformMetricSeries(value, - deviationPercentage, - outlierProbability, - outlierDeviationLowerPercentage, - outlierDeviationHigherPercentage, - outliersAboveValue); - - return metricSeries.getSeries(n); - } - - - /** - * /** - * Returns a normally distributed series. - * - * @param n size of the data series - * @param mean mean of the distribution - * @param sd sd of the distribution - * @param outlierProbability sd of the distribution - * @param outlierDeviationSDTimesLower Lower Limit of the outlier with respect to times sdev from the mean. - * @param outlierDeviationSDTimesHigher Higher Limit of the outlier with respect to times sdev from the mean. - * @param outlierOnRightEnd Outlier should be on the right end or the left end. - * @return normal series - */ - public static double[] createNormalSeries(int n, - double mean, - double sd, - double outlierProbability, - double outlierDeviationSDTimesLower, - double outlierDeviationSDTimesHigher, - boolean outlierOnRightEnd) { - - - NormalMetricSeries metricSeries = new NormalMetricSeries(mean, - sd, - outlierProbability, - outlierDeviationSDTimesLower, - outlierDeviationSDTimesHigher, - outlierOnRightEnd); - - return metricSeries.getSeries(n); - } - - - /** - * Returns a monotonically increasing / decreasing series - * - * @param n size of the data series - * @param startValue Start value of the monotonic sequence - * @param slope direction of monotonicity m > 0 for increasing and m < 0 for decreasing. - * @param deviationPercentage The allowed deviation % on either side of the current 'y' value. For example, if current value = 10 according to slope, and deviation % is 0.1, the series values lie between 0.9 to 1.1. - * @param outlierProbability The probability of finding an outlier in the series. - * @param outlierDeviationLowerPercentage min percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 30%, the outlier is 7 and 13. - * @param outlierDeviationHigherPercentage max percentage outlier should be away from the current 'y' value in % terms. if value = 10 and outlierDeviationPercentage = 60%, the outlier is 4 and 16. - * @param outliersAboveValue Outlier should be greater or smaller than the 'y' value. - * @return - */ - public static double[] createMonotonicSeries(int n, - double startValue, - double slope, - double deviationPercentage, - double outlierProbability, - double outlierDeviationLowerPercentage, - double outlierDeviationHigherPercentage, - boolean outliersAboveValue) { - - MonotonicMetricSeries metricSeries = new MonotonicMetricSeries(startValue, - slope, - deviationPercentage, - outlierProbability, - outlierDeviationLowerPercentage, - outlierDeviationHigherPercentage, - outliersAboveValue); - - return metricSeries.getSeries(n); - } - - - /** - * Returns a dual band series (lower and higher) - * - * @param n size of the data series - * @param lowBandValue lower band value - * @param lowBandDeviationPercentage lower band deviation - * @param lowBandPeriodSize lower band - * @param highBandValue high band centre value - * @param highBandDeviationPercentage high band deviation. - * @param highBandPeriodSize high band size - * @return - */ - public static double[] getDualBandSeries(int n, - double lowBandValue, - double lowBandDeviationPercentage, - int lowBandPeriodSize, - double highBandValue, - double highBandDeviationPercentage, - int highBandPeriodSize) { - - DualBandMetricSeries metricSeries = new DualBandMetricSeries(lowBandValue, - lowBandDeviationPercentage, - lowBandPeriodSize, - highBandValue, - highBandDeviationPercentage, - highBandPeriodSize); - - return metricSeries.getSeries(n); - } - - /** - * Returns a step function series. - * - * @param n size of the data series - * @param startValue start steady value - * @param steadyValueDeviationPercentage required devation in the steady state value - * @param steadyPeriodSlope direction of monotonicity m > 0 for increasing and m < 0 for decreasing, m = 0 no increase or decrease. - * @param steadyPeriodMinSize min size for step period - * @param steadyPeriodMaxSize max size for step period. - * @param stepChangePercentage Increase / decrease in steady state to denote a step in terms of deviation percentage from the last value. - * @param upwardStep upward or downward step. - * @return - */ - public static double[] getStepFunctionSeries(int n, - double startValue, - double steadyValueDeviationPercentage, - double steadyPeriodSlope, - int steadyPeriodMinSize, - int steadyPeriodMaxSize, - double stepChangePercentage, - boolean upwardStep) { - - StepFunctionMetricSeries metricSeries = new StepFunctionMetricSeries(startValue, - steadyValueDeviationPercentage, - steadyPeriodSlope, - steadyPeriodMinSize, - steadyPeriodMaxSize, - stepChangePercentage, - upwardStep); - - return metricSeries.getSeries(n); - } - - /** - * Series with small period of turbulence and then back to steady. - * - * @param n size of the data series - * @param steadyStateValue steady state center value - * @param steadyStateDeviationPercentage steady state deviation in percentage - * @param turbulentPeriodDeviationLowerPercentage turbulent state lower limit in terms of percentage from centre value. - * @param turbulentPeriodDeviationHigherPercentage turbulent state higher limit in terms of percentage from centre value. - * @param turbulentPeriodLength turbulent period length (number of points) - * @param turbulentStatePosition Where the turbulent state should be 0 - at the beginning, 1 - in the middle (25% - 50% of the series), 2 - at the end of the series. - * @return - */ - public static double[] getSteadySeriesWithTurbulentPeriod(int n, - double steadyStateValue, - double steadyStateDeviationPercentage, - double turbulentPeriodDeviationLowerPercentage, - double turbulentPeriodDeviationHigherPercentage, - int turbulentPeriodLength, - int turbulentStatePosition - ) { - - - SteadyWithTurbulenceMetricSeries metricSeries = new SteadyWithTurbulenceMetricSeries(n, - steadyStateValue, - steadyStateDeviationPercentage, - turbulentPeriodDeviationLowerPercentage, - turbulentPeriodDeviationHigherPercentage, - turbulentPeriodLength, - turbulentStatePosition); - - return metricSeries.getSeries(n); - } - - - public static double[] generateSeries(String type, int n, Map<String, String> configs) { - - double[] series; - switch (type) { - - case "normal": - series = createNormalSeries(n, - Double.parseDouble(configs.getOrDefault("mean", "0")), - Double.parseDouble(configs.getOrDefault("sd", "1")), - Double.parseDouble(configs.getOrDefault("outlierProbability", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")), - Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true"))); - break; - - case "uniform": - series = createUniformSeries(n, - Double.parseDouble(configs.getOrDefault("value", "10")), - Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierProbability", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")), - Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true"))); - break; - - case "monotonic": - series = createMonotonicSeries(n, - Double.parseDouble(configs.getOrDefault("startValue", "10")), - Double.parseDouble(configs.getOrDefault("slope", "0")), - Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierProbability", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")), - Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true"))); - break; - - case "dualband": - series = getDualBandSeries(n, - Double.parseDouble(configs.getOrDefault("lowBandValue", "10")), - Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")), - Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")), - Double.parseDouble(configs.getOrDefault("highBandValue", "10")), - Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")), - Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0"))); - break; - - case "step": - series = getStepFunctionSeries(n, - Double.parseDouble(configs.getOrDefault("startValue", "10")), - Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")), - Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")), - Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")), - Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")), - Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true"))); - break; - - case "turbulence": - series = getSteadySeriesWithTurbulentPeriod(n, - Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")), - Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")), - Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")), - Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")), - Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0"))); - break; - - default: - series = createNormalSeries(n, - 0, - 1, - 0, - 0, - 0, - true); - } - return series; - } - - public static AbstractMetricSeries generateSeries(String type, Map<String, String> configs) { - - AbstractMetricSeries series; - switch (type) { - - case "normal": - series = new NormalMetricSeries(Double.parseDouble(configs.getOrDefault("mean", "0")), - Double.parseDouble(configs.getOrDefault("sd", "1")), - Double.parseDouble(configs.getOrDefault("outlierProbability", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesLower", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationSDTimesHigher", "0")), - Boolean.parseBoolean(configs.getOrDefault("outlierOnRightEnd", "true"))); - break; - - case "uniform": - series = new UniformMetricSeries( - Double.parseDouble(configs.getOrDefault("value", "10")), - Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierProbability", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")), - Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true"))); - break; - - case "monotonic": - series = new MonotonicMetricSeries( - Double.parseDouble(configs.getOrDefault("startValue", "10")), - Double.parseDouble(configs.getOrDefault("slope", "0")), - Double.parseDouble(configs.getOrDefault("deviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierProbability", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationLowerPercentage", "0")), - Double.parseDouble(configs.getOrDefault("outlierDeviationHigherPercentage", "0")), - Boolean.parseBoolean(configs.getOrDefault("outliersAboveValue", "true"))); - break; - - case "dualband": - series = new DualBandMetricSeries( - Double.parseDouble(configs.getOrDefault("lowBandValue", "10")), - Double.parseDouble(configs.getOrDefault("lowBandDeviationPercentage", "0")), - Integer.parseInt(configs.getOrDefault("lowBandPeriodSize", "0")), - Double.parseDouble(configs.getOrDefault("highBandValue", "10")), - Double.parseDouble(configs.getOrDefault("highBandDeviationPercentage", "0")), - Integer.parseInt(configs.getOrDefault("highBandPeriodSize", "0"))); - break; - - case "step": - series = new StepFunctionMetricSeries( - Double.parseDouble(configs.getOrDefault("startValue", "10")), - Double.parseDouble(configs.getOrDefault("steadyValueDeviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("steadyPeriodSlope", "0")), - Integer.parseInt(configs.getOrDefault("steadyPeriodMinSize", "0")), - Integer.parseInt(configs.getOrDefault("steadyPeriodMaxSize", "0")), - Double.parseDouble(configs.getOrDefault("stepChangePercentage", "0")), - Boolean.parseBoolean(configs.getOrDefault("upwardStep", "true"))); - break; - - case "turbulence": - series = new SteadyWithTurbulenceMetricSeries( - Integer.parseInt(configs.getOrDefault("approxSeriesLength", "100")), - Double.parseDouble(configs.getOrDefault("steadyStateValue", "10")), - Double.parseDouble(configs.getOrDefault("steadyStateDeviationPercentage", "0")), - Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationLowerPercentage", "0")), - Double.parseDouble(configs.getOrDefault("turbulentPeriodDeviationHigherPercentage", "10")), - Integer.parseInt(configs.getOrDefault("turbulentPeriodLength", "0")), - Integer.parseInt(configs.getOrDefault("turbulentStatePosition", "0"))); - break; - - default: - series = new NormalMetricSeries(0, - 1, - 0, - 0, - 0, - true); - } - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java deleted file mode 100644 index a883d08..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/MonotonicMetricSeries.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Random; - -public class MonotonicMetricSeries implements AbstractMetricSeries { - - double startValue = 0.0; - double slope = 0.5; - double deviationPercentage = 0.0; - double outlierProbability = 0.0; - double outlierDeviationLowerPercentage = 0.0; - double outlierDeviationHigherPercentage = 0.0; - boolean outliersAboveValue = true; - - Random random = new Random(); - double nonOutlierProbability; - - // y = mx + c - double y; - double m; - double x; - double c; - - public MonotonicMetricSeries(double startValue, - double slope, - double deviationPercentage, - double outlierProbability, - double outlierDeviationLowerPercentage, - double outlierDeviationHigherPercentage, - boolean outliersAboveValue) { - this.startValue = startValue; - this.slope = slope; - this.deviationPercentage = deviationPercentage; - this.outlierProbability = outlierProbability; - this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage; - this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage; - this.outliersAboveValue = outliersAboveValue; - init(); - } - - private void init() { - y = startValue; - m = slope; - x = 1; - c = y - (m * x); - nonOutlierProbability = 1.0 - outlierProbability; - } - - @Override - public double nextValue() { - - double value; - double probability = random.nextDouble(); - - y = m * x + c; - if (probability <= nonOutlierProbability) { - double valueDeviationLowerLimit = y - deviationPercentage * y; - double valueDeviationHigherLimit = y + deviationPercentage * y; - value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble(); - } else { - if (outliersAboveValue) { - double outlierLowerLimit = y + outlierDeviationLowerPercentage * y; - double outlierUpperLimit = y + outlierDeviationHigherPercentage * y; - value = outlierLowerLimit + (outlierUpperLimit - outlierLowerLimit) * random.nextDouble(); - } else { - double outlierLowerLimit = y - outlierDeviationLowerPercentage * y; - double outlierUpperLimit = y - outlierDeviationHigherPercentage * y; - value = outlierUpperLimit + (outlierLowerLimit - outlierUpperLimit) * random.nextDouble(); - } - } - x++; - return value; - } - - @Override - public double[] getSeries(int n) { - double[] series = new double[n]; - for (int i = 0; i < n; i++) { - series[i] = nextValue(); - } - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java deleted file mode 100644 index cc83d2c..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/NormalMetricSeries.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Random; - -public class NormalMetricSeries implements AbstractMetricSeries { - - double mean = 0.0; - double sd = 1.0; - double outlierProbability = 0.0; - double outlierDeviationSDTimesLower = 0.0; - double outlierDeviationSDTimesHigher = 0.0; - boolean outlierOnRightEnd = true; - - Random random = new Random(); - double nonOutlierProbability; - - - public NormalMetricSeries(double mean, - double sd, - double outlierProbability, - double outlierDeviationSDTimesLower, - double outlierDeviationSDTimesHigher, - boolean outlierOnRightEnd) { - this.mean = mean; - this.sd = sd; - this.outlierProbability = outlierProbability; - this.outlierDeviationSDTimesLower = outlierDeviationSDTimesLower; - this.outlierDeviationSDTimesHigher = outlierDeviationSDTimesHigher; - this.outlierOnRightEnd = outlierOnRightEnd; - init(); - } - - private void init() { - nonOutlierProbability = 1.0 - outlierProbability; - } - - @Override - public double nextValue() { - - double value; - double probability = random.nextDouble(); - - if (probability <= nonOutlierProbability) { - value = random.nextGaussian() * sd + mean; - } else { - if (outlierOnRightEnd) { - value = mean + (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd; - } else { - value = mean - (outlierDeviationSDTimesLower + (outlierDeviationSDTimesHigher - outlierDeviationSDTimesLower) * random.nextDouble()) * sd; - } - } - return value; - } - - @Override - public double[] getSeries(int n) { - double[] series = new double[n]; - for (int i = 0; i < n; i++) { - series[i] = nextValue(); - } - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java deleted file mode 100644 index c4ed3ba..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/SteadyWithTurbulenceMetricSeries.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Random; - -public class SteadyWithTurbulenceMetricSeries implements AbstractMetricSeries { - - double steadyStateValue = 0.0; - double steadyStateDeviationPercentage = 0.0; - double turbulentPeriodDeviationLowerPercentage = 0.3; - double turbulentPeriodDeviationHigherPercentage = 0.5; - int turbulentPeriodLength = 5; - int turbulentStatePosition = 1; - int approximateSeriesLength = 10; - - Random random = new Random(); - double valueDeviationLowerLimit; - double valueDeviationHigherLimit; - double tPeriodLowerLimit; - double tPeriodUpperLimit; - int tPeriodStartIndex = 0; - int index = 0; - - public SteadyWithTurbulenceMetricSeries(int approximateSeriesLength, - double steadyStateValue, - double steadyStateDeviationPercentage, - double turbulentPeriodDeviationLowerPercentage, - double turbulentPeriodDeviationHigherPercentage, - int turbulentPeriodLength, - int turbulentStatePosition) { - this.approximateSeriesLength = approximateSeriesLength; - this.steadyStateValue = steadyStateValue; - this.steadyStateDeviationPercentage = steadyStateDeviationPercentage; - this.turbulentPeriodDeviationLowerPercentage = turbulentPeriodDeviationLowerPercentage; - this.turbulentPeriodDeviationHigherPercentage = turbulentPeriodDeviationHigherPercentage; - this.turbulentPeriodLength = turbulentPeriodLength; - this.turbulentStatePosition = turbulentStatePosition; - init(); - } - - private void init() { - - if (turbulentStatePosition == 1) { - tPeriodStartIndex = (int) (0.25 * approximateSeriesLength + (0.25 * approximateSeriesLength * random.nextDouble())); - } else if (turbulentStatePosition == 2) { - tPeriodStartIndex = approximateSeriesLength - turbulentPeriodLength; - } - - valueDeviationLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue; - valueDeviationHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue; - - tPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue; - tPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue; - } - - @Override - public double nextValue() { - - double value; - - if (index >= tPeriodStartIndex && index <= (tPeriodStartIndex + turbulentPeriodLength)) { - value = tPeriodLowerLimit + (tPeriodUpperLimit - tPeriodLowerLimit) * random.nextDouble(); - } else { - value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble(); - } - index++; - return value; - } - - @Override - public double[] getSeries(int n) { - - double[] series = new double[n]; - int turbulentPeriodStartIndex = 0; - - if (turbulentStatePosition == 1) { - turbulentPeriodStartIndex = (int) (0.25 * n + (0.25 * n * random.nextDouble())); - } else if (turbulentStatePosition == 2) { - turbulentPeriodStartIndex = n - turbulentPeriodLength; - } - - double valueDevLowerLimit = steadyStateValue - steadyStateDeviationPercentage * steadyStateValue; - double valueDevHigherLimit = steadyStateValue + steadyStateDeviationPercentage * steadyStateValue; - - double turbulentPeriodLowerLimit = steadyStateValue + turbulentPeriodDeviationLowerPercentage * steadyStateValue; - double turbulentPeriodUpperLimit = steadyStateValue + turbulentPeriodDeviationHigherPercentage * steadyStateValue; - - for (int i = 0; i < n; i++) { - if (i >= turbulentPeriodStartIndex && i < (turbulentPeriodStartIndex + turbulentPeriodLength)) { - series[i] = turbulentPeriodLowerLimit + (turbulentPeriodUpperLimit - turbulentPeriodLowerLimit) * random.nextDouble(); - } else { - series[i] = valueDevLowerLimit + (valueDevHigherLimit - valueDevLowerLimit) * random.nextDouble(); - } - } - - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java deleted file mode 100644 index d5beb48..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/StepFunctionMetricSeries.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Random; - -public class StepFunctionMetricSeries implements AbstractMetricSeries { - - double startValue = 0.0; - double steadyValueDeviationPercentage = 0.0; - double steadyPeriodSlope = 0.5; - int steadyPeriodMinSize = 10; - int steadyPeriodMaxSize = 20; - double stepChangePercentage = 0.0; - boolean upwardStep = true; - - Random random = new Random(); - - // y = mx + c - double y; - double m; - double x; - double c; - int currentStepSize; - int currentIndex; - - public StepFunctionMetricSeries(double startValue, - double steadyValueDeviationPercentage, - double steadyPeriodSlope, - int steadyPeriodMinSize, - int steadyPeriodMaxSize, - double stepChangePercentage, - boolean upwardStep) { - this.startValue = startValue; - this.steadyValueDeviationPercentage = steadyValueDeviationPercentage; - this.steadyPeriodSlope = steadyPeriodSlope; - this.steadyPeriodMinSize = steadyPeriodMinSize; - this.steadyPeriodMaxSize = steadyPeriodMaxSize; - this.stepChangePercentage = stepChangePercentage; - this.upwardStep = upwardStep; - init(); - } - - private void init() { - y = startValue; - m = steadyPeriodSlope; - x = 1; - c = y - (m * x); - - currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble()); - currentIndex = 0; - } - - @Override - public double nextValue() { - - double value = 0.0; - - if (currentIndex < currentStepSize) { - y = m * x + c; - double valueDeviationLowerLimit = y - steadyValueDeviationPercentage * y; - double valueDeviationHigherLimit = y + steadyValueDeviationPercentage * y; - value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble(); - x++; - currentIndex++; - } - - if (currentIndex == currentStepSize) { - currentIndex = 0; - currentStepSize = (int) (steadyPeriodMinSize + (steadyPeriodMaxSize - steadyPeriodMinSize) * random.nextDouble()); - if (upwardStep) { - y = y + stepChangePercentage * y; - } else { - y = y - stepChangePercentage * y; - } - x = 1; - c = y - (m * x); - } - - return value; - } - - @Override - public double[] getSeries(int n) { - double[] series = new double[n]; - for (int i = 0; i < n; i++) { - series[i] = nextValue(); - } - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java deleted file mode 100644 index a2b0eea..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/java/org/apache/ambari/metrics/alertservice/seriesgenerator/UniformMetricSeries.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.metrics.alertservice.seriesgenerator; - -import java.util.Random; - -public class UniformMetricSeries implements AbstractMetricSeries { - - double value = 0.0; - double deviationPercentage = 0.0; - double outlierProbability = 0.0; - double outlierDeviationLowerPercentage = 0.0; - double outlierDeviationHigherPercentage = 0.0; - boolean outliersAboveValue= true; - - Random random = new Random(); - double valueDeviationLowerLimit; - double valueDeviationHigherLimit; - double outlierLeftLowerLimit; - double outlierLeftHigherLimit; - double outlierRightLowerLimit; - double outlierRightUpperLimit; - double nonOutlierProbability; - - - public UniformMetricSeries(double value, - double deviationPercentage, - double outlierProbability, - double outlierDeviationLowerPercentage, - double outlierDeviationHigherPercentage, - boolean outliersAboveValue) { - this.value = value; - this.deviationPercentage = deviationPercentage; - this.outlierProbability = outlierProbability; - this.outlierDeviationLowerPercentage = outlierDeviationLowerPercentage; - this.outlierDeviationHigherPercentage = outlierDeviationHigherPercentage; - this.outliersAboveValue = outliersAboveValue; - init(); - } - - private void init() { - valueDeviationLowerLimit = value - deviationPercentage * value; - valueDeviationHigherLimit = value + deviationPercentage * value; - - outlierLeftLowerLimit = value - outlierDeviationHigherPercentage * value; - outlierLeftHigherLimit = value - outlierDeviationLowerPercentage * value; - outlierRightLowerLimit = value + outlierDeviationLowerPercentage * value; - outlierRightUpperLimit = value + outlierDeviationHigherPercentage * value; - - nonOutlierProbability = 1.0 - outlierProbability; - } - - @Override - public double nextValue() { - - double value; - double probability = random.nextDouble(); - - if (probability <= nonOutlierProbability) { - value = valueDeviationLowerLimit + (valueDeviationHigherLimit - valueDeviationLowerLimit) * random.nextDouble(); - } else { - if (!outliersAboveValue) { - value = outlierLeftLowerLimit + (outlierLeftHigherLimit - outlierLeftLowerLimit) * random.nextDouble(); - } else { - value = outlierRightLowerLimit + (outlierRightUpperLimit - outlierRightLowerLimit) * random.nextDouble(); - } - } - return value; - } - - @Override - public double[] getSeries(int n) { - double[] series = new double[n]; - for (int i = 0; i < n; i++) { - series[i] = nextValue(); - } - return series; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R deleted file mode 100644 index 0b66095..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/ema.R +++ /dev/null @@ -1,96 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# EMA <- w * EMA + (1 - w) * x -# EMS <- sqrt( w * EMS^2 + (1 - w) * (x - EMA)^2 ) -# Alarm = abs(x - EMA) > n * EMS - -ema_global <- function(train_data, test_data, w, n) { - -# res <- get_data(url) -# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) -# names(data) <- c("TS", res$metrics[[1]]$metricname) -# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2] -# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ] - - anomalies <- data.frame() - ema <- 0 - ems <- 0 - - #Train Step - for (x in train_data) { - ema <- w*ema + (1-w)*x - ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2) - } - - for ( i in 1:length(test_data[,1])) { - x <- test_data[i,2] - if (abs(x - ema) > n*ems) { - anomaly <- c(as.numeric(test_data[i,1]), x) - # print (anomaly) - anomalies <- rbind(anomalies, anomaly) - } - ema <- w*ema + (1-w)*x - ems <- sqrt(w* ems^2 + (1 - w)*(x - ema)^2) - } - - if(length(anomalies) > 0) { - names(anomalies) <- c("TS", "Value") - } - return (anomalies) -} - -ema_daily <- function(train_data, test_data, w, n) { - -# res <- get_data(url) -# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) -# names(data) <- c("TS", res$metrics[[1]]$metricname) -# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), ] -# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ] - - anomalies <- data.frame() - ema <- vector("numeric", 7) - ems <- vector("numeric", 7) - - #Train Step - for ( i in 1:length(train_data[,1])) { - x <- train_data[i,2] - time <- as.POSIXlt(as.numeric(train_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT") - index <- time$wday - ema[index] <- w*ema[index] + (1-w)*x - ems[index] <- sqrt(w* ems[index]^2 + (1 - w)*(x - ema[index])^2) - } - - for ( i in 1:length(test_data[,1])) { - x <- test_data[i,2] - time <- as.POSIXlt(as.numeric(test_data[i,1])/1000, origin = "1970-01-01" ,tz = "GMT") - index <- time$wday - - if (abs(x - ema[index+1]) > n*ems[index+1]) { - anomaly <- c(as.numeric(test_data[i,1]), x) - # print (anomaly) - anomalies <- rbind(anomalies, anomaly) - } - ema[index+1] <- w*ema[index+1] + (1-w)*x - ems[index+1] <- sqrt(w* ems[index+1]^2 + (1 - w)*(x - ema[index+1])^2) - } - - if(length(anomalies) > 0) { - names(anomalies) <- c("TS", "Value") - } - return(anomalies) -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r deleted file mode 100644 index bca3366..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/hsdev.r +++ /dev/null @@ -1,67 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -hsdev_daily <- function(train_data, test_data, n, num_historic_periods, interval, period) { - - #res <- get_data(url) - #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) - #names(data) <- c("TS", res$metrics[[1]]$metricname) - anomalies <- data.frame() - - granularity <- train_data[2,1] - train_data[1,1] - test_start <- test_data[1,1] - test_end <- test_data[length(test_data[1,]),1] - train_start <- test_start - num_historic_periods*period - # round to start of day - train_start <- train_start - (train_start %% interval) - - time <- as.POSIXlt(as.numeric(test_data[1,1])/1000, origin = "1970-01-01" ,tz = "GMT") - test_data_day <- time$wday - - h_data <- c() - for ( i in length(train_data[,1]):1) { - ts <- train_data[i,1] - if ( ts < train_start) { - break - } - time <- as.POSIXlt(as.numeric(ts)/1000, origin = "1970-01-01" ,tz = "GMT") - if (time$wday == test_data_day) { - x <- train_data[i,2] - h_data <- c(h_data, x) - } - } - - if (length(h_data) < 2*length(test_data[,1])) { - cat ("\nNot enough training data") - return (anomalies) - } - - past_median <- median(h_data) - past_sd <- sd(h_data) - curr_median <- median(test_data[,2]) - - if (abs(curr_median - past_median) > n * past_sd) { - anomaly <- c(test_start, test_end, curr_median, past_median, past_sd) - anomalies <- rbind(anomalies, anomaly) - } - - if(length(anomalies) > 0) { - names(anomalies) <- c("TS Start", "TS End", "Current Median", "Past Median", "Past SD") - } - - return (anomalies) -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R deleted file mode 100644 index 8956400..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/iforest.R +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ams_iforest <- function(url, train_start, train_end, test_start, test_end, threshold_score) { - - res <- get_data(url) - num_metrics <- length(res$metrics) - anomalies <- data.frame() - - metricname <- res$metrics[[1]]$metricname - data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) - names(data) <- c("TS", res$metrics[[1]]$metricname) - - for (i in 2:num_metrics) { - metricname <- res$metrics[[i]]$metricname - df <- data.frame(as.numeric(names(res$metrics[[i]]$metrics)), as.numeric(res$metrics[[i]]$metrics)) - names(df) <- c("TS", res$metrics[[i]]$metricname) - data <- merge(data, df) - } - - algo_data <- data[ which(df$TS >= train_start & df$TS <= train_end) , ][c(1:num_metrics+1)] - iForest <- IsolationTrees(algo_data) - test_data <- data[ which(df$TS >= test_start & df$TS <= test_end) , ] - - if_res <- AnomalyScore(test_data[c(1:num_metrics+1)], iForest) - for (i in 1:length(if_res$outF)) { - index <- test_start+i-1 - if (if_res$outF[i] > threshold_score) { - anomaly <- c(test_data[i,1], if_res$outF[i], if_res$pathLength[i]) - anomalies <- rbind(anomalies, anomaly) - } - } - - if(length(anomalies) > 0) { - names(anomalies) <- c("TS", "Anomaly Score", "Path length") - } - return (anomalies) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r deleted file mode 100644 index f22bc15..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/kstest.r +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ams_ks <- function(train_data, test_data, p_value) { - -# res <- get_data(url) -# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) -# names(data) <- c("TS", res$metrics[[1]]$metricname) -# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2] -# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), 2] - - anomalies <- data.frame() - res <- ks.test(train_data[,2], test_data[,2]) - - if (res[2] < p_value) { - anomaly <- c(test_data[1,1], test_data[length(test_data),1], res[1], res[2]) - anomalies <- rbind(anomalies, anomaly) - } - - if(length(anomalies) > 0) { - names(anomalies) <- c("TS Start", "TS End", "D", "p-value") - } - return (anomalies) -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R deleted file mode 100644 index 7650356..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/test.R +++ /dev/null @@ -1,85 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -tukeys_anomalies <- data.frame() -ema_global_anomalies <- data.frame() -ema_daily_anomalies <- data.frame() -ks_anomalies <- data.frame() -hsdev_anomalies <- data.frame() - -init <- function() { - tukeys_anomalies <- data.frame() - ema_global_anomalies <- data.frame() - ema_daily_anomalies <- data.frame() - ks_anomalies <- data.frame() - hsdev_anomalies <- data.frame() -} - -test_methods <- function(data) { - - init() - #res <- get_data(url) - #data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) - #names(data) <- c("TS", res$metrics[[1]]$metricname) - - limit <- data[length(data[,1]),1] - step <- data[2,1] - data[1,1] - - train_start <- data[1,1] - train_end <- get_next_day_boundary(train_start, step, limit) - test_start <- train_end + step - test_end <- get_next_day_boundary(test_start, step, limit) - i <- 1 - day <- 24*60*60*1000 - - while (test_start < limit) { - - print (i) - i <- i + 1 - train_data <- data[which(data$TS >= train_start & data$TS <= train_end),] - test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ] - - #tukeys_anomalies <<- rbind(tukeys_anomalies, ams_tukeys(train_data, test_data, 3)) - #ema_global_anomalies <<- rbind(ema_global_anomalies, ema_global(train_data, test_data, 0.9, 3)) - #ema_daily_anomalies <<- rbind(ema_daily_anomalies, ema_daily(train_data, test_data, 0.9, 3)) - #ks_anomalies <<- rbind(ks_anomalies, ams_ks(train_data, test_data, 0.05)) - hsdev_train_data <- data[which(data$TS < test_start),] - hsdev_anomalies <<- rbind(hsdev_anomalies, hsdev_daily(hsdev_train_data, test_data, 3, 3, day, 7*day)) - - train_start <- test_start - train_end <- get_next_day_boundary(train_start, step, limit) - test_start <- train_end + step - test_end <- get_next_day_boundary(test_start, step, limit) - } - return (hsdev_anomalies) -} - -get_next_day_boundary <- function(start, step, limit) { - - if (start > limit) { - return (-1) - } - - while (start <= limit) { - if (((start %% (24*60*60*1000)) - 28800000) == 0) { - return (start) - } - start <- start + step - } - return (start) -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r deleted file mode 100644 index 0312226..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/R-scripts/tukeys.r +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -ams_tukeys <- function(train_data, test_data, n) { - -# res <- get_data(url) -# data <- data.frame(as.numeric(names(res$metrics[[1]]$metrics)), as.numeric(res$metrics[[1]]$metrics)) -# names(data) <- c("TS", res$metrics[[1]]$metricname) -# train_data <- data[which(data$TS >= train_start & data$TS <= train_end), 2] -# test_data <- data[which(data$TS >= test_start & data$TS <= test_end), ] - - anomalies <- data.frame() - quantiles <- quantile(train_data[,2]) - iqr <- quantiles[4] - quantiles[2] - niqr <- 0 - - for ( i in 1:length(test_data[,1])) { - x <- test_data[i,2] - lb <- quantiles[2] - n*iqr - ub <- quantiles[4] + n*iqr - if ( (x < lb) || (x > ub) ) { - if (iqr != 0) { - if (x < lb) { - niqr <- (quantiles[2] - x) / iqr - } else { - niqr <- (x - quantiles[4]) / iqr - } - } - anomaly <- c(test_data[i,1], x, niqr) - anomalies <- rbind(anomalies, anomaly) - } - } - if(length(anomalies) > 0) { - names(anomalies) <- c("TS", "Value", "niqr") - } - return (anomalies) -} http://git-wip-us.apache.org/repos/asf/ambari/blob/4613b471/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties b/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties deleted file mode 100644 index ab106c4..0000000 --- a/ambari-metrics/ambari-metrics-anomaly-detector/src/main/resources/input-config.properties +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -appIds=HOST - -collectorHost=localhost -collectorPort=6188 -collectorProtocol=http - -zkQuorum=localhost:2181 - -ambariServerHost=localhost -clusterName=c1 - -emaW=0.8 -emaN=3 -tukeysN=3 -pointInTimeTestInterval=300000 -pointInTimeTrainInterval=900000 - -ksTestInterval=600000 -ksTrainInterval=600000 -hsdevNhp=3 -hsdevInterval=1800000; - -skipMetricPatterns=sdisk*,cpu_sintr*,proc*,disk*,boottime -hosts=avijayan-ad-1.openstacklocal \ No newline at end of file
