This is an automated email from the ASF dual-hosted git repository. jihao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push: new f50e87f [TE] Model evaluator interface & Implementation (#4209) f50e87f is described below commit f50e87f14c3756f1673bc2c5d8f8b8d4cc1f74d1 Author: Jihao Zhang <jihzh...@linkedin.com> AuthorDate: Fri May 17 14:12:52 2019 -0700 [TE] Model evaluator interface & Implementation (#4209) - The boilerplate code to retrieve the evaluations in detection components, including dataProvider, InputDataFetcher, etc. - Define the model evaluator interface, which read the evaluation metrics and evaluate the detection model status. - Implement the MAPE percentage change evaluator, which compare the mean MAPE over past 7 days and past 30 days, if the up percentage change is more than a certain threshold, returns a bad detection status. --- .../dashboard/resources/v2/AnomaliesResource.java | 6 +- .../pinot/thirdeye/detection/DataProvider.java | 14 +++ .../thirdeye/detection/DefaultDataProvider.java | 31 +++++- .../detection/DefaultInputDataFetcher.java | 6 +- .../detection/DetectionPipelineTaskRunner.java | 7 +- .../thirdeye/detection/DetectionResource.java | 5 +- .../detection/alert/DetectionAlertTaskFactory.java | 6 +- .../MapeAveragePercentageChangeModelEvaluator.java | 124 +++++++++++++++++++++ .../finetune/GridSearchTuningAlgorithm.java | 20 ++-- .../onboard/YamlOnboardingTaskRunner.java | 5 +- ...eAveragePercentageChangeModelEvaluatorSpec.java | 38 +++++++ .../detection/spi/components/ModelEvaluator.java | 41 +++++++ .../detection/spi/model/EvaluationSlice.java | 68 +++++++++++ .../thirdeye/detection/spi/model/InputData.java | 12 +- .../detection/spi/model/InputDataSpec.java | 34 ++++-- .../detection/spi/model/ModelEvaluationResult.java | 39 +++++++ .../thirdeye/detection/spi/model/ModelStatus.java | 30 +++++ .../validators/DetectionConfigValidator.java | 1 + .../thirdeye/detection/yaml/YamlResource.java | 5 +- .../pinot/thirdeye/detection/DataProviderTest.java | 6 +- .../pinot/thirdeye/detection/MockDataProvider.java | 25 +++++ .../MapePercentageChangeModelEvaluatorTest.java | 82 ++++++++++++++ 22 files changed, 571 insertions(+), 34 deletions(-) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java index 461ffe3..fbc7920 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java @@ -82,6 +82,7 @@ import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.GroupedAnomalyResultsManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; @@ -155,6 +156,7 @@ public class AnomaliesResource { private final DetectionConfigManager detectionDAO; private final EventManager eventDAO; private final MergedAnomalyResultManager anomalyDAO; + private final EvaluationManager evaluationDAO; private final ExecutorService threadPool; private final AlertFilterFactory alertFilterFactory; @@ -177,6 +179,7 @@ public class AnomaliesResource { this.anomalyFunctionFactory = anomalyFunctionFactory; this.eventDAO = DAORegistry.getInstance().getEventDAO(); this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO(); + this.evaluationDAO = DAO_REGISTRY.getEvaluationManager(); QueryCache queryCache = ThirdEyeCacheRegistry.getInstance().getQueryCache(); LoadingCache<String, Long> maxTimeCache = ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache(); @@ -185,7 +188,8 @@ public class AnomaliesResource { this.aggregationLoader = new DefaultAggregationLoader(this.metricConfigDAO, this.datasetConfigDAO, queryCache, maxTimeCache); this.loader = new DetectionPipelineLoader(); - this.provider = new DefaultDataProvider(this.metricConfigDAO, this.datasetConfigDAO, this.eventDAO, this.anomalyDAO, this.timeSeriesLoader, this.aggregationLoader, this.loader); + this.provider = new DefaultDataProvider(this.metricConfigDAO, this.datasetConfigDAO, this.eventDAO, this.anomalyDAO, this.evaluationDAO, + this.timeSeriesLoader, this.aggregationLoader, this.loader); } @GET diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java index 4d80598..fb6e902 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java @@ -23,14 +23,17 @@ import com.google.common.collect.Multimap; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.commons.lang.NotImplementedException; import org.apache.pinot.thirdeye.dataframe.DataFrame; import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; import org.apache.pinot.thirdeye.datalayer.dto.EventDTO; import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; +import org.apache.pinot.thirdeye.detection.spi.model.EvaluationSlice; import org.apache.pinot.thirdeye.detection.spi.model.EventSlice; @@ -136,4 +139,15 @@ public interface DataProvider { * @throws Exception */ DetectionPipeline loadPipeline(DetectionConfigDTO config, long start, long end) throws Exception; + + /** + * Returns a multimap of evaluations (keyed by the evaluations slice) for a given set of evaluations slices. + * + * @see Evaluation + * + * @param evaluationSlices the evaluation slices + * @param configId configId + * @return a multimap of evaluations (keyed by the evaluations slice) + */ + Multimap<EvaluationSlice, EvaluationDTO> fetchEvaluations(Collection<EvaluationSlice> evaluationSlices, long configId); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java index 81b5bd1..4d890bf 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java @@ -45,11 +45,13 @@ import org.apache.pinot.thirdeye.dataframe.DataFrame; import org.apache.pinot.thirdeye.dataframe.LongSeries; import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; import org.apache.pinot.thirdeye.datalayer.dto.EventDTO; import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; @@ -58,6 +60,7 @@ import org.apache.pinot.thirdeye.datasource.comparison.Row; import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader; import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader; import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; +import org.apache.pinot.thirdeye.detection.spi.model.EvaluationSlice; import org.apache.pinot.thirdeye.detection.spi.model.EventSlice; import org.joda.time.DateTimeZone; import org.slf4j.Logger; @@ -76,19 +79,20 @@ public class DefaultDataProvider implements DataProvider { private final DatasetConfigManager datasetDAO; private final EventManager eventDAO; private final MergedAnomalyResultManager anomalyDAO; + private final EvaluationManager evaluationDAO; private final TimeSeriesLoader timeseriesLoader; private final AggregationLoader aggregationLoader; private final DetectionPipelineLoader loader; private static LoadingCache<MetricSlice, DataFrame> DETECTION_TIME_SERIES_CACHE; - public DefaultDataProvider(MetricConfigManager metricDAO, DatasetConfigManager datasetDAO, EventManager eventDAO, - MergedAnomalyResultManager anomalyDAO, TimeSeriesLoader timeseriesLoader, AggregationLoader aggregationLoader, - DetectionPipelineLoader loader) { + MergedAnomalyResultManager anomalyDAO, EvaluationManager evaluationDAO, TimeSeriesLoader timeseriesLoader, + AggregationLoader aggregationLoader, DetectionPipelineLoader loader) { this.metricDAO = metricDAO; this.datasetDAO = datasetDAO; this.eventDAO = eventDAO; this.anomalyDAO = anomalyDAO; + this.evaluationDAO = evaluationDAO; this.timeseriesLoader = timeseriesLoader; this.aggregationLoader = aggregationLoader; this.loader = loader; @@ -316,6 +320,27 @@ public class DefaultDataProvider implements DataProvider { return this.metricDAO.findByMetricAndDataset(metricName, datasetName); } + @Override + public Multimap<EvaluationSlice, EvaluationDTO> fetchEvaluations(Collection<EvaluationSlice> slices, long configId) { + Multimap<EvaluationSlice, EvaluationDTO> output = ArrayListMultimap.create(); + for (EvaluationSlice slice : slices) { + List<Predicate> predicates = new ArrayList<>(); + if (slice.getEnd() >= 0) + predicates.add(Predicate.LT("startTime", slice.getEnd())); + if (slice.getStart() >= 0) + predicates.add(Predicate.GT("endTime", slice.getStart())); + if (predicates.isEmpty()) + throw new IllegalArgumentException("Must provide at least one of start, or end"); + + if (configId >= 0) { + predicates.add(Predicate.EQ("detectionConfigId", configId)); + } + List<EvaluationDTO> evaluations = this.evaluationDAO.findByPredicate(AND(predicates)); + output.putAll(slice, evaluations.stream().filter(slice::match).collect(Collectors.toList())); + } + return output; + } + private static Predicate AND(Collection<Predicate> predicates) { return Predicate.AND(predicates.toArray(new Predicate[predicates.size()])); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultInputDataFetcher.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultInputDataFetcher.java index 43c3db4..3c72011 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultInputDataFetcher.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultInputDataFetcher.java @@ -23,10 +23,12 @@ import com.google.common.collect.Multimap; import org.apache.pinot.thirdeye.dataframe.DataFrame; import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; import org.apache.pinot.thirdeye.datalayer.dto.EventDTO; import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; +import org.apache.pinot.thirdeye.detection.spi.model.EvaluationSlice; import org.apache.pinot.thirdeye.detection.spi.model.EventSlice; import org.apache.pinot.thirdeye.detection.spi.model.InputData; import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec; @@ -59,10 +61,10 @@ public class DefaultInputDataFetcher implements InputDataFetcher { Multimap<EventSlice, EventDTO> events = provider.fetchEvents(inputDataSpec.getEventSlices()); Map<Long, MetricConfigDTO> metrics = provider.fetchMetrics(inputDataSpec.getMetricIds()); Map<String, DatasetConfigDTO> datasets = provider.fetchDatasets(inputDataSpec.getDatasetNames()); - + Multimap<EvaluationSlice, EvaluationDTO> evaluations = provider.fetchEvaluations(inputDataSpec.getEvaluationSlices(), configId); Map<Long, DatasetConfigDTO> datasetForMetricId = fetchDatasetForMetricId(inputDataSpec.getMetricIdsForDatasets()); Map<InputDataSpec.MetricAndDatasetName, MetricConfigDTO> metricForMetricAndDatasetName = fetchMetricForDatasetAndMetricNames(inputDataSpec.getMetricAndDatasetNames()); - return new InputData(inputDataSpec, timeseries, aggregates, existingAnomalies, events, metrics, datasets, datasetForMetricId, metricForMetricAndDatasetName); + return new InputData(inputDataSpec, timeseries, aggregates, existingAnomalies, events, metrics, datasets, evaluations, datasetForMetricId, metricForMetricAndDatasetName); } private Map<InputDataSpec.MetricAndDatasetName, MetricConfigDTO> fetchMetricForDatasetAndMetricNames(Collection<InputDataSpec.MetricAndDatasetName> metricNameAndDatasetNames){ diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java index 42dec01..716917b 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java @@ -19,6 +19,8 @@ package org.apache.pinot.thirdeye.detection; +import java.util.Collections; +import java.util.List; import org.apache.pinot.thirdeye.anomaly.task.TaskContext; import org.apache.pinot.thirdeye.anomaly.task.TaskInfo; import org.apache.pinot.thirdeye.anomaly.task.TaskResult; @@ -39,8 +41,6 @@ import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader; import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader; import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader; import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader; -import java.util.Collections; -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +78,7 @@ public class DetectionPipelineTaskRunner implements TaskRunner { ThirdEyeCacheRegistry.getInstance().getQueryCache(), ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache()); - this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, this.anomalyDAO, + this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, this.anomalyDAO, this.evaluationDAO, timeseriesLoader, aggregationLoader, this.loader); } @@ -143,4 +143,5 @@ public class DetectionPipelineTaskRunner implements TaskRunner { ThirdeyeMetricsUtil.detectionTaskSuccessCounter.inc(); } } + } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java index 9558aad..cb7e761 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java @@ -55,6 +55,7 @@ import org.apache.pinot.thirdeye.dashboard.resources.v2.rootcause.AnomalyEventFo import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; @@ -98,6 +99,7 @@ public class DetectionResource { private final DetectionPipelineLoader loader; private final DataProvider provider; private final DetectionConfigManager configDAO; + private final EvaluationManager evaluationDAO; private final DetectionAlertConfigManager detectionAlertConfigDAO; public DetectionResource() { @@ -107,6 +109,7 @@ public class DetectionResource { this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO(); this.configDAO = DAORegistry.getInstance().getDetectionConfigManager(); this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager(); + this.evaluationDAO = DAORegistry.getInstance().getEvaluationManager(); TimeSeriesLoader timeseriesLoader = new DefaultTimeSeriesLoader(metricDAO, datasetDAO, ThirdEyeCacheRegistry.getInstance().getQueryCache()); @@ -117,7 +120,7 @@ public class DetectionResource { this.loader = new DetectionPipelineLoader(); - this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, timeseriesLoader, aggregationLoader, loader); + this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, evaluationDAO, timeseriesLoader, aggregationLoader, loader); } @Path("/{id}") diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java index 0b752c6..0737ab3 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java @@ -22,10 +22,12 @@ package org.apache.pinot.thirdeye.detection.alert; import com.google.common.base.Preconditions; import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; import org.apache.pinot.thirdeye.datasource.DAORegistry; import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry; import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader; @@ -61,13 +63,13 @@ public class DetectionAlertTaskFactory { MetricConfigManager metricDAO = DAO_REGISTRY.getMetricConfigDAO(); DatasetConfigManager datasetDAO = DAO_REGISTRY.getDatasetConfigDAO(); MergedAnomalyResultManager anomalyMergedResultDAO = DAO_REGISTRY.getMergedAnomalyResultDAO(); - + EvaluationManager evaluationDAO = DAO_REGISTRY.getEvaluationManager(); TimeSeriesLoader timeseriesLoader = new DefaultTimeSeriesLoader(metricDAO, datasetDAO, ThirdEyeCacheRegistry.getInstance().getQueryCache()); AggregationLoader aggregationLoader = new DefaultAggregationLoader(metricDAO, datasetDAO, ThirdEyeCacheRegistry.getInstance().getQueryCache(), ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache()); - this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyMergedResultDAO, + this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyMergedResultDAO, evaluationDAO, timeseriesLoader, aggregationLoader, new DetectionPipelineLoader()); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MapeAveragePercentageChangeModelEvaluator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MapeAveragePercentageChangeModelEvaluator.java new file mode 100644 index 0000000..cff9d7d --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MapeAveragePercentageChangeModelEvaluator.java @@ -0,0 +1,124 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.components; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; +import org.apache.pinot.thirdeye.datalayer.pojo.EvaluationBean; +import org.apache.pinot.thirdeye.detection.InputDataFetcher; +import org.apache.pinot.thirdeye.detection.spec.MapeAveragePercentageChangeModelEvaluatorSpec; +import org.apache.pinot.thirdeye.detection.spi.components.ModelEvaluator; +import org.apache.pinot.thirdeye.detection.spi.model.EvaluationSlice; +import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec; +import org.apache.pinot.thirdeye.detection.spi.model.ModelEvaluationResult; +import org.apache.pinot.thirdeye.detection.spi.model.ModelStatus; +import org.joda.time.Instant; + + +/** + * Monitor the recent mean MAPE in last 7 days, and compare that with the mean MAPE for the last 30 days. + * If the percentage change dropped to a certain threshold for a metric urn, return a bad model status to trigger + * auto configuration. + */ +public class MapeAveragePercentageChangeModelEvaluator implements ModelEvaluator<MapeAveragePercentageChangeModelEvaluatorSpec> { + private static final int MAPE_LOOK_BACK_DAYS_RECENT = 7; + private static final int MAPE_LOOK_BACK_DAYS_BASELINE = 30; + + private InputDataFetcher dataFetcher; + private double threshold; + + @Override + public ModelEvaluationResult evaluateModel(Instant evaluationTimeStamp) { + EvaluationSlice evaluationSlice = + new EvaluationSlice().withStartTime(evaluationTimeStamp.toDateTime().minusDays(MAPE_LOOK_BACK_DAYS_BASELINE).getMillis()) + .withEndTime(evaluationTimeStamp.getMillis()); + // fetch evaluations + Collection<EvaluationDTO> evaluations = + this.dataFetcher.fetchData(new InputDataSpec().withEvaluationSlices(Collections.singleton(evaluationSlice))) + .getEvaluations() + .get(evaluationSlice); + + Collection<EvaluationDTO> recentEvaluations = getEvaluationsWithinDays(evaluations, evaluationTimeStamp, + MAPE_LOOK_BACK_DAYS_RECENT); + Collection<EvaluationDTO> baselineEvaluations = getEvaluationsWithinDays(evaluations, evaluationTimeStamp, + MAPE_LOOK_BACK_DAYS_BASELINE); + + if (recentEvaluations.isEmpty() || recentEvaluations.containsAll(baselineEvaluations)) { + // data is insufficient for performing evaluations + return new ModelEvaluationResult(ModelStatus.UNKNOWN); + } + + // calculate past 7 day mean MAPE for each metric urn and rules + Map<String, Double> recentMeanMapeForMetricUrnsAndRules = getMeanMapeForEachMetricUrnAndRule(recentEvaluations); + + // calculate past 30 day mean MAPE for each metric urn and rules + Map<String, Double> baselineMeanMapeForMetricUrnsAndRules = getMeanMapeForEachMetricUrnAndRule(baselineEvaluations); + + // evaluate for each metric urn + Map<String, Boolean> evaluationResultForMetricUrnsAndRules = recentMeanMapeForMetricUrnsAndRules.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + // compare the MAPE percentage change to threshold + recentMeanMape -> recentMeanMape.getValue() / baselineMeanMapeForMetricUrnsAndRules.get(recentMeanMape.getKey()) - 1 <= threshold)); + + if (evaluationResultForMetricUrnsAndRules.values().stream().allMatch(result -> result)) { + // if all metric urn's status is good, return overall good status + return new ModelEvaluationResult(ModelStatus.GOOD); + } + return new ModelEvaluationResult(ModelStatus.BAD); + } + + /** + * Filter the evaluations to return only the past number days. + * @param evaluations evaluations + * @param evaluationTimeStamp the time stamp for evaluations + * @param days look back number of days + * @return the filtered collection of evaluationDTOs + */ + private Collection<EvaluationDTO> getEvaluationsWithinDays(Collection<EvaluationDTO> evaluations, + Instant evaluationTimeStamp, int days) { + return evaluations.stream() + .filter(eval -> evaluationTimeStamp.toDateTime().minusDays(days).getMillis() < eval.getStartTime()) + .collect(Collectors.toSet()); + } + + /** + * calculate the mean MAPE for each metric urn based on the available evaluations over the past numbe of days + * @param evaluations the available evaluations + * @return the mean MAPE keyed by metric urns + */ + private Map<String, Double> getMeanMapeForEachMetricUrnAndRule(Collection<EvaluationDTO> evaluations) { + return + evaluations.stream().collect( + Collectors.groupingBy(e -> String.format("%s:%s", e.getMetricUrn(), e.getDetectorName()), Collectors.averagingDouble(EvaluationBean::getMape))); + } + + @Override + public void init(MapeAveragePercentageChangeModelEvaluatorSpec spec, InputDataFetcher dataFetcher) { + this.dataFetcher = dataFetcher; + this.threshold = spec.getThreshold(); + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/finetune/GridSearchTuningAlgorithm.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/finetune/GridSearchTuningAlgorithm.java index 30d67ac..a574b24 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/finetune/GridSearchTuningAlgorithm.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/finetune/GridSearchTuningAlgorithm.java @@ -24,7 +24,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; @@ -36,19 +44,12 @@ import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader; import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader; import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader; import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader; -import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; import org.apache.pinot.thirdeye.detection.DataProvider; import org.apache.pinot.thirdeye.detection.DefaultDataProvider; import org.apache.pinot.thirdeye.detection.DetectionPipeline; import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader; import org.apache.pinot.thirdeye.detection.DetectionPipelineResult; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +81,7 @@ public class GridSearchTuningAlgorithm implements TuningAlgorithm { MetricConfigManager metricDAO = DAORegistry.getInstance().getMetricConfigDAO(); DatasetConfigManager datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO(); EventManager eventDAO = DAORegistry.getInstance().getEventDAO(); + EvaluationManager evaluationDAO = DAORegistry.getInstance().getEvaluationManager(); this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO(); TimeSeriesLoader timeseriesLoader = @@ -89,7 +91,7 @@ public class GridSearchTuningAlgorithm implements TuningAlgorithm { new DefaultAggregationLoader(metricDAO, datasetDAO, ThirdEyeCacheRegistry.getInstance().getQueryCache(), ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache()); - this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, timeseriesLoader, aggregationLoader, loader); + this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, evaluationDAO, timeseriesLoader, aggregationLoader, loader); this.scores = new HashMap<>(); this.results = new LinkedHashMap<>(); this.scoreFunction = new TimeBucketF1ScoreFunction(); diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java index 5730b41..5835238 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java @@ -30,6 +30,7 @@ import org.apache.pinot.thirdeye.anomaly.task.TaskRunner; import org.apache.pinot.thirdeye.constant.AnomalyResultSource; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; @@ -62,6 +63,7 @@ public class YamlOnboardingTaskRunner implements TaskRunner { private static final Logger LOG = LoggerFactory.getLogger(YamlOnboardingTaskRunner.class); private final DetectionConfigManager detectionDAO; private final MergedAnomalyResultManager anomalyDAO; + private final EvaluationManager evaluationDAO; private final DetectionPipelineLoader loader; private final DataProvider provider; private final YamlDetectionTranslatorLoader translatorLoader; @@ -72,6 +74,7 @@ public class YamlOnboardingTaskRunner implements TaskRunner { this.loader = new DetectionPipelineLoader(); this.detectionDAO = DAORegistry.getInstance().getDetectionConfigManager(); this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO(); + this.evaluationDAO = DAORegistry.getInstance().getEvaluationManager(); this.translatorLoader = new YamlDetectionTranslatorLoader(); this.yaml = new Yaml(); @@ -88,7 +91,7 @@ public class YamlOnboardingTaskRunner implements TaskRunner { ThirdEyeCacheRegistry.getInstance().getQueryCache(), ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache()); - this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, this.anomalyDAO, + this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, this.anomalyDAO, this.evaluationDAO, timeseriesLoader, aggregationLoader, this.loader); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MapeAveragePercentageChangeModelEvaluatorSpec.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MapeAveragePercentageChangeModelEvaluatorSpec.java new file mode 100644 index 0000000..f1edea1 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MapeAveragePercentageChangeModelEvaluatorSpec.java @@ -0,0 +1,38 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.spec; + +/** + * The spec class for MAPE change evaluator + */ +public class MapeAveragePercentageChangeModelEvaluatorSpec extends AbstractSpec { + private double threshold = 0.1; // default threshold to 10% + + public double getThreshold() { + return threshold; + } + + public void setThreshold(double threshold) { + this.threshold = threshold; + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/ModelEvaluator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/ModelEvaluator.java new file mode 100644 index 0000000..1cb20d9 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/ModelEvaluator.java @@ -0,0 +1,41 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.spi.components; + +import org.apache.pinot.thirdeye.detection.spec.AbstractSpec; +import org.apache.pinot.thirdeye.detection.spi.model.ModelEvaluationResult; +import org.joda.time.Instant; + + +/** + * The interface for model evaluator. + * @param <T> the spec class for this model evaluator + */ +public interface ModelEvaluator<T extends AbstractSpec> extends BaseComponent<T> { + /** + * Evaluate the current detection model. + * @param evaluationTimeStamp the time stamp when the evaluation is run. + * @return a model evaluation result + */ + ModelEvaluationResult evaluateModel(Instant evaluationTimeStamp); +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/EvaluationSlice.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/EvaluationSlice.java new file mode 100644 index 0000000..318f214 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/EvaluationSlice.java @@ -0,0 +1,68 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.spi.model; + +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; + + +/** + * Selector for evaluations based on (optionally) start time and end time. + */ +public class EvaluationSlice { + private final long start; + private final long end; + + private EvaluationSlice(long start, long end) { + this.start = start; + this.end = end; + } + + public EvaluationSlice() { + // -1 means match any + this(-1, -1); + } + + public EvaluationSlice withStartTime(long startTime) { + return new EvaluationSlice(startTime, this.end); + } + + public EvaluationSlice withEndTime(long endTime) { + return new EvaluationSlice(this.start, endTime); + } + + public long getStart() { + return start; + } + + public long getEnd() { + return end; + } + + public boolean match(EvaluationDTO evaluationDTO) { + if (this.start >= 0 && evaluationDTO.getEndTime() <= this.start) + return false; + if (this.end >= 0 && evaluationDTO.getStartTime() >= this.end) + return false; + return true; + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputData.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputData.java index 1f8e4af..1642fa7 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputData.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputData.java @@ -19,10 +19,12 @@ package org.apache.pinot.thirdeye.detection.spi.model; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.apache.pinot.thirdeye.dataframe.DataFrame; import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; import org.apache.pinot.thirdeye.datalayer.dto.EventDTO; import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; @@ -42,6 +44,7 @@ public class InputData { final Multimap<EventSlice, EventDTO> events; final Map<Long, MetricConfigDTO> metrics; final Map<String, DatasetConfigDTO> datasets; + final Multimap<EvaluationSlice, EvaluationDTO> evaluations; /** * The data set config dtos for metric ids @@ -66,11 +69,13 @@ public class InputData { this.datasets = Collections.emptyMap(); this.datasetForMetricId = Collections.emptyMap(); this.metricForMetricAndDatasetNames = Collections.emptyMap(); + this.evaluations = ArrayListMultimap.create(); } public InputData(InputDataSpec spec, Map<MetricSlice, DataFrame> timeseries, Map<MetricSlice, DataFrame> aggregates, Multimap<AnomalySlice, MergedAnomalyResultDTO> anomalies, Multimap<EventSlice, EventDTO> events, - Map<Long, MetricConfigDTO> metrics, Map<String, DatasetConfigDTO> datasets, Map<Long, DatasetConfigDTO> datasetForMetricId, + Map<Long, MetricConfigDTO> metrics, Map<String, DatasetConfigDTO> datasets, + Multimap<EvaluationSlice, EvaluationDTO> evaluations, Map<Long, DatasetConfigDTO> datasetForMetricId, Map<InputDataSpec.MetricAndDatasetName, MetricConfigDTO> metricForMetricAndDatasetNames) { this.dataSpec = spec; this.timeseries = timeseries; @@ -79,6 +84,7 @@ public class InputData { this.events = events; this.metrics = metrics; this.datasets = datasets; + this.evaluations = evaluations; this.datasetForMetricId = datasetForMetricId; this.metricForMetricAndDatasetNames = metricForMetricAndDatasetNames; } @@ -118,4 +124,8 @@ public class InputData { public Map<InputDataSpec.MetricAndDatasetName, MetricConfigDTO> getMetricForMetricAndDatasetNames() { return metricForMetricAndDatasetNames; } + + public Multimap<EvaluationSlice, EvaluationDTO> getEvaluations() { + return evaluations; + } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputDataSpec.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputDataSpec.java index f77265e..c4ba20b 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputDataSpec.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/InputDataSpec.java @@ -53,6 +53,12 @@ public class InputDataSpec { final Collection<EventSlice> eventSlices; /* + Specs for evaluations. Describe what evaluations to fetch. + Each slice defines the time range and detection config id of the evaluations to fetch. + */ + final Collection<EvaluationSlice> evaluationSlices; + + /* Metric ids to fetch the MetricConfigDTO for. */ final Collection<Long> metricIds; @@ -91,23 +97,27 @@ public class InputDataSpec { } public InputDataSpec() { - this.timeseriesSlices = Collections.emptyList(); - this.aggregateSlices = Collections.emptyList(); - this.anomalySlices = Collections.emptyList(); - this.eventSlices = Collections.emptyList(); - this.metricIds = Collections.emptyList(); - this.datasetNames = Collections.emptyList(); - this.metricIdsForDatasets = Collections.emptyList(); - this.metricAndDatasetNames = Collections.emptyList(); + this(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList()); } public InputDataSpec(Collection<MetricSlice> timeseriesSlices, Collection<MetricSlice> aggregateSlices, Collection<AnomalySlice> anomalySlices, Collection<EventSlice> eventSlices, Collection<Long> metricIds, Collection<String> datasetNames, Collection<Long> metricIdsForDatasets, Collection<MetricAndDatasetName> metricAndDatasetNames) { + this(timeseriesSlices, aggregateSlices, anomalySlices, eventSlices, Collections.emptyList(), metricIds, + datasetNames, metricIdsForDatasets, metricAndDatasetNames); + } + + public InputDataSpec(Collection<MetricSlice> timeseriesSlices, Collection<MetricSlice> aggregateSlices, + Collection<AnomalySlice> anomalySlices, Collection<EventSlice> eventSlices, + Collection<EvaluationSlice> evaluationSlices, Collection<Long> metricIds, Collection<String> datasetNames, + Collection<Long> metricIdsForDatasets, Collection<MetricAndDatasetName> metricAndDatasetNames) { this.timeseriesSlices = timeseriesSlices; this.aggregateSlices = aggregateSlices; this.anomalySlices = anomalySlices; this.eventSlices = eventSlices; + this.evaluationSlices = evaluationSlices; this.metricIds = metricIds; this.datasetNames = datasetNames; this.metricIdsForDatasets = metricIdsForDatasets; @@ -138,6 +148,10 @@ public class InputDataSpec { return datasetNames; } + public Collection<EvaluationSlice> getEvaluationSlices() { + return evaluationSlices; + } + public Collection<Long> getMetricIdsForDatasets() { return metricIdsForDatasets; } @@ -162,6 +176,10 @@ public class InputDataSpec { return new InputDataSpec(this.timeseriesSlices, this.aggregateSlices, this.anomalySlices, eventSlices, this.metricIds, this.datasetNames, this.metricIdsForDatasets, this.metricAndDatasetNames); } + public InputDataSpec withEvaluationSlices(Collection<EvaluationSlice> evaluationSlices) { + return new InputDataSpec(this.timeseriesSlices, this.aggregateSlices, this.anomalySlices, this.eventSlices, evaluationSlices, this.metricIds, this.datasetNames, this.metricIdsForDatasets, this.metricAndDatasetNames); + } + public InputDataSpec withMetricIds(Collection<Long> metricIds) { return new InputDataSpec(this.timeseriesSlices, this.aggregateSlices, this.anomalySlices, eventSlices, metricIds, this.datasetNames, this.metricIdsForDatasets, this.metricAndDatasetNames); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/ModelEvaluationResult.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/ModelEvaluationResult.java new file mode 100644 index 0000000..851b2ac --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/ModelEvaluationResult.java @@ -0,0 +1,39 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.spi.model; + +/** + * The model valuation result + */ +public class ModelEvaluationResult { + // the overall model status + private final ModelStatus status; + + public ModelStatus getStatus() { + return status; + } + + public ModelEvaluationResult(ModelStatus status) { + this.status = status; + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/ModelStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/ModelStatus.java new file mode 100644 index 0000000..b823e0a --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/ModelStatus.java @@ -0,0 +1,30 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.spi.model; + +/** + * The detection model status. + */ +public enum ModelStatus { + GOOD, BAD, UNKNOWN +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java index fef9934..94ba5ff 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java @@ -57,6 +57,7 @@ public class DetectionConfigValidator implements ConfigValidator<DetectionConfig this.provider = new DefaultDataProvider(metricDAO, datasetDAO, DAORegistry.getInstance().getEventDAO(), DAORegistry.getInstance().getMergedAnomalyResultDAO(), + DAORegistry.getInstance().getEvaluationManager(), timeseriesLoader, aggregationLoader, loader); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java index 64f6577..db765fd 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java @@ -54,6 +54,7 @@ import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; @@ -110,6 +111,7 @@ public class YamlResource { private final DatasetConfigManager datasetDAO; private final EventManager eventDAO; private final MergedAnomalyResultManager anomalyDAO; + private final EvaluationManager evaluationDAO; private final TaskManager taskDAO; private final DetectionPipelineLoader loader; private final Yaml yaml; @@ -126,6 +128,7 @@ public class YamlResource { this.eventDAO = DAORegistry.getInstance().getEventDAO(); this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO(); this.taskDAO = DAORegistry.getInstance().getTaskDAO(); + this.evaluationDAO = DAORegistry.getInstance().getEvaluationManager(); this.yaml = new Yaml(); TimeSeriesLoader timeseriesLoader = @@ -137,7 +140,7 @@ public class YamlResource { this.loader = new DetectionPipelineLoader(); - this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, timeseriesLoader, aggregationLoader, loader); + this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, evaluationDAO, timeseriesLoader, aggregationLoader, loader); } public DetectionConfigDTO translateToDetectionConfig(Map<String, Object> yamlConfig) throws Exception { diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java index e12c4ba..d3a65d7 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java @@ -33,6 +33,7 @@ import org.apache.pinot.thirdeye.common.dimension.DimensionMap; import org.apache.pinot.thirdeye.dataframe.DataFrame; import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; +import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager; import org.apache.pinot.thirdeye.datalayer.bao.EventManager; import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; @@ -65,6 +66,7 @@ public class DataProviderTest { private MergedAnomalyResultManager anomalyDAO; private MetricConfigManager metricDAO; private DatasetConfigManager datasetDAO; + private EvaluationManager evaluationDAO; private QueryCache cache; private TimeSeriesLoader timeseriesLoader; @@ -86,7 +88,7 @@ public class DataProviderTest { this.anomalyDAO = reg.getMergedAnomalyResultDAO(); this.metricDAO = reg.getMetricConfigDAO(); this.datasetDAO = reg.getDatasetConfigDAO(); - + this.evaluationDAO = reg.getEvaluationManager(); // events this.eventIds = new ArrayList<>(); this.eventIds.add(this.eventDAO.save(makeEvent(3600000L, 7200000L))); @@ -140,7 +142,7 @@ public class DataProviderTest { this.timeseriesLoader = new DefaultTimeSeriesLoader(this.metricDAO, this.datasetDAO, this.cache); // provider - this.provider = new DefaultDataProvider(this.metricDAO, this.datasetDAO, this.eventDAO, this.anomalyDAO, + this.provider = new DefaultDataProvider(this.metricDAO, this.datasetDAO, this.eventDAO, this.anomalyDAO, this.evaluationDAO, this.timeseriesLoader, null, null); } diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java index 27483b3..3af49e8 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java @@ -28,10 +28,12 @@ import org.apache.pinot.thirdeye.dataframe.Series; import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; import org.apache.pinot.thirdeye.datalayer.dto.EventDTO; import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; +import org.apache.pinot.thirdeye.detection.spi.model.EvaluationSlice; import org.apache.pinot.thirdeye.detection.spi.model.EventSlice; import java.util.ArrayList; import java.util.Collection; @@ -54,6 +56,7 @@ public class MockDataProvider implements DataProvider { private List<MergedAnomalyResultDTO> anomalies; private List<MetricConfigDTO> metrics; private List<DatasetConfigDTO> datasets; + private List<EvaluationDTO> evaluations; private DetectionPipelineLoader loader; public MockDataProvider() { @@ -168,6 +171,20 @@ public class MockDataProvider implements DataProvider { } @Override + public Multimap<EvaluationSlice, EvaluationDTO> fetchEvaluations(Collection<EvaluationSlice> slices, + long configId) { + Multimap<EvaluationSlice, EvaluationDTO> result = ArrayListMultimap.create(); + for (EvaluationSlice slice : slices) { + for (EvaluationDTO evaluation :this.evaluations) { + if (slice.match(evaluation) && evaluation.getDetectionConfigId() == configId) { + result.put(slice, evaluation); + } + } + } + return result; + } + + @Override public Map<Long, MetricConfigDTO> fetchMetrics(Collection<Long> ids) { Map<Long, MetricConfigDTO> result = new HashMap<>(); for (Long id : ids) { @@ -271,6 +288,14 @@ public class MockDataProvider implements DataProvider { return this; } + public List<EvaluationDTO> getEvaluations() { + return evaluations; + } + + public void setEvaluations(List<EvaluationDTO> evaluations) { + this.evaluations = evaluations; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MapePercentageChangeModelEvaluatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MapePercentageChangeModelEvaluatorTest.java new file mode 100644 index 0000000..e7e0a55 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MapePercentageChangeModelEvaluatorTest.java @@ -0,0 +1,82 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, + * * software distributed under the License is distributed on an + * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * * KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations + * * under the License. + * + * + */ + +package org.apache.pinot.thirdeye.detection.components; + +import java.util.Arrays; +import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO; +import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher; +import org.apache.pinot.thirdeye.detection.InputDataFetcher; +import org.apache.pinot.thirdeye.detection.MockDataProvider; +import org.apache.pinot.thirdeye.detection.spec.MapeAveragePercentageChangeModelEvaluatorSpec; +import org.apache.pinot.thirdeye.detection.spi.model.ModelEvaluationResult; +import org.apache.pinot.thirdeye.detection.spi.model.ModelStatus; +import org.joda.time.Instant; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + + +public class MapePercentageChangeModelEvaluatorTest { + private InputDataFetcher dataFetcher; + + @BeforeMethod + public void setUp() { + MockDataProvider dataProvider = new MockDataProvider(); + long mockDetectionConfigId = 100L; + String mockMetricUrn = "thirdeye:metric:1"; + EvaluationDTO eval1 = makeMockEvaluationDTO(mockDetectionConfigId, mockMetricUrn, 1557187200000L,1557273600000L, 0.06); + EvaluationDTO eval2 = makeMockEvaluationDTO(mockDetectionConfigId, mockMetricUrn, 1555368321000L,1555454721000L, 0.055); + dataProvider.setEvaluations(Arrays.asList(eval1, eval2)); + dataFetcher = new DefaultInputDataFetcher(dataProvider, mockDetectionConfigId); + } + + private EvaluationDTO makeMockEvaluationDTO(long mockDetectionConfigId, String mockMetricUrn, long start, long end, double mape) { + EvaluationDTO eval = new EvaluationDTO(); + eval.setStartTime(start); + eval.setEndTime(end); + eval.setMetricUrn(mockMetricUrn); + eval.setMape(mape); + eval.setDetectionConfigId(mockDetectionConfigId); + return eval; + } + + @Test + public void testEvaluateModelGood() { + MapeAveragePercentageChangeModelEvaluatorSpec spec = new MapeAveragePercentageChangeModelEvaluatorSpec(); + spec.setThreshold(0.1); + MapeAveragePercentageChangeModelEvaluator evaluator = new MapeAveragePercentageChangeModelEvaluator(); + evaluator.init(spec, dataFetcher); + ModelEvaluationResult result = evaluator.evaluateModel(Instant.parse("2019-05-08T20:00:00.000Z")); + Assert.assertEquals(result.getStatus(), ModelStatus.GOOD); + } + + @Test + public void testEvaluateModelBad() { + MapeAveragePercentageChangeModelEvaluatorSpec spec = new MapeAveragePercentageChangeModelEvaluatorSpec(); + spec.setThreshold(0.01); + MapeAveragePercentageChangeModelEvaluator evaluator = new MapeAveragePercentageChangeModelEvaluator(); + evaluator.init(spec, dataFetcher); + ModelEvaluationResult result = evaluator.evaluateModel(Instant.parse("2019-05-08T20:00:00.000Z")); + Assert.assertEquals(result.getStatus(), ModelStatus.BAD); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org