This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push: new dab44a6 [TE] Remove migrator, partial reference and legacy alert (#4124) dab44a6 is described below commit dab44a696cea604f0e369d88ebb241a47da1ab2e Author: Akshay Rai <akshayra...@gmail.com> AuthorDate: Tue Apr 16 15:14:22 2019 -0700 [TE] Remove migrator, partial reference and legacy alert (#4124) --- .../dashboard/ThirdEyeDashboardApplication.java | 6 - .../datalayer/pojo/DetectionAlertConfigBean.java | 14 +- .../datasource/sql/SqlResponseCacheLoader.java | 2 +- .../sql/resources/SqlDataSourceResource.java | 2 +- .../pinot/thirdeye/detection/DataProvider.java | 12 - .../thirdeye/detection/DefaultDataProvider.java | 5 - .../detection/DetectionMigrationResource.java | 730 --------------------- .../alert/StatefulDetectionAlertFilter.java | 6 +- .../detection/alert/filter/LegacyAlertFilter.java | 143 ---- .../yaml/YamlDetectionAlertConfigTranslator.java | 3 - .../thirdeye/detection/yaml/YamlResource.java | 1 - .../detection/DetectionMigrationResourceTest.java | 294 --------- .../pinot/thirdeye/detection/MockDataProvider.java | 6 - .../alert/filter/LegacyAlertFilterTest.java | 110 ---- .../ToAllRecipientsDetectionAlertFilterTest.java | 10 - .../YamlDetectionAlertConfigTranslatorTest.java | 1 - 16 files changed, 5 insertions(+), 1340 deletions(-) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java index 5c7357d..76b5403 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java @@ -71,7 +71,6 @@ import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader; import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader; import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader; import org.apache.pinot.thirdeye.datasource.sql.resources.SqlDataSourceResource; -import org.apache.pinot.thirdeye.detection.DetectionMigrationResource; import org.apache.pinot.thirdeye.detection.DetectionResource; import org.apache.pinot.thirdeye.detection.annotation.DetectionConfigurationResource; import org.apache.pinot.thirdeye.detection.yaml.YamlResource; @@ -173,11 +172,6 @@ public class ThirdEyeDashboardApplication env.jersey().register(new ThirdEyeResource()); env.jersey().register(new DataResource(anomalyFunctionFactory, alertFilterFactory)); env.jersey().register(new AnomaliesResource(anomalyFunctionFactory, alertFilterFactory)); - env.jersey().register(new DetectionMigrationResource( - DAO_REGISTRY.getAnomalyFunctionDAO(), DAO_REGISTRY.getAlertConfigDAO(), DAO_REGISTRY.getApplicationDAO(), - DAO_REGISTRY.getMetricConfigDAO(), DAO_REGISTRY.getDetectionConfigManager(), - DAO_REGISTRY.getDetectionAlertConfigManager(), DAO_REGISTRY.getDatasetConfigDAO(), - DAO_REGISTRY.getMergedAnomalyResultDAO())); env.jersey().register(new OnboardResource(config)); env.jersey().register(new EntityMappingResource()); env.jersey().register(new OnboardDatasetMetricResource()); diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java index d05e624..8bbda0e 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java @@ -40,7 +40,6 @@ public class DetectionAlertConfigBean extends AbstractBean { String cronExpression; String application; String yaml; - boolean onlyFetchLegacyAnomalies; Map<String, Map<String, Object>> alertSchemes; Map<String, Map<String, Object>> alertSuppressors; @@ -53,14 +52,6 @@ public class DetectionAlertConfigBean extends AbstractBean { Map<String, String> refLinks; - public boolean isOnlyFetchLegacyAnomalies() { - return onlyFetchLegacyAnomalies; - } - - public void setOnlyFetchLegacyAnomalies(boolean onlyFetchLegacyAnomalies) { - this.onlyFetchLegacyAnomalies = onlyFetchLegacyAnomalies; - } - public boolean isActive() { return active; } @@ -179,13 +170,12 @@ public class DetectionAlertConfigBean extends AbstractBean { && subjectType == that.subjectType && Objects.equals(vectorClocks, that.vectorClocks) && Objects.equals( highWaterMark, that.highWaterMark) && Objects.equals(properties, that.properties) && Objects.equals(alertSchemes, that.alertSchemes) && Objects.equals(alertSuppressors, that.alertSuppressors) - && Objects.equals(refLinks, that.refLinks) && onlyFetchLegacyAnomalies == that.onlyFetchLegacyAnomalies - && Objects.equals(yaml, that.yaml); + && Objects.equals(refLinks, that.refLinks) && Objects.equals(yaml, that.yaml); } @Override public int hashCode() { return Objects.hash(active, name, from, cronExpression, application, subjectType, vectorClocks, - highWaterMark, properties, alertSchemes, alertSuppressors, refLinks, onlyFetchLegacyAnomalies, yaml); + highWaterMark, properties, alertSchemes, alertSuppressors, refLinks, yaml); } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java index ad4e61a..9f1953b 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java @@ -19,7 +19,7 @@ package org.apache.pinot.thirdeye.datasource.sql; -import com.facebook.presto.jdbc.internal.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.cache.CacheLoader; import java.io.File; import java.sql.Connection; diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java index fba4b0a..4a61a86 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java @@ -1,6 +1,6 @@ package org.apache.pinot.thirdeye.datasource.sql.resources; -import com.facebook.presto.jdbc.internal.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectMapper; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java index d51187b..4d80598 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java @@ -69,18 +69,6 @@ public interface DataProvider { Map<MetricSlice, DataFrame> fetchAggregates(Collection<MetricSlice> slices, List<String> dimensions); /** - * Returns a multimap of anomalies (keyed by slice) generated by the legacy detection pipeline. - * - * @see MergedAnomalyResultDTO - * @see AnomalySlice - * - * @param slices anomaly slice - * @param configId configId - * @return multimap of anomalies (keyed by slice) - */ - Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchLegacyAnomalies(Collection<AnomalySlice> slices, long configId); - - /** * Returns a multimap of anomalies (keyed by slice) for a given set of slices. * * @see MergedAnomalyResultDTO diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java index 0a2e200..a512974 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java @@ -247,11 +247,6 @@ public class DefaultDataProvider implements DataProvider { } @Override - public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchLegacyAnomalies(Collection<AnomalySlice> slices, long configId) { - return fetchAnomalies(slices, configId, true); - } - - @Override public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchAnomalies(Collection<AnomalySlice> slices, long configId) { return fetchAnomalies(slices, configId, false); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java deleted file mode 100644 index 08fe6ae..0000000 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java +++ /dev/null @@ -1,730 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.thirdeye.detection; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Multimap; -import com.wordnik.swagger.annotations.Api; -import com.wordnik.swagger.annotations.ApiOperation; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.xml.bind.ValidationException; -import org.apache.commons.lang3.StringUtils; -import org.apache.pinot.thirdeye.anomaly.detection.AnomalyDetectionInputContextBuilder; -import org.apache.pinot.thirdeye.api.Constants; -import org.apache.pinot.thirdeye.common.dimension.DimensionMap; -import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager; -import org.apache.pinot.thirdeye.datalayer.bao.ApplicationManager; -import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; -import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; -import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO; -import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO; -import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; -import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; -import org.apache.pinot.thirdeye.datalayer.util.Predicate; -import org.apache.pinot.thirdeye.detection.alert.filter.ToAllRecipientsDetectionAlertFilter; -import org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator; -import org.apache.pinot.thirdeye.detection.yaml.YamlResource; -import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity; -import org.joda.time.Period; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; - -import static org.apache.pinot.thirdeye.anomaly.merge.AnomalyMergeStrategy.*; -import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*; - - -/** - * The Detection migration resource. - */ -@Path("/migrate") -@Api(tags = {Constants.YAML_TAG}) -public class DetectionMigrationResource { - private static final Logger LOGGER = LoggerFactory.getLogger(DetectionMigrationResource.class); - private static final String PROP_WINDOW_DELAY = "windowDelay"; - private static final String PROP_WINDOW_DELAY_UNIT = "windowDelayUnit"; - private static final String PROP_WINDOW_SIZE = "windowSize"; - private static final String PROP_WINDOW_UNIT = "windowUnit"; - - static final String MIGRATED_TAG = "_thirdeye_migrated"; - - private final AnomalyFunctionManager anomalyFunctionDAO; - private final DetectionConfigManager detectionConfigDAO; - private final DetectionAlertConfigManager detectionAlertConfigDAO; - private final DatasetConfigManager datasetConfigDAO; - private final MergedAnomalyResultManager mergedAnomalyResultDAO; - private final AlertConfigManager alertConfigDAO; - private final ApplicationManager appDAO; - private final MetricConfigManager metricConfigDAO; - private final Yaml yaml; - - /** - * Instantiates a new Detection migration resource. - */ - public DetectionMigrationResource( - AnomalyFunctionManager anomalyFunctionDAO, - AlertConfigManager alertConfigDAO, - ApplicationManager appDAO, - MetricConfigManager metricConfigDAO, - DetectionConfigManager detectionConfigDAO, - DetectionAlertConfigManager detectionAlertConfigDAO, - DatasetConfigManager datasetConfigDAO, - MergedAnomalyResultManager mergedAnomalyResultDAO) { - this.anomalyFunctionDAO = anomalyFunctionDAO; - this.detectionConfigDAO = detectionConfigDAO; - this.detectionAlertConfigDAO = detectionAlertConfigDAO; - this.alertConfigDAO = alertConfigDAO; - this.appDAO = appDAO; - this.metricConfigDAO = metricConfigDAO; - this.datasetConfigDAO = datasetConfigDAO; - this.mergedAnomalyResultDAO = mergedAnomalyResultDAO; - DumperOptions options = new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - options.setPrettyFlow(true); - this.yaml = new Yaml(options); - } - - private Map<String, Object> translateAnomalyFunctionToYaml(AnomalyFunctionDTO anomalyFunctionDTO) { - Map<String, Object> yamlConfigs = new LinkedHashMap<>(); - yamlConfigs.put("detectionName", anomalyFunctionDTO.getFunctionName()); - yamlConfigs.put("description", "Please update description - If this alert fires then it means so-and-so and check so-and-so for irregularities"); - yamlConfigs.put("metric", anomalyFunctionDTO.getMetric()); - yamlConfigs.put("active", anomalyFunctionDTO.getIsActive()); - yamlConfigs.put("dataset", anomalyFunctionDTO.getCollection()); - yamlConfigs.put("pipelineType", "Composite"); - if (StringUtils.isNotBlank(anomalyFunctionDTO.getExploreDimensions())) { - // dimension explore and data filter - yamlConfigs.put("dimensionExploration", - getDimensionExplorationParams(anomalyFunctionDTO)); - } - if (anomalyFunctionDTO.getFilters() != null){ - yamlConfigs.put("filters", - AnomalyDetectionInputContextBuilder.getFiltersForFunction(anomalyFunctionDTO.getFilters()).asMap()); - } - - Map<String, Object> ruleYaml = new LinkedHashMap<>(); - - // detection - if (anomalyFunctionDTO.getType().equals("WEEK_OVER_WEEK_RULE")){ - // wo1w change detector - ruleYaml.put("detection", Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type", "PERCENTAGE_RULE", - "params", getPercentageChangeRuleDetectorParams(anomalyFunctionDTO)))); - } else if (anomalyFunctionDTO.getType().equals("MIN_MAX_THRESHOLD")){ - // threshold detector - ruleYaml.put("detection", Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type", "THRESHOLD", - "params", getMinMaxThresholdRuleDetectorParams(anomalyFunctionDTO)))); - } else{ - // algorithm detector - Map<String, Object> detectionProperties = new HashMap<>(); - if (anomalyFunctionDTO.getWindowDelay() != 0) { - detectionProperties.put(PROP_WINDOW_DELAY, anomalyFunctionDTO.getWindowDelay()); - detectionProperties.put(PROP_WINDOW_DELAY_UNIT, anomalyFunctionDTO.getWindowDelayUnit().toString()); - } - detectionProperties.put("name", "detection_rule1"); - detectionProperties.put("type", "MIGRATED_ALGORITHM"); - detectionProperties.put("params", getAlgorithmDetectorParams(anomalyFunctionDTO)); - detectionProperties.put(PROP_WINDOW_SIZE, anomalyFunctionDTO.getWindowSize()); - detectionProperties.put(PROP_WINDOW_UNIT, anomalyFunctionDTO.getWindowUnit().toString()); - detectionProperties.put("bucketPeriod", getBucketPeriod(anomalyFunctionDTO)); - - ruleYaml.put("detection", Collections.singletonList(detectionProperties)); - } - - // filters - Map<String, String> alertFilter = anomalyFunctionDTO.getAlertFilter(); - - if (alertFilter != null && !alertFilter.isEmpty()){ - Map<String, Object> filterYaml = new LinkedHashMap<>(); - if (!alertFilter.containsKey("thresholdField")) { - // algorithm alert filter - filterYaml = ImmutableMap.of("name", "filter_rule1", "type", "MIGRATED_ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO)); - } else { - // threshold filter migrate to rule filters - // site wide impact filter migrate to rule based swi filter - if (anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("impactToGlobal")){ - filterYaml.put("type", "SITEWIDE_IMPACT_FILTER"); - filterYaml.put("name", "filter_rule1"); - filterYaml.put("params", getSiteWideImpactFilterParams(anomalyFunctionDTO)); - } - // weight filter migrate to rule based percentage change filter - if (anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("weight")){ - filterYaml.put("name", "filter_rule1"); - filterYaml.put("type", "PERCENTAGE_CHANGE_FILTER"); - filterYaml.put("params", getPercentageChangeFilterParams(anomalyFunctionDTO)); - } - } - ruleYaml.put("filter", Collections.singletonList(filterYaml)); - } - - yamlConfigs.put("rules", Collections.singletonList(ruleYaml)); - - // merger configs - if (anomalyFunctionDTO.getAnomalyMergeConfig() != null ) { - Map<String, Object> mergerYaml = new LinkedHashMap<>(); - if (anomalyFunctionDTO.getAnomalyMergeConfig().getMergeStrategy() == FUNCTION_DIMENSIONS){ - mergerYaml.put("maxGap", anomalyFunctionDTO.getAnomalyMergeConfig().getSequentialAllowedGap()); - mergerYaml.put("maxDuration", anomalyFunctionDTO.getAnomalyMergeConfig().getMaxMergeDurationLength()); - } - yamlConfigs.put("merger", mergerYaml); - } - - return yamlConfigs; - } - - private Map<String, Object> getDimensionExplorationParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> dimensionExploreYaml = new LinkedHashMap<>(); - dimensionExploreYaml.put("dimensions", Collections.singletonList(functionDTO.getExploreDimensions())); - if (functionDTO.getDataFilter() != null && !functionDTO.getDataFilter().isEmpty() && functionDTO.getDataFilter().get("type").equals("average_threshold")) { - // migrate average threshold data filter - dimensionExploreYaml.put("dimensionFilterMetric", functionDTO.getDataFilter().get("metricName")); - dimensionExploreYaml.put("minValue", Double.valueOf(functionDTO.getDataFilter().get("threshold"))); - dimensionExploreYaml.put("minLiveZone", functionDTO.getDataFilter().get("minLiveZone")); - } - if (functionDTO.getType().equals("MIN_MAX_THRESHOLD")){ - // migrate volume threshold - Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties()); - if (properties.containsKey("averageVolumeThreshold")){ - dimensionExploreYaml.put("minValue", properties.getProperty("averageVolumeThreshold")); - } - } - return dimensionExploreYaml; - } - - private Map<String, Object> getPercentageChangeFilterParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> filterYamlParams = new LinkedHashMap<>(); - filterYamlParams.put("threshold", Math.abs(Double.valueOf(functionDTO.getAlertFilter().get("maxThreshold")))); - filterYamlParams.put("pattern", "up_or_down"); - return filterYamlParams; - } - - private Map<String, Object> getSiteWideImpactFilterParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> filterYamlParams = new LinkedHashMap<>(); - filterYamlParams.put("threshold", Math.abs(Double.valueOf(functionDTO.getAlertFilter().get("maxThreshold")))); - filterYamlParams.put("pattern", "up_or_down"); - filterYamlParams.put("sitewideMetricName", functionDTO.getGlobalMetric()); - filterYamlParams.put("sitewideCollection", functionDTO.getCollection()); - if (StringUtils.isNotBlank(functionDTO.getGlobalMetricFilters())) { - filterYamlParams.put("filters", - AnomalyDetectionInputContextBuilder.getFiltersForFunction(functionDTO.getGlobalMetricFilters()).asMap()); - } - return filterYamlParams; - } - - private Map<String, Object> getAlertFilterParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> filterYamlParams = new LinkedHashMap<>(); - Map<String, Object> params = new HashMap<>(); - filterYamlParams.put("configuration", params); - params.putAll(functionDTO.getAlertFilter()); - params.put("bucketPeriod", getBucketPeriod(functionDTO)); - params.put("timeZone", getTimezone(functionDTO)); - return filterYamlParams; - } - - private String getTimezone(AnomalyFunctionDTO functionDTO) { - DatasetConfigDTO datasetConfigDTO = this.datasetConfigDAO.findByDataset(functionDTO.getCollection()); - return datasetConfigDTO.getTimezone(); - } - - private String getBucketPeriod(AnomalyFunctionDTO functionDTO) { - if (functionDTO.getBucketUnit().equals(TimeUnit.DAYS)){ - return Period.days(functionDTO.getBucketSize()).toString(); - } - return new Period(TimeUnit.MILLISECONDS.convert(functionDTO.getBucketSize(), functionDTO.getBucketUnit())).toString(); - } - - private Map<String, Object> getPercentageChangeRuleDetectorParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> detectorYaml = new LinkedHashMap<>(); - Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties()); - double threshold = Double.valueOf(properties.getProperty("changeThreshold")); - if (properties.containsKey("changeThreshold")){ - detectorYaml.put("percentageChange", Math.abs(threshold)); - if (threshold > 0){ - detectorYaml.put("pattern", "UP"); - } else { - detectorYaml.put("pattern", "DOWN"); - } - } - return detectorYaml; - } - - private Map<String, Object> getMinMaxThresholdRuleDetectorParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> detectorYaml = new LinkedHashMap<>(); - Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties()); - if (properties.containsKey("min")){ - detectorYaml.put("min", properties.getProperty("min")); - } - if (properties.containsKey("max")){ - detectorYaml.put("max", properties.getProperty("max")); - } - return detectorYaml; - } - - private Map<String, Object> getAlgorithmDetectorParams(AnomalyFunctionDTO functionDTO) { - Map<String, Object> detectorYaml = new LinkedHashMap<>(); - Map<String, Object> params = new LinkedHashMap<>(); - detectorYaml.put("configuration", params); - Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties()); - for (Map.Entry<Object, Object> property : properties.entrySet()) { - params.put((String) property.getKey(), property.getValue()); - } - params.put("variables.bucketPeriod", getBucketPeriod(functionDTO)); - params.put("variables.timeZone", getTimezone(functionDTO)); - return detectorYaml; - } - - long migrateLegacyAnomalyFunction(long anomalyFunctionId) throws ValidationException { - AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionId); - - // Verify if function is still valid - validateFunction(anomalyFunctionDTO); - - return migrateLegacyAnomalyFunction(anomalyFunctionDTO); - } - - private long migrateLegacyAnomalyFunction(AnomalyFunctionDTO anomalyFunctionDTO) { - DetectionConfigDTO detectionConfig; - - LOGGER.info(String.format("[MIG] Migrating anomaly function %d %s", anomalyFunctionDTO.getId(), - anomalyFunctionDTO.getFunctionName())); - - // Check if this anomaly function is already migrated - if (anomalyFunctionDTO.getFunctionName().contains(MIGRATED_TAG)) { - LOGGER.info(String.format("[MIG] Anomaly function %d is already migrated.", anomalyFunctionDTO.getId())); - - // Fetch the migrated config id and return - String funcName = anomalyFunctionDTO.getFunctionName(); - return Long.parseLong(funcName.substring(funcName.lastIndexOf("_") + 1, funcName.length())); - } - - // Migrate anomaly function config to the detection config by converting to YAML and then to Detection Config - try { - Map<String, Object> detectionYAMLMap = translateAnomalyFunctionToYaml(anomalyFunctionDTO); - detectionConfig = new YamlResource().translateToDetectionConfig(detectionYAMLMap); - Preconditions.checkNotNull(detectionConfig); - } catch (Exception e) { - throw new RuntimeException("Error translating anomaly function config to the detection config" + e.getMessage()); - } - - // Save the migrated anomaly function - detectionConfigDAO.save(detectionConfig); - if (detectionConfig.getId() == null) { - throw new RuntimeException("Error saving the new detection config."); - } - - // Hack to retain Anomaly function owner - detectionConfig.setCreatedBy(anomalyFunctionDTO.getCreatedBy()); - detectionConfigDAO.update(detectionConfig); - if (detectionConfig.getId() == null) { - throw new RuntimeException("Error saving the new detection config after updating the owner."); - } - - // Point all the associated anomalies to the migrated anomaly function. - List<MergedAnomalyResultDTO> mergedAnomalyResultDTOS = mergedAnomalyResultDAO.findByPredicate(Predicate.EQ("functionId", anomalyFunctionDTO.getId())); - for (MergedAnomalyResultDTO anomaly : mergedAnomalyResultDTOS) { - // Drop the baseline and current values from the anomalies. - if (anomaly.getProperties() != null) { - anomaly.getProperties().remove("anomalyTimelinesView"); - } - anomaly.setMetricUrn(buildMetricUrn(anomaly)); - anomaly.setDetectionConfigId(detectionConfig.getId()); - int affectedRows = mergedAnomalyResultDAO.update(anomaly); - if (affectedRows == 0) { - throw new RuntimeException("Failed to update the anomaly " + anomaly.getId() + " with the new detection id" - + " for anomaly function " + detectionConfig.getId()); - } - } - - // Mark the old anomaly function as migrated - anomalyFunctionDTO.setActive(false); - anomalyFunctionDTO.setFunctionName(anomalyFunctionDTO.getFunctionName() + MIGRATED_TAG + "_" + detectionConfig.getId()); - int affectedRows = this.anomalyFunctionDAO.update(anomalyFunctionDTO); - if (affectedRows == 0) { - throw new RuntimeException("Anomaly function migrated successfully but failed to disable and update the" - + " migration status of the old anomaly function. Recommend doing it manually. Migrated detection id " - + detectionConfig.getId()); - } - - LOGGER.info(String.format("[MIG] Successfully migrated anomaly function %d %s", anomalyFunctionDTO.getId(), - anomalyFunctionDTO.getFunctionName())); - return detectionConfig.getId(); - } - - private String buildMetricUrn(MergedAnomalyResultDTO anomaly) { - try { - DimensionMap dimensionMap = anomaly.getDimensions(); - Multimap<String, String> filters = ArrayListMultimap.create(); - for (DimensionMap.Entry<String, String> entry : dimensionMap.entrySet()) { - filters.put(entry.getKey(), entry.getValue()); - } - MetricEntity me = MetricEntity.fromMetric(1.0, metricConfigDAO.findByMetricAndDataset(anomaly.getMetric(), anomaly.getCollection()).getId(), filters); - return me.getUrn(); - } catch (Exception e) { - throw new RuntimeException("Resolve metric urn failed for anomaly " + anomaly.getId(), e); - } - } - - - private void migrateLegacyNotification(AlertConfigDTO alertConfigDTO) { - int anomalyFailureCount = 0; - int anomalyWarningCount = 0; - String alertName = alertConfigDTO.getName(); - - LOGGER.info(String.format("[MIG] Migrating alert %d %s", alertConfigDTO.getId(), alertName)); - - // Skip if the alert is already migrated - if (alertConfigDTO.getName().contains(MIGRATED_TAG)) { - LOGGER.info(String.format("[MIG] Alert %d is already migrated. Skipping!", alertConfigDTO.getId())); - return; - } - - // Migrate all the subscribed anomaly functions. Note that this will update the state of old anomaly functions. - List<Long> detectionIds = ConfigUtils.getLongs(alertConfigDTO.getEmailConfig().getFunctionIds()); - List<Long> filteredIds = new ArrayList<>(); - for (long detectionId : detectionIds) { - try { - migrateLegacyAnomalyFunction(detectionId); - filteredIds.add(detectionId); - } catch (ValidationException e) { - anomalyWarningCount++; - // Ignore those anomaly functions which are pointing to invalid entities - LOGGER.warn("[MIG] Validation error while migrating anomaly function {}. Error ", detectionId, e.getMessage()); - } catch (Exception e) { - anomalyFailureCount++; - LOGGER.error("[MIG] Error while migrating anomaly function {}. Error ", detectionId, e); - } - } - alertConfigDTO.getEmailConfig().setFunctionIds(filteredIds); - - // Translate the old alert and capture the state. - Map<String, Object> detectionAlertYaml = translateAlertToYaml(alertConfigDTO); - - // Migrate the alert/notification group - DetectionAlertConfigDTO alertConfig = new YamlDetectionAlertConfigTranslator(detectionConfigDAO).translate(detectionAlertYaml); - List<DetectionAlertConfigDTO> alertDTOs = detectionAlertConfigDAO.findByPredicate(Predicate.EQ("name", alertConfig.getName())); - if (!alertDTOs.isEmpty()) { - LOGGER.warn("[MIG] Looks like this alert was already migrated. old id = " + alertConfig.getId() + " new id = " - + alertDTOs.get(0).getId()); - } else { - detectionAlertConfigDAO.save(alertConfig); - if (alertConfig.getId() == null) { - throw new RuntimeException("Error while saving the migrated alert config for " + alertName); - } - - // Hack to retain subscription group owner - alertConfig.setCreatedBy(alertConfigDTO.getCreatedBy()); - detectionAlertConfigDAO.update(alertConfig); - if (alertConfig.getId() == null) { - throw new RuntimeException("Error saving the migrated alert config after updating the owner."); - } - } - - // Update migration status and disable the old alert - alertConfigDTO.setName(alertName + MIGRATED_TAG + "_" + alertConfig.getId()); - alertConfigDTO.setActive(false); - int affectedRows = alertConfigDAO.update(alertConfigDTO); - if (affectedRows == 0) { - throw new RuntimeException( - "Alert migrated successfully but failed to disable and update the migration status" + " of the old alert." - + " Migrated alert id " + alertConfig.getId()); - } - - if (anomalyFailureCount == 0 && anomalyWarningCount == 0) { - LOGGER.info(String.format("[MIG] Successfully migrated alert %d %s", alertConfigDTO.getId(), alertName)); - } else { - throw new RuntimeException("Failures/Warnings found. anomalyFailureCount " + anomalyFailureCount + " and" - + " anomalyWarningCount " + anomalyWarningCount); - } - } - - private void validateFunction(AnomalyFunctionDTO functionDTO) throws ValidationException { - if (functionDTO == null) { - throw new ValidationException("Couldn't find anomaly function."); - } - - List<DatasetConfigDTO> datasetConfigDTOs = this.datasetConfigDAO.findByPredicate(Predicate.EQ("dataset", functionDTO.getCollection())); - if (datasetConfigDTOs.isEmpty()) { - throw new ValidationException("Dataset cannot be found for anomaly function " + functionDTO.getId()); - } - - List<MetricConfigDTO> metricConfigDTOs = this.metricConfigDAO.findByPredicate(Predicate.AND( - Predicate.EQ("name", functionDTO.getMetric()), - Predicate.EQ("dataset", functionDTO.getCollection()))); - if (metricConfigDTOs.isEmpty()) { - throw new ValidationException("Metric cannot be found for anomaly function " + functionDTO.getId()); - } - } - - Map<String, Object> translateAlertToYaml(AlertConfigDTO alertConfigDTO) { - Map<String, Object> yamlConfigs = new LinkedHashMap<>(); - - yamlConfigs.put(PROP_SUBS_GROUP_NAME, alertConfigDTO.getName()); - yamlConfigs.put(PROP_CRON, alertConfigDTO.getCronExpression()); - yamlConfigs.put(PROP_ACTIVE, alertConfigDTO.isActive()); - yamlConfigs.put(PROP_APPLICATION, alertConfigDTO.getApplication()); - yamlConfigs.put(PROP_EMAIL_SUBJECT_TYPE, alertConfigDTO.getSubjectType().name()); - yamlConfigs.put(PROP_FROM, alertConfigDTO.getFromAddress()); - - yamlConfigs.put(PROP_TYPE, "DEFAULT_ALERTER_PIPELINE"); - - Map<String, Object> recipients = new LinkedHashMap<>(); - recipients.put("to", ConfigUtils.getList(alertConfigDTO.getReceiverAddresses().getTo())); - recipients.put("cc", ConfigUtils.getList(alertConfigDTO.getReceiverAddresses().getCc())); - recipients.put("bcc", ConfigUtils.getList(alertConfigDTO.getReceiverAddresses().getBcc())); - yamlConfigs.put(PROP_RECIPIENTS, recipients); - - List<Map<String, Object>> schemes = new ArrayList<>(); - Map<String, Object> emailScheme = new LinkedHashMap<>(); - emailScheme.put(PROP_TYPE, "EMAIL"); - schemes.add(emailScheme); - yamlConfigs.put(PROP_ALERT_SCHEMES, schemes); - - List<String> detectionNames = new ArrayList<>(); - List<Long> detectionIds = alertConfigDTO.getEmailConfig().getFunctionIds(); - for (Long id : detectionIds) { - List<AnomalyFunctionDTO> functionDTOS = this.anomalyFunctionDAO.findByPredicate(Predicate.EQ("baseId", id)); - if (functionDTOS.isEmpty()) { - LOGGER.warn("[MIG] Anomaly function " + id + " cannot be found while generating notification YAML from legacy notification."); - // Ignore missing anomaly functions - continue; - } - - String functionName = functionDTOS.get(0).getFunctionName(); - if (functionName.contains(MIGRATED_TAG)) { - functionName = functionName.substring(0, functionName.lastIndexOf(MIGRATED_TAG)); - } - detectionNames.add(functionName); - } - yamlConfigs.put(PROP_DETECTION_NAMES, detectionNames); - - return yamlConfigs; - } - - @GET - @Produces(MediaType.TEXT_PLAIN) - @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation("migrate a function") - @Path("/legacy-anomaly-function-to-yaml/{id}") - public Response getYamlFromLegacyAnomalyFunction(@PathParam("id") long anomalyFunctionID) { - AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionID); - if (anomalyFunctionDTO == null) { - return Response.status(Response.Status.BAD_REQUEST) - .entity(ImmutableMap.of("message", "Legacy Anomaly function cannot be found for id "+ anomalyFunctionID)) - .build(); - } - return Response.ok(this.yaml.dump(translateAnomalyFunctionToYaml(anomalyFunctionDTO))).build(); - } - - @GET - @Produces(MediaType.TEXT_PLAIN) - @Consumes(MediaType.APPLICATION_JSON) - @Path("/legacy-alert-to-yaml/{id}") - public Response getYamlFromLegacyAlert(@PathParam("id") long alertId) { - AlertConfigDTO alertConfigDTO = this.alertConfigDAO.findById(alertId); - if (alertConfigDTO == null) { - return Response.status(Response.Status.BAD_REQUEST) - .entity(ImmutableMap.of("message", "Legacy alert cannot be found for ID "+ alertId)) - .build(); - } - return Response.ok(this.yaml.dump(translateAlertToYaml(alertConfigDTO))).build(); - } - - @POST - @ApiOperation("migrate an application") - @Path("/application/{name}") - public Response migrateApplication(@PathParam("name") String application) { - List<AlertConfigDTO> alertConfigDTOList = alertConfigDAO.findByPredicate(Predicate.EQ("application", application)); - Map<String, String> responseMessage = new HashMap<>(); - - for (AlertConfigDTO alertConfigDTO : alertConfigDTOList) { - try { - migrateLegacyNotification(alertConfigDTO); - } catch (Exception e) { - // Skip migrating this alert and move on to the next - LOGGER.error("[MIG] Failed to migrate alert ID {} name {}. Exception {}", alertConfigDTO.getId(), alertConfigDTO.getName(), e); - responseMessage.put("Status of alert " + alertConfigDTO.getId(), - String.format("Failed to migrate alert ID %d with name %s due to %s", alertConfigDTO.getId(), - alertConfigDTO.getName(), e.getMessage())); - } - } - - if (responseMessage.isEmpty()) { - LOGGER.info("[MIG] Application " + application + " has been successfully migrated"); - return Response.ok("Application " + application + " has been successfully migrated").build(); - } else { - LOGGER.error("[MIG] Found errors while migrating application " + application); - return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build(); - } - } - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @Path("/anomaly-function/{id}") - public Response migrateAnomalyFunction(@PathParam("id") long anomalyFunctionId) throws Exception { - return Response.ok(migrateLegacyAnomalyFunction(anomalyFunctionId)).build(); - } - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @Path("/unsubscribed-anomaly-functions") - public Response migrateAnomalyFunction() throws Exception { - List<AnomalyFunctionDTO> anomalyFunctionDTOs = anomalyFunctionDAO.findAll(); - Map<String, String> responseMessage = new HashMap<>(); - - for (AnomalyFunctionDTO func : anomalyFunctionDTOs) { - if (func.getFunctionName().contains(MIGRATED_TAG)) { - LOGGER.info("[MIG] Function already migrated. Name " + func.getFunctionName()); - continue; - } - - try { - validateFunction(func); - } catch (ValidationException e) { - LOGGER.info("[MIG] Function failed validation. Name " + func.getFunctionName() + " Error : " + e.getMessage()); - responseMessage.put("Failed to migrate " + func.getId(), String.format("Validation Error : %s", e.getMessage())); - continue; - } - - try { - migrateLegacyAnomalyFunction(func); - } catch (Exception e) { - // Skip migrating this function and move on to the next - responseMessage.put("Failed to migrate " + func.getId(), String.format("Error : %s", e.getMessage())); - } - } - - return Response.ok(responseMessage).build(); - } - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation("migrate all applications") - @Path("/applications") - public Response migrateApplication() { - List<ApplicationDTO> applicationDTOS = this.appDAO.findAll(); - Map<String, String> responseMessage = new HashMap<>(); - - for (ApplicationDTO app : applicationDTOS) { - try { - Response response = migrateApplication(app.getApplication()); - if (response.getStatusInfo() != Response.Status.OK) { - throw new RuntimeException("Found " + ConfigUtils.getMap(response.getEntity()).size() + " issues while migrating alerts."); - } - } catch (Exception e) { - // Skip migrating this application - LOGGER.error("[MIG] Failed to migrate application {}. Exception {}", app.getApplication(), e); - responseMessage.put("Status of application " + app.getApplication(), - String.format("Failed to migrate application %s due to %s", app.getApplication(), e.getMessage())); - } - } - - if (responseMessage.isEmpty()) { - LOGGER.info("[MIG] Successfully migrated all the applications"); - return Response.ok("All applications have been successfully migrated").build(); - } else { - LOGGER.error("[MIG] Errors found while migrating application. Errors:\n" + responseMessage); - return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build(); - } - } - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation("migrate the partially migrated alerts") - @Path("/partial-alerts") - public Response migratePartialAlerts() { - List<DetectionAlertConfigDTO> subsGroups = this.detectionAlertConfigDAO.findAll(); - Map<String, String> responseMessage = new HashMap<>(); - - for (DetectionAlertConfigDTO subsGroup : subsGroups) { - try { - if (subsGroup.isOnlyFetchLegacyAnomalies()) { - Set<Long> detectionIds = new HashSet<>(); - // Update and point these to the new detection ids - if (subsGroup.getVectorClocks() != null) { - Map<Long, Long> migratedVectorClock = new HashMap<>(); - Map<Long, Long> legacyVectorClock = subsGroup.getVectorClocks(); - for (long id : legacyVectorClock.keySet()) { - long migratedId = migrateLegacyAnomalyFunction(id); - detectionIds.add(migratedId); - migratedVectorClock.put(migratedId, legacyVectorClock.get(id)); - } - subsGroup.setVectorClocks(migratedVectorClock); - } - subsGroup.getProperties().put(PROP_DETECTION_CONFIG_IDS, detectionIds); - - // Remove Alert Filters! These are migrated and part of the detection yaml - subsGroup.getProperties().remove("legacyAlertFilterConfigs"); - subsGroup.getProperties().remove("legacyAlertFilterClassName"); - - subsGroup.getProperties().put("className", ToAllRecipientsDetectionAlertFilter.class.getName()); - - int detectionAlertConfigId = this.detectionAlertConfigDAO.update(subsGroup); - if (detectionAlertConfigId <= 0) { - throw new RuntimeException("Failed to update the detection alert config."); - } - } - } catch (Exception e) { - // Skip migrating this partial migrated alert - LOGGER.error("[MIG] Failed to migrate partial subscription group id {} name {}. Exception {}", subsGroup.getId(), subsGroup.getName(), e); - responseMessage.put("Status of subscription group " + subsGroup.getName(), - String.format("Failed to migrate subscription group %s due to %s", subsGroup.getName(), e.getMessage())); - } - } - - if (responseMessage.isEmpty()) { - LOGGER.info("[MIG] Successfully migrated all the partially migrated alerts"); - return Response.ok("All partial alerts have been successfully migrated").build(); - } else { - LOGGER.error("[MIG] Errors found while migrating partially migrated alerts. Errors:\n" + responseMessage); - return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build(); - } - } -} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java index 21bea17..7226e75 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java @@ -57,11 +57,7 @@ public abstract class StatefulDetectionAlertFilter extends DetectionAlertFilter AnomalySlice slice = new AnomalySlice().withStart(startTime).withEnd(this.endTime); Collection<MergedAnomalyResultDTO> candidates; - if (this.config.isOnlyFetchLegacyAnomalies()) { - candidates = this.provider.fetchLegacyAnomalies(Collections.singletonList(slice), functionId).get(slice); - } else { - candidates = this.provider.fetchAnomalies(Collections.singletonList(slice), functionId).get(slice); - } + candidates = this.provider.fetchAnomalies(Collections.singletonList(slice), functionId).get(slice); Collection<MergedAnomalyResultDTO> anomalies = Collections2.filter(candidates, new Predicate<MergedAnomalyResultDTO>() { diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilter.java deleted file mode 100644 index e96ab88..0000000 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilter.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.thirdeye.detection.alert.filter; - -import com.google.common.base.Predicate; -import com.google.common.collect.Collections2; -import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; -import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients; -import org.apache.pinot.thirdeye.detection.annotation.AlertFilter; -import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice; -import org.apache.pinot.thirdeye.detection.ConfigUtils; -import org.apache.pinot.thirdeye.detection.DataProvider; -import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilter; -import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterResult; -import org.apache.pinot.thirdeye.detector.email.filter.BaseAlertFilter; -import org.apache.pinot.thirdeye.detector.email.filter.DummyAlertFilter; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nullable; -import org.apache.commons.collections.MapUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -@AlertFilter(type = "LEGACY_ALERTER_PIPELINE") -public class LegacyAlertFilter extends DetectionAlertFilter { - private final static Logger LOG = LoggerFactory.getLogger(LegacyAlertFilter.class); - - private static final String PROP_LEGACY_ALERT_FILTER_CONFIGS = "legacyAlertFilterConfigs"; - private static final String PROP_LEGACY_ALERT_FILTER_CLASS_NAME = "legacyAlertFilterClassName"; - private static final String PROP_DETECTION_CONFIG_IDS = "detectionConfigIds"; - private static final String PROP_RECIPIENTS = "recipients"; - private static final String PROP_TO = "to"; - private static final String PROP_CC = "cc"; - private static final String PROP_BCC = "bcc"; - private static final String PROP_SEND_ONCE = "sendOnce"; - - private final List<Long> detectionConfigIds; - private final Map<Long, Long> vectorClocks; - private final boolean sendOnce; - - public LegacyAlertFilter(DataProvider provider, DetectionAlertConfigDTO config, long endTime) throws Exception { - super(provider, config, endTime); - - this.detectionConfigIds = ConfigUtils.getLongs(this.config.getProperties().get(PROP_DETECTION_CONFIG_IDS)); - this.vectorClocks = this.config.getVectorClocks(); - this.sendOnce = MapUtils.getBoolean(this.config.getProperties(), PROP_SEND_ONCE, true); - } - - @Override - public DetectionAlertFilterResult run() throws Exception { - DetectionAlertFilterResult result = new DetectionAlertFilterResult(); - - Map<String, Set<String>> recipientsMap = ConfigUtils.getMap(this.config.getProperties().get(PROP_RECIPIENTS)); - Set<String> to = (recipientsMap.get(PROP_TO) == null) ? Collections.emptySet() : new HashSet<>(recipientsMap.get(PROP_TO)); - Set<String> cc = (recipientsMap.get(PROP_CC) == null) ? Collections.emptySet() : new HashSet<>(recipientsMap.get(PROP_CC)); - Set<String> bcc = (recipientsMap.get(PROP_BCC) == null) ? Collections.emptySet() : new HashSet<>(recipientsMap.get(PROP_BCC)); - DetectionAlertFilterRecipients recipients = new DetectionAlertFilterRecipients(to, cc, bcc); - - Map<String, Object> alertFilterConfig = MapUtils.getMap(config.getProperties(), PROP_LEGACY_ALERT_FILTER_CONFIGS); - if (alertFilterConfig == null || alertFilterConfig.size() == 0) { - LOG.warn("alertFilterConfig is null or empty in notification group {}", this.config.getId()); - } - - for (Long functionId : this.detectionConfigIds) { - long startTime = MapUtils.getLong(this.vectorClocks, functionId, 0L); - - AnomalySlice slice = new AnomalySlice() - .withStart(startTime) - .withEnd(this.endTime); - - Collection<MergedAnomalyResultDTO> candidates; - if (this.config.isOnlyFetchLegacyAnomalies()) { - candidates = this.provider.fetchLegacyAnomalies(Collections.singletonList(slice), functionId).get(slice); - } else { - candidates = this.provider.fetchAnomalies(Collections.singletonList(slice), functionId).get(slice); - } - - BaseAlertFilter alertFilter = new DummyAlertFilter(); - if (config.getProperties().containsKey(PROP_LEGACY_ALERT_FILTER_CLASS_NAME)) { - String className = MapUtils.getString(config.getProperties(), PROP_LEGACY_ALERT_FILTER_CLASS_NAME); - alertFilter = (BaseAlertFilter) Class.forName(className).newInstance(); - Map<String, String> params = MapUtils.getMap(alertFilterConfig, functionId.toString()); - if (params == null) { - LOG.warn("AlertFilter cannot be found for function {} in notification group {}", functionId, this.config.getId()); - } - - alertFilter.setParameters(params); - } - - BaseAlertFilter finalAlertFilter = alertFilter; - final long minId = getMinId(this.config.getHighWaterMark()); - Collection<MergedAnomalyResultDTO> anomalies = - Collections2.filter(candidates, new Predicate<MergedAnomalyResultDTO>() { - @Override - public boolean apply(@Nullable MergedAnomalyResultDTO mergedAnomaly) { - return mergedAnomaly != null - && !mergedAnomaly.isChild() - && finalAlertFilter.isQualified(mergedAnomaly) - && (mergedAnomaly.getId() == null || mergedAnomaly.getId() >= minId); - } - }); - - if (result.getResult().isEmpty()) { - result.addMapping(recipients, new HashSet<>(anomalies)); - } else { - result.getResult().get(recipients).addAll(anomalies); - } - } - - return result; - } - - private long getMinId(long highWaterMark) { - if (this.sendOnce) { - return highWaterMark + 1; - } else { - return 0; - } - } -} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java index 9d36de9..9f6d8d9 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java @@ -171,9 +171,6 @@ public class YamlDetectionAlertConfigTranslator { alertConfigDTO.setCronExpression(MapUtils.getString(yamlAlertConfig, PROP_CRON, CRON_SCHEDULE_DEFAULT)); alertConfigDTO.setActive(MapUtils.getBooleanValue(yamlAlertConfig, PROP_ACTIVE, true)); - // TODO: Remove all references to onlyFetchLegacyAnomalies after migration - alertConfigDTO.setOnlyFetchLegacyAnomalies(MapUtils.getBooleanValue(yamlAlertConfig, PROP_ONLY_FETCH_LEGACY_ANOMALIES, false)); - alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.valueOf( (String) MapUtils.getObject(yamlAlertConfig, PROP_EMAIL_SUBJECT_TYPE, AlertConfigBean.SubjectType.METRICS.name()))); diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java index 98945a5..c3a1a5a 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java @@ -471,7 +471,6 @@ public class YamlResource { oldAlertConfig.setActive(newAlertConfig.isActive()); oldAlertConfig.setAlertSchemes(newAlertConfig.getAlertSchemes()); oldAlertConfig.setAlertSuppressors(newAlertConfig.getAlertSuppressors()); - oldAlertConfig.setOnlyFetchLegacyAnomalies(newAlertConfig.isOnlyFetchLegacyAnomalies()); oldAlertConfig.setProperties(newAlertConfig.getProperties()); return oldAlertConfig; diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java deleted file mode 100644 index 4b796ed..0000000 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.thirdeye.detection; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import javax.ws.rs.core.Response; -import org.apache.commons.collections.MapUtils; -import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager; -import org.apache.pinot.thirdeye.datalayer.bao.ApplicationManager; -import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase; -import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; -import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager; -import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager; -import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO; -import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO; -import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; -import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO; -import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean; -import org.apache.pinot.thirdeye.datasource.DAORegistry; -import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients; -import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionAlertRegistry; -import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry; -import org.apache.pinot.thirdeye.detection.components.PercentageChangeRuleDetector; -import org.apache.pinot.thirdeye.detection.components.RuleBaselineProvider; -import org.apache.pinot.thirdeye.detection.components.ThresholdRuleAnomalyFilter; -import org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector; -import org.apache.pinot.thirdeye.detection.yaml.CompositePipelineConfigTranslator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import static org.apache.pinot.thirdeye.detection.DetectionMigrationResource.*; -import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*; - - -public class DetectionMigrationResourceTest { - private static final Logger LOGGER = LoggerFactory.getLogger(DetectionMigrationResourceTest.class); - - private DAOTestBase testDAOProvider; - private MetricConfigManager metricDAO; - private DatasetConfigManager datasetDAO; - private MergedAnomalyResultManager anomalyDAO; - private AnomalyFunctionManager anomalyFunctionDAO; - private AlertConfigManager alertConfigDAO; - private ApplicationManager applicationDAO; - private DetectionConfigManager detectionConfigDAO; - private DetectionAlertConfigManager detectionAlertConfigDAO; - - private DetectionMigrationResource migrationResource; - - private static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - @BeforeMethod(alwaysRun = true) - public void setup() { - this.testDAOProvider = DAOTestBase.getInstance(); - this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO(); - this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO(); - this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO(); - this.anomalyFunctionDAO = DAORegistry.getInstance().getAnomalyFunctionDAO(); - this.alertConfigDAO = DAORegistry.getInstance().getAlertConfigDAO(); - this.detectionConfigDAO = DAORegistry.getInstance().getDetectionConfigManager(); - this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager(); - this.applicationDAO = DAORegistry.getInstance().getApplicationDAO(); - - migrationResource = new DetectionMigrationResource( - anomalyFunctionDAO, alertConfigDAO, applicationDAO, metricDAO, detectionConfigDAO, detectionAlertConfigDAO, datasetDAO, anomalyDAO); - - DetectionRegistry.registerYamlConvertor(CompositePipelineConfigTranslator.class.getName(), "COMPOSITE"); - DetectionRegistry.registerComponent(PercentageChangeRuleDetector.class.getName(), "PERCENTAGE_RULE"); - DetectionRegistry.registerComponent(RuleBaselineProvider.class.getName(), "RULE_BASELINE"); - DetectionRegistry.registerComponent(ThresholdRuleDetector.class.getName(), "THRESHOLD"); - DetectionRegistry.registerComponent(ThresholdRuleAnomalyFilter.class.getName(), "THRESHOLD_RULE_FILTER"); - - DetectionAlertRegistry.getInstance().registerAlertScheme("EMAIL", "EmailClass"); - DetectionAlertRegistry.getInstance().registerAlertFilter("DEFAULT_ALERTER_PIPELINE", "RECIPIENTClass"); - - MetricConfigDTO metricConfigDTO = new MetricConfigDTO(); - metricConfigDTO.setName("test_metric"); - metricConfigDTO.setDataset("test_collection"); - metricConfigDTO.setActive(true); - metricConfigDTO.setAlias("test_collection::test_metric"); - metricDAO.save(metricConfigDTO); - - DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO(); - datasetConfigDTO.setDataset("test_collection"); - datasetConfigDTO.setNonAdditiveBucketSize(1); - datasetConfigDTO.setNonAdditiveBucketUnit(TimeUnit.DAYS); - datasetDAO.save(datasetConfigDTO); - } - - @AfterMethod(alwaysRun = true) - public void cleanup() { - this.testDAOProvider.cleanup(); - } - - @Test - public void testTranslateAlertToYaml() throws Exception { - AnomalyFunctionDTO anomalyFunction1 = new AnomalyFunctionDTO(); - anomalyFunction1.setFunctionName("function1"); - long id1 = anomalyFunctionDAO.save(anomalyFunction1); - - AnomalyFunctionDTO anomalyFunction2 = new AnomalyFunctionDTO(); - anomalyFunction2.setFunctionName("function2"); - long id2 = anomalyFunctionDAO.save(anomalyFunction2); - - AlertConfigDTO alertConfigDTO = new AlertConfigDTO(); - alertConfigDTO.setActive(true); - alertConfigDTO.setName("test_notification"); - alertConfigDTO.setApplication("test_application"); - alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.ALERT); - alertConfigDTO.setCronExpression("0 0 14 * * ? *"); - alertConfigDTO.setFromAddress("t...@thirdeye.com"); - alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients( - Collections.singleton("to@test"), - Collections.singleton("cc@test"), - Collections.singleton("bcc@test"))); - - AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig(); - emailConfig.setFunctionIds(Arrays.asList(id1, id2)); - emailConfig.setAnomalyWatermark(9999); - alertConfigDTO.setEmailConfig(emailConfig); - - Map<String, Object> yamlMap = migrationResource.translateAlertToYaml(alertConfigDTO); - - Assert.assertTrue(MapUtils.getBoolean(yamlMap, "active")); - Assert.assertEquals(yamlMap.get(PROP_SUBS_GROUP_NAME), "test_notification"); - Assert.assertEquals(yamlMap.get(PROP_CRON), "0 0 14 * * ? *"); - Assert.assertEquals(yamlMap.get(PROP_APPLICATION), "test_application"); - Assert.assertEquals(yamlMap.get(PROP_EMAIL_SUBJECT_TYPE), AlertConfigBean.SubjectType.ALERT.name()); - Assert.assertEquals(yamlMap.get(PROP_FROM), "t...@thirdeye.com"); - Assert.assertEquals(yamlMap.get(PROP_TYPE), "DEFAULT_ALERTER_PIPELINE"); - Assert.assertEquals(ConfigUtils.getList(yamlMap.get(PROP_DETECTION_NAMES)).size(), 2); - Assert.assertNotNull(yamlMap.get(PROP_ALERT_SCHEMES)); - Assert.assertEquals(ConfigUtils.getList(yamlMap.get(PROP_ALERT_SCHEMES)).size(), 1); - Assert.assertEquals(ConfigUtils.getMap(ConfigUtils.getList(yamlMap.get(PROP_ALERT_SCHEMES)).get(0)).get(PROP_TYPE), "EMAIL"); - Assert.assertNotNull(yamlMap.get(PROP_RECIPIENTS)); - Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("to")), Collections.singletonList("to@test")); - Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("cc")), Collections.singletonList("cc@test")); - Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("bcc")), Collections.singletonList("bcc@test")); - } - - @Test - public void testMigrateWoWAnomalyFunction() throws Exception { - AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-1.json"), AnomalyFunctionDTO.class); - long oldID = anomalyFunctionDAO.save(actual); - - AnomalyFunctionDTO legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID); - legacyAnomalyFunction.setCreatedBy("test_user"); - anomalyFunctionDAO.update(legacyAnomalyFunction); - - MergedAnomalyResultDTO mergedAnomalyResultDTO = new MergedAnomalyResultDTO(); - mergedAnomalyResultDTO.setFunction(actual); - mergedAnomalyResultDTO.setMetric("test_metric"); - mergedAnomalyResultDTO.setCollection("test_collection"); - anomalyDAO.save(mergedAnomalyResultDTO); - - Response responseId = migrationResource.migrateAnomalyFunction(oldID); - long newID = (long) responseId.getEntity(); - - DetectionConfigDTO migratedAnomalyFunction = detectionConfigDAO.findById(newID); - - legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID); - - // Verify if the migration status is updated correctly and the old detection is disabled. - Assert.assertEquals(legacyAnomalyFunction.getFunctionName(), "test_function_thirdeye_migrated_" + newID); - Assert.assertFalse(legacyAnomalyFunction.getIsActive()); - - // Verify if the anomaly is pointing to the migrated anomaly function - List<MergedAnomalyResultDTO> mergedAnomalyResultDTOList = anomalyDAO.findAll(); - Assert.assertEquals(mergedAnomalyResultDTOList.get(0).getDetectionConfigId().longValue(), newID); - - // Assert the migrated object - DetectionConfigDTO expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("migrated-detection-config-1.json"), DetectionConfigDTO.class); - expected.setId(migratedAnomalyFunction.getId()); - expected.setVersion(migratedAnomalyFunction.getVersion()); - expected.setCreatedBy(migratedAnomalyFunction.getCreatedBy()); - expected.setUpdatedBy(migratedAnomalyFunction.getUpdatedBy()); - expected.setLastTimestamp(migratedAnomalyFunction.getLastTimestamp()); - - Assert.assertEquals(migratedAnomalyFunction, expected); - } - - @Test - public void testMinMaxAnomalyFunction() throws Exception { - AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-2.json"), AnomalyFunctionDTO.class); - long oldID = anomalyFunctionDAO.save(actual); - - MergedAnomalyResultDTO mergedAnomalyResultDTO = new MergedAnomalyResultDTO(); - mergedAnomalyResultDTO.setFunction(actual); - mergedAnomalyResultDTO.setMetric("test_metric"); - mergedAnomalyResultDTO.setCollection("test_collection"); - anomalyDAO.save(mergedAnomalyResultDTO); - - Response responseId = migrationResource.migrateAnomalyFunction(oldID); - long newID = (long) responseId.getEntity(); - - AnomalyFunctionDTO legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID); - DetectionConfigDTO migratedAnomalyFunction = detectionConfigDAO.findById(newID); - - // Verify if the migration status is updated correctly and the old detection is disabled. - Assert.assertEquals(legacyAnomalyFunction.getFunctionName(), "test_function_thirdeye_migrated_" + newID); - Assert.assertFalse(legacyAnomalyFunction.getIsActive()); - - // Verify if the anomaly is pointing to the migrated anomaly function - List<MergedAnomalyResultDTO> mergedAnomalyResultDTOList = anomalyDAO.findAll(); - Assert.assertEquals(mergedAnomalyResultDTOList.get(0).getDetectionConfigId().longValue(), newID); - - // Assert the migrated object - DetectionConfigDTO expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("migrated-detection-config-2.json"), DetectionConfigDTO.class); - expected.setId(migratedAnomalyFunction.getId()); - expected.setVersion(migratedAnomalyFunction.getVersion()); - expected.setCreatedBy(migratedAnomalyFunction.getCreatedBy()); - expected.setUpdatedBy(migratedAnomalyFunction.getUpdatedBy()); - expected.setLastTimestamp(migratedAnomalyFunction.getLastTimestamp()); - Assert.assertEquals(migratedAnomalyFunction, expected); - } - - @Test - public void testMigrateApplication() throws Exception { - AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-2.json"), AnomalyFunctionDTO.class); - long id1 = anomalyFunctionDAO.save(actual); - - AlertConfigDTO alertConfigDTO = new AlertConfigDTO(); - alertConfigDTO.setActive(true); - alertConfigDTO.setName("test_notification"); - alertConfigDTO.setApplication("test_application"); - alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.ALERT); - alertConfigDTO.setCronExpression("0 0 14 * * ? *"); - alertConfigDTO.setFromAddress("t...@thirdeye.com"); - alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients( - Collections.singleton("to@test"), - Collections.singleton("cc@test"), - Collections.singleton("bcc@test"))); - AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig(); - emailConfig.setFunctionIds(Arrays.asList(id1)); - emailConfig.setAnomalyWatermark(9999); - alertConfigDTO.setEmailConfig(emailConfig); - alertConfigDAO.save(alertConfigDTO); - - AlertConfigDTO alertConfigDTOMigrated = new AlertConfigDTO(); - alertConfigDTOMigrated.setActive(false); - alertConfigDTOMigrated.setName("test_notification" + MIGRATED_TAG); - alertConfigDTOMigrated.setApplication("test_application"); - alertConfigDTOMigrated.setSubjectType(AlertConfigBean.SubjectType.ALERT); - alertConfigDTOMigrated.setCronExpression("0 0 14 * * ? *"); - alertConfigDTOMigrated.setFromAddress("t...@thirdeye.com"); - alertConfigDTOMigrated.setReceiverAddresses(new DetectionAlertFilterRecipients( - Collections.singleton("to@test"), - Collections.singleton("cc@test"), - Collections.singleton("bcc@test"))); - alertConfigDTO.setEmailConfig(emailConfig); - alertConfigDAO.save(alertConfigDTOMigrated); - - ApplicationDTO applicationDTO = new ApplicationDTO(); - applicationDTO.setApplication("test_application"); - applicationDTO.setRecipients("t...@thirdeye.com"); - applicationDAO.save(applicationDTO); - - Response response = migrationResource.migrateApplication("test_application"); - - Assert.assertEquals(response.getEntity(), "Application test_application has been successfully migrated"); - } -} diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java index 17497b7..27483b3 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java @@ -149,12 +149,6 @@ public class MockDataProvider implements DataProvider { } @Override - public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchLegacyAnomalies(Collection<AnomalySlice> slices, - long configId) { - return fetchAnomalies(slices, configId, true); - } - - @Override public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchAnomalies(Collection<AnomalySlice> slices, long configId) { return fetchAnomalies(slices, configId, false); diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilterTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilterTest.java deleted file mode 100644 index c2c9b01..0000000 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilterTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-c...@linkedin.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.pinot.thirdeye.detection.alert.filter; - -import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO; -import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO; -import org.apache.pinot.thirdeye.detection.DataProvider; -import org.apache.pinot.thirdeye.detection.MockDataProvider; -import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterResult; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.testng.Assert; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import static org.apache.pinot.thirdeye.detection.DetectionTestUtils.*; - - -public class LegacyAlertFilterTest { - private static final String PROP_DETECTION_CONFIG_IDS = "detectionConfigIds"; - private static final List<Long> PROP_ID_VALUE = Arrays.asList(1001L, 1002L); - private static final String PROP_LEGACY_ALERT_FILTER_CONFIG = "legacyAlertFilterConfig"; - private static final String PROP_LEGACY_ALERT_FILTER_CLASS_NAME = "legacyAlertFilterClassName"; - private static final Set<String> TO_RECIPIENTS_VALUES = new HashSet<>(Arrays.asList("t...@example.com", "myt...@example.org")); - private static final Set<String> CC_RECIPIENTS_VALUES = new HashSet<>(Arrays.asList("iamcc@host.domain", "iamcc2@host.domain")); - private static final Set<String> BCC_RECIPIENTS_VALUES = new HashSet<>(Arrays.asList("iambcc@host.domain")); - private static final String PROP_RECIPIENTS = "recipients"; - - private List<MergedAnomalyResultDTO> detectedAnomalies; - private LegacyAlertFilter legacyAlertFilter; - private LegacyAlertFilter legacyAlertFilterOnLegacyAnomalies; - private Map<String, Set<String>> recipientsMap; - - @BeforeMethod - public void beforeMethod() throws Exception { - this.detectedAnomalies = new ArrayList<>(); - this.detectedAnomalies.add(makeAnomaly(1001L, 1500, 2000)); - this.detectedAnomalies.add(makeAnomaly(1001L, 0, 1000)); - this.detectedAnomalies.add(makeAnomaly(1002L, 0, 1000)); - this.detectedAnomalies.add(makeAnomaly(1002L, 1100, 1500)); - this.detectedAnomalies.add(makeAnomaly(1002L, 3333, 9999)); - this.detectedAnomalies.add(makeAnomaly(1003L, 1100, 1500)); - - // Anomalies generated by legacy pipeline - this.detectedAnomalies.add(makeAnomaly(null, 1000L, 1100, 1500)); - this.detectedAnomalies.add(makeAnomaly(null, 1002L, 0, 1000)); - this.detectedAnomalies.add(makeAnomaly(null, 1002L, 1100, 2000)); - - - DataProvider mockDataProvider = new MockDataProvider().setAnomalies(this.detectedAnomalies); - - DetectionAlertConfigDTO detectionAlertConfig = createDetectionAlertConfig(); - this.legacyAlertFilter = new LegacyAlertFilter(mockDataProvider, detectionAlertConfig, 2500L); - - DetectionAlertConfigDTO detectionAlertConfigLegacyAnomalies = createDetectionAlertConfig(); - detectionAlertConfigLegacyAnomalies.setOnlyFetchLegacyAnomalies(true); - this.legacyAlertFilterOnLegacyAnomalies = new LegacyAlertFilter(mockDataProvider, detectionAlertConfigLegacyAnomalies, 2500L); - - this.recipientsMap = new HashMap<>(); - recipientsMap.put("to", TO_RECIPIENTS_VALUES); - recipientsMap.put("cc", CC_RECIPIENTS_VALUES); - recipientsMap.put("bcc", BCC_RECIPIENTS_VALUES); - } - - private DetectionAlertConfigDTO createDetectionAlertConfig() { - DetectionAlertConfigDTO detectionAlertConfig = new DetectionAlertConfigDTO(); - Map<String, Object> properties = new HashMap<>(); - properties.put(PROP_DETECTION_CONFIG_IDS, PROP_ID_VALUE); - properties.put(PROP_LEGACY_ALERT_FILTER_CLASS_NAME, "org.apache.pinot.thirdeye.detector.email.filter.DummyAlertFilter"); - properties.put(PROP_LEGACY_ALERT_FILTER_CONFIG, ""); - properties.put(PROP_RECIPIENTS, recipientsMap); - detectionAlertConfig.setHighWaterMark(0L); - detectionAlertConfig.setProperties(properties); - detectionAlertConfig.setVectorClocks(new HashMap<Long, Long>()); - - return detectionAlertConfig; - } - - @Test - public void testRun() throws Exception { - DetectionAlertFilterResult result = this.legacyAlertFilter.run(); - Assert.assertEquals(result.getAllAnomalies(), new HashSet<>(this.detectedAnomalies.subList(0, 4))); - } - - @Test - public void testFetchingLegacyAnomalies() throws Exception { - DetectionAlertFilterResult result = this.legacyAlertFilterOnLegacyAnomalies.run(); - Assert.assertEquals(result.getAllAnomalies().size(), 2); - Assert.assertEquals(result.getAllAnomalies(), new HashSet<>(this.detectedAnomalies.subList(7, 9))); - } -} diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java index 6896820..138d054 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java @@ -82,7 +82,6 @@ public class ToAllRecipientsDetectionAlertFilterTest { this.alertConfig = createDetectionAlertConfig(); this.alertConfigForLegacyAnomalies = createDetectionAlertConfig(); - this.alertConfigForLegacyAnomalies.setOnlyFetchLegacyAnomalies(true); } private DetectionAlertConfigDTO createDetectionAlertConfig() { @@ -113,15 +112,6 @@ public class ToAllRecipientsDetectionAlertFilterTest { } @Test - public void testGetAlertFilterResultForLegacyAnomalies() throws Exception { - this.alertFilter = new ToAllRecipientsDetectionAlertFilter(this.provider, this.alertConfigForLegacyAnomalies,2500L); - - DetectionAlertFilterResult result = this.alertFilter.run(); - Assert.assertEquals(result.getResult().get(RECIPIENTS).size(), 2); - Assert.assertEquals(result.getResult().get(RECIPIENTS), new HashSet<>(this.detectedAnomalies.subList(7, 9))); - } - - @Test public void testAlertFilterFeedback() throws Exception { this.alertConfig.getProperties().put(PROP_DETECTION_CONFIG_IDS, Collections.singletonList(1003L)); this.alertFilter = new ToAllRecipientsDetectionAlertFilter(this.provider, this.alertConfig,2500L); diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java index e9eb0b0..cb57d1c 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java @@ -68,7 +68,6 @@ public class YamlDetectionAlertConfigTranslatorTest { DetectionAlertConfigDTO alertConfig = new YamlDetectionAlertConfigTranslator(this.detectionConfigManager).translate(alertYamlConfigs); Assert.assertTrue(alertConfig.isActive()); - Assert.assertFalse(alertConfig.isOnlyFetchLegacyAnomalies()); Assert.assertEquals(alertConfig.getName(), "test_group_name"); Assert.assertEquals(alertConfig.getApplication(), "test_application"); Assert.assertEquals(alertConfig.getFrom(), "thirdeye@thirdeye"); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org