jihaozh closed pull request #3597: [TE] detection - yaml validation
URL: https://github.com/apache/incubator-pinot/pull/3597
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
index bbaac7fb4a..b19913ff85 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
@@ -26,4 +26,5 @@
public static final String RCA_TAG = "Rootcause";
public static final String DASHBOARD_TAG = "Dashboard";
public static final String ONBOARD_TAG = "Onboard";
+ public static final String YAML_TAG = "Yaml";
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
index 946c9c72f8..06ad3ca1ba 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
@@ -19,7 +19,10 @@
import com.google.inject.Singleton;
import com.linkedin.thirdeye.datalayer.bao.DetectionConfigManager;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
+import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.datalayer.pojo.DetectionConfigBean;
+import com.linkedin.thirdeye.datalayer.pojo.MergedAnomalyResultBean;
+import java.util.Collections;
@Singleton
@@ -28,5 +31,37 @@ public DetectionConfigManagerImpl() {
super(DetectionConfigDTO.class, DetectionConfigBean.class);
}
+ @Override
+ public int update(DetectionConfigDTO detectionConfigDTO) {
+ if (detectionConfigDTO.getId() == null) {
+ Long id = save(detectionConfigDTO);
+ if (id > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ } else {
+ DetectionConfigBean detectionConfigBean =
convertDetectionConfigDTO2Bean(detectionConfigDTO);
+ return genericPojoDao.update(detectionConfigBean);
+ }
+ }
+
+ @Override
+ public Long save(DetectionConfigDTO detectionConfigDTO) {
+ if (detectionConfigDTO.getId() != null) {
+ //TODO: throw exception and force the caller to call update instead
+ update(detectionConfigDTO);
+ return detectionConfigDTO.getId();
+ }
+ DetectionConfigBean detectionConfigBean =
convertDetectionConfigDTO2Bean(detectionConfigDTO);
+ Long id = genericPojoDao.put(detectionConfigBean);
+ detectionConfigDTO.setId(id);
+ return id;
+ }
+ DetectionConfigBean convertDetectionConfigDTO2Bean(DetectionConfigDTO
detectionConfigDTO){
+ detectionConfigDTO.setComponents(Collections.emptyMap());
+ DetectionConfigBean bean = convertDTO2Bean(detectionConfigDTO,
DetectionConfigBean.class);
+ return bean;
+ }
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
index c53af497b9..eee8f0c218 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
@@ -115,21 +115,20 @@ public String migrateToYaml(@QueryParam("id") long
anomalyFunctionId) throws Exc
}
Map<String, Object> ruleYaml = new LinkedHashMap<>();
- ruleYaml.put("name", "myRule");
// detection
if (anomalyFunctionDTO.getType().equals("WEEK_OVER_WEEK_RULE")){
// wo1w change detector
- ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("type", "PERCENTAGE_RULE",
+ ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type",
"PERCENTAGE_RULE",
"params",
getPercentageChangeRuleDetectorParams(anomalyFunctionDTO))));
} else if (anomalyFunctionDTO.getType().equals("MIN_MAX_THRESHOLD")){
// threshold detector
- ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("type", "THRESHOLD",
+ ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type",
"THRESHOLD",
"params",
getMinMaxThresholdRuleDetectorParams(anomalyFunctionDTO))));
} else{
// algorithm detector
ruleYaml.put("detection", Collections.singletonList(
- ImmutableMap.of("type", "ALGORITHM", "params",
getAlgorithmDetectorParams(anomalyFunctionDTO),
+ ImmutableMap.of("name", "detection_rule1", "type", "ALGORITHM",
"params", getAlgorithmDetectorParams(anomalyFunctionDTO),
PROP_WINDOW_SIZE, anomalyFunctionDTO.getWindowSize(),
PROP_WINDOW_UNIT,
anomalyFunctionDTO.getWindowUnit().toString())));
}
@@ -141,16 +140,18 @@ public String migrateToYaml(@QueryParam("id") long
anomalyFunctionId) throws Exc
Map<String, Object> filterYaml = new LinkedHashMap<>();
if (!alertFilter.containsKey("thresholdField")) {
// algorithm alert filter
- filterYaml = ImmutableMap.of("type", "ALGORITHM_FILTER", "params",
getAlertFilterParams(anomalyFunctionDTO));
+ filterYaml = ImmutableMap.of("name", "filter_rule1", "type",
"ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO));
} else {
// threshold filter migrate to rule filters
// site wide impact filter migrate to rule based swi filter
if
(anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("impactToGlobal")){
filterYaml.put("type", "SITEWIDE_IMPACT_FILTER");
+ filterYaml.put("name", "filter_rule1");
filterYaml.put("params",
getSiteWideImpactFilterParams(anomalyFunctionDTO));
}
// weight filter migrate to rule based percentage change filter
if
(anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("weight")){
+ filterYaml.put("name", "filter_rule1");
filterYaml.put("type", "PERCENTAGE_CHANGE_FILTER");
filterYaml.put("params",
getPercentageChangeFilterParams(anomalyFunctionDTO));
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
index 776b427a2c..b07e4e969f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
@@ -67,7 +67,8 @@ protected DetectionPipeline(DataProvider provider,
DetectionConfigDTO config, lo
try {
this.initComponents();
} catch (Exception e) {
- LOG.error("initialize components failed", e);
+ LOG.error("Initialize components failed", e);
+ throw new IllegalArgumentException("Initialize components failed. Please
check rule parameters. " + e.getMessage());
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
index e5800b6a68..610ac39cf8 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
@@ -1,5 +1,6 @@
package com.linkedin.thirdeye.detection.components;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
@@ -92,6 +93,8 @@ public boolean isQualified(MergedAnomalyResultDTO anomaly) {
public void init(SitewideImpactRuleAnomalyFilterSpec spec, InputDataFetcher
dataFetcher) {
this.dataFetcher = dataFetcher;
this.threshold = spec.getThreshold();
+ Preconditions.checkArgument(Math.abs(this.threshold) <= 1, "Site wide
impact threshold should be less or equal than 1");
+
this.pattern = Pattern.valueOf(spec.getPattern().toUpperCase());
// customize baseline offset
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
index 138ca5045b..a304d682a5 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
@@ -20,7 +20,7 @@
private String timezone = "UTC";
private double threshold = Double.NaN;
private String offset;
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
public String getTimezone() {
return timezone;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
index 26f5d662fe..e7b573ceb9 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
@@ -20,7 +20,7 @@
private double absoluteChange = Double.NaN;
private String offset = "wo1w";
private String timezone = "UTC";
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
public double getAbsoluteChange() {
return absoluteChange;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
index 3717bb6bba..fbb7885724 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
@@ -20,7 +20,7 @@
private String timezone = "UTC";
private double threshold = Double.NaN;
private String offset;
- private String pattern;
+ private String pattern= "UP_OR_DOWN";
public String getTimezone() {
return timezone;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
index d58533e39d..e59d14c3d3 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
@@ -20,7 +20,7 @@
private double percentageChange = Double.NaN;
private String offset = "wo1w";
private String timezone = "UTC";
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
public String getPattern() {
return pattern;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
index c0090fa618..5d1231732d 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
@@ -9,7 +9,7 @@
private String timezone = "UTC";
private double threshold = Double.NaN;
private String offset;
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
private String sitewideMetricName;
private String sitewideCollection;
private Map<String, Collection<String>> filters = new HashMap<>();
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
index df08645a7d..a3a560f1f5 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
@@ -40,7 +40,6 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +50,7 @@
* of filling baseline & current values and inject detector, metric urn. Each
detector has a separate baseline filling merge wrapper
*/
public class BaselineFillingMergeWrapper extends MergeWrapper {
- private static final Logger LOG =
LoggerFactory.getLogger(MergeWrapper.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(BaselineFillingMergeWrapper.class);
private static final String PROP_BASELINE_PROVIDER = "baselineValueProvider";
private static final String PROP_CURRENT_PROVIDER = "currentValueProvider";
@@ -59,10 +58,11 @@
private static final String PROP_BASELINE_PROVIDER_COMPONENT_NAME =
"baselineProviderComponentName";
private static final String PROP_DETECTOR = "detector";
private static final String PROP_DETECTOR_COMPONENT_NAME =
"detectorComponentName";
+ private static final String DEFAULT_WOW_BASELINE_PROVIDER_NAME =
"DEFAULT_WOW";
private BaselineProvider baselineValueProvider; // optionally configure a
baseline value loader
private BaselineProvider currentValueProvider;
- private String baselineProviderComponentKey;
+ private String baselineProviderComponentName;
private String detectorComponentName;
private String metricUrn;
@@ -71,10 +71,19 @@ public BaselineFillingMergeWrapper(DataProvider provider,
DetectionConfigDTO con
super(provider, config, startTime, endTime);
if (config.getProperties().containsKey(PROP_BASELINE_PROVIDER)) {
- this.baselineProviderComponentKey =
DetectionUtils.getComponentName(MapUtils.getString(config.getProperties(),
PROP_BASELINE_PROVIDER));
-
Preconditions.checkArgument(this.config.getComponents().containsKey(this.baselineProviderComponentKey));
- this.baselineValueProvider = (BaselineProvider)
this.config.getComponents().get(this.baselineProviderComponentKey);
+ this.baselineProviderComponentName =
DetectionUtils.getComponentName(MapUtils.getString(config.getProperties(),
PROP_BASELINE_PROVIDER));
+
Preconditions.checkArgument(this.config.getComponents().containsKey(this.baselineProviderComponentName));
+ this.baselineValueProvider = (BaselineProvider)
this.config.getComponents().get(this.baselineProviderComponentName);
+ } else {
+ // default baseline provider, use wo1w
+ this.baselineValueProvider = new RuleBaselineProvider();
+ this.baselineProviderComponentName = DEFAULT_WOW_BASELINE_PROVIDER_NAME;
+ RuleBaselineProviderSpec spec = new RuleBaselineProviderSpec();
+ spec.setOffset("wo1w");
+ InputDataFetcher dataFetcher = new
DefaultInputDataFetcher(this.provider, this.config.getId());
+ this.baselineValueProvider.init(spec, dataFetcher);
}
+
if (config.getProperties().containsKey(PROP_CURRENT_PROVIDER)) {
String detectorReferenceKey =
DetectionUtils.getComponentName(MapUtils.getString(config.getProperties(),
currentValueProvider));
Preconditions.checkArgument(this.config.getComponents().containsKey(detectorReferenceKey));
@@ -158,7 +167,7 @@ public BaselineFillingMergeWrapper(DataProvider provider,
DetectionConfigDTO con
anomaly.setAvgCurrentVal(this.currentValueProvider.computePredictedAggregates(slice,
aggregationFunction));
if (this.baselineValueProvider != null) {
anomaly.setAvgBaselineVal(this.baselineValueProvider.computePredictedAggregates(slice,
aggregationFunction));
- anomaly.getProperties().put(PROP_BASELINE_PROVIDER_COMPONENT_NAME,
this.baselineProviderComponentKey);
+ anomaly.getProperties().put(PROP_BASELINE_PROVIDER_COMPONENT_NAME,
this.baselineProviderComponentName);
anomaly.setWeight(calculateWeight(anomaly));
}
} catch (Exception e) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index 9ffc5c28c5..9bba54f9dd 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -16,6 +16,7 @@
package com.linkedin.thirdeye.detection.wrapper;
+import com.google.common.collect.Collections2;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.detection.DataProvider;
@@ -87,7 +88,11 @@ public ChildKeepingMergeWrapper(DataProvider provider,
DetectionConfigDTO config
}
}
- return super.fillCurrentAndBaselineValue(output);
+ // refill current and baseline values for parent anomalies
+ Collection<MergedAnomalyResultDTO> parentAnomalies =
+ Collections2.filter(output, mergedAnomaly -> mergedAnomaly != null &&
!mergedAnomaly.getChildren().isEmpty());
+ super.fillCurrentAndBaselineValue(new ArrayList<>(parentAnomalies));
+ return output;
}
private MergedAnomalyResultDTO copyAnomalyInfo(MergedAnomalyResultDTO
anomaly, MergedAnomalyResultDTO newAnomaly) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 45cad5bfca..2007265f59 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -129,7 +129,6 @@
private static final String PROP_RULES = "rules";
private static final String PROP_NESTED = "nested";
private static final String PROP_BASELINE_PROVIDER = "baselineValueProvider";
- private static final String PROP_NAME = "name";
private static final String PROP_DETECTOR = "detector";
private static final String PROP_MOVING_WINDOW_DETECTION =
"isMovingWindowDetection";
private static final String PROP_WINDOW_DELAY = "windowDelay";
@@ -139,6 +138,8 @@
private static final String PROP_FREQUENCY = "frequency";
private static final String PROP_MERGER = "merger";
private static final String PROP_TIMEZONE = "timezone";
+ private static final String PROP_NAME = "name";
+ private static final String DEFAULT_BASELINE_PROVIDER_YAML_TYPE =
"RULE_BASELINE";
private static final DetectionRegistry DETECTION_REGISTRY =
DetectionRegistry.getInstance();
private static final Map<String, String> DETECTOR_TO_BASELINE =
ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE");
@@ -149,10 +150,11 @@
private DatasetConfigDTO datasetConfig;
private String metricUrn;
private Map<String, Object> mergerProperties = new HashMap<>();
- private Map<String, Collection<String>> filterMaps;
+ protected final org.yaml.snakeyaml.Yaml yaml;
public CompositePipelineConfigTranslator(Map<String, Object> yamlConfig,
DataProvider provider) {
super(yamlConfig, provider);
+ this.yaml = new org.yaml.snakeyaml.Yaml();
}
@Override
@@ -175,17 +177,16 @@ YamlTranslationResult translateYaml() {
List<Map<String, Object>> ruleYamls = getList(yamlConfig.get(PROP_RULES));
List<Map<String, Object>> nestedPipelines = new ArrayList<>();
for (Map<String, Object> ruleYaml : ruleYamls) {
- String ruleName = MapUtils.getString(ruleYaml, PROP_NAME);
List<Map<String, Object>> filterYamls =
ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
List<Map<String, Object>> detectionYamls =
ConfigUtils.getList(ruleYaml.get(PROP_DETECTION));
- List<Map<String, Object>> detectionProperties =
buildListOfMergeWrapperProperties(ruleName, detectionYamls);
+ List<Map<String, Object>> detectionProperties =
buildListOfMergeWrapperProperties(detectionYamls);
if (filterYamls == null || filterYamls.isEmpty()) {
nestedPipelines.addAll(detectionProperties);
} else {
List<Map<String, Object>> filterNestedProperties = detectionProperties;
for (Map<String, Object> filterProperties : filterYamls) {
filterNestedProperties =
buildFilterWrapperProperties(AnomalyFilterWrapper.class.getName(),
filterProperties,
- filterNestedProperties, ruleName);
+ filterNestedProperties);
}
nestedPipelines.addAll(filterNestedProperties);
}
@@ -213,21 +214,21 @@ YamlTranslationResult translateYaml() {
return dimensionWrapperProperties;
}
- private List<Map<String, Object>> buildListOfMergeWrapperProperties(String
ruleName,
+ private List<Map<String, Object>> buildListOfMergeWrapperProperties(
List<Map<String, Object>> yamlConfigs) {
List<Map<String, Object>> properties = new ArrayList<>();
for (Map<String, Object> yamlConfig : yamlConfigs) {
- properties.add(buildMergeWrapperProperties(ruleName, yamlConfig));
+ properties.add(buildMergeWrapperProperties(yamlConfig));
}
return properties;
}
- private Map<String, Object> buildMergeWrapperProperties(String ruleName,
Map<String, Object> yamlConfig) {
+ private Map<String, Object> buildMergeWrapperProperties(Map<String, Object>
yamlConfig) {
String detectorType = MapUtils.getString(yamlConfig, PROP_TYPE);
- long id = MapUtils.getLong(yamlConfig, "id", 0L);
+ String name = MapUtils.getString(yamlConfig, PROP_NAME);
Map<String, Object> nestedProperties = new HashMap<>();
nestedProperties.put(PROP_CLASS_NAME,
AnomalyDetectorWrapper.class.getName());
- String detectorKey = makeComponentKey(ruleName, detectorType, id);
+ String detectorKey = makeComponentKey(detectorType, name);
fillInWindowSizeAndUnit(nestedProperties, yamlConfig, detectorType);
@@ -236,11 +237,11 @@ YamlTranslationResult translateYaml() {
Map<String, Object> properties = new HashMap<>();
properties.put(PROP_CLASS_NAME,
BaselineFillingMergeWrapper.class.getName());
properties.put(PROP_NESTED, Collections.singletonList(nestedProperties));
- String baselineProviderType = "RULE_BASELINE";
+ String baselineProviderType = DEFAULT_BASELINE_PROVIDER_YAML_TYPE;
if (DETECTOR_TO_BASELINE.containsKey(detectorType)) {
baselineProviderType = DETECTOR_TO_BASELINE.get(detectorType);
}
- String baselineProviderKey = makeComponentKey(ruleName,
baselineProviderType, id);
+ String baselineProviderKey = makeComponentKey(baselineProviderType, name);
properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
properties.put(PROP_DETECTOR, detectorKey);
buildComponentSpec(yamlConfig, baselineProviderType, baselineProviderKey);
@@ -291,7 +292,7 @@ private void fillInWindowSizeAndUnit(Map<String, Object>
properties, Map<String,
}
private List<Map<String, Object>> buildFilterWrapperProperties(String
wrapperClassName,
- Map<String, Object> yamlConfig, List<Map<String, Object>>
nestedProperties, String ruleName) {
+ Map<String, Object> yamlConfig, List<Map<String, Object>>
nestedProperties) {
if (yamlConfig == null || yamlConfig.isEmpty()) {
return nestedProperties;
}
@@ -299,9 +300,9 @@ private void fillInWindowSizeAndUnit(Map<String, Object>
properties, Map<String,
if (wrapperProperties.isEmpty()) {
return Collections.emptyList();
}
- long id = MapUtils.getLong(yamlConfig, "id", 0L);
+ String name = MapUtils.getString(yamlConfig, PROP_NAME);
String filterType = MapUtils.getString(yamlConfig, PROP_TYPE);
- String filterKey = makeComponentKey(ruleName, filterType, id);
+ String filterKey = makeComponentKey(filterType, name);
wrapperProperties.put(PROP_FILTER, filterKey);
buildComponentSpec(yamlConfig, filterType, filterKey);
@@ -361,7 +362,10 @@ private void buildComponentSpec(Map<String, Object>
yamlConfig, String type, Str
String componentClassName = DETECTION_REGISTRY.lookup(type);
Map<String, Object> componentSpecs = new HashMap<>();
componentSpecs.put(PROP_CLASS_NAME, componentClassName);
- Map<String, Object> params = MapUtils.getMap(yamlConfig, PROP_PARAMS);
+ Map<String, Object> params = new HashMap<>();
+ if (yamlConfig.containsKey(PROP_PARAMS)){
+ params = MapUtils.getMap(yamlConfig, PROP_PARAMS);
+ }
if (DETECTION_REGISTRY.isTunable(componentClassName)) {
try {
@@ -399,8 +403,8 @@ private Tunable getTunable(String componentClassName,
Map<String, Object> params
return tunable;
}
- private String makeComponentKey(String name, String type, long id) {
- return "$" + name + ":" + type + ":" + id;
+ private String makeComponentKey(String type, String name) {
+ return "$" + name + ":" + type;
}
@Override
@@ -409,26 +413,41 @@ protected void validateYAML(Map<String, Object>
yamlConfig) {
Preconditions.checkArgument(yamlConfig.containsKey(PROP_METRIC), "Property
missing " + PROP_METRIC);
Preconditions.checkArgument(yamlConfig.containsKey(PROP_DATASET),
"Property missing " + PROP_DATASET);
Preconditions.checkArgument(yamlConfig.containsKey(PROP_RULES), "Property
missing " + PROP_RULES);
+ if (existingConfig != null) {
+ Map<String, Object> existingYamlConfig = (Map<String, Object>)
this.yaml.load(existingConfig.getYaml());
+ Preconditions.checkArgument(MapUtils.getString(yamlConfig,
PROP_METRIC).equals(MapUtils.getString(existingYamlConfig, PROP_METRIC)),
"metric name cannot be modified");
+ Preconditions.checkArgument(MapUtils.getString(yamlConfig,
PROP_DATASET).equals(MapUtils.getString(existingYamlConfig, PROP_DATASET)),
"dataset name cannot be modified");
+ }
Set<String> names = new HashSet<>();
List<Map<String, Object>> ruleYamls = getList(yamlConfig.get(PROP_RULES));
for (int i = 0; i < ruleYamls.size(); i++) {
Map<String, Object> ruleYaml = ruleYamls.get(i);
- Preconditions.checkArgument(ruleYaml.containsKey(PROP_NAME), "In rule
No." + (i + 1) + ", rule name property missing. ");
- String name = MapUtils.getString(ruleYaml, PROP_NAME);
- Preconditions.checkArgument(!names.contains(name), "In rule No." + (i +
1) + ", found duplicated rule name: " + name, ". Rule name must be unique.");
- names.add(name);
Preconditions.checkArgument(ruleYaml.containsKey(PROP_DETECTION),
- "In rule No." + (i + 1) + ", detection stage property missing. ");
+ "In rule No." + (i + 1) + ", detection rule is missing. ");
List<Map<String, Object>> detectionStageYamls =
ConfigUtils.getList(ruleYaml.get(PROP_DETECTION));
+ // check each detection rule
for (Map<String, Object> detectionStageYaml : detectionStageYamls) {
Preconditions.checkArgument(detectionStageYaml.containsKey(PROP_TYPE),
- "In rule No." + (i + 1) + ", detection stage type missing. ");
+ "In rule No." + (i + 1) + ", a detection rule type is missing. ");
+ String type = MapUtils.getString(detectionStageYaml, PROP_TYPE);
+ String name = MapUtils.getString(detectionStageYaml, PROP_NAME);
+ Preconditions.checkNotNull(name, "In rule No." + (i + 1) + ", a
detection rule name for type " + type + " is missing");
+ Preconditions.checkArgument(!names.contains(name), "In rule No." + (i
+ 1) +
+ ", found duplicate rule name, rule name must be unique." );
+ Preconditions.checkArgument(!name.contains(":"), "Sorry, rule name
cannot contain \':\'");
}
if (ruleYaml.containsKey(PROP_FILTER)) {
- List<Map<String, Object>> filterStageYamls =
ConfigUtils.getList(MapUtils.getMap(ruleYaml, PROP_FILTER));
+ List<Map<String, Object>> filterStageYamls =
ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
+ // check each filter rule
for (Map<String, Object> filterStageYaml : filterStageYamls) {
Preconditions.checkArgument(filterStageYaml.containsKey(PROP_TYPE),
- "In rule No." + (i + 1) + ", filter stage type missing. ");
+ "In rule No." + (i + 1) + ", a filter rule type is missing. ");
+ String type = MapUtils.getString(filterStageYaml, PROP_TYPE);
+ String name = MapUtils.getString(filterStageYaml, PROP_NAME);
+ Preconditions.checkNotNull(name, "In rule No." + (i + 1) + ", a
filter rule name for type " + type + " is missing");
+ Preconditions.checkArgument(!names.contains(name), "In rule No." +
(i + 1) +
+ ", found duplicate rule name, rule name must be unique." );
+ Preconditions.checkArgument(!name.contains(":"), "Sorry, rule name
cannot contain \':\'");
}
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
index 4e748b5ed4..29257dcd86 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
@@ -3,7 +3,6 @@
import com.google.common.base.Preconditions;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
import com.linkedin.thirdeye.detection.DataProvider;
-import com.linkedin.thirdeye.detection.DetectionPipeline;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
index 0d6b2319a6..e25a7b59ff 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
@@ -1,5 +1,6 @@
package com.linkedin.thirdeye.detection.yaml;
+import com.google.common.base.Preconditions;
import com.linkedin.thirdeye.detection.DataProvider;
import com.linkedin.thirdeye.detection.annotation.DetectionRegistry;
import java.lang.reflect.Constructor;
@@ -13,6 +14,7 @@
private static DetectionRegistry DETECTION_REGISTRY =
DetectionRegistry.getInstance();
public YamlDetectionConfigTranslator from(Map<String, Object> yamlConfig,
DataProvider provider) throws Exception {
+ Preconditions.checkArgument(yamlConfig.containsKey(PROP_PIPELINE_TYPE),
"Pipeline type is missing.");
String className =
DETECTION_REGISTRY.lookupYamlConverter(yamlConfig.get(PROP_PIPELINE_TYPE).toString().toUpperCase());
Constructor<?> constructor =
Class.forName(className).getConstructor(Map.class, DataProvider.class);
return (YamlDetectionConfigTranslator) constructor.newInstance(yamlConfig,
provider);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
index 56c01b2557..e13717956c 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
@@ -1,8 +1,8 @@
package com.linkedin.thirdeye.detection.yaml;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
+import com.linkedin.thirdeye.api.Constants;
import com.linkedin.thirdeye.datalayer.bao.DatasetConfigManager;
import com.linkedin.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import com.linkedin.thirdeye.datalayer.bao.DetectionConfigManager;
@@ -22,7 +22,10 @@
import com.linkedin.thirdeye.detection.DataProvider;
import com.linkedin.thirdeye.detection.DefaultDataProvider;
import com.linkedin.thirdeye.detection.DetectionPipelineLoader;
+import com.wordnik.swagger.annotations.Api;
+import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -33,18 +36,22 @@
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@Path("/yaml")
+@Api(tags = {Constants.YAML_TAG})
public class YamlResource {
protected static final Logger LOG =
LoggerFactory.getLogger(YamlResource.class);
@@ -52,7 +59,6 @@
private static final String PROP_TYPE = "type";
private static final String PROP_DETECTION_CONFIG_ID = "detectionConfigIds";
- private static final Yaml YAML = new Yaml();
private final DetectionConfigManager detectionConfigDAO;
private final DetectionAlertConfigManager detectionAlertConfigDAO;
@@ -64,6 +70,7 @@
private final EventManager eventDAO;
private final MergedAnomalyResultManager anomalyDAO;
private final DetectionPipelineLoader loader;
+ private final Yaml yaml;
public YamlResource() {
this.detectionConfigDAO =
DAORegistry.getInstance().getDetectionConfigManager();
@@ -74,6 +81,7 @@ public YamlResource() {
this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
this.eventDAO = DAORegistry.getInstance().getEventDAO();
this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
+ this.yaml = new Yaml();
TimeSeriesLoader timeseriesLoader =
new DefaultTimeSeriesLoader(metricDAO, datasetDAO,
ThirdEyeCacheRegistry.getInstance().getQueryCache());
@@ -90,48 +98,111 @@ public YamlResource() {
/**
Set up a detection pipeline using a YAML config
@param payload YAML config string
- @return a message contains the saved detection config id & detection alert
id
+ @param startTime tuning window start time for tunable components
+ @param endTime tuning window end time for tunable components
+ @return a message contains the saved detection config id
*/
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.TEXT_PLAIN)
- public Response setUpDetectionPipeline(@ApiParam("payload") String payload,
- @QueryParam("startTime") long startTime, @QueryParam("endTime") long
endTime) throws Exception {
- if (Strings.isNullOrEmpty(payload)) {
- throw new IllegalArgumentException("Empty Payload");
- }
- Map<String, Object> yamlConfig = (Map<String, Object>)
this.YAML.load(payload);
-
- Preconditions.checkArgument(yamlConfig.containsKey(PROP_NAME), "missing "
+ PROP_NAME);
- // retrieve id if detection config already exists
- List<DetectionConfigDTO> detectionConfigDTOs =
- this.detectionConfigDAO.findByPredicate(Predicate.EQ("name",
MapUtils.getString(yamlConfig, PROP_NAME)));
- DetectionConfigDTO existingDetectionConfig = null;
- if (!detectionConfigDTOs.isEmpty()) {
- existingDetectionConfig = detectionConfigDTOs.get(0);
- }
+ @ApiOperation("Set up a detection pipeline using a YAML config")
+ public Response setUpDetectionPipeline(
+ @ApiParam("yaml config") String payload,
+ @ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
+ @ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
+ String errorMessage;
+ try {
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "Empty
payload");
+ Map<String, Object> yamlConfig = (Map<String, Object>)
this.yaml.load(payload);
+
+ // retrieve id if detection config already exists
+ List<DetectionConfigDTO> detectionConfigDTOs =
+ this.detectionConfigDAO.findByPredicate(Predicate.EQ("name",
MapUtils.getString(yamlConfig, PROP_NAME)));
+ DetectionConfigDTO existingDetectionConfig = null;
+ if (!detectionConfigDTOs.isEmpty()) {
+ existingDetectionConfig = detectionConfigDTOs.get(0);
+ }
+
+ YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
+ DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
+ .withExistingDetectionConfig(existingDetectionConfig)
+ .generateDetectionConfig();
+ detectionConfig.setYaml(payload);
+ validatePipeline(detectionConfig);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
- YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
- DetectionConfigDTO detectionConfig;
- try{
- detectionConfig = translator.withTrainingWindow(startTime,
endTime).withExistingDetectionConfig(existingDetectionConfig).generateDetectionConfig();
+ return Response.ok(detectionConfig).build();
+ } catch (InvocationTargetException e){
+ // exception thrown in validate pipeline via reflection
+ LOG.error("Validate pipeline error", e);
+ errorMessage = e.getCause().getMessage();
} catch (Exception e) {
LOG.error("yaml translation error", e);
- return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", e.getMessage())).build();
+ errorMessage = e.getMessage();
}
- detectionConfig.setYaml(payload);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
-
- // optionally set up an alerter for the detection pipeline
- if (yamlConfig.containsKey("alert")) {
- Map<String, Object> alertYaml = MapUtils.getMap(yamlConfig, "alert");
- DetectionAlertConfigDTO alertConfigDTO =
getDetectionAlertConfig(alertYaml, detectionConfigId);
- Long detectionAlertConfigId =
this.detectionAlertConfigDAO.save(alertConfigDTO);
- Preconditions.checkNotNull(detectionAlertConfigId, "Save detection
alerter config failed");
+ return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", errorMessage)).build();
+ }
+
+ /**
+ Edit a detection pipeline using a YAML config
+ @param payload YAML config string
+ @param id the detection config id to edit
+ @param startTime tuning window start time for tunable components
+ @param endTime tuning window end time for tunable components
+ @return a message contains the saved detection config id
+ */
+ @PUT
+ @Path("/{id}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.TEXT_PLAIN)
+ @ApiOperation("Edit a detection pipeline using a YAML config")
+ public Response editDetectionPipeline(
+ @ApiParam("yaml config") String payload,
+ @ApiParam("the detection config id to edit") @PathParam("id") long id,
+ @ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
+ @ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
+ String errorMessage;
+ try {
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "Empty
payload");
+ Map<String, Object> yamlConfig = (Map<String, Object>)
this.yaml.load(payload);
+
+ DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(id);
+ Preconditions.checkArgument(existingDetectionConfig != null, "Existing
detection config " + id + " not found");
+
+ YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
+ DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
+ .withExistingDetectionConfig(existingDetectionConfig)
+ .generateDetectionConfig();
+ detectionConfig.setYaml(payload);
+ validatePipeline(detectionConfig);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+
+ return Response.ok(detectionConfig).build();
+ } catch (InvocationTargetException e){
+ // exception thrown in validate pipeline via reflection
+ LOG.error("Validate pipeline error", e);
+ errorMessage = e.getCause().getMessage();
+ } catch (Exception e) {
+ LOG.error("yaml translation error", e);
+ errorMessage = e.getMessage();
}
+ return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", errorMessage)).build();
+ }
- return Response.ok(detectionConfig).build();
+
+ /*
+ * Init the pipeline to check if detection pipeline property is valid
semantically.
+ */
+ private void validatePipeline(DetectionConfigDTO detectionConfig) throws
Exception {
+ Long id = detectionConfig.getId();
+ // swap out id
+ detectionConfig.setId(-1L);
+ // try to load the detection pipeline and init all the components
+ this.loader.from(provider, detectionConfig, 0, 0);
+ // set id back
+ detectionConfig.setId(id);
}
/**
@@ -195,7 +266,7 @@ private DetectionAlertConfigDTO
getDetectionAlertConfig(Map<String, Object> aler
for (DetectionConfigDTO detectionConfigDTO : detectionConfigDTOs) {
if (detectionConfigDTO.getYaml() != null) {
Map<String, Object> yamlObject = new HashMap<>();
- yamlObject.putAll((Map<? extends String, ?>)
this.YAML.load(detectionConfigDTO.getYaml()));
+ yamlObject.putAll((Map<? extends String, ?>)
this.yaml.load(detectionConfigDTO.getYaml()));
yamlObject.put("id", detectionConfigDTO.getId());
yamlObject.put("isActive", detectionConfigDTO.isActive());
yamlObject.put("createdBy", detectionConfigDTO.getCreatedBy());
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
index 0e04d11207..6f634c9d65 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
@@ -49,7 +49,7 @@ public void setUp() {
this.properties.put(PROP_METRIC_URN, "thirdeye:metric:1");
this.properties.put("specs", this.stageSpecs);
this.config = new DetectionConfigDTO();
-
+ this.config.setId(-1L);
this.config.setProperties(properties);
this.provider = new MockDataProvider();
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
index 12cb7c4992..72acffac57 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
@@ -79,7 +79,7 @@ public void beforeMethod() throws Exception {
this.config = new DetectionConfigDTO();
this.config.setProperties(properties);
-
+ this.config.setId(-1L);
this.provider = new MockDataProvider()
.setTimeseries(timeseries)
.setMetrics(Collections.singletonList(metricConfigDTO));
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
index 0d6891e5b6..960d4a8826 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
@@ -54,6 +54,7 @@ public void setUp() {
this.config = new DetectionConfigDTO();
this.config.setComponents(ImmutableMap.of("testDetector", new
ThresholdRuleDetector()));
this.config.setProperties(properties);
+ this.config.setId(-1L);
this.provider = new MockDataProvider();
MetricConfigDTO metric = new MetricConfigDTO();
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
index 481c6f0995..ac73ad431e 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
@@ -97,7 +97,9 @@ public void beforeMethod() {
@Test
public void testMergerCurrentAndBaselineLoading() throws Exception {
MergedAnomalyResultDTO anomaly = makeAnomaly(3000, 3600);
- anomaly.setProperties(ImmutableMap.of("detectorComponentName",
"testDetector"));
+ Map<String, String> anomalyProperties = new HashMap<>();
+ anomalyProperties.put("detectorComponentName", "testDetector");
+ anomaly.setProperties(anomalyProperties);
anomaly.setMetricUrn("thirdeye:metric:1");
Map<MetricSlice, DataFrame> aggregates = new HashMap<>();
@@ -127,6 +129,8 @@ public void testMergerCurrentAndBaselineLoading() throws
Exception {
Assert.assertTrue(anomalyResults.contains(anomaly));
Assert.assertEquals(anomalyResults.get(0).getAvgBaselineVal(), 100.0);
Assert.assertEquals(anomalyResults.get(0).getAvgCurrentVal(), 100.0);
+
Assert.assertEquals(anomalyResults.get(0).getProperties().get("detectorComponentName"),
"testDetector");
+
Assert.assertEquals(anomalyResults.get(0).getProperties().get("baselineProviderComponentName"),
"baseline");
}
}
\ No newline at end of file
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
index 65fb1b5979..75335eb1b3 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
@@ -7,33 +7,33 @@
"className" :
"com.linkedin.thirdeye.detection.algorithm.DimensionWrapper",
"metricUrn" : "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested" : [ {
- "filter" : "$rule1:THRESHOLD_RULE_FILTER:1",
+ "filter" : "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested" : [ {
- "filter" : "$rule1:THRESHOLD_RULE_FILTER:0",
+ "filter" : "$thresholdFilter_1:THRESHOLD_RULE_FILTER",
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested" : [ {
- "baselineValueProvider" : "$rule1:RULE_BASELINE:0",
+ "baselineValueProvider" : "$maxThreshold_1:RULE_BASELINE",
"className" :
"com.linkedin.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap" : 0,
"nested" : [ {
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
- "detector" : "$rule1:THRESHOLD:0",
+ "detector" : "$maxThreshold_1:THRESHOLD",
"maxDuration" : 100
} ]
} ]
}, {
- "filter" : "$rule2:THRESHOLD_RULE_FILTER:0",
+ "filter" : "$thresholdFilter_3:THRESHOLD_RULE_FILTER",
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested" : [ {
- "baselineValueProvider" : "$rule2:RULE_BASELINE:0",
+ "baselineValueProvider" : "$maxThreshold_2:RULE_BASELINE",
"className" :
"com.linkedin.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap" : 0,
"nested" : [ {
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
- "detector" : "$rule2:THRESHOLD:0",
+ "detector" : "$maxThreshold_2:THRESHOLD",
"maxDuration" : 100
} ]
} ],
@@ -43,33 +43,33 @@
"maxDuration" : 100
},
"components" : {
- "rule1:RULE_BASELINE:0" : {
+ "maxThreshold_2:THRESHOLD" : {
+ "max" : 100,
+ "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
+ },
+ "maxThreshold_1:THRESHOLD" : {
+ "max" : 100,
+ "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
+ },
+ "maxThreshold_2:RULE_BASELINE" : {
"max" : 100,
"className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
},
- "rule1:THRESHOLD_RULE_FILTER:0" : {
+ "thresholdFilter_1:THRESHOLD_RULE_FILTER" : {
"min" : 50,
"className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
},
- "rule1:THRESHOLD_RULE_FILTER:1" : {
+ "thresholdFilter_2:THRESHOLD_RULE_FILTER" : {
"min" : 50,
"className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
},
- "rule2:THRESHOLD_RULE_FILTER:0" : {
+ "thresholdFilter_3:THRESHOLD_RULE_FILTER" : {
"min" : 50,
"className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
},
- "rule2:THRESHOLD:0" : {
- "max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
- },
- "rule2:RULE_BASELINE:0" : {
+ "maxThreshold_1:RULE_BASELINE" : {
"max" : 100,
"className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
- },
- "rule1:THRESHOLD:0" : {
- "max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
}
},
"cron" : "0 0 14 * * ? *"
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
index bd43106dde..91c176d552 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
@@ -6,25 +6,25 @@
"className" :
"com.linkedin.thirdeye.detection.algorithm.DimensionWrapper",
"metricUrn" : "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested" : [ {
- "baselineValueProvider" : "$rule1:RULE_BASELINE:0",
+ "baselineValueProvider" : "$rule1:RULE_BASELINE",
"className" :
"com.linkedin.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested" : [ {
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
- "detector" : "$rule1:THRESHOLD:0"
+ "detector" : "$rule1:THRESHOLD"
} ],
"minContribution" : 0.05,
"dimensions" : [ "D1", "D2" ]
} ]
},
"components" : {
- "rule1:RULE_BASELINE:0" : {
+ "rule1:THRESHOLD" : {
"max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
+ "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
},
- "rule1:THRESHOLD:0" : {
+ "rule1:RULE_BASELINE" : {
"max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
+ "className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
}
},
"cron" : "0 0 14 * * ? *"
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
index 49c6463d35..911d0d9ac4 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
@@ -14,27 +14,30 @@ dimensionExploration:
- D2
minContribution: 0.05
rules:
-- name: rule1
- detection:
+- detection:
- type: THRESHOLD
+ name: maxThreshold_1
params:
max: 100
filter:
- type: THRESHOLD_RULE_FILTER
- id: 0
+ name: thresholdFilter_1
params:
min: 50
- type: THRESHOLD_RULE_FILTER
- id: 1
+ name: thresholdFilter_2
params:
min: 50
+
- name: rule2
detection:
- type: THRESHOLD
+ name: maxThreshold_2
params:
max: 100
filter:
- type: THRESHOLD_RULE_FILTER
+ name: thresholdFilter_3
params:
min: 50
merger:
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
index 815a77f3c2..323325e0af 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
@@ -14,9 +14,9 @@ dimensionExploration:
- D2
minContribution: 0.05
rules:
-- name: rule1
- detection:
+- detection:
- type: THRESHOLD
+ name: rule1
params:
max: 100
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
