This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a7a1745 [TE] detection - yaml validation (#3597)
a7a1745 is described below
commit a7a1745b6845252410fe7f0f4bff602d474df227
Author: Jihao Zhang <[email protected]>
AuthorDate: Mon Dec 10 14:05:38 2018 -0800
[TE] detection - yaml validation (#3597)
- Detection yaml config validation, both syntactically and semantically
- Yaml edit endpoint
- Use wow baseline if baseline provider not specified.
---
.../java/com/linkedin/thirdeye/api/Constants.java | 1 +
.../bao/jdbc/DetectionConfigManagerImpl.java | 35 +++++
.../detection/DetectionMigrationResource.java | 11 +-
.../thirdeye/detection/DetectionPipeline.java | 3 +-
.../SitewideImpactRuleAnomalyFilter.java | 3 +
.../spec/AbsoluteChangeRuleAnomalyFilterSpec.java | 2 +-
.../spec/AbsoluteChangeRuleDetectorSpec.java | 2 +-
.../PercentageChangeRuleAnomalyFilterSpec.java | 2 +-
.../spec/PercentageChangeRuleDetectorSpec.java | 2 +-
.../spec/SitewideImpactRuleAnomalyFilterSpec.java | 2 +-
.../wrapper/BaselineFillingMergeWrapper.java | 23 +++-
.../wrapper/ChildKeepingMergeWrapper.java | 7 +-
.../yaml/CompositePipelineConfigTranslator.java | 71 +++++++----
.../yaml/YamlDetectionConfigTranslator.java | 1 -
.../yaml/YamlDetectionTranslatorLoader.java | 2 +
.../thirdeye/detection/yaml/YamlResource.java | 141 ++++++++++++++++-----
.../stage/AnomalyDetectionStageWrapperTest.java | 2 +-
.../stage/BaselineRuleDetectionStageTest.java | 2 +-
.../wrapper/AnomalyDetectorWrapperTest.java | 1 +
.../wrapper/BaselineFillingMergeWrapperTest.java | 6 +-
.../compositePipelineTranslatorTestResult-1.json | 40 +++---
.../compositePipelineTranslatorTestResult-2.json | 12 +-
.../thirdeye/detection/yaml/pipeline-config-1.yaml | 11 +-
.../thirdeye/detection/yaml/pipeline-config-2.yaml | 4 +-
24 files changed, 270 insertions(+), 116 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
index bbaac7f..b19913f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
@@ -26,4 +26,5 @@ public class Constants {
public static final String RCA_TAG = "Rootcause";
public static final String DASHBOARD_TAG = "Dashboard";
public static final String ONBOARD_TAG = "Onboard";
+ public static final String YAML_TAG = "Yaml";
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
index 946c9c7..06ad3ca 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/DetectionConfigManagerImpl.java
@@ -19,7 +19,10 @@ package com.linkedin.thirdeye.datalayer.bao.jdbc;
import com.google.inject.Singleton;
import com.linkedin.thirdeye.datalayer.bao.DetectionConfigManager;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
+import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.datalayer.pojo.DetectionConfigBean;
+import com.linkedin.thirdeye.datalayer.pojo.MergedAnomalyResultBean;
+import java.util.Collections;
@Singleton
@@ -28,5 +31,37 @@ public class DetectionConfigManagerImpl extends
AbstractManagerImpl<DetectionCon
super(DetectionConfigDTO.class, DetectionConfigBean.class);
}
+ @Override
+ public int update(DetectionConfigDTO detectionConfigDTO) {
+ if (detectionConfigDTO.getId() == null) {
+ Long id = save(detectionConfigDTO);
+ if (id > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ } else {
+ DetectionConfigBean detectionConfigBean =
convertDetectionConfigDTO2Bean(detectionConfigDTO);
+ return genericPojoDao.update(detectionConfigBean);
+ }
+ }
+
+ @Override
+ public Long save(DetectionConfigDTO detectionConfigDTO) {
+ if (detectionConfigDTO.getId() != null) {
+ //TODO: throw exception and force the caller to call update instead
+ update(detectionConfigDTO);
+ return detectionConfigDTO.getId();
+ }
+ DetectionConfigBean detectionConfigBean =
convertDetectionConfigDTO2Bean(detectionConfigDTO);
+ Long id = genericPojoDao.put(detectionConfigBean);
+ detectionConfigDTO.setId(id);
+ return id;
+ }
+ DetectionConfigBean convertDetectionConfigDTO2Bean(DetectionConfigDTO
detectionConfigDTO){
+ detectionConfigDTO.setComponents(Collections.emptyMap());
+ DetectionConfigBean bean = convertDTO2Bean(detectionConfigDTO,
DetectionConfigBean.class);
+ return bean;
+ }
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
index c53af49..eee8f0c 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionMigrationResource.java
@@ -115,21 +115,20 @@ public class DetectionMigrationResource {
}
Map<String, Object> ruleYaml = new LinkedHashMap<>();
- ruleYaml.put("name", "myRule");
// detection
if (anomalyFunctionDTO.getType().equals("WEEK_OVER_WEEK_RULE")){
// wo1w change detector
- ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("type", "PERCENTAGE_RULE",
+ ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type",
"PERCENTAGE_RULE",
"params",
getPercentageChangeRuleDetectorParams(anomalyFunctionDTO))));
} else if (anomalyFunctionDTO.getType().equals("MIN_MAX_THRESHOLD")){
// threshold detector
- ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("type", "THRESHOLD",
+ ruleYaml.put("detection",
Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type",
"THRESHOLD",
"params",
getMinMaxThresholdRuleDetectorParams(anomalyFunctionDTO))));
} else{
// algorithm detector
ruleYaml.put("detection", Collections.singletonList(
- ImmutableMap.of("type", "ALGORITHM", "params",
getAlgorithmDetectorParams(anomalyFunctionDTO),
+ ImmutableMap.of("name", "detection_rule1", "type", "ALGORITHM",
"params", getAlgorithmDetectorParams(anomalyFunctionDTO),
PROP_WINDOW_SIZE, anomalyFunctionDTO.getWindowSize(),
PROP_WINDOW_UNIT,
anomalyFunctionDTO.getWindowUnit().toString())));
}
@@ -141,16 +140,18 @@ public class DetectionMigrationResource {
Map<String, Object> filterYaml = new LinkedHashMap<>();
if (!alertFilter.containsKey("thresholdField")) {
// algorithm alert filter
- filterYaml = ImmutableMap.of("type", "ALGORITHM_FILTER", "params",
getAlertFilterParams(anomalyFunctionDTO));
+ filterYaml = ImmutableMap.of("name", "filter_rule1", "type",
"ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO));
} else {
// threshold filter migrate to rule filters
// site wide impact filter migrate to rule based swi filter
if
(anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("impactToGlobal")){
filterYaml.put("type", "SITEWIDE_IMPACT_FILTER");
+ filterYaml.put("name", "filter_rule1");
filterYaml.put("params",
getSiteWideImpactFilterParams(anomalyFunctionDTO));
}
// weight filter migrate to rule based percentage change filter
if
(anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("weight")){
+ filterYaml.put("name", "filter_rule1");
filterYaml.put("type", "PERCENTAGE_CHANGE_FILTER");
filterYaml.put("params",
getPercentageChangeFilterParams(anomalyFunctionDTO));
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
index 776b427..b07e4e9 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionPipeline.java
@@ -67,7 +67,8 @@ public abstract class DetectionPipeline {
try {
this.initComponents();
} catch (Exception e) {
- LOG.error("initialize components failed", e);
+ LOG.error("Initialize components failed", e);
+ throw new IllegalArgumentException("Initialize components failed. Please
check rule parameters. " + e.getMessage());
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
index e5800b6..610ac39 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
@@ -1,5 +1,6 @@
package com.linkedin.thirdeye.detection.components;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
@@ -92,6 +93,8 @@ public class SitewideImpactRuleAnomalyFilter implements
AnomalyFilter<SitewideIm
public void init(SitewideImpactRuleAnomalyFilterSpec spec, InputDataFetcher
dataFetcher) {
this.dataFetcher = dataFetcher;
this.threshold = spec.getThreshold();
+ Preconditions.checkArgument(Math.abs(this.threshold) <= 1, "Site wide
impact threshold should be less or equal than 1");
+
this.pattern = Pattern.valueOf(spec.getPattern().toUpperCase());
// customize baseline offset
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
index 138ca50..a304d68 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleAnomalyFilterSpec.java
@@ -20,7 +20,7 @@ public class AbsoluteChangeRuleAnomalyFilterSpec extends
AbstractSpec {
private String timezone = "UTC";
private double threshold = Double.NaN;
private String offset;
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
public String getTimezone() {
return timezone;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
index 26f5d66..e7b573c 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
@@ -20,7 +20,7 @@ public class AbsoluteChangeRuleDetectorSpec extends
AbstractSpec {
private double absoluteChange = Double.NaN;
private String offset = "wo1w";
private String timezone = "UTC";
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
public double getAbsoluteChange() {
return absoluteChange;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
index 3717bb6..fbb7885 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleAnomalyFilterSpec.java
@@ -20,7 +20,7 @@ public class PercentageChangeRuleAnomalyFilterSpec extends
AbstractSpec {
private String timezone = "UTC";
private double threshold = Double.NaN;
private String offset;
- private String pattern;
+ private String pattern= "UP_OR_DOWN";
public String getTimezone() {
return timezone;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
index d58533e..e59d14c 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
@@ -20,7 +20,7 @@ public class PercentageChangeRuleDetectorSpec extends
AbstractSpec {
private double percentageChange = Double.NaN;
private String offset = "wo1w";
private String timezone = "UTC";
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
public String getPattern() {
return pattern;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
index c0090fa..5d12317 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
@@ -9,7 +9,7 @@ public class SitewideImpactRuleAnomalyFilterSpec extends
AbstractSpec {
private String timezone = "UTC";
private double threshold = Double.NaN;
private String offset;
- private String pattern;
+ private String pattern = "UP_OR_DOWN";
private String sitewideMetricName;
private String sitewideCollection;
private Map<String, Collection<String>> filters = new HashMap<>();
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
index df08645..a3a560f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapper.java
@@ -40,7 +40,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +50,7 @@ import org.slf4j.LoggerFactory;
* of filling baseline & current values and inject detector, metric urn. Each
detector has a separate baseline filling merge wrapper
*/
public class BaselineFillingMergeWrapper extends MergeWrapper {
- private static final Logger LOG =
LoggerFactory.getLogger(MergeWrapper.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(BaselineFillingMergeWrapper.class);
private static final String PROP_BASELINE_PROVIDER = "baselineValueProvider";
private static final String PROP_CURRENT_PROVIDER = "currentValueProvider";
@@ -59,10 +58,11 @@ public class BaselineFillingMergeWrapper extends
MergeWrapper {
private static final String PROP_BASELINE_PROVIDER_COMPONENT_NAME =
"baselineProviderComponentName";
private static final String PROP_DETECTOR = "detector";
private static final String PROP_DETECTOR_COMPONENT_NAME =
"detectorComponentName";
+ private static final String DEFAULT_WOW_BASELINE_PROVIDER_NAME =
"DEFAULT_WOW";
private BaselineProvider baselineValueProvider; // optionally configure a
baseline value loader
private BaselineProvider currentValueProvider;
- private String baselineProviderComponentKey;
+ private String baselineProviderComponentName;
private String detectorComponentName;
private String metricUrn;
@@ -71,10 +71,19 @@ public class BaselineFillingMergeWrapper extends
MergeWrapper {
super(provider, config, startTime, endTime);
if (config.getProperties().containsKey(PROP_BASELINE_PROVIDER)) {
- this.baselineProviderComponentKey =
DetectionUtils.getComponentName(MapUtils.getString(config.getProperties(),
PROP_BASELINE_PROVIDER));
-
Preconditions.checkArgument(this.config.getComponents().containsKey(this.baselineProviderComponentKey));
- this.baselineValueProvider = (BaselineProvider)
this.config.getComponents().get(this.baselineProviderComponentKey);
+ this.baselineProviderComponentName =
DetectionUtils.getComponentName(MapUtils.getString(config.getProperties(),
PROP_BASELINE_PROVIDER));
+
Preconditions.checkArgument(this.config.getComponents().containsKey(this.baselineProviderComponentName));
+ this.baselineValueProvider = (BaselineProvider)
this.config.getComponents().get(this.baselineProviderComponentName);
+ } else {
+ // default baseline provider, use wo1w
+ this.baselineValueProvider = new RuleBaselineProvider();
+ this.baselineProviderComponentName = DEFAULT_WOW_BASELINE_PROVIDER_NAME;
+ RuleBaselineProviderSpec spec = new RuleBaselineProviderSpec();
+ spec.setOffset("wo1w");
+ InputDataFetcher dataFetcher = new
DefaultInputDataFetcher(this.provider, this.config.getId());
+ this.baselineValueProvider.init(spec, dataFetcher);
}
+
if (config.getProperties().containsKey(PROP_CURRENT_PROVIDER)) {
String detectorReferenceKey =
DetectionUtils.getComponentName(MapUtils.getString(config.getProperties(),
currentValueProvider));
Preconditions.checkArgument(this.config.getComponents().containsKey(detectorReferenceKey));
@@ -158,7 +167,7 @@ public class BaselineFillingMergeWrapper extends
MergeWrapper {
anomaly.setAvgCurrentVal(this.currentValueProvider.computePredictedAggregates(slice,
aggregationFunction));
if (this.baselineValueProvider != null) {
anomaly.setAvgBaselineVal(this.baselineValueProvider.computePredictedAggregates(slice,
aggregationFunction));
- anomaly.getProperties().put(PROP_BASELINE_PROVIDER_COMPONENT_NAME,
this.baselineProviderComponentKey);
+ anomaly.getProperties().put(PROP_BASELINE_PROVIDER_COMPONENT_NAME,
this.baselineProviderComponentName);
anomaly.setWeight(calculateWeight(anomaly));
}
} catch (Exception e) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index 9ffc5c2..9bba54f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -16,6 +16,7 @@
package com.linkedin.thirdeye.detection.wrapper;
+import com.google.common.collect.Collections2;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.detection.DataProvider;
@@ -87,7 +88,11 @@ public class ChildKeepingMergeWrapper extends
BaselineFillingMergeWrapper {
}
}
- return super.fillCurrentAndBaselineValue(output);
+ // refill current and baseline values for parent anomalies
+ Collection<MergedAnomalyResultDTO> parentAnomalies =
+ Collections2.filter(output, mergedAnomaly -> mergedAnomaly != null &&
!mergedAnomaly.getChildren().isEmpty());
+ super.fillCurrentAndBaselineValue(new ArrayList<>(parentAnomalies));
+ return output;
}
private MergedAnomalyResultDTO copyAnomalyInfo(MergedAnomalyResultDTO
anomaly, MergedAnomalyResultDTO newAnomaly) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index b9f6710..2288c4b 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -129,7 +129,6 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
private static final String PROP_RULES = "rules";
private static final String PROP_NESTED = "nested";
private static final String PROP_BASELINE_PROVIDER = "baselineValueProvider";
- private static final String PROP_NAME = "name";
private static final String PROP_DETECTOR = "detector";
private static final String PROP_MOVING_WINDOW_DETECTION =
"isMovingWindowDetection";
private static final String PROP_WINDOW_DELAY = "windowDelay";
@@ -139,6 +138,8 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
private static final String PROP_FREQUENCY = "frequency";
private static final String PROP_MERGER = "merger";
private static final String PROP_TIMEZONE = "timezone";
+ private static final String PROP_NAME = "name";
+ private static final String DEFAULT_BASELINE_PROVIDER_YAML_TYPE =
"RULE_BASELINE";
private static final DetectionRegistry DETECTION_REGISTRY =
DetectionRegistry.getInstance();
private static final Map<String, String> DETECTOR_TO_BASELINE =
ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE");
@@ -149,10 +150,11 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
private DatasetConfigDTO datasetConfig;
private String metricUrn;
private Map<String, Object> mergerProperties = new HashMap<>();
- private Map<String, Collection<String>> filterMaps;
+ protected final org.yaml.snakeyaml.Yaml yaml;
public CompositePipelineConfigTranslator(Map<String, Object> yamlConfig,
DataProvider provider) {
super(yamlConfig, provider);
+ this.yaml = new org.yaml.snakeyaml.Yaml();
}
@Override
@@ -175,17 +177,16 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
List<Map<String, Object>> ruleYamls = getList(yamlConfig.get(PROP_RULES));
List<Map<String, Object>> nestedPipelines = new ArrayList<>();
for (Map<String, Object> ruleYaml : ruleYamls) {
- String ruleName = MapUtils.getString(ruleYaml, PROP_NAME);
List<Map<String, Object>> filterYamls =
ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
List<Map<String, Object>> detectionYamls =
ConfigUtils.getList(ruleYaml.get(PROP_DETECTION));
- List<Map<String, Object>> detectionProperties =
buildListOfMergeWrapperProperties(ruleName, detectionYamls);
+ List<Map<String, Object>> detectionProperties =
buildListOfMergeWrapperProperties(detectionYamls);
if (filterYamls == null || filterYamls.isEmpty()) {
nestedPipelines.addAll(detectionProperties);
} else {
List<Map<String, Object>> filterNestedProperties = detectionProperties;
for (Map<String, Object> filterProperties : filterYamls) {
filterNestedProperties =
buildFilterWrapperProperties(AnomalyFilterWrapper.class.getName(),
filterProperties,
- filterNestedProperties, ruleName);
+ filterNestedProperties);
}
nestedPipelines.addAll(filterNestedProperties);
}
@@ -213,21 +214,21 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
return dimensionWrapperProperties;
}
- private List<Map<String, Object>> buildListOfMergeWrapperProperties(String
ruleName,
+ private List<Map<String, Object>> buildListOfMergeWrapperProperties(
List<Map<String, Object>> yamlConfigs) {
List<Map<String, Object>> properties = new ArrayList<>();
for (Map<String, Object> yamlConfig : yamlConfigs) {
- properties.add(buildMergeWrapperProperties(ruleName, yamlConfig));
+ properties.add(buildMergeWrapperProperties(yamlConfig));
}
return properties;
}
- private Map<String, Object> buildMergeWrapperProperties(String ruleName,
Map<String, Object> yamlConfig) {
+ private Map<String, Object> buildMergeWrapperProperties(Map<String, Object>
yamlConfig) {
String detectorType = MapUtils.getString(yamlConfig, PROP_TYPE);
- long id = MapUtils.getLong(yamlConfig, "id", 0L);
+ String name = MapUtils.getString(yamlConfig, PROP_NAME);
Map<String, Object> nestedProperties = new HashMap<>();
nestedProperties.put(PROP_CLASS_NAME,
AnomalyDetectorWrapper.class.getName());
- String detectorKey = makeComponentKey(ruleName, detectorType, id);
+ String detectorKey = makeComponentKey(detectorType, name);
fillInWindowSizeAndUnit(nestedProperties, yamlConfig, detectorType);
@@ -236,11 +237,11 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
Map<String, Object> properties = new HashMap<>();
properties.put(PROP_CLASS_NAME,
BaselineFillingMergeWrapper.class.getName());
properties.put(PROP_NESTED, Collections.singletonList(nestedProperties));
- String baselineProviderType = "RULE_BASELINE";
+ String baselineProviderType = DEFAULT_BASELINE_PROVIDER_YAML_TYPE;
if (DETECTOR_TO_BASELINE.containsKey(detectorType)) {
baselineProviderType = DETECTOR_TO_BASELINE.get(detectorType);
}
- String baselineProviderKey = makeComponentKey(ruleName,
baselineProviderType, id);
+ String baselineProviderKey = makeComponentKey(baselineProviderType, name);
properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
properties.put(PROP_DETECTOR, detectorKey);
buildComponentSpec(yamlConfig, baselineProviderType, baselineProviderKey);
@@ -291,7 +292,7 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
}
private List<Map<String, Object>> buildFilterWrapperProperties(String
wrapperClassName,
- Map<String, Object> yamlConfig, List<Map<String, Object>>
nestedProperties, String ruleName) {
+ Map<String, Object> yamlConfig, List<Map<String, Object>>
nestedProperties) {
if (yamlConfig == null || yamlConfig.isEmpty()) {
return nestedProperties;
}
@@ -299,9 +300,9 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
if (wrapperProperties.isEmpty()) {
return Collections.emptyList();
}
- long id = MapUtils.getLong(yamlConfig, "id", 0L);
+ String name = MapUtils.getString(yamlConfig, PROP_NAME);
String filterType = MapUtils.getString(yamlConfig, PROP_TYPE);
- String filterKey = makeComponentKey(ruleName, filterType, id);
+ String filterKey = makeComponentKey(filterType, name);
wrapperProperties.put(PROP_FILTER, filterKey);
buildComponentSpec(yamlConfig, filterType, filterKey);
@@ -361,7 +362,10 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
String componentClassName = DETECTION_REGISTRY.lookup(type);
Map<String, Object> componentSpecs = new HashMap<>();
componentSpecs.put(PROP_CLASS_NAME, componentClassName);
- Map<String, Object> params = MapUtils.getMap(yamlConfig, PROP_PARAMS);
+ Map<String, Object> params = new HashMap<>();
+ if (yamlConfig.containsKey(PROP_PARAMS)){
+ params = MapUtils.getMap(yamlConfig, PROP_PARAMS);
+ }
if (DETECTION_REGISTRY.isTunable(componentClassName)) {
try {
@@ -399,8 +403,8 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
return tunable;
}
- private String makeComponentKey(String name, String type, long id) {
- return "$" + name + ":" + type + ":" + id;
+ private String makeComponentKey(String type, String name) {
+ return "$" + name + ":" + type;
}
@Override
@@ -409,26 +413,41 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
Preconditions.checkArgument(yamlConfig.containsKey(PROP_METRIC), "Property
missing " + PROP_METRIC);
Preconditions.checkArgument(yamlConfig.containsKey(PROP_DATASET),
"Property missing " + PROP_DATASET);
Preconditions.checkArgument(yamlConfig.containsKey(PROP_RULES), "Property
missing " + PROP_RULES);
+ if (existingConfig != null) {
+ Map<String, Object> existingYamlConfig = (Map<String, Object>)
this.yaml.load(existingConfig.getYaml());
+ Preconditions.checkArgument(MapUtils.getString(yamlConfig,
PROP_METRIC).equals(MapUtils.getString(existingYamlConfig, PROP_METRIC)),
"metric name cannot be modified");
+ Preconditions.checkArgument(MapUtils.getString(yamlConfig,
PROP_DATASET).equals(MapUtils.getString(existingYamlConfig, PROP_DATASET)),
"dataset name cannot be modified");
+ }
Set<String> names = new HashSet<>();
List<Map<String, Object>> ruleYamls = getList(yamlConfig.get(PROP_RULES));
for (int i = 0; i < ruleYamls.size(); i++) {
Map<String, Object> ruleYaml = ruleYamls.get(i);
- Preconditions.checkArgument(ruleYaml.containsKey(PROP_NAME), "In rule
No." + (i + 1) + ", rule name property missing. ");
- String name = MapUtils.getString(ruleYaml, PROP_NAME);
- Preconditions.checkArgument(!names.contains(name), "In rule No." + (i +
1) + ", found duplicated rule name: " + name, ". Rule name must be unique.");
- names.add(name);
Preconditions.checkArgument(ruleYaml.containsKey(PROP_DETECTION),
- "In rule No." + (i + 1) + ", detection stage property missing. ");
+ "In rule No." + (i + 1) + ", detection rule is missing. ");
List<Map<String, Object>> detectionStageYamls =
ConfigUtils.getList(ruleYaml.get(PROP_DETECTION));
+ // check each detection rule
for (Map<String, Object> detectionStageYaml : detectionStageYamls) {
Preconditions.checkArgument(detectionStageYaml.containsKey(PROP_TYPE),
- "In rule No." + (i + 1) + ", detection stage type missing. ");
+ "In rule No." + (i + 1) + ", a detection rule type is missing. ");
+ String type = MapUtils.getString(detectionStageYaml, PROP_TYPE);
+ String name = MapUtils.getString(detectionStageYaml, PROP_NAME);
+ Preconditions.checkNotNull(name, "In rule No." + (i + 1) + ", a
detection rule name for type " + type + " is missing");
+ Preconditions.checkArgument(!names.contains(name), "In rule No." + (i
+ 1) +
+ ", found duplicate rule name, rule name must be unique." );
+ Preconditions.checkArgument(!name.contains(":"), "Sorry, rule name
cannot contain \':\'");
}
if (ruleYaml.containsKey(PROP_FILTER)) {
- List<Map<String, Object>> filterStageYamls =
ConfigUtils.getList(MapUtils.getMap(ruleYaml, PROP_FILTER));
+ List<Map<String, Object>> filterStageYamls =
ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
+ // check each filter rule
for (Map<String, Object> filterStageYaml : filterStageYamls) {
Preconditions.checkArgument(filterStageYaml.containsKey(PROP_TYPE),
- "In rule No." + (i + 1) + ", filter stage type missing. ");
+ "In rule No." + (i + 1) + ", a filter rule type is missing. ");
+ String type = MapUtils.getString(filterStageYaml, PROP_TYPE);
+ String name = MapUtils.getString(filterStageYaml, PROP_NAME);
+ Preconditions.checkNotNull(name, "In rule No." + (i + 1) + ", a
filter rule name for type " + type + " is missing");
+ Preconditions.checkArgument(!names.contains(name), "In rule No." +
(i + 1) +
+ ", found duplicate rule name, rule name must be unique." );
+ Preconditions.checkArgument(!name.contains(":"), "Sorry, rule name
cannot contain \':\'");
}
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
index 4e748b5..29257dc 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
@@ -3,7 +3,6 @@ package com.linkedin.thirdeye.detection.yaml;
import com.google.common.base.Preconditions;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
import com.linkedin.thirdeye.detection.DataProvider;
-import com.linkedin.thirdeye.detection.DetectionPipeline;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
index acea6a9..ca2e87f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
@@ -1,5 +1,6 @@
package com.linkedin.thirdeye.detection.yaml;
+import com.google.common.base.Preconditions;
import com.linkedin.thirdeye.detection.DataProvider;
import com.linkedin.thirdeye.detection.annotation.registry.DetectionRegistry;
import java.lang.reflect.Constructor;
@@ -13,6 +14,7 @@ public class YamlDetectionTranslatorLoader {
private static DetectionRegistry DETECTION_REGISTRY =
DetectionRegistry.getInstance();
public YamlDetectionConfigTranslator from(Map<String, Object> yamlConfig,
DataProvider provider) throws Exception {
+ Preconditions.checkArgument(yamlConfig.containsKey(PROP_PIPELINE_TYPE),
"Pipeline type is missing.");
String className =
DETECTION_REGISTRY.lookupYamlConverter(yamlConfig.get(PROP_PIPELINE_TYPE).toString().toUpperCase());
Constructor<?> constructor =
Class.forName(className).getConstructor(Map.class, DataProvider.class);
return (YamlDetectionConfigTranslator) constructor.newInstance(yamlConfig,
provider);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
index 5ae893a..6ffd36a 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
@@ -1,8 +1,8 @@
package com.linkedin.thirdeye.detection.yaml;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
+import com.linkedin.thirdeye.api.Constants;
import com.linkedin.thirdeye.datalayer.bao.DatasetConfigManager;
import com.linkedin.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import com.linkedin.thirdeye.datalayer.bao.DetectionConfigManager;
@@ -22,7 +22,10 @@ import com.linkedin.thirdeye.detection.ConfigUtils;
import com.linkedin.thirdeye.detection.DataProvider;
import com.linkedin.thirdeye.detection.DefaultDataProvider;
import com.linkedin.thirdeye.detection.DetectionPipelineLoader;
+import com.wordnik.swagger.annotations.Api;
+import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -33,18 +36,22 @@ import java.util.Set;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@Path("/yaml")
+@Api(tags = {Constants.YAML_TAG})
public class YamlResource {
protected static final Logger LOG =
LoggerFactory.getLogger(YamlResource.class);
@@ -52,7 +59,6 @@ public class YamlResource {
private static final String PROP_TYPE = "type";
private static final String PROP_DETECTION_CONFIG_ID = "detectionConfigIds";
- private static final Yaml YAML = new Yaml();
private final DetectionConfigManager detectionConfigDAO;
private final DetectionAlertConfigManager detectionAlertConfigDAO;
@@ -64,6 +70,7 @@ public class YamlResource {
private final EventManager eventDAO;
private final MergedAnomalyResultManager anomalyDAO;
private final DetectionPipelineLoader loader;
+ private final Yaml yaml;
public YamlResource() {
this.detectionConfigDAO =
DAORegistry.getInstance().getDetectionConfigManager();
@@ -74,6 +81,7 @@ public class YamlResource {
this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
this.eventDAO = DAORegistry.getInstance().getEventDAO();
this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
+ this.yaml = new Yaml();
TimeSeriesLoader timeseriesLoader =
new DefaultTimeSeriesLoader(metricDAO, datasetDAO,
ThirdEyeCacheRegistry.getInstance().getQueryCache());
@@ -90,48 +98,111 @@ public class YamlResource {
/**
Set up a detection pipeline using a YAML config
@param payload YAML config string
- @return a message contains the saved detection config id & detection alert
id
+ @param startTime tuning window start time for tunable components
+ @param endTime tuning window end time for tunable components
+ @return a message contains the saved detection config id
*/
@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.TEXT_PLAIN)
- public Response setUpDetectionPipeline(@ApiParam("payload") String payload,
- @QueryParam("startTime") long startTime, @QueryParam("endTime") long
endTime) throws Exception {
- if (Strings.isNullOrEmpty(payload)) {
- throw new IllegalArgumentException("Empty Payload");
- }
- Map<String, Object> yamlConfig = (Map<String, Object>)
this.YAML.load(payload);
-
- Preconditions.checkArgument(yamlConfig.containsKey(PROP_NAME), "missing "
+ PROP_NAME);
- // retrieve id if detection config already exists
- List<DetectionConfigDTO> detectionConfigDTOs =
- this.detectionConfigDAO.findByPredicate(Predicate.EQ("name",
MapUtils.getString(yamlConfig, PROP_NAME)));
- DetectionConfigDTO existingDetectionConfig = null;
- if (!detectionConfigDTOs.isEmpty()) {
- existingDetectionConfig = detectionConfigDTOs.get(0);
- }
+ @ApiOperation("Set up a detection pipeline using a YAML config")
+ public Response setUpDetectionPipeline(
+ @ApiParam("yaml config") String payload,
+ @ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
+ @ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
+ String errorMessage;
+ try {
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "Empty
payload");
+ Map<String, Object> yamlConfig = (Map<String, Object>)
this.yaml.load(payload);
+
+ // retrieve id if detection config already exists
+ List<DetectionConfigDTO> detectionConfigDTOs =
+ this.detectionConfigDAO.findByPredicate(Predicate.EQ("name",
MapUtils.getString(yamlConfig, PROP_NAME)));
+ DetectionConfigDTO existingDetectionConfig = null;
+ if (!detectionConfigDTOs.isEmpty()) {
+ existingDetectionConfig = detectionConfigDTOs.get(0);
+ }
+
+ YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
+ DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
+ .withExistingDetectionConfig(existingDetectionConfig)
+ .generateDetectionConfig();
+ detectionConfig.setYaml(payload);
+ validatePipeline(detectionConfig);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
- YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
- DetectionConfigDTO detectionConfig;
- try{
- detectionConfig = translator.withTrainingWindow(startTime,
endTime).withExistingDetectionConfig(existingDetectionConfig).generateDetectionConfig();
+ return Response.ok(detectionConfig).build();
+ } catch (InvocationTargetException e){
+ // exception thrown in validate pipeline via reflection
+ LOG.error("Validate pipeline error", e);
+ errorMessage = e.getCause().getMessage();
} catch (Exception e) {
LOG.error("yaml translation error", e);
- return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", e.getMessage())).build();
+ errorMessage = e.getMessage();
}
- detectionConfig.setYaml(payload);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
-
- // optionally set up an alerter for the detection pipeline
- if (yamlConfig.containsKey("alert")) {
- Map<String, Object> alertYaml = MapUtils.getMap(yamlConfig, "alert");
- DetectionAlertConfigDTO alertConfigDTO =
getDetectionAlertConfig(alertYaml, detectionConfigId);
- Long detectionAlertConfigId =
this.detectionAlertConfigDAO.save(alertConfigDTO);
- Preconditions.checkNotNull(detectionAlertConfigId, "Save detection
alerter config failed");
+ return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", errorMessage)).build();
+ }
+
+ /**
+ Edit a detection pipeline using a YAML config
+ @param payload YAML config string
+ @param id the detection config id to edit
+ @param startTime tuning window start time for tunable components
+ @param endTime tuning window end time for tunable components
+ @return a message contains the saved detection config id
+ */
+ @PUT
+ @Path("/{id}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.TEXT_PLAIN)
+ @ApiOperation("Edit a detection pipeline using a YAML config")
+ public Response editDetectionPipeline(
+ @ApiParam("yaml config") String payload,
+ @ApiParam("the detection config id to edit") @PathParam("id") long id,
+ @ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
+ @ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
+ String errorMessage;
+ try {
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "Empty
payload");
+ Map<String, Object> yamlConfig = (Map<String, Object>)
this.yaml.load(payload);
+
+ DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(id);
+ Preconditions.checkArgument(existingDetectionConfig != null, "Existing
detection config " + id + " not found");
+
+ YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
+ DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
+ .withExistingDetectionConfig(existingDetectionConfig)
+ .generateDetectionConfig();
+ detectionConfig.setYaml(payload);
+ validatePipeline(detectionConfig);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+
+ return Response.ok(detectionConfig).build();
+ } catch (InvocationTargetException e){
+ // exception thrown in validate pipeline via reflection
+ LOG.error("Validate pipeline error", e);
+ errorMessage = e.getCause().getMessage();
+ } catch (Exception e) {
+ LOG.error("yaml translation error", e);
+ errorMessage = e.getMessage();
}
+ return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", errorMessage)).build();
+ }
- return Response.ok(detectionConfig).build();
+
+ /*
+ * Init the pipeline to check if detection pipeline property is valid
semantically.
+ */
+ private void validatePipeline(DetectionConfigDTO detectionConfig) throws
Exception {
+ Long id = detectionConfig.getId();
+ // swap out id
+ detectionConfig.setId(-1L);
+ // try to load the detection pipeline and init all the components
+ this.loader.from(provider, detectionConfig, 0, 0);
+ // set id back
+ detectionConfig.setId(id);
}
/**
@@ -195,7 +266,7 @@ public class YamlResource {
for (DetectionConfigDTO detectionConfigDTO : detectionConfigDTOs) {
if (detectionConfigDTO.getYaml() != null) {
Map<String, Object> yamlObject = new HashMap<>();
- yamlObject.putAll((Map<? extends String, ?>)
this.YAML.load(detectionConfigDTO.getYaml()));
+ yamlObject.putAll((Map<? extends String, ?>)
this.yaml.load(detectionConfigDTO.getYaml()));
yamlObject.put("id", detectionConfigDTO.getId());
yamlObject.put("isActive", detectionConfigDTO.isActive());
yamlObject.put("createdBy", detectionConfigDTO.getCreatedBy());
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
index 0e04d11..6f634c9 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/AnomalyDetectionStageWrapperTest.java
@@ -49,7 +49,7 @@ public class AnomalyDetectionStageWrapperTest {
this.properties.put(PROP_METRIC_URN, "thirdeye:metric:1");
this.properties.put("specs", this.stageSpecs);
this.config = new DetectionConfigDTO();
-
+ this.config.setId(-1L);
this.config.setProperties(properties);
this.provider = new MockDataProvider();
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
index 12cb7c4..72acffa 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/algorithm/stage/BaselineRuleDetectionStageTest.java
@@ -79,7 +79,7 @@ public class BaselineRuleDetectionStageTest {
this.config = new DetectionConfigDTO();
this.config.setProperties(properties);
-
+ this.config.setId(-1L);
this.provider = new MockDataProvider()
.setTimeseries(timeseries)
.setMetrics(Collections.singletonList(metricConfigDTO));
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
index 0d6891e..960d4a8 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
@@ -54,6 +54,7 @@ public class AnomalyDetectorWrapperTest {
this.config = new DetectionConfigDTO();
this.config.setComponents(ImmutableMap.of("testDetector", new
ThresholdRuleDetector()));
this.config.setProperties(properties);
+ this.config.setId(-1L);
this.provider = new MockDataProvider();
MetricConfigDTO metric = new MetricConfigDTO();
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
index 481c6f0..ac73ad4 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/wrapper/BaselineFillingMergeWrapperTest.java
@@ -97,7 +97,9 @@ public class BaselineFillingMergeWrapperTest {
@Test
public void testMergerCurrentAndBaselineLoading() throws Exception {
MergedAnomalyResultDTO anomaly = makeAnomaly(3000, 3600);
- anomaly.setProperties(ImmutableMap.of("detectorComponentName",
"testDetector"));
+ Map<String, String> anomalyProperties = new HashMap<>();
+ anomalyProperties.put("detectorComponentName", "testDetector");
+ anomaly.setProperties(anomalyProperties);
anomaly.setMetricUrn("thirdeye:metric:1");
Map<MetricSlice, DataFrame> aggregates = new HashMap<>();
@@ -127,6 +129,8 @@ public class BaselineFillingMergeWrapperTest {
Assert.assertTrue(anomalyResults.contains(anomaly));
Assert.assertEquals(anomalyResults.get(0).getAvgBaselineVal(), 100.0);
Assert.assertEquals(anomalyResults.get(0).getAvgCurrentVal(), 100.0);
+
Assert.assertEquals(anomalyResults.get(0).getProperties().get("detectorComponentName"),
"testDetector");
+
Assert.assertEquals(anomalyResults.get(0).getProperties().get("baselineProviderComponentName"),
"baseline");
}
}
\ No newline at end of file
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
index 65fb1b5..75335eb 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
@@ -7,33 +7,33 @@
"className" :
"com.linkedin.thirdeye.detection.algorithm.DimensionWrapper",
"metricUrn" : "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested" : [ {
- "filter" : "$rule1:THRESHOLD_RULE_FILTER:1",
+ "filter" : "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested" : [ {
- "filter" : "$rule1:THRESHOLD_RULE_FILTER:0",
+ "filter" : "$thresholdFilter_1:THRESHOLD_RULE_FILTER",
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested" : [ {
- "baselineValueProvider" : "$rule1:RULE_BASELINE:0",
+ "baselineValueProvider" : "$maxThreshold_1:RULE_BASELINE",
"className" :
"com.linkedin.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap" : 0,
"nested" : [ {
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
- "detector" : "$rule1:THRESHOLD:0",
+ "detector" : "$maxThreshold_1:THRESHOLD",
"maxDuration" : 100
} ]
} ]
}, {
- "filter" : "$rule2:THRESHOLD_RULE_FILTER:0",
+ "filter" : "$thresholdFilter_3:THRESHOLD_RULE_FILTER",
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested" : [ {
- "baselineValueProvider" : "$rule2:RULE_BASELINE:0",
+ "baselineValueProvider" : "$maxThreshold_2:RULE_BASELINE",
"className" :
"com.linkedin.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap" : 0,
"nested" : [ {
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
- "detector" : "$rule2:THRESHOLD:0",
+ "detector" : "$maxThreshold_2:THRESHOLD",
"maxDuration" : 100
} ]
} ],
@@ -43,33 +43,33 @@
"maxDuration" : 100
},
"components" : {
- "rule1:RULE_BASELINE:0" : {
+ "maxThreshold_2:THRESHOLD" : {
+ "max" : 100,
+ "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
+ },
+ "maxThreshold_1:THRESHOLD" : {
+ "max" : 100,
+ "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
+ },
+ "maxThreshold_2:RULE_BASELINE" : {
"max" : 100,
"className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
},
- "rule1:THRESHOLD_RULE_FILTER:0" : {
+ "thresholdFilter_1:THRESHOLD_RULE_FILTER" : {
"min" : 50,
"className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
},
- "rule1:THRESHOLD_RULE_FILTER:1" : {
+ "thresholdFilter_2:THRESHOLD_RULE_FILTER" : {
"min" : 50,
"className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
},
- "rule2:THRESHOLD_RULE_FILTER:0" : {
+ "thresholdFilter_3:THRESHOLD_RULE_FILTER" : {
"min" : 50,
"className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
},
- "rule2:THRESHOLD:0" : {
- "max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
- },
- "rule2:RULE_BASELINE:0" : {
+ "maxThreshold_1:RULE_BASELINE" : {
"max" : 100,
"className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
- },
- "rule1:THRESHOLD:0" : {
- "max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
}
},
"cron" : "0 0 14 * * ? *"
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
index bd43106..91c176d 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
@@ -6,25 +6,25 @@
"className" :
"com.linkedin.thirdeye.detection.algorithm.DimensionWrapper",
"metricUrn" : "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested" : [ {
- "baselineValueProvider" : "$rule1:RULE_BASELINE:0",
+ "baselineValueProvider" : "$rule1:RULE_BASELINE",
"className" :
"com.linkedin.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested" : [ {
"className" :
"com.linkedin.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
- "detector" : "$rule1:THRESHOLD:0"
+ "detector" : "$rule1:THRESHOLD"
} ],
"minContribution" : 0.05,
"dimensions" : [ "D1", "D2" ]
} ]
},
"components" : {
- "rule1:RULE_BASELINE:0" : {
+ "rule1:THRESHOLD" : {
"max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
+ "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
},
- "rule1:THRESHOLD:0" : {
+ "rule1:RULE_BASELINE" : {
"max" : 100,
- "className" :
"com.linkedin.thirdeye.detection.components.ThresholdRuleDetector"
+ "className" :
"com.linkedin.thirdeye.detection.components.RuleBaselineProvider"
}
},
"cron" : "0 0 14 * * ? *"
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
index 49c6463..911d0d9 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-1.yaml
@@ -14,27 +14,30 @@ dimensionExploration:
- D2
minContribution: 0.05
rules:
-- name: rule1
- detection:
+- detection:
- type: THRESHOLD
+ name: maxThreshold_1
params:
max: 100
filter:
- type: THRESHOLD_RULE_FILTER
- id: 0
+ name: thresholdFilter_1
params:
min: 50
- type: THRESHOLD_RULE_FILTER
- id: 1
+ name: thresholdFilter_2
params:
min: 50
+
- name: rule2
detection:
- type: THRESHOLD
+ name: maxThreshold_2
params:
max: 100
filter:
- type: THRESHOLD_RULE_FILTER
+ name: thresholdFilter_3
params:
min: 50
merger:
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
index 815a77f..323325e 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
+++
b/thirdeye/thirdeye-pinot/src/test/resources/com/linkedin/thirdeye/detection/yaml/pipeline-config-2.yaml
@@ -14,9 +14,9 @@ dimensionExploration:
- D2
minContribution: 0.05
rules:
-- name: rule1
- detection:
+- detection:
- type: THRESHOLD
+ name: rule1
params:
max: 100
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
