This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c674e72 [TE] adjustment on Yaml configurations (#4122)
c674e72 is described below
commit c674e72ee107a46795e24b3b024821fafe3d867d
Author: Xiaohui Sun <[email protected]>
AuthorDate: Tue Apr 16 17:21:14 2019 -0700
[TE] adjustment on Yaml configurations (#4122)
* [TE] adjustment on Yaml configurations
* [TE] remove excessive lines
* [TE] Fix integration test in frontend
* [TE] Fix integration test in CompositePipelineConfigTranslatorTest
---
thirdeye/thirdeye-frontend/app/utils/constants.js | 36 +++++++++++-----------
.../pods/components/yaml-editor/component-test.js | 2 +-
.../yaml/CompositePipelineConfigTranslator.java | 4 ++-
.../yaml/YamlDetectionConfigTranslator.java | 21 ++++++++++++-
.../yaml/YamlDetectionTranslatorLoader.java | 5 ++-
.../thirdeye/detection/yaml/YamlResource.java | 4 +++
.../compositePipelineTranslatorTestResult-1.json | 2 ++
.../compositePipelineTranslatorTestResult-2.json | 1 +
8 files changed, 53 insertions(+), 22 deletions(-)
diff --git a/thirdeye/thirdeye-frontend/app/utils/constants.js
b/thirdeye/thirdeye-frontend/app/utils/constants.js
index afbc378..fbfc3c4 100644
--- a/thirdeye/thirdeye-frontend/app/utils/constants.js
+++ b/thirdeye/thirdeye-frontend/app/utils/constants.js
@@ -5,34 +5,34 @@ export const deleteProps = {
credentials: 'include'
};
-export const yamlAlertProps = `# Below is a sample template for setting up a
WoW percentage rule. You may refer the documentation for more examples and
update the fields accordingly.
+export const yamlAlertProps = `# Below is a sample template. You may refer the
documentation for more examples and update the fields accordingly.
-# Give a name for this anomaly detection pipeline (should be unique)
+# Give a name for this anomaly detection pipeline (should be unique).
detectionName: name_of_the_detection
-# Tell the alert recipients what it means if this alert is fired
+# Tell the alert recipients what it means if this alert is fired.
description: If this alert fires then it means so-and-so and check so-and-so
for irregularities
-# The metric you want to do anomaly detection on. You may type a few
characters and look ahead (ctrl + space) to auto-fill
+# The metric you want to do anomaly detection on. You may type a few
characters and look ahead (ctrl + space) to auto-fill.
metric: metric_name
-# The dataset or UMP table name to which the metric belongs. Look ahead should
auto populate this field
+# The dataset or UMP table name to which the metric belongs. Look ahead should
auto populate this field.
dataset: dataset_name
-# ThirdEye pipeline type. Only "Composite" is supported now.
-pipelineType: Composite
-
-# Configure multiple rules. ThirdEye supports single or a list of rules
combined together with "OR" relationship
-rules:
-- detection: # Eg. Detect anomalies if the week over week
change of this metric is more than 10%
- - name: detection_rule_1 # Give a unique name for this detection rule.
- type: PERCENTAGE_RULE # Configure the detection type here. See doc
for more details.
+rules: # Can configure multiple rules with "OR"
relationship.
+- detection:
+ - name: detection_rule_1
+ type: ALGORITHM # Configure the detection type here. See doc
for more details.
params: # The parameters for this rule. Different
rules have different params.
- offset: wo1w # Compare current value with last week.
(Values supported - wo1w, wo2w, median3w etc)
- percentageChange: 0.1 # The threshold above which you want to be
alerted.
- pattern: UP_OR_DOWN # Alert when value goes up or down by the
configured threshold. (Values supported - UP, DOWN, UP_OR_DOWN)
-
-# Refer to examples and documentation for exploring other types of detection
rules, algorithms and filters with more detailed settings.
+ configuration:
+ bucketPeriod: P1D # Use PT1H for hourly and PT5M for minute
level (ingraph metrics) data.
+ pValueThreshold: 0.05 # Higher value means more sensitive to small
changes.
+ filter: # Filter out anomalies detected by rules to
reduce noise.
+ - name: filter_rule_1
+ type: PERCENTAGE_CHANGE_FILTER
+ params:
+ pattern: UP_OR_DOWN # Other patterns: "UP","DOWN".
+ threshold: 0.05 # Filter out all changes less than 5%
compared to baseline.
`;
export const yamlAlertSettings = `# Below is a sample subscription group
template. You may refer the documentation and update accordingly.
diff --git
a/thirdeye/thirdeye-frontend/tests/integration/pods/components/yaml-editor/component-test.js
b/thirdeye/thirdeye-frontend/tests/integration/pods/components/yaml-editor/component-test.js
index 086d781..ef4037f 100644
---
a/thirdeye/thirdeye-frontend/tests/integration/pods/components/yaml-editor/component-test.js
+++
b/thirdeye/thirdeye-frontend/tests/integration/pods/components/yaml-editor/component-test.js
@@ -47,7 +47,7 @@ module('Integration | Component | yaml-editor',
function(hooks) {
test(`displays default yaml file of alert configuration in create mode`,
async function(assert) {
- const defaultText = '# Below is a sample template for setting up a WoW
percentage rule. You may refer the documentation for more examples and update
the fields accordingly.';
+ const defaultText = '# Below is a sample template. You may refer the
documentation for more examples and update the fields accordingly.';
await render(hbs`
{{yaml-editor
isEditMode=false
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 4d0e42e..18c9f1a 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -197,7 +197,7 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
this.yaml = new org.yaml.snakeyaml.Yaml();
this.metricConfig =
this.dataProvider.fetchMetric(MapUtils.getString(yamlConfig, PROP_METRIC),
MapUtils.getString(yamlConfig, PROP_DATASET));
- Preconditions.checkNotNull(this.metricConfig, "Metric not found");
+ Preconditions.checkNotNull(this.metricConfig, "metric not found");
this.datasetConfig =
this.dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
.get(metricConfig.getDataset());
@@ -311,6 +311,8 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
properties.put(PROP_WINDOW_UNIT, TimeUnit.HOURS);
}
}
+ // set default bucketPeriod
+ properties.put(PROP_BUCKET_PERIOD,
this.datasetConfig.bucketTimeGranularity().toPeriod().toString());
// override from yaml
if (yamlConfig.containsKey(PROP_WINDOW_SIZE)) {
properties.put(PROP_MOVING_WINDOW_DETECTION, true);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
index 0cdb18d..54d766f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionConfigTranslator.java
@@ -38,6 +38,7 @@ import org.yaml.snakeyaml.Yaml;
public abstract class YamlDetectionConfigTranslator {
protected static final Logger LOG =
LoggerFactory.getLogger(YamlDetectionConfigTranslator.class);
private static final String PROP_NAME = "detectionName";
+ private static final String PROP_FILTER = "filter";
private static final String PROP_DESC_NAME = "description";
private static final String PROP_ACTIVE = "active";
@@ -108,6 +109,24 @@ public abstract class YamlDetectionConfigTranslator {
* @param yamlConfig yamlConfiguration to be checked
*/
protected void validateYAML(Map<String, Object> yamlConfig) {
- Preconditions.checkArgument(yamlConfig.containsKey(PROP_NAME), "Property
missing " + PROP_NAME);
+ validatePropertyExists(PROP_NAME);
+ validateFilter();
+ }
+
+ /**
+ * Validate property name exists.
+ * @param propName The property name to validate.
+ */
+ private void validatePropertyExists(String propName) {
+ Preconditions.checkArgument(yamlConfig.containsKey(propName), "Property
missing " + propName);
+ }
+
+ /**
+ * Validate filter is set in correct level.
+ */
+ private void validateFilter() {
+ if (yamlConfig.containsKey(PROP_FILTER)) {
+ throw new IllegalArgumentException("Filter should be set under rules");
+ }
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
index 6bbf0e3..151aa17 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
@@ -31,10 +31,13 @@ import java.util.Map;
*/
public class YamlDetectionTranslatorLoader {
private static final String PROP_PIPELINE_TYPE= "pipelineType";
+ private static final String DEFAULT_PIPELINE_TYPE= "Composite";
private static DetectionRegistry DETECTION_REGISTRY =
DetectionRegistry.getInstance();
public YamlDetectionConfigTranslator from(Map<String, Object> yamlConfig,
DataProvider provider) throws Exception {
- Preconditions.checkArgument(yamlConfig.containsKey(PROP_PIPELINE_TYPE),
"Pipeline type is missing.");
+ if (!yamlConfig.containsKey(PROP_PIPELINE_TYPE)) {
+ yamlConfig.put(PROP_PIPELINE_TYPE, DEFAULT_PIPELINE_TYPE);
+ }
String className =
DETECTION_REGISTRY.lookupYamlConverter(yamlConfig.get(PROP_PIPELINE_TYPE).toString().toUpperCase());
Constructor<?> constructor =
Class.forName(className).getConstructor(Map.class, DataProvider.class);
return (YamlDetectionConfigTranslator) constructor.newInstance(yamlConfig,
provider);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index c3a1a5a..81451e0 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap;
import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -576,6 +577,9 @@ public class YamlResource {
LOG.warn("Validation error while running preview with payload " +
payload, e);
responseMessage.put("message", "Validation Error! " + e.getMessage());
return Response.serverError().entity(responseMessage).build();
+ } catch (InvocationTargetException e) {
+ responseMessage.put("message", "Failed to run the preview due to " +
e.getTargetException().getMessage());
+ return Response.serverError().entity(responseMessage).build();
} catch (Exception e) {
LOG.error("Error running preview with payload " + payload, e);
responseMessage.put("message", "Failed to run the preview due to " +
e.getMessage());
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
index 2ff1fbc..4630cd8 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-1.json
@@ -17,6 +17,7 @@
"className" :
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap" : 0,
"nested" : [ {
+ "bucketPeriod":"P1D",
"className" :
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
"detector" : "$maxThreshold_1:THRESHOLD",
@@ -31,6 +32,7 @@
"className" :
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap" : 0,
"nested" : [ {
+ "bucketPeriod":"P1D",
"className" :
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
"detector" : "$maxThreshold_2:THRESHOLD",
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
index a254ac1..67292a5 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/compositePipelineTranslatorTestResult-2.json
@@ -9,6 +9,7 @@
"baselineValueProvider" : "$rule1:RULE_BASELINE",
"className" :
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested" : [ {
+ "bucketPeriod":"P1D",
"className" :
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
} ],
"detector" : "$rule1:THRESHOLD"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
