This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a123c60 [TE] add labeler into yaml (#6007)
a123c60 is described below
commit a123c60f63e3fc78fb691b380097aaf1f8c4d4b8
Author: Vincent Chen <[email protected]>
AuthorDate: Tue Sep 15 15:40:34 2020 -0700
[TE] add labeler into yaml (#6007)
This PR is second PR for severity-based alert feature, including the logic
of parsing labeler configuration and constructing the detection pipelines based
on the YAML.
---
.../components/ThresholdSeverityLabeler.java | 16 ++---
.../thirdeye/detection/spi/components/Labeler.java | 9 ++-
.../detection/wrapper/AnomalyLabelerWrapper.java | 13 +++-
.../wrapper/ChildKeepingMergeWrapper.java | 6 +-
.../yaml/translator/DetectionConfigTranslator.java | 54 ++++++++------
.../builder/DataQualityPropertiesBuilder.java | 2 +-
.../builder/DetectionConfigPropertiesBuilder.java | 34 +++++----
.../builder/DetectionPropertiesBuilder.java | 23 ++++--
.../detection/detection-config-schema.json | 3 +
.../thirdeye/detection/DetectionTestUtils.java | 16 +++--
.../components/ThresholdSeverityLabelerTest.java | 2 +-
.../wrapper/ChildKeepingMergeWrapperTest.java | 24 +++++++
.../translator/DetectionConfigTranslatorTest.java | 3 +
.../compositePipelineTranslatorTestResult-1.json | 84 +++++++++++++++-------
.../yaml/translator/pipeline-config-1.yaml | 19 +++++
15 files changed, 223 insertions(+), 85 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
index ff04015..5f31ccf 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
@@ -19,6 +19,7 @@
package org.apache.pinot.thirdeye.detection.components;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -46,7 +47,8 @@ public class ThresholdSeverityLabeler implements
Labeler<SeverityThresholdLabele
private TreeMap<AnomalySeverity, Threshold> severityMap;
@Override
- public void label(List<MergedAnomalyResultDTO> anomalies) {
+ public Map<MergedAnomalyResultDTO, AnomalySeverity>
label(List<MergedAnomalyResultDTO> anomalies) {
+ Map<MergedAnomalyResultDTO, AnomalySeverity> res = new HashMap<>();
for (MergedAnomalyResultDTO anomaly : anomalies) {
double currVal = anomaly.getAvgCurrentVal();
double baseVal = anomaly.getAvgBaselineVal();
@@ -59,18 +61,12 @@ public class ThresholdSeverityLabeler implements
Labeler<SeverityThresholdLabele
long duration = anomaly.getEndTime() - anomaly.getStartTime();
for (Map.Entry<AnomalySeverity, Threshold> entry :
severityMap.entrySet()) {
if (deviation >= entry.getValue().change || duration >=
entry.getValue().duration) {
- if (anomaly.getSeverityLabel() != entry.getKey()) {
- // find the severity from highest to lowest
- if (anomaly.getId() != null &&
anomaly.getSeverityLabel().compareTo(entry.getKey()) > 0) {
- // only set renotify if the anomaly exists and its severity gets
higher
- anomaly.setRenotify(true);
- }
- anomaly.setSeverityLabel(entry.getKey());
- break;
- }
+ res.put(anomaly, entry.getKey());
+ break;
}
}
}
+ return res;
}
@Override
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
index 61541a4..77e4316 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
@@ -20,14 +20,17 @@
package org.apache.pinot.thirdeye.detection.spi.components;
import java.util.List;
+import java.util.Map;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
public interface Labeler <T extends AbstractSpec> extends BaseComponent<T> {
/**
- * add or modify labels of anomalies in place
- * @param anomalies
+ * Calculate the severity for list of anomalies
+ * @param anomalies input anoamlies
+ * @return mapping from anomaly to severity
*/
- void label(List<MergedAnomalyResultDTO> anomalies);
+ Map<MergedAnomalyResultDTO, AnomalySeverity>
label(List<MergedAnomalyResultDTO> anomalies);
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
index 5d59e9b..9237fb6 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.collections4.MapUtils;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
@@ -76,7 +77,17 @@ public class AnomalyLabelerWrapper extends DetectionPipeline
{
diagnostics.putAll(intermediate.getDiagnostics());
anomalies.addAll(intermediate.getAnomalies());
}
- this.labeler.label(anomalies);
+ Map<MergedAnomalyResultDTO, AnomalySeverity> res =
this.labeler.label(anomalies);
+ for (MergedAnomalyResultDTO anomaly : anomalies) {
+ AnomalySeverity newSeverity = res.getOrDefault(anomaly,
AnomalySeverity.DEFAULT);
+ if (anomaly.getSeverityLabel() != newSeverity) {
+ if (anomaly.getId() != null &&
anomaly.getSeverityLabel().compareTo(newSeverity) > 0) {
+ // only set renotify if the anomaly exists and its severity gets
higher
+ anomaly.setRenotify(true);
+ }
+ anomaly.setSeverityLabel(newSeverity);
+ }
+ }
return new DetectionPipelineResult(anomalies,
DetectionUtils.consolidateNestedLastTimeStamps(lastTimeStamps),
predictionResults, evaluations).setDiagnostics(diagnostics);
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index 064e4f1..d808d68 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -118,7 +118,11 @@ public class ChildKeepingMergeWrapper extends
BaselineFillingMergeWrapper {
// merge the anomaly's properties into parent
ThirdEyeUtils.mergeAnomalyProperties(parent.getProperties(),
anomaly.getProperties());
-
+ // merge the anomaly severity
+ if (parent.getSeverityLabel().compareTo(anomaly.getSeverityLabel()) >
0) {
+ // set the highest severity
+ parent.setSeverityLabel(anomaly.getSeverityLabel());
+ }
if (anomaly.getChildren().isEmpty()) {
parent.getChildren().add(anomaly);
} else {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index 6fcddef..5f3bce2 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -70,6 +70,12 @@ import
org.apache.pinot.thirdeye.detection.yaml.translator.builder.DetectionProp
* |Rule | |Algorithm |
* |Filters | |Filters |
* +-----+----+ +-----+----+
+ * | |
+ * v v
+ * +-----+----+ +-----+----+
+ * |Rule | |Algorithm |
+ * |Labelers | |Labelers |
+ * +-----+----+ +-----+----+
* +-------+--------+
* |
* +-----v-----+
@@ -86,27 +92,33 @@ import
org.apache.pinot.thirdeye.detection.yaml.translator.builder.DetectionProp
*
* This translator translates a yaml that describes the above detection flow
* to the flowing wrapper structure to be execute in the detection pipeline.
- * +-----------------------------------------+
- * | Merger |
- * | +--------------------------------------+
- * | | Dimension Exploration & Filter ||
- * | | +---------------------------------+ ||
- * | | | Filters | ||
- * | | | +--------Merger--------------+ | ||
- * | | | | Rule detections | | ||
- * | | | +----------------------------+ | ||
- * | | +---------------------------------+ ||
- * | | +---------------------------------+ ||
- * | | | Filters | ||
- * | | | +-----------Merger------------+ | ||
- * | | | | +---------------------------| | ||
- * | | | | | Algorithm detection | | ||
- * | | | | +---------------------------| | ||
- * | | | +-----------------------------| | ||
- * | | +---------------------------------+ ||
- * | +--------------------------------------|
- * | |--------------------------------------|
- * +-----------------------------------------+
+ * +-------------------------------------------------+
+ * | Merger |
+ * | +-------------------------------------------+ |
+ * | | Dimension Exploration & Filter | |
+ * | | | |
+ * | | +-------------------------------------+ | |
+ * | | | Labelers | | |
+ * | | | +---------------------------------+ | | |
+ * | | | | Filters | | | |
+ * | | | | +--------Merger--------------+ | | | |
+ * | | | | | Rule detections | | | | |
+ * | | | | +----------------------------+ | | | |
+ * | | | +---------------------------------+ | | |
+ * | | +-------------------------------------+ | |
+ * | | | |
+ * | | +-------------------------------------+ | |
+ * | | | Labelers | | |
+ * | | | +---------------------------------+ | | |
+ * | | | | Filters | | | |
+ * | | | | | +---------Merger------------+ | | | |
+ * | | | | | | Algorithm detection | | | | |
+ * | | | | | +---------------------------+ | | | |
+ * | | | +---------------------------------+ | | |
+ * | | +-------------------------------------+ | |
+ * | | | |
+ * | +-------------------------------------------+ |
+ * +-------------------------------------------------+
*
*/
public class DetectionConfigTranslator extends
ConfigTranslator<DetectionConfigDTO, DetectionConfigValidator> {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
index 897a1f7..2c62909 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
@@ -131,7 +131,7 @@ public class DataQualityPropertiesBuilder extends
DetectionConfigPropertiesBuild
String qualityRefKey = makeComponentRefKey(qualityType, name);
properties.put(PROP_QUALITY_CHECK, qualityRefKey);
- buildComponentSpec(metricUrn, yamlConfig, qualityType, qualityRefKey);
+ buildComponentSpec(metricUrn, yamlConfig, qualityRefKey);
properties.putAll(mergerProperties);
return properties;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
index 1d15653..8a7799e 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
@@ -60,6 +60,7 @@ public abstract class DetectionConfigPropertiesBuilder {
static final String PROP_NESTED_METRIC_URNS = "nestedMetricUrns";
static final String PROP_RULES = "rules";
static final String PROP_GROUPER = "grouper";
+ static final String PROP_LABELER = "labeler";
static final String PROP_NESTED = "nested";
static final String PROP_BASELINE_PROVIDER = "baselineValueProvider";
static final String PROP_DETECTOR = "detector";
@@ -121,12 +122,11 @@ public abstract class DetectionConfigPropertiesBuilder {
properties.put(PROP_NESTED, nestedProps);
properties.put(PROP_SUB_ENTITY_NAME, entityName);
- String grouperType = MapUtils.getString(grouperYaml, PROP_TYPE);
- String grouperName = MapUtils.getString(grouperYaml, PROP_NAME);
- String grouperRefKey = makeComponentRefKey(grouperType, grouperName);
+ String grouperRefKey = makeComponentRefKey(
+ MapUtils.getString(grouperYaml, PROP_TYPE),
MapUtils.getString(grouperYaml, PROP_NAME));
properties.put(PROP_GROUPER, grouperRefKey);
- buildComponentSpec(metricUrn, grouperYaml, grouperType, grouperRefKey);
+ buildComponentSpec(metricUrn, grouperYaml, grouperRefKey);
return properties;
}
@@ -140,19 +140,29 @@ public abstract class DetectionConfigPropertiesBuilder {
if (wrapperProperties.isEmpty()) {
return Collections.emptyList();
}
- String name = MapUtils.getString(yamlConfig, PROP_NAME);
- String filterType = MapUtils.getString(yamlConfig, PROP_TYPE);
- String filterRefKey = makeComponentRefKey(filterType, name);
+ String filterRefKey = makeComponentRefKey(
+ MapUtils.getString(yamlConfig, PROP_TYPE),
MapUtils.getString(yamlConfig, PROP_NAME));
wrapperProperties.put(PROP_FILTER, filterRefKey);
- buildComponentSpec(metricUrn, yamlConfig, filterType, filterRefKey);
+ buildComponentSpec(metricUrn, yamlConfig, filterRefKey);
return Collections.singletonList(wrapperProperties);
}
- void buildComponentSpec(String metricUrn, Map<String, Object> yamlConfig,
String type, String componentRefKey) {
- Map<String, Object> componentSpecs = new HashMap<>();
+ Map<String, Object> buildLabelerWrapperProperties(String metricUrn, String
wrapperClassName,
+ Map<String, Object> yamlConfig, List<Map<String, Object>>
nestedProperties) {
+ Map<String, Object> wrapperProperties =
buildWrapperProperties(wrapperClassName, nestedProperties);
+ String labelerRefKey = makeComponentRefKey(
+ MapUtils.getString(yamlConfig, PROP_TYPE),
MapUtils.getString(yamlConfig, PROP_NAME));
+ wrapperProperties.put(PROP_LABELER, labelerRefKey);
+ buildComponentSpec(metricUrn, yamlConfig, labelerRefKey);
+ return wrapperProperties;
+ }
+
- String componentClassName = DETECTION_REGISTRY.lookup(type);
+ void buildComponentSpec(String metricUrn, Map<String, Object> yamlConfig,
String componentRefKey) {
+ Map<String, Object> componentSpecs = new HashMap<>();
+ String componentKey = DetectionUtils.getComponentKey(componentRefKey);
+ String componentClassName =
DETECTION_REGISTRY.lookup(DetectionUtils.getComponentType(componentKey));
componentSpecs.put(PROP_CLASS_NAME, componentClassName);
if (metricUrn != null) {
componentSpecs.put(PROP_METRIC_URN, metricUrn);
@@ -168,7 +178,7 @@ public abstract class DetectionConfigPropertiesBuilder {
componentSpecs.putAll(params);
}
-
metricAttributesMap.addComponent(DetectionUtils.getComponentKey(componentRefKey),
componentSpecs);
+ metricAttributesMap.addComponent(componentKey, componentSpecs);
}
Map<String, Object> compositePropertyBuilderHelper(List<Map<String, Object>>
nestedPropertiesList,
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
index 664e40c..f4c15fd 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
@@ -37,6 +37,7 @@ import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper;
+import org.apache.pinot.thirdeye.detection.wrapper.AnomalyLabelerWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.EntityAnomalyMergeWrapper;
@@ -81,15 +82,26 @@ public class DetectionPropertiesBuilder extends
DetectionConfigPropertiesBuilder
subEntityName, metricUrn, detectionYamls, mergerProperties,
datasetConfigDTO.bucketTimeGranularity());
List<Map<String, Object>> filterYamls =
ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
- if (filterYamls.isEmpty()) {
+ List<Map<String, Object>> labelerYamls =
ConfigUtils.getList(ruleYaml.get(PROP_LABELER));
+ if (filterYamls.isEmpty() && labelerYamls.isEmpty()) {
+ // output detection properties if neither filter and labeler is
configured
nestedPipelines.addAll(detectionProperties);
} else {
+ // wrap detection properties around with filter properties if a filter
is configured
List<Map<String, Object>> filterNestedProperties = detectionProperties;
for (Map<String, Object> filterProperties : filterYamls) {
filterNestedProperties = buildFilterWrapperProperties(metricUrn,
AnomalyFilterWrapper.class.getName(), filterProperties,
filterNestedProperties);
}
- nestedPipelines.addAll(filterNestedProperties);
+ if (labelerYamls.isEmpty()) {
+ // output filter properties if no labeler is configured
+ nestedPipelines.addAll(filterNestedProperties);
+ } else {
+ // wrap filter properties around with labeler properties if a
labeler is configured
+ nestedPipelines.add(
+ buildLabelerWrapperProperties(metricUrn,
AnomalyLabelerWrapper.class.getName(), labelerYamls.get(0),
+ filterNestedProperties));
+ }
}
}
@@ -157,7 +169,7 @@ public class DetectionPropertiesBuilder extends
DetectionConfigPropertiesBuilder
fillInDetectorWrapperProperties(nestedProperties, yamlConfig,
detectorType, datasetTimegranularity);
- buildComponentSpec(metricUrn, yamlConfig, detectorType, detectorRefKey);
+ buildComponentSpec(metricUrn, yamlConfig, detectorRefKey);
Map<String, Object> properties = new HashMap<>();
properties.put(PROP_CLASS_NAME,
BaselineFillingMergeWrapper.class.getName());
@@ -169,9 +181,8 @@ public class DetectionPropertiesBuilder extends
DetectionConfigPropertiesBuilder
// if the detector implements the baseline provider interface, use it to
generate baseline
properties.put(PROP_BASELINE_PROVIDER, detectorRefKey);
} else {
- String baselineProviderType = DEFAULT_BASELINE_PROVIDER_YAML_TYPE;
- String baselineProviderKey = makeComponentRefKey(baselineProviderType,
name);
- buildComponentSpec(metricUrn, yamlConfig, baselineProviderType,
baselineProviderKey);
+ String baselineProviderKey =
makeComponentRefKey(DEFAULT_BASELINE_PROVIDER_YAML_TYPE, name);
+ buildComponentSpec(metricUrn, yamlConfig, baselineProviderKey);
properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
}
properties.putAll(mergerProperties);
diff --git
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
index 59ee067..e5a3d7e 100644
---
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
+++
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
@@ -225,6 +225,9 @@
},
"quality": {
"$ref": "#/definitions/genericComponent"
+ },
+ "labeler": {
+ "$ref": "#/definitions/genericComponent"
}
},
"additionalProperties": false
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
index 8a9eac6..0ad231d 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
@@ -22,6 +22,7 @@ package org.apache.pinot.thirdeye.detection;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import java.util.HashMap;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import java.util.Collections;
@@ -34,7 +35,8 @@ public class DetectionTestUtils {
private static final Long PROP_ID_VALUE = 1000L;
public static MergedAnomalyResultDTO makeAnomaly(Long configId, Long
legacyFunctionId, long start, long end,
- String metric, String dataset, Map<String, String> dimensions,
Map<String, String> props) {
+ String metric, String dataset, Map<String, String> dimensions,
Map<String, String> props,
+ AnomalySeverity severity) {
MergedAnomalyResultDTO anomaly = new MergedAnomalyResultDTO();
anomaly.setDetectionConfigId(configId);
anomaly.setStartTime(start);
@@ -43,6 +45,7 @@ public class DetectionTestUtils {
anomaly.setCollection(dataset);
anomaly.setFunctionId(legacyFunctionId);
anomaly.setProperties(props);
+ anomaly.setSeverityLabel(severity);
Multimap<String, String> filters = HashMultimap.create();
for (Map.Entry<String, String> dimension : dimensions.entrySet()) {
@@ -60,7 +63,7 @@ public class DetectionTestUtils {
public static MergedAnomalyResultDTO makeAnomaly(Long configId, long start,
long end, String metric, String dataset,
Map<String, String> dimensions) {
return DetectionTestUtils.makeAnomaly(configId, null, start, end, metric,
dataset, dimensions,
- new HashMap<>());
+ new HashMap<>(), AnomalySeverity.DEFAULT);
}
public static MergedAnomalyResultDTO setAnomalyId(MergedAnomalyResultDTO
anomaly, long id) {
@@ -74,7 +77,7 @@ public class DetectionTestUtils {
public static MergedAnomalyResultDTO makeAnomaly(Long configId, Long
legacyFuncId, long start, long end) {
return DetectionTestUtils.makeAnomaly(configId, legacyFuncId, start, end,
null, null,
- Collections.emptyMap(), new HashMap<>());
+ Collections.emptyMap(), new HashMap<>(), AnomalySeverity.DEFAULT);
}
public static MergedAnomalyResultDTO makeAnomaly(Long configId, long start,
long end) {
@@ -122,7 +125,12 @@ public class DetectionTestUtils {
public static MergedAnomalyResultDTO makeAnomalyWithProps(long start, long
end, Map<String, String> props) {
return DetectionTestUtils.makeAnomaly(PROP_ID_VALUE, null, start, end,
null, null,
- Collections.emptyMap(), props);
+ Collections.emptyMap(), props, AnomalySeverity.DEFAULT);
+ }
+
+ public static MergedAnomalyResultDTO makeAnomaly(long start, long end,
AnomalySeverity severity) {
+ return DetectionTestUtils.makeAnomaly(PROP_ID_VALUE, null, start, end,
null, null,
+ Collections.emptyMap(), Collections.emptyMap(), severity);
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
index 459386b..0b7067b 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
@@ -166,7 +166,7 @@ public class ThresholdSeverityLabelerTest {
severityMap.put(AnomalySeverity.HIGH.toString(), new Threshold(0.15,
2000));
severityMap.put(AnomalySeverity.MEDIUM.toString(), new Threshold(0.10,
1500));
this.specs.put("severity", severityMap);
- MergedAnomalyResultDTO anomaly = DetectionTestUtils.makeAnomaly(4000L,
6000L, METRIC_URN, 2700, 3000);
+ MergedAnomalyResultDTO anomaly = DetectionTestUtils.makeAnomaly(4200L,
6000L, METRIC_URN, 2700, 3000);
anomaly.setSeverityLabel(AnomalySeverity.HIGH);
anomaly.setId(125L);
this.anomalies.set(this.anomalies.size() - 1, anomaly);
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
index 7703209..69b74db 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
@@ -18,6 +18,7 @@ package org.apache.pinot.thirdeye.detection.wrapper;
import com.google.common.collect.ImmutableSet;
import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.DataProvider;
@@ -350,4 +351,27 @@ public class ChildKeepingMergeWrapperTest {
Assert.assertTrue(output.getAnomalies().contains(makeAnomaly(2800, 3600,
null, 3, 2)));
Assert.assertTrue(output.getAnomalies().contains(makeAnomaly(3500, 3800,
null, 1, 2)));
}
+
+ @Test
+ public void testMergerDifferentSeverity() throws Exception {
+ this.config.getProperties().put(PROP_MAX_GAP, 200);
+ this.config.getProperties().put(PROP_MAX_DURATION, 1250);
+
+ this.outputs.add(new MockPipelineOutput(Arrays.asList(
+ makeAnomaly(2800, 3800, AnomalySeverity.MEDIUM),
+ makeAnomaly(3500, 3600, AnomalySeverity.CRITICAL)
+ ), 3700));
+
+ Map<String, Object> nestedProperties = new HashMap<>();
+ nestedProperties.put(PROP_CLASS_NAME, "none");
+ nestedProperties.put(PROP_METRIC_URN, "thirdeye:metric:3");
+
+ this.nestedProperties.add(nestedProperties);
+
+ this.wrapper = new ChildKeepingMergeWrapper(this.provider, this.config,
1000, 4000);
+ DetectionPipelineResult output = this.wrapper.run();
+
+ Assert.assertEquals(output.getAnomalies().size(), 4);
+ Assert.assertEquals(output.getAnomalies().get(3).getSeverityLabel(),
AnomalySeverity.CRITICAL);
+ }
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
index 4ebb897..6f7d292 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
@@ -1,6 +1,7 @@
package org.apache.pinot.thirdeye.detection.yaml.translator;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.io.IOUtils;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
@@ -19,6 +20,7 @@ import
org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.detection.components.ThresholdSeverityLabeler;
import
org.apache.pinot.thirdeye.detection.validators.ConfigValidationException;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
@@ -71,6 +73,7 @@ public class DetectionConfigTranslatorTest {
DetectionRegistry.registerComponent(ThresholdRuleAnomalyFilter.class.getName(),
"THRESHOLD_RULE_FILTER");
DetectionRegistry.registerComponent(RuleBaselineProvider.class.getName(),
"RULE_BASELINE");
DetectionRegistry.registerComponent(MockGrouper.class.getName(),
"MOCK_GROUPER");
+
DetectionRegistry.registerComponent(ThresholdSeverityLabeler.class.getName(),
"THRESHOLD_SEVERITY_LABELER");
this.provider = new
MockDataProvider().setMetrics(Collections.singletonList(metricConfig)).setDatasets(Collections.singletonList(datasetConfigDTO));
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
index e3bfaa5..f55fbda 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
+++
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
@@ -24,48 +24,60 @@
"metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested": [
{
- "filter": "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
- "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
- "nested": [
+ "className" :
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyLabelerWrapper",
+ "labeler" : "$labeler_1:THRESHOLD_SEVERITY_LABELER",
+ "nested" : [
{
- "filter":
"$thresholdFilter_1:THRESHOLD_RULE_FILTER",
+ "filter":
"$thresholdFilter_2:THRESHOLD_RULE_FILTER",
"className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested": [
{
- "baselineValueProvider":
"$maxThreshold_1:THRESHOLD",
- "className":
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "maxGap": 0,
+ "filter":
"$thresholdFilter_1:THRESHOLD_RULE_FILTER",
+ "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested": [
{
- "bucketPeriod": "P1D",
- "subEntityName": "testPipeline",
- "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ "baselineValueProvider":
"$maxThreshold_1:THRESHOLD",
+ "className":
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+ "maxGap": 0,
+ "nested": [
+ {
+ "bucketPeriod": "P1D",
+ "subEntityName": "testPipeline",
+ "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }
+ ],
+ "detector": "$maxThreshold_1:THRESHOLD",
+ "maxDuration": 100000000
}
- ],
- "detector": "$maxThreshold_1:THRESHOLD",
- "maxDuration": 100000000
+ ]
}
]
}
]
},
{
- "filter": "$thresholdFilter_3:THRESHOLD_RULE_FILTER",
- "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
- "nested": [
+ "className" :
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyLabelerWrapper",
+ "labeler" : "$labeler_2:THRESHOLD_SEVERITY_LABELER",
+ "nested" : [
{
- "baselineValueProvider":
"$maxThreshold_2:THRESHOLD",
- "className":
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "maxGap": 0,
+ "filter":
"$thresholdFilter_3:THRESHOLD_RULE_FILTER",
+ "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested": [
{
- "bucketPeriod": "P1D",
- "subEntityName": "testPipeline",
- "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ "baselineValueProvider":
"$maxThreshold_2:THRESHOLD",
+ "className":
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+ "maxGap": 0,
+ "nested": [
+ {
+ "bucketPeriod": "P1D",
+ "subEntityName": "testPipeline",
+ "className":
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }
+ ],
+ "detector": "$maxThreshold_2:THRESHOLD",
+ "maxDuration": 100000000
}
- ],
- "detector": "$maxThreshold_2:THRESHOLD",
- "maxDuration": 100000000
+ ]
}
]
}
@@ -88,10 +100,32 @@
"maxDuration": 100000000
},
"components": {
+ "labeler_1:THRESHOLD_SEVERITY_LABELER" : {
+ "severity" : {
+ "critical" : {
+ "change" : 0.5,
+ "duration" : 43200000
+ },
+ "high" : {
+ "change" : 0.3,
+ "duration" : 21600000
+ }
+ },
+ "className" :
"org.apache.pinot.thirdeye.detection.components.ThresholdSeverityLabeler"
+ },
"maxThreshold_2:THRESHOLD": {
"max": 100,
"className":
"org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector"
},
+ "labeler_2:THRESHOLD_SEVERITY_LABELER" : {
+ "severity" : {
+ "high" : {
+ "change" : 0.2,
+ "duration" : 43200000
+ }
+ },
+ "className" :
"org.apache.pinot.thirdeye.detection.components.ThresholdSeverityLabeler"
+ },
"thresholdFilter_2:THRESHOLD_RULE_FILTER": {
"min": 50,
"className":
"org.apache.pinot.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
index 5d2feae..4646287 100644
---
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
+++
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
@@ -29,6 +29,17 @@ rules:
name: thresholdFilter_2
params:
min: 50
+ labeler:
+ - type: THRESHOLD_SEVERITY_LABELER
+ name: labeler_1
+ params:
+ severity:
+ critical:
+ change: 0.5
+ duration: 43200000
+ high:
+ change: 0.3
+ duration: 21600000
- detection:
- type: THRESHOLD
@@ -40,6 +51,14 @@ rules:
name: thresholdFilter_3
params:
min: 50
+ labeler:
+ - type: THRESHOLD_SEVERITY_LABELER
+ name: labeler_2
+ params:
+ severity:
+ high:
+ change: 0.2
+ duration: 43200000
merger:
maxGap: 0
maxDuration: 100000000
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
