This is an automated email from the ASF dual-hosted git repository.

jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new a123c60  [TE] add labeler into yaml (#6007)
a123c60 is described below

commit a123c60f63e3fc78fb691b380097aaf1f8c4d4b8
Author: Vincent Chen <jianc...@linkedin.com>
AuthorDate: Tue Sep 15 15:40:34 2020 -0700

    [TE] add labeler into yaml (#6007)
    
    This PR is second PR for severity-based alert feature, including the logic 
of parsing labeler configuration and constructing the detection pipelines based 
on the YAML.
---
 .../components/ThresholdSeverityLabeler.java       | 16 ++---
 .../thirdeye/detection/spi/components/Labeler.java |  9 ++-
 .../detection/wrapper/AnomalyLabelerWrapper.java   | 13 +++-
 .../wrapper/ChildKeepingMergeWrapper.java          |  6 +-
 .../yaml/translator/DetectionConfigTranslator.java | 54 ++++++++------
 .../builder/DataQualityPropertiesBuilder.java      |  2 +-
 .../builder/DetectionConfigPropertiesBuilder.java  | 34 +++++----
 .../builder/DetectionPropertiesBuilder.java        | 23 ++++--
 .../detection/detection-config-schema.json         |  3 +
 .../thirdeye/detection/DetectionTestUtils.java     | 16 +++--
 .../components/ThresholdSeverityLabelerTest.java   |  2 +-
 .../wrapper/ChildKeepingMergeWrapperTest.java      | 24 +++++++
 .../translator/DetectionConfigTranslatorTest.java  |  3 +
 .../compositePipelineTranslatorTestResult-1.json   | 84 +++++++++++++++-------
 .../yaml/translator/pipeline-config-1.yaml         | 19 +++++
 15 files changed, 223 insertions(+), 85 deletions(-)

diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
index ff04015..5f31ccf 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabeler.java
@@ -19,6 +19,7 @@
 
 package org.apache.pinot.thirdeye.detection.components;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -46,7 +47,8 @@ public class ThresholdSeverityLabeler implements 
Labeler<SeverityThresholdLabele
   private TreeMap<AnomalySeverity, Threshold> severityMap;
 
   @Override
-  public void label(List<MergedAnomalyResultDTO> anomalies) {
+  public Map<MergedAnomalyResultDTO, AnomalySeverity> 
label(List<MergedAnomalyResultDTO> anomalies) {
+    Map<MergedAnomalyResultDTO, AnomalySeverity> res = new HashMap<>();
     for (MergedAnomalyResultDTO anomaly : anomalies) {
       double currVal = anomaly.getAvgCurrentVal();
       double baseVal = anomaly.getAvgBaselineVal();
@@ -59,18 +61,12 @@ public class ThresholdSeverityLabeler implements 
Labeler<SeverityThresholdLabele
       long duration = anomaly.getEndTime() - anomaly.getStartTime();
       for (Map.Entry<AnomalySeverity, Threshold> entry : 
severityMap.entrySet()) {
         if (deviation >= entry.getValue().change || duration >= 
entry.getValue().duration) {
-          if (anomaly.getSeverityLabel() != entry.getKey()) {
-            // find the severity from highest to lowest
-            if (anomaly.getId() != null && 
anomaly.getSeverityLabel().compareTo(entry.getKey()) > 0) {
-              // only set renotify if the anomaly exists and its severity gets 
higher
-              anomaly.setRenotify(true);
-            }
-            anomaly.setSeverityLabel(entry.getKey());
-            break;
-          }
+          res.put(anomaly, entry.getKey());
+          break;
         }
       }
     }
+    return res;
   }
 
   @Override
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
index 61541a4..77e4316 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/Labeler.java
@@ -20,14 +20,17 @@
 package org.apache.pinot.thirdeye.detection.spi.components;
 
 import java.util.List;
+import java.util.Map;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
 import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
 import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
 
 
 public interface Labeler <T extends AbstractSpec> extends BaseComponent<T> {
   /**
-   * add or modify labels of anomalies in place
-   * @param anomalies
+   * Calculate the severity for list of anomalies
+   * @param anomalies input anoamlies
+   * @return mapping from anomaly to severity
    */
-  void label(List<MergedAnomalyResultDTO> anomalies);
+  Map<MergedAnomalyResultDTO, AnomalySeverity> 
label(List<MergedAnomalyResultDTO> anomalies);
 }
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
index 5d59e9b..9237fb6 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyLabelerWrapper.java
@@ -27,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import org.apache.commons.collections4.MapUtils;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
 import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
@@ -76,7 +77,17 @@ public class AnomalyLabelerWrapper extends DetectionPipeline 
{
       diagnostics.putAll(intermediate.getDiagnostics());
       anomalies.addAll(intermediate.getAnomalies());
     }
-    this.labeler.label(anomalies);
+    Map<MergedAnomalyResultDTO, AnomalySeverity> res = 
this.labeler.label(anomalies);
+    for (MergedAnomalyResultDTO anomaly : anomalies) {
+      AnomalySeverity newSeverity = res.getOrDefault(anomaly, 
AnomalySeverity.DEFAULT);
+      if (anomaly.getSeverityLabel() != newSeverity) {
+        if (anomaly.getId() != null && 
anomaly.getSeverityLabel().compareTo(newSeverity) > 0) {
+          // only set renotify if the anomaly exists and its severity gets 
higher
+          anomaly.setRenotify(true);
+        }
+        anomaly.setSeverityLabel(newSeverity);
+      }
+    }
     return new DetectionPipelineResult(anomalies, 
DetectionUtils.consolidateNestedLastTimeStamps(lastTimeStamps),
         predictionResults, evaluations).setDiagnostics(diagnostics);
   }
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index 064e4f1..d808d68 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -118,7 +118,11 @@ public class ChildKeepingMergeWrapper extends 
BaselineFillingMergeWrapper {
 
         // merge the anomaly's properties into parent
         ThirdEyeUtils.mergeAnomalyProperties(parent.getProperties(), 
anomaly.getProperties());
-
+        // merge the anomaly severity
+        if (parent.getSeverityLabel().compareTo(anomaly.getSeverityLabel()) > 
0) {
+          // set the highest severity
+          parent.setSeverityLabel(anomaly.getSeverityLabel());
+        }
         if (anomaly.getChildren().isEmpty()) {
           parent.getChildren().add(anomaly);
         } else {
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index 6fcddef..5f3bce2 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -70,6 +70,12 @@ import 
org.apache.pinot.thirdeye.detection.yaml.translator.builder.DetectionProp
  * |Rule      |     |Algorithm |
  * |Filters   |     |Filters   |
  * +-----+----+     +-----+----+
+ *       |                |
+ *       v                v
+ * +-----+----+     +-----+----+
+ * |Rule      |     |Algorithm |
+ * |Labelers  |     |Labelers  |
+ * +-----+----+     +-----+----+
  *       +-------+--------+
  *               |
  *         +-----v-----+
@@ -86,27 +92,33 @@ import 
org.apache.pinot.thirdeye.detection.yaml.translator.builder.DetectionProp
  *
  * This translator translates a yaml that describes the above detection flow
  * to the flowing wrapper structure to be execute in the detection pipeline.
- * +-----------------------------------------+
- * |  Merger                                 |
- * |  +--------------------------------------+
- * |  |   Dimension Exploration & Filter    ||
- * |  | +---------------------------------+ ||
- * |  | |        Filters                  | ||
- * |  | |  +--------Merger--------------+ | ||
- * |  | |  |  Rule detections           | | ||
- * |  | |  +----------------------------+ | ||
- * |  | +---------------------------------+ ||
- * |  | +---------------------------------+ ||
- * |  | |              Filters            | ||
- * |  | | +-----------Merger------------+ | ||
- * |  | | | +---------------------------| | ||
- * |  | | | | Algorithm detection       | | ||
- * |  | | | +---------------------------| | ||
- * |  | | +-----------------------------| | ||
- * |  | +---------------------------------+ ||
- * |  +--------------------------------------|
- * |  |--------------------------------------|
- * +-----------------------------------------+
+ * +-------------------------------------------------+
+ * |  Merger                                         |
+ * |  +-------------------------------------------+  |
+ * |  |       Dimension Exploration & Filter      |  |
+ * |  |                                           |  |
+ * |  |  +-------------------------------------+  |  |
+ * |  |  |            Labelers                 |  |  |
+ * |  |  | +---------------------------------+ |  |  |
+ * |  |  | |          Filters                | |  |  |
+ * |  |  | |  +--------Merger--------------+ | |  |  |
+ * |  |  | |  |  Rule detections           | | |  |  |
+ * |  |  | |  +----------------------------+ | |  |  |
+ * |  |  | +---------------------------------+ |  |  |
+ * |  |  +-------------------------------------+  |  |
+ * |  |                                           |  |
+ * |  |  +-------------------------------------+  |  |
+ * |  |  |               Labelers              |  |  |
+ * |  |  | +---------------------------------+ |  |  |
+ * |  |  | |              Filters            | |  |  |
+ * |  |  | | | +---------Merger------------+ | |  |  |
+ * |  |  | | | | Algorithm detection       | | |  |  |
+ * |  |  | | | +---------------------------+ | |  |  |
+ * |  |  | +---------------------------------+ |  |  |
+ * |  |  +-------------------------------------+  |  |
+ * |  |                                           |  |
+ * |  +-------------------------------------------+  |
+ * +-------------------------------------------------+
  *
  */
 public class DetectionConfigTranslator extends 
ConfigTranslator<DetectionConfigDTO, DetectionConfigValidator> {
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
index 897a1f7..2c62909 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DataQualityPropertiesBuilder.java
@@ -131,7 +131,7 @@ public class DataQualityPropertiesBuilder extends 
DetectionConfigPropertiesBuild
     String qualityRefKey = makeComponentRefKey(qualityType, name);
     properties.put(PROP_QUALITY_CHECK, qualityRefKey);
 
-    buildComponentSpec(metricUrn, yamlConfig, qualityType, qualityRefKey);
+    buildComponentSpec(metricUrn, yamlConfig, qualityRefKey);
 
     properties.putAll(mergerProperties);
     return properties;
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
index 1d15653..8a7799e 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionConfigPropertiesBuilder.java
@@ -60,6 +60,7 @@ public abstract class DetectionConfigPropertiesBuilder {
   static final String PROP_NESTED_METRIC_URNS = "nestedMetricUrns";
   static final String PROP_RULES = "rules";
   static final String PROP_GROUPER = "grouper";
+  static final String PROP_LABELER = "labeler";
   static final String PROP_NESTED = "nested";
   static final String PROP_BASELINE_PROVIDER = "baselineValueProvider";
   static final String PROP_DETECTOR = "detector";
@@ -121,12 +122,11 @@ public abstract class DetectionConfigPropertiesBuilder {
     properties.put(PROP_NESTED, nestedProps);
     properties.put(PROP_SUB_ENTITY_NAME, entityName);
 
-    String grouperType = MapUtils.getString(grouperYaml, PROP_TYPE);
-    String grouperName = MapUtils.getString(grouperYaml, PROP_NAME);
-    String grouperRefKey = makeComponentRefKey(grouperType, grouperName);
+    String grouperRefKey = makeComponentRefKey(
+        MapUtils.getString(grouperYaml, PROP_TYPE), 
MapUtils.getString(grouperYaml, PROP_NAME));
     properties.put(PROP_GROUPER, grouperRefKey);
 
-    buildComponentSpec(metricUrn, grouperYaml, grouperType, grouperRefKey);
+    buildComponentSpec(metricUrn, grouperYaml, grouperRefKey);
 
     return properties;
   }
@@ -140,19 +140,29 @@ public abstract class DetectionConfigPropertiesBuilder {
     if (wrapperProperties.isEmpty()) {
       return Collections.emptyList();
     }
-    String name = MapUtils.getString(yamlConfig, PROP_NAME);
-    String filterType = MapUtils.getString(yamlConfig, PROP_TYPE);
-    String filterRefKey = makeComponentRefKey(filterType, name);
+    String filterRefKey = makeComponentRefKey(
+        MapUtils.getString(yamlConfig, PROP_TYPE), 
MapUtils.getString(yamlConfig, PROP_NAME));
     wrapperProperties.put(PROP_FILTER, filterRefKey);
-    buildComponentSpec(metricUrn, yamlConfig, filterType, filterRefKey);
+    buildComponentSpec(metricUrn, yamlConfig, filterRefKey);
 
     return Collections.singletonList(wrapperProperties);
   }
 
-  void buildComponentSpec(String metricUrn, Map<String, Object> yamlConfig, 
String type, String componentRefKey) {
-    Map<String, Object> componentSpecs = new HashMap<>();
+  Map<String, Object> buildLabelerWrapperProperties(String metricUrn, String 
wrapperClassName,
+      Map<String, Object> yamlConfig, List<Map<String, Object>> 
nestedProperties) {
+    Map<String, Object> wrapperProperties = 
buildWrapperProperties(wrapperClassName, nestedProperties);
+    String labelerRefKey = makeComponentRefKey(
+        MapUtils.getString(yamlConfig, PROP_TYPE), 
MapUtils.getString(yamlConfig, PROP_NAME));
+    wrapperProperties.put(PROP_LABELER, labelerRefKey);
+    buildComponentSpec(metricUrn, yamlConfig, labelerRefKey);
+    return wrapperProperties;
+  }
+
 
-    String componentClassName = DETECTION_REGISTRY.lookup(type);
+  void buildComponentSpec(String metricUrn, Map<String, Object> yamlConfig, 
String componentRefKey) {
+    Map<String, Object> componentSpecs = new HashMap<>();
+    String componentKey = DetectionUtils.getComponentKey(componentRefKey);
+    String componentClassName = 
DETECTION_REGISTRY.lookup(DetectionUtils.getComponentType(componentKey));
     componentSpecs.put(PROP_CLASS_NAME, componentClassName);
     if (metricUrn != null) {
       componentSpecs.put(PROP_METRIC_URN, metricUrn);
@@ -168,7 +178,7 @@ public abstract class DetectionConfigPropertiesBuilder {
       componentSpecs.putAll(params);
     }
 
-    
metricAttributesMap.addComponent(DetectionUtils.getComponentKey(componentRefKey),
 componentSpecs);
+    metricAttributesMap.addComponent(componentKey, componentSpecs);
   }
 
   Map<String, Object> compositePropertyBuilderHelper(List<Map<String, Object>> 
nestedPropertiesList,
diff --git 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
index 664e40c..f4c15fd 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
+++ 
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/builder/DetectionPropertiesBuilder.java
@@ -37,6 +37,7 @@ import org.apache.pinot.thirdeye.detection.DataProvider;
 import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
 import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
 import org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper;
+import org.apache.pinot.thirdeye.detection.wrapper.AnomalyLabelerWrapper;
 import org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper;
 import org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper;
 import org.apache.pinot.thirdeye.detection.wrapper.EntityAnomalyMergeWrapper;
@@ -81,15 +82,26 @@ public class DetectionPropertiesBuilder extends 
DetectionConfigPropertiesBuilder
           subEntityName, metricUrn, detectionYamls, mergerProperties, 
datasetConfigDTO.bucketTimeGranularity());
 
       List<Map<String, Object>> filterYamls = 
ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
-      if (filterYamls.isEmpty()) {
+      List<Map<String, Object>> labelerYamls = 
ConfigUtils.getList(ruleYaml.get(PROP_LABELER));
+      if (filterYamls.isEmpty() && labelerYamls.isEmpty()) {
+        // output detection properties if neither filter and labeler is 
configured
         nestedPipelines.addAll(detectionProperties);
       } else {
+        // wrap detection properties around with filter properties if a filter 
is configured
         List<Map<String, Object>> filterNestedProperties = detectionProperties;
         for (Map<String, Object> filterProperties : filterYamls) {
           filterNestedProperties = buildFilterWrapperProperties(metricUrn, 
AnomalyFilterWrapper.class.getName(), filterProperties,
               filterNestedProperties);
         }
-        nestedPipelines.addAll(filterNestedProperties);
+        if (labelerYamls.isEmpty()) {
+          // output filter properties if no labeler is configured
+          nestedPipelines.addAll(filterNestedProperties);
+        } else {
+          // wrap filter properties around with labeler properties if a 
labeler is configured
+          nestedPipelines.add(
+              buildLabelerWrapperProperties(metricUrn, 
AnomalyLabelerWrapper.class.getName(), labelerYamls.get(0),
+                  filterNestedProperties));
+        }
       }
     }
 
@@ -157,7 +169,7 @@ public class DetectionPropertiesBuilder extends 
DetectionConfigPropertiesBuilder
 
     fillInDetectorWrapperProperties(nestedProperties, yamlConfig, 
detectorType, datasetTimegranularity);
 
-    buildComponentSpec(metricUrn, yamlConfig, detectorType, detectorRefKey);
+    buildComponentSpec(metricUrn, yamlConfig, detectorRefKey);
 
     Map<String, Object> properties = new HashMap<>();
     properties.put(PROP_CLASS_NAME, 
BaselineFillingMergeWrapper.class.getName());
@@ -169,9 +181,8 @@ public class DetectionPropertiesBuilder extends 
DetectionConfigPropertiesBuilder
       // if the detector implements the baseline provider interface, use it to 
generate baseline
       properties.put(PROP_BASELINE_PROVIDER, detectorRefKey);
     } else {
-      String baselineProviderType = DEFAULT_BASELINE_PROVIDER_YAML_TYPE;
-      String baselineProviderKey = makeComponentRefKey(baselineProviderType, 
name);
-      buildComponentSpec(metricUrn, yamlConfig, baselineProviderType, 
baselineProviderKey);
+      String baselineProviderKey = 
makeComponentRefKey(DEFAULT_BASELINE_PROVIDER_YAML_TYPE, name);
+      buildComponentSpec(metricUrn, yamlConfig, baselineProviderKey);
       properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
     }
     properties.putAll(mergerProperties);
diff --git 
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
 
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
index 59ee067..e5a3d7e 100644
--- 
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
+++ 
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
@@ -225,6 +225,9 @@
           },
           "quality": {
             "$ref": "#/definitions/genericComponent"
+          },
+          "labeler": {
+            "$ref": "#/definitions/genericComponent"
           }
         },
         "additionalProperties": false
diff --git 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
index 8a9eac6..0ad231d 100644
--- 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
+++ 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
@@ -22,6 +22,7 @@ package org.apache.pinot.thirdeye.detection;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import java.util.HashMap;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
 import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
 import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
 import java.util.Collections;
@@ -34,7 +35,8 @@ public class DetectionTestUtils {
   private static final Long PROP_ID_VALUE = 1000L;
 
   public static MergedAnomalyResultDTO makeAnomaly(Long configId, Long 
legacyFunctionId, long start, long end,
-      String metric, String dataset, Map<String, String> dimensions, 
Map<String, String> props) {
+      String metric, String dataset, Map<String, String> dimensions, 
Map<String, String> props,
+      AnomalySeverity severity) {
     MergedAnomalyResultDTO anomaly = new MergedAnomalyResultDTO();
     anomaly.setDetectionConfigId(configId);
     anomaly.setStartTime(start);
@@ -43,6 +45,7 @@ public class DetectionTestUtils {
     anomaly.setCollection(dataset);
     anomaly.setFunctionId(legacyFunctionId);
     anomaly.setProperties(props);
+    anomaly.setSeverityLabel(severity);
 
     Multimap<String, String> filters = HashMultimap.create();
     for (Map.Entry<String, String> dimension : dimensions.entrySet()) {
@@ -60,7 +63,7 @@ public class DetectionTestUtils {
   public static MergedAnomalyResultDTO makeAnomaly(Long configId, long start, 
long end, String metric, String dataset,
       Map<String, String> dimensions) {
     return DetectionTestUtils.makeAnomaly(configId, null, start, end, metric, 
dataset, dimensions,
-        new HashMap<>());
+        new HashMap<>(), AnomalySeverity.DEFAULT);
   }
 
   public static MergedAnomalyResultDTO setAnomalyId(MergedAnomalyResultDTO 
anomaly, long id) {
@@ -74,7 +77,7 @@ public class DetectionTestUtils {
 
   public static MergedAnomalyResultDTO makeAnomaly(Long configId, Long 
legacyFuncId, long start, long end) {
     return DetectionTestUtils.makeAnomaly(configId, legacyFuncId, start, end, 
null, null,
-        Collections.emptyMap(), new HashMap<>());
+        Collections.emptyMap(), new HashMap<>(), AnomalySeverity.DEFAULT);
   }
 
   public static MergedAnomalyResultDTO makeAnomaly(Long configId, long start, 
long end) {
@@ -122,7 +125,12 @@ public class DetectionTestUtils {
 
   public static MergedAnomalyResultDTO makeAnomalyWithProps(long start, long 
end, Map<String, String> props) {
     return DetectionTestUtils.makeAnomaly(PROP_ID_VALUE, null, start, end, 
null, null,
-        Collections.emptyMap(), props);
+        Collections.emptyMap(), props, AnomalySeverity.DEFAULT);
+  }
+
+  public static MergedAnomalyResultDTO makeAnomaly(long start, long end, 
AnomalySeverity severity) {
+    return DetectionTestUtils.makeAnomaly(PROP_ID_VALUE, null, start, end, 
null, null,
+        Collections.emptyMap(), Collections.emptyMap(), severity);
   }
 
 }
diff --git 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
index 459386b..0b7067b 100644
--- 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
+++ 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdSeverityLabelerTest.java
@@ -166,7 +166,7 @@ public class ThresholdSeverityLabelerTest {
     severityMap.put(AnomalySeverity.HIGH.toString(), new Threshold(0.15, 
2000));
     severityMap.put(AnomalySeverity.MEDIUM.toString(), new Threshold(0.10, 
1500));
     this.specs.put("severity", severityMap);
-    MergedAnomalyResultDTO anomaly = DetectionTestUtils.makeAnomaly(4000L, 
6000L, METRIC_URN, 2700, 3000);
+    MergedAnomalyResultDTO anomaly = DetectionTestUtils.makeAnomaly(4200L, 
6000L, METRIC_URN, 2700, 3000);
     anomaly.setSeverityLabel(AnomalySeverity.HIGH);
     anomaly.setId(125L);
     this.anomalies.set(this.anomalies.size() - 1, anomaly);
diff --git 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
index 7703209..69b74db 100644
--- 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
+++ 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
@@ -18,6 +18,7 @@ package org.apache.pinot.thirdeye.detection.wrapper;
 
 import com.google.common.collect.ImmutableSet;
 import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.anomaly.AnomalySeverity;
 import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
 import org.apache.pinot.thirdeye.detection.DataProvider;
@@ -350,4 +351,27 @@ public class ChildKeepingMergeWrapperTest {
     Assert.assertTrue(output.getAnomalies().contains(makeAnomaly(2800, 3600, 
null, 3, 2)));
     Assert.assertTrue(output.getAnomalies().contains(makeAnomaly(3500, 3800, 
null, 1, 2)));
   }
+
+  @Test
+  public void testMergerDifferentSeverity() throws Exception {
+    this.config.getProperties().put(PROP_MAX_GAP, 200);
+    this.config.getProperties().put(PROP_MAX_DURATION, 1250);
+
+    this.outputs.add(new MockPipelineOutput(Arrays.asList(
+        makeAnomaly(2800, 3800, AnomalySeverity.MEDIUM),
+        makeAnomaly(3500, 3600, AnomalySeverity.CRITICAL)
+    ), 3700));
+
+    Map<String, Object> nestedProperties = new HashMap<>();
+    nestedProperties.put(PROP_CLASS_NAME, "none");
+    nestedProperties.put(PROP_METRIC_URN, "thirdeye:metric:3");
+
+    this.nestedProperties.add(nestedProperties);
+
+    this.wrapper = new ChildKeepingMergeWrapper(this.provider, this.config, 
1000, 4000);
+    DetectionPipelineResult output = this.wrapper.run();
+
+    Assert.assertEquals(output.getAnomalies().size(), 4);
+    Assert.assertEquals(output.getAnomalies().get(3).getSeverityLabel(), 
AnomalySeverity.CRITICAL);
+  }
 }
diff --git 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
index 4ebb897..6f7d292 100644
--- 
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
+++ 
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
@@ -1,6 +1,7 @@
 package org.apache.pinot.thirdeye.detection.yaml.translator;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.io.IOUtils;
 import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
@@ -19,6 +20,7 @@ import 
org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.detection.components.ThresholdSeverityLabeler;
 import 
org.apache.pinot.thirdeye.detection.validators.ConfigValidationException;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
@@ -71,6 +73,7 @@ public class DetectionConfigTranslatorTest {
     
DetectionRegistry.registerComponent(ThresholdRuleAnomalyFilter.class.getName(), 
"THRESHOLD_RULE_FILTER");
     DetectionRegistry.registerComponent(RuleBaselineProvider.class.getName(), 
"RULE_BASELINE");
     DetectionRegistry.registerComponent(MockGrouper.class.getName(), 
"MOCK_GROUPER");
+    
DetectionRegistry.registerComponent(ThresholdSeverityLabeler.class.getName(), 
"THRESHOLD_SEVERITY_LABELER");
     this.provider = new 
MockDataProvider().setMetrics(Collections.singletonList(metricConfig)).setDatasets(Collections.singletonList(datasetConfigDTO));
   }
 
diff --git 
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
 
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
index e3bfaa5..f55fbda 100644
--- 
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
+++ 
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
@@ -24,48 +24,60 @@
                     "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
                     "nested": [
                       {
-                        "filter": "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
-                        "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
-                        "nested": [
+                        "className" : 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyLabelerWrapper",
+                        "labeler" : "$labeler_1:THRESHOLD_SEVERITY_LABELER",
+                        "nested" : [
                           {
-                            "filter": 
"$thresholdFilter_1:THRESHOLD_RULE_FILTER",
+                            "filter": 
"$thresholdFilter_2:THRESHOLD_RULE_FILTER",
                             "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
                             "nested": [
                               {
-                                "baselineValueProvider": 
"$maxThreshold_1:THRESHOLD",
-                                "className": 
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
-                                "maxGap": 0,
+                                "filter": 
"$thresholdFilter_1:THRESHOLD_RULE_FILTER",
+                                "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
                                 "nested": [
                                   {
-                                    "bucketPeriod": "P1D",
-                                    "subEntityName": "testPipeline",
-                                    "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+                                    "baselineValueProvider": 
"$maxThreshold_1:THRESHOLD",
+                                    "className": 
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+                                    "maxGap": 0,
+                                    "nested": [
+                                      {
+                                        "bucketPeriod": "P1D",
+                                        "subEntityName": "testPipeline",
+                                        "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+                                      }
+                                    ],
+                                    "detector": "$maxThreshold_1:THRESHOLD",
+                                    "maxDuration": 100000000
                                   }
-                                ],
-                                "detector": "$maxThreshold_1:THRESHOLD",
-                                "maxDuration": 100000000
+                                ]
                               }
                             ]
                           }
                         ]
                       },
                       {
-                        "filter": "$thresholdFilter_3:THRESHOLD_RULE_FILTER",
-                        "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
-                        "nested": [
+                        "className" : 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyLabelerWrapper",
+                        "labeler" : "$labeler_2:THRESHOLD_SEVERITY_LABELER",
+                        "nested" : [
                           {
-                            "baselineValueProvider": 
"$maxThreshold_2:THRESHOLD",
-                            "className": 
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
-                            "maxGap": 0,
+                            "filter": 
"$thresholdFilter_3:THRESHOLD_RULE_FILTER",
+                            "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
                             "nested": [
                               {
-                                "bucketPeriod": "P1D",
-                                "subEntityName": "testPipeline",
-                                "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+                                "baselineValueProvider": 
"$maxThreshold_2:THRESHOLD",
+                                "className": 
"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+                                "maxGap": 0,
+                                "nested": [
+                                  {
+                                    "bucketPeriod": "P1D",
+                                    "subEntityName": "testPipeline",
+                                    "className": 
"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+                                  }
+                                ],
+                                "detector": "$maxThreshold_2:THRESHOLD",
+                                "maxDuration": 100000000
                               }
-                            ],
-                            "detector": "$maxThreshold_2:THRESHOLD",
-                            "maxDuration": 100000000
+                            ]
                           }
                         ]
                       }
@@ -88,10 +100,32 @@
     "maxDuration": 100000000
   },
   "components": {
+    "labeler_1:THRESHOLD_SEVERITY_LABELER" : {
+      "severity" : {
+        "critical" : {
+          "change" : 0.5,
+          "duration" : 43200000
+        },
+        "high" : {
+          "change" : 0.3,
+          "duration" : 21600000
+        }
+      },
+      "className" : 
"org.apache.pinot.thirdeye.detection.components.ThresholdSeverityLabeler"
+    },
     "maxThreshold_2:THRESHOLD": {
       "max": 100,
       "className": 
"org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector"
     },
+    "labeler_2:THRESHOLD_SEVERITY_LABELER" : {
+      "severity" : {
+        "high" : {
+          "change" : 0.2,
+          "duration" : 43200000
+        }
+      },
+      "className" : 
"org.apache.pinot.thirdeye.detection.components.ThresholdSeverityLabeler"
+    },
     "thresholdFilter_2:THRESHOLD_RULE_FILTER": {
       "min": 50,
       "className": 
"org.apache.pinot.thirdeye.detection.components.ThresholdRuleAnomalyFilter"
diff --git 
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
 
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
index 5d2feae..4646287 100644
--- 
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
+++ 
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/pipeline-config-1.yaml
@@ -29,6 +29,17 @@ rules:
     name: thresholdFilter_2
     params:
       min: 50
+  labeler:
+    - type: THRESHOLD_SEVERITY_LABELER
+      name: labeler_1
+      params:
+        severity:
+          critical:
+            change: 0.5
+            duration: 43200000
+          high:
+            change: 0.3
+            duration: 21600000
 
 - detection:
   - type: THRESHOLD
@@ -40,6 +51,14 @@ rules:
     name: thresholdFilter_3
     params:
       min: 50
+  labeler:
+    - type: THRESHOLD_SEVERITY_LABELER
+      name: labeler_2
+      params:
+        severity:
+          high:
+            change: 0.2
+            duration: 43200000
 merger:
   maxGap: 0
   maxDuration: 100000000


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org

Reply via email to