This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5f195b6 [TE] detection - yaml translation/migration changes (#3767)
5f195b6 is described below
commit 5f195b6c3594f0e48f37e1a7298b46dd318d4928
Author: Jihao Zhang <[email protected]>
AuthorDate: Thu Jan 31 11:34:38 2019 -0800
[TE] detection - yaml translation/migration changes (#3767)
Do not trigger tune for migrated anomaly functions
Default tuning window
---
.../detection/DetectionMigrationResource.java | 4 +-
.../annotation/registry/DetectionRegistry.java | 4 +-
.../yaml/CompositePipelineConfigTranslator.java | 51 +++++++++++++-------
.../thirdeye/detection/yaml/YamlResource.java | 55 ++++++++++++----------
4 files changed, 68 insertions(+), 46 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
index afdfbd8..bc51de7 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
@@ -144,7 +144,7 @@ public class DetectionMigrationResource {
} else{
// algorithm detector
ruleYaml.put("detection", Collections.singletonList(
- ImmutableMap.of("name", "detection_rule1", "type", "ALGORITHM",
"params", getAlgorithmDetectorParams(anomalyFunctionDTO),
+ ImmutableMap.of("name", "detection_rule1", "type",
"MIGRATED_ALGORITHM", "params", getAlgorithmDetectorParams(anomalyFunctionDTO),
PROP_WINDOW_SIZE, anomalyFunctionDTO.getWindowSize(),
PROP_WINDOW_UNIT,
anomalyFunctionDTO.getWindowUnit().toString())));
}
@@ -156,7 +156,7 @@ public class DetectionMigrationResource {
Map<String, Object> filterYaml = new LinkedHashMap<>();
if (!alertFilter.containsKey("thresholdField")) {
// algorithm alert filter
- filterYaml = ImmutableMap.of("name", "filter_rule1", "type",
"ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO));
+ filterYaml = ImmutableMap.of("name", "filter_rule1", "type",
"MIGRATED_ALGORITHM_FILTER", "params",
getAlertFilterParams(anomalyFunctionDTO));
} else {
// threshold filter migrate to rule filters
// site wide impact filter migrate to rule based swi filter
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
index 8890ddd..0d310ac 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
@@ -117,8 +117,8 @@ public class DetectionRegistry {
* @return component class name
*/
public String lookup(String type) {
- Preconditions.checkArgument(REGISTRY_MAP.containsKey(type.toUpperCase()),
type + " not found in registry");
- return MapUtils.getString(REGISTRY_MAP.get(type.toUpperCase()),
KEY_CLASS_NAME);
+ Preconditions.checkArgument(REGISTRY_MAP.containsKey(type), type + " not
found in registry");
+ return MapUtils.getString(REGISTRY_MAP.get(type), KEY_CLASS_NAME);
}
/**
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 00b3a7c..0151c08 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -24,34 +24,35 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.collections.MapUtils;
import org.apache.pinot.thirdeye.api.TimeGranularity;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher;
+import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.InputDataFetcher;
import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
-import
org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
import org.apache.pinot.thirdeye.detection.annotation.Yaml;
+import
org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
import org.apache.pinot.thirdeye.detection.spi.components.Tunable;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper;
-import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.collections.MapUtils;
+import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
@@ -158,10 +159,23 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
private static final String PROP_MERGER = "merger";
private static final String PROP_TIMEZONE = "timezone";
private static final String PROP_NAME = "name";
+ private static final String DEFAULT_TIMEZONE = "America/Los_Angeles";
private static final String DEFAULT_BASELINE_PROVIDER_YAML_TYPE =
"RULE_BASELINE";
private static final DetectionRegistry DETECTION_REGISTRY =
DetectionRegistry.getInstance();
- private static final Map<String, String> DETECTOR_TO_BASELINE =
ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE");
+ static {
+ // do not tune for alerts migrated from legacy anomaly function.
+
DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibAlertFilter",
+ "MIGRATED_ALGORITHM_FILTER");
+
DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibAnomalyDetector",
+ "MIGRATED_ALGORITHM");
+
DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibBaselineProvider",
+ "MIGRATED_ALGORITHM_BASELINE");
+ }
+ private static final Set<String> TUNING_OFF_COMPONENTS =
+ ImmutableSet.of("MIGRATED_ALGORITHM_FILTER", "MIGRATED_ALGORITHM",
"MIGRATED_ALGORITHM_BASELINE");
+ private static final Map<String, String> DETECTOR_TO_BASELINE =
+ ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE", "MIGRATED_ALGORITHM",
"MIGRATED_ALGORITHM_BASELINE");
private static final Set<String> MOVING_WINDOW_DETECTOR_TYPES =
ImmutableSet.of("ALGORITHM");
private final Map<String, Object> components = new HashMap<>();
@@ -386,7 +400,7 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
params = MapUtils.getMap(yamlConfig, PROP_PARAMS);
}
- if (DETECTION_REGISTRY.isTunable(componentClassName)) {
+ if (!TUNING_OFF_COMPONENTS.contains(type) &&
DETECTION_REGISTRY.isTunable(componentClassName)) {
try {
componentSpecs.putAll(getTunedSpecs(componentName, componentClassName,
params));
} catch (Exception e) {
@@ -400,10 +414,15 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
private Map<String, Object> getTunedSpecs(String componentName, String
componentClassName, Map<String, Object> params)
throws Exception {
- long configId = this.existingConfig == null ? -1 :
this.existingConfig.getId();
+ long configId = this.existingConfig == null ? 0 :
this.existingConfig.getId();
InputDataFetcher dataFetcher = new
DefaultInputDataFetcher(this.dataProvider, configId);
Tunable tunable = getTunable(componentClassName, params, dataFetcher);
- Interval window = new Interval(this.startTime, this.endTime,
DateTimeZone.forID(this.datasetConfig.getTimezone()));
+
+ // round to daily boundary
+ DateTimeZone timezone =
DateTimeZone.forID(this.datasetConfig.getTimezone() == null ? DEFAULT_TIMEZONE
: this.datasetConfig.getTimezone());
+ DateTime start = new DateTime(this.startTime,
timezone).withTimeAtStartOfDay();
+ DateTime end = new DateTime(this.endTime,
timezone).withTimeAtStartOfDay();
+ Interval window = new Interval(start, end);
Map<String, Object> existingComponentSpec =
this.existingComponentSpecs.containsKey(componentName) ?
MapUtils.getMap(this.existingComponentSpecs,
componentName) : Collections.emptyMap();
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 57014af..d87386c 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -22,33 +22,8 @@ package org.apache.pinot.thirdeye.detection.yaml;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import javax.ws.rs.DefaultValue;
-import org.apache.pinot.thirdeye.api.Constants;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.util.Predicate;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
-import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
-import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
-import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
-import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
-import org.apache.pinot.thirdeye.detection.DataProvider;
-import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
-import org.apache.pinot.thirdeye.detection.DetectionPipeline;
-import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import com.wordnik.swagger.annotations.Api;
-import com.wordnik.swagger.annotations.ApiImplicitParam;
-import com.wordnik.swagger.annotations.ApiImplicitParams;
import com.wordnik.swagger.annotations.ApiOperation;
-import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
-import
org.apache.pinot.thirdeye.detection.validators.DetectionAlertConfigValidator;
import com.wordnik.swagger.annotations.ApiParam;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
@@ -57,6 +32,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@@ -69,6 +45,28 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.thirdeye.api.Constants;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.util.Predicate;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
+import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
+import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
+import org.apache.pinot.thirdeye.detection.DataProvider;
+import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
+import org.apache.pinot.thirdeye.detection.DetectionPipeline;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
+import
org.apache.pinot.thirdeye.detection.validators.DetectionAlertConfigValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -129,7 +127,6 @@ public class YamlResource {
@ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) throws Exception{
Map<String, String> yamls = OBJECT_MAPPER.readValue(payload, Map.class);
-
if (StringUtils.isBlank(payload)){
return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"Empty payload")).build();
}
@@ -200,6 +197,12 @@ public class YamlResource {
*/
private DetectionConfigDTO buildDetectionConfigFromYaml(long startTime, long
endTime, Map<String, Object> yamlConfig,
DetectionConfigDTO existingDetectionConfig, Map<String, String>
responseMessage) {
+ if (startTime == 0L && endTime == 0L) {
+ // default tuning window 28 days
+ endTime = System.currentTimeMillis();
+ startTime = endTime - TimeUnit.DAYS.toMillis(28);
+ }
+
try{
YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
