This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 859c68c [TE] Tuning of Composite alert - Store metric refs in
component spec (#4359)
859c68c is described below
commit 859c68c7f92280292bb6f570b6f58708bc53ef20
Author: Akshay Rai <[email protected]>
AuthorDate: Tue Jun 25 08:29:02 2019 -0700
[TE] Tuning of Composite alert - Store metric refs in component spec (#4359)
---
.../thirdeye/detection/DetectionPipeline.java | 2 +-
.../detection/yaml/DetectionConfigTuner.java | 32 ++++++----------------
.../yaml/translator/DetectionConfigTranslator.java | 6 +++-
3 files changed, 15 insertions(+), 25 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
index 51406b8..425a246 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
@@ -100,7 +100,7 @@ public abstract class DetectionPipeline {
for (String componentKey : componentSpecs.keySet()) {
Map<String, Object> componentSpec =
ConfigUtils.getMap(componentSpecs.get(componentKey));
for (Map.Entry<String, Object> entry : componentSpec.entrySet()){
- if (DetectionUtils.isReferenceName(entry.getValue().toString())) {
+ if (entry.getValue() != null &&
DetectionUtils.isReferenceName(entry.getValue().toString())) {
componentSpec.put(entry.getKey(),
instancesMap.get(DetectionUtils.getComponentKey(entry.getValue().toString())));
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
index c05f2af..9a96527 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
@@ -77,25 +77,11 @@ public class DetectionConfigTuner {
private final DetectionConfigDTO detectionConfig;
private final DataProvider dataProvider;
- private final DatasetConfigDTO datasetConfig;
- private final String metricUrn;
public DetectionConfigTuner(DetectionConfigDTO config, DataProvider
dataProvider) {
Preconditions.checkNotNull(config);
this.detectionConfig = config;
this.dataProvider = dataProvider;
-
- Map<String, Object> yamlConfig = ConfigUtils.getMap(new
org.yaml.snakeyaml.Yaml().load(config.getYaml()));
-
- MetricConfigDTO metricConfig = dataProvider.fetchMetric(
- MapUtils.getString(yamlConfig, PROP_METRIC),
- MapUtils.getString(yamlConfig, PROP_DATASET));
- Preconditions.checkNotNull(metricConfig, "metric not found");
- this.datasetConfig =
dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
- .get(metricConfig.getDataset());
- Preconditions.checkNotNull(this.datasetConfig, "dataset not found");
-
- this.metricUrn =
MetricEntity.fromMetric(ConfigUtils.getMap(yamlConfig.get(PROP_FILTERS)),
metricConfig.getId()).getUrn();
}
/**
@@ -117,17 +103,19 @@ public class DetectionConfigTuner {
Tunable tunable = instantiateTunable(componentClassName, yamlParams,
dataFetcher);
// round to daily boundary
- DateTimeZone timezone =
DateTimeZone.forID(this.datasetConfig.getTimezone() == null ? DEFAULT_TIMEZONE
: this.datasetConfig.getTimezone());
+ String metricName = componentProps.get(PROP_METRIC).toString();
+ String datasetName = componentProps.get(PROP_DATASET).toString();
+ MetricConfigDTO metricConfig = dataProvider.fetchMetric(metricName,
datasetName);
+ DatasetConfigDTO datasetConfig =
dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
+ .get(metricConfig.getDataset());
+ String metricUrn =
MetricEntity.fromMetric(ConfigUtils.getMap(componentProps.get(PROP_FILTERS)),
metricConfig.getId()).getUrn();
+ DateTimeZone timezone = DateTimeZone.forID(datasetConfig.getTimezone() ==
null ? DEFAULT_TIMEZONE : datasetConfig.getTimezone());
DateTime start = new DateTime(startTime, timezone).withTimeAtStartOfDay();
DateTime end = new DateTime(endTime, timezone).withTimeAtStartOfDay();
Interval window = new Interval(start, end);
// TODO: if dimension drill down applied, pass in the metric urn of top
dimension
- tunedSpec.putAll(tunable.tune(componentProps, window, this.metricUrn));
-
- // Hack to retain the raw yaml parameters.
- // The tunable requires raw yaml params and previously tuned params to
generate fresh params
- tunedSpec.put(PROP_YAML_PARAMS, yamlParams);
+ tunedSpec.putAll(tunable.tune(componentProps, window, metricUrn));
return tunedSpec;
}
@@ -162,15 +150,13 @@ public class DetectionConfigTuner {
String type = DetectionUtils.getComponentType(componentKey);
if (!TURNOFF_TUNING_COMPONENTS.contains(type) &&
DETECTION_REGISTRY.isTunable(componentClassName)) {
try {
- tunedComponentProps.put(PROP_CLASS_NAME, componentClassName);
tunedComponentProps.putAll(getTunedSpecs(existingComponentProps,
tuningWindowStart, tuningWindowEnd));
} catch (Exception e) {
LOG.error("Tuning failed for component " + type, e);
}
- } else {
- tunedComponentProps.putAll(existingComponentProps);
}
+ tunedComponentProps.putAll(existingComponentProps);
tunedComponentSpecs.put(componentKey, tunedComponentProps);
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index 9a5c5cb..ab29b4a 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -40,7 +40,6 @@ import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
import
org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
-import org.apache.pinot.thirdeye.detection.components.MockGrouper;
import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper;
@@ -133,6 +132,8 @@ public class DetectionConfigTranslator extends
ConfigTranslator<DetectionConfigD
private static final String PROP_TYPE = "type";
private static final String PROP_CLASS_NAME = "className";
private static final String PROP_PARAMS = "params";
+ private static final String PROP_METRIC = "metric";
+ private static final String PROP_DATASET = "dataset";
private static final String PROP_METRIC_URN = "metricUrn";
private static final String PROP_DIMENSION_FILTER_METRIC =
"dimensionFilterMetric";
private static final String PROP_NESTED_METRIC_URNS = "nestedMetricUrns";
@@ -470,6 +471,9 @@ public class DetectionConfigTranslator extends
ConfigTranslator<DetectionConfigD
String componentClassName = DETECTION_REGISTRY.lookup(type);
componentSpecs.put(PROP_CLASS_NAME, componentClassName);
+ componentSpecs.put(PROP_METRIC, MapUtils.getString(yamlConfig,
PROP_METRIC));
+ componentSpecs.put(PROP_DATASET, MapUtils.getString(yamlConfig,
PROP_DATASET));
+ componentSpecs.put(PROP_FILTERS, MapUtils.getString(yamlConfig,
PROP_FILTERS));
Map<String, Object> params =
ConfigUtils.getMap(yamlConfig.get(PROP_PARAMS));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
