This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 67b729d [TE] yaml - more validation on max duration (#3982)
67b729d is described below
commit 67b729d4b0935e652be31d5a3fff025b9f361204
Author: Jihao Zhang <[email protected]>
AuthorDate: Mon Mar 18 12:28:26 2019 -0700
[TE] yaml - more validation on max duration (#3982)
---
.../yaml/CompositePipelineConfigTranslator.java | 29 +++++++++++-----------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 13ffbdf..657efa6 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -182,19 +182,17 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
private static final Set<String> MOVING_WINDOW_DETECTOR_TYPES =
ImmutableSet.of("ALGORITHM", "MIGRATED_ALGORITHM");
private final Map<String, Object> components = new HashMap<>();
- private MetricConfigDTO metricConfig;
- private DatasetConfigDTO datasetConfig;
- private String metricUrn;
- private Map<String, Object> mergerProperties = new HashMap<>();
+ private final MetricConfigDTO metricConfig;
+ private final DatasetConfigDTO datasetConfig;
+ private final String metricUrn;
+ private final Map<String, Object> mergerProperties;
+ // metric dimension filter maps
+ private final Map<String, Collection<String>> filterMaps;
protected final org.yaml.snakeyaml.Yaml yaml;
public CompositePipelineConfigTranslator(Map<String, Object> yamlConfig,
DataProvider provider) {
super(yamlConfig, provider);
this.yaml = new org.yaml.snakeyaml.Yaml();
- }
-
- @Override
- YamlTranslationResult translateYaml() {
this.metricConfig =
this.dataProvider.fetchMetric(MapUtils.getString(yamlConfig, PROP_METRIC),
MapUtils.getString(yamlConfig, PROP_DATASET));
Preconditions.checkNotNull(this.metricConfig, "Metric not found");
@@ -202,12 +200,13 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
this.datasetConfig =
this.dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
.get(metricConfig.getDataset());
Preconditions.checkNotNull(this.datasetConfig, "dataset not found");
-
- // if user set merger properties
this.mergerProperties = MapUtils.getMap(yamlConfig, PROP_MERGER, new
HashMap());
-
- Map<String, Collection<String>> filterMaps = MapUtils.getMap(yamlConfig,
PROP_FILTERS);
+ this.filterMaps = MapUtils.getMap(yamlConfig, PROP_FILTERS);
this.metricUrn = buildMetricUrn(filterMaps, this.metricConfig.getId());
+ }
+
+ @Override
+ YamlTranslationResult translateYaml() {
String detectionCronInYaml = MapUtils.getString(yamlConfig, PROP_CRON);
String cron = (detectionCronInYaml == null) ? buildCron() :
detectionCronInYaml;
@@ -228,14 +227,14 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
nestedPipelines.addAll(filterNestedProperties);
}
}
- Map<String, Object> dimensionWrapperProperties =
buildDimensionWrapperProperties(filterMaps);
+ Map<String, Object> dimensionWrapperProperties =
buildDimensionWrapperProperties();
Map<String, Object> properties =
buildWrapperProperties(ChildKeepingMergeWrapper.class.getName(),
Collections.singletonList(
buildWrapperProperties(DimensionWrapper.class.getName(),
nestedPipelines, dimensionWrapperProperties)), this.mergerProperties);
return new
YamlTranslationResult().withProperties(properties).withComponents(this.components).withCron(cron);
}
- private Map<String, Object> buildDimensionWrapperProperties(Map<String,
Collection<String>> filterMaps) {
+ private Map<String, Object> buildDimensionWrapperProperties() {
Map<String, Object> dimensionWrapperProperties = new HashMap<>();
dimensionWrapperProperties.put(PROP_NESTED_METRIC_URNS,
Collections.singletonList(this.metricUrn));
if (yamlConfig.containsKey(PROP_DIMENSION_EXPLORATION)) {
@@ -470,7 +469,7 @@ public class CompositePipelineConfigTranslator extends
YamlDetectionConfigTransl
// Safety condition: Validate if maxDuration is greater than 15 minutes
Map<String, Object> mergerProperties = MapUtils.getMap(yamlConfig,
PROP_MERGER, new HashMap());
if (mergerProperties.get(PROP_MAX_DURATION) != null) {
- Preconditions.checkArgument(MapUtils.getLong(mergerProperties,
PROP_MAX_DURATION) >= TimeUnit.MINUTES.toMillis(15),
+ Preconditions.checkArgument(MapUtils.getLong(mergerProperties,
PROP_MAX_DURATION) >= datasetConfig.bucketTimeGranularity().toMillis(),
"The maxDuration field set is not acceptable. Please check the the
document and set it correctly.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
