This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b29ad2e [TE] Add validations for the threshold rule anomaly filter
(#5278)
b29ad2e is described below
commit b29ad2ed7a9e2cb19f744f1860dc5392fb4c8ae8
Author: Jihao Zhang <[email protected]>
AuthorDate: Wed Apr 22 11:28:29 2020 -0700
[TE] Add validations for the threshold rule anomaly filter (#5278)
The threshold rule anomaly filter is used for filtering the anomaly based
on the current value in the anomaly duration.
However, if the metric is not aggregated by SUM or COUNT, the user may set
the wrong parameter and cause the system to filter out the anomaly based on the
wrong value. This commit fixes the issue.
---
.../components/ThresholdRuleAnomalyFilter.java | 32 ++++++++++++++--------
1 file changed, 21 insertions(+), 11 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
index 0f2e7b4..68a1369 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
@@ -19,26 +19,20 @@
package org.apache.pinot.thirdeye.detection.components;
+import java.util.Collections;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.thirdeye.dataframe.DataFrame;
-import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.constant.MetricAggFunction;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.detection.InputDataFetcher;
import org.apache.pinot.thirdeye.detection.annotation.Components;
import org.apache.pinot.thirdeye.detection.annotation.DetectionTag;
-import org.apache.pinot.thirdeye.detection.annotation.Param;
-import org.apache.pinot.thirdeye.detection.annotation.PresentationOption;
import org.apache.pinot.thirdeye.detection.spec.ThresholdRuleFilterSpec;
import org.apache.pinot.thirdeye.detection.spi.components.AnomalyFilter;
-import org.apache.pinot.thirdeye.detection.spi.model.InputData;
import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
-import java.util.Collections;
-import java.util.Map;
import org.joda.time.Interval;
-import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
-
/**
* This threshold rule filter stage filters the anomalies if either the min or
max thresholds do not pass.
@@ -52,13 +46,23 @@ public class ThresholdRuleAnomalyFilter implements
AnomalyFilter<ThresholdRuleFi
private double maxValueDaily;
private double maxValue;
private double minValue;
+ private InputDataFetcher dataFetcher;
+
@Override
public boolean isQualified(MergedAnomalyResultDTO anomaly) {
+ MetricEntity me = MetricEntity.fromURN(anomaly.getMetricUrn());
+ MetricConfigDTO metric = dataFetcher.fetchData(new
InputDataSpec().withMetricIds(Collections.singleton(me.getId())))
+ .getMetrics()
+ .get(me.getId());
double currentValue = anomaly.getAvgCurrentVal();
Interval anomalyInterval = new Interval(anomaly.getStartTime(),
anomaly.getEndTime());
- double hourlyMultiplier = TimeUnit.HOURS.toMillis(1) / (double)
anomalyInterval.toDurationMillis();
- double dailyMultiplier = TimeUnit.DAYS.toMillis(1) / (double)
anomalyInterval.toDurationMillis();
+ // apply multiplier if the metric is aggregated by SUM or COUNT
+ double hourlyMultiplier =
+ isAdditive(metric) ? (TimeUnit.HOURS.toMillis(1) / (double)
anomalyInterval.toDurationMillis()) : 1.0;
+ double dailyMultiplier =
+ isAdditive(metric) ? (TimeUnit.DAYS.toMillis(1) / (double)
anomalyInterval.toDurationMillis()) : 1.0;
+
if (!Double.isNaN(this.minValue) && currentValue < this.minValue
|| !Double.isNaN(this.maxValue) && currentValue > this.maxValue) {
return false;
@@ -78,6 +82,11 @@ public class ThresholdRuleAnomalyFilter implements
AnomalyFilter<ThresholdRuleFi
return true;
}
+ private boolean isAdditive(MetricConfigDTO metric) {
+ MetricAggFunction aggFunction = metric.getDefaultAggFunction();
+ return aggFunction.equals(MetricAggFunction.SUM) ||
aggFunction.equals(MetricAggFunction.COUNT);
+ }
+
@Override
public void init(ThresholdRuleFilterSpec spec, InputDataFetcher dataFetcher)
{
this.minValueHourly = spec.getMinValueHourly();
@@ -86,5 +95,6 @@ public class ThresholdRuleAnomalyFilter implements
AnomalyFilter<ThresholdRuleFi
this.maxValueDaily = spec.getMaxValueDaily();
this.maxValue = spec.getMaxValue();
this.minValue = spec.getMinValue();
+ this.dataFetcher = dataFetcher;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
