This is an automated email from the ASF dual-hosted git repository. xhsun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push: new 440a3a3 [TE] Endpoint for upper/lower bounds (#4160) 440a3a3 is described below commit 440a3a30b66c7b0a2bcc07fd1935a4201fbcabd7 Author: Xiaohui Sun <xh...@linkedin.com> AuthorDate: Wed Apr 24 13:28:18 2019 -0700 [TE] Endpoint for upper/lower bounds (#4160) --- .../thirdeye/dataframe/util/DataFrameUtils.java | 1 + .../thirdeye/detection/spi/model/TimeSeries.java | 4 + .../thirdeye/detection/yaml/YamlResource.java | 93 ++++++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java index 3a66a79..15b810d 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java @@ -61,6 +61,7 @@ import org.joda.time.PeriodType; */ public class DataFrameUtils { public static final String COL_TIME = "timestamp"; + // baseline value public static final String COL_VALUE = "value"; public static final String COL_CURRENT = "current"; public static final String COL_UPPER_BOUND = "upper_bound"; diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java index 8758c19..a513037 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java @@ -81,6 +81,10 @@ public class TimeSeries { return this.df.getDoubles(DataFrameUtils.COL_CURRENT); } + public LongSeries getTime() { + return this.df.getLongs(DataFrameUtils.COL_TIME); + } + public DoubleSeries getPredictedBaseline() { return this.df.getDoubles(DataFrameUtils.COL_VALUE); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java index 81451e0..c4e1160 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java @@ -31,8 +31,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import javax.validation.constraints.NotNull; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -48,6 +51,7 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.thirdeye.anomaly.task.TaskConstants; import org.apache.pinot.thirdeye.api.Constants; +import org.apache.pinot.thirdeye.dataframe.util.MetricSlice; import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager; import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager; @@ -72,12 +76,16 @@ import org.apache.pinot.thirdeye.detection.DetectionPipeline; import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader; import org.apache.pinot.thirdeye.detection.DetectionPipelineResult; import org.apache.pinot.thirdeye.detection.onboard.YamlOnboardingTaskInfo; +import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider; +import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries; import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator; import org.apache.pinot.thirdeye.detection.validators.SubscriptionConfigValidator; +import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; +import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*; import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*; @@ -630,6 +638,91 @@ public class YamlResource { return Response.ok(result).build(); } + @POST + @Path("/preview/baseline") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.TEXT_PLAIN) + @ApiOperation("Get baseline from YAML configuration") + /* TODO: Will return baseline from yamlPreviewApi together with detection in the future. */ + public Response yamlPreviewBaselineApi( + @QueryParam("start") long start, + @QueryParam("end") long end, + @QueryParam("urn") @NotNull String urn, + @QueryParam("tuningStart") long tuningStart, + @QueryParam("tuningEnd") long tuningEnd, + @ApiParam("jsonPayload") String payload, + @QueryParam("ruleName") String ruleName) { + try { + Preconditions.checkArgument(StringUtils.isNotBlank(payload), "The Yaml Payload in the request is empty."); + + // Translate config from YAML to detection config (JSON) + Map<String, Object> newDetectionConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(payload))); + DetectionConfigDTO detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, newDetectionConfigMap, null); + Preconditions.checkNotNull(detectionConfig); + detectionConfig.setId(Long.MAX_VALUE); + + // There is a side effect to update detectionConfig when loading the pipeline. + this.loader.from(this.provider, detectionConfig, start, end); + TimeSeries baseline = getBaseline(detectionConfig, start, end, urn, ruleName); + + return Response.ok(makeTimeSeriesMap(baseline)).build(); + } catch (Exception e) { + LOG.error("Error getting baseline with payload " + payload, e); + } + return Response.ok().build(); + } + + /** + * Returns a map of time/baseline/current/upper/lower time series derived from the TimeSeries. + * + * @param baseline Baseline values. + * @return map of time/baseline/current/upper/lower time series. + */ + private static Map<String, List<? extends Number>> makeTimeSeriesMap(TimeSeries baseline) { + Map<String, List<? extends Number>> output = new HashMap<>(); + // time and baseline are mandatory + output.put(COL_TIME, baseline.getTime().toList()); + output.put(COL_VALUE, baseline.getPredictedBaseline().toList()); + if (baseline.getDataFrame().contains(COL_CURRENT)) { + output.put(COL_CURRENT, baseline.getCurrent().toList()); + } + if (baseline.getDataFrame().contains(COL_UPPER_BOUND)) { + output.put(COL_UPPER_BOUND, baseline.getPredictedUpperBound().toList()); + } + if (baseline.getDataFrame().contains(COL_LOWER_BOUND)) { + output.put(COL_LOWER_BOUND, baseline.getPredictedLowerBound().toList()); + } + return output; + } + + /** + * Get the baselines for metric urn. + * If there are multiple rules return the first rule's baseline. + * TODO: The baseline should be calculated together with detection in the future. + * + * @param detectionConfig The detection configuration. + * @param start Start time for baseline calculation. + * @param end End time for baseline calculation. + * @param urn The metric urn. + * @param rule The rule name. If not provided then find the first rule. + * @return The baseline for the urn. + */ + private TimeSeries getBaseline(DetectionConfigDTO detectionConfig, long start, long end, String urn, String rule) { + MetricEntity metric = MetricEntity.fromURN(urn); + MetricSlice slice = MetricSlice.from(metric.getId(), start, end, metric.getFilters(), MetricSlice.NATIVE_GRANULARITY); + + Optional<BaselineProvider> provider = detectionConfig.getComponents().entrySet().stream() + .filter(x -> x.getValue() instanceof BaselineProvider && (rule.isEmpty() || x.getKey().startsWith(rule))) + .map(x -> (BaselineProvider) x.getValue()) + .findFirst(); + + if (provider.isPresent()) { + return provider.get().computePredictedTimeSeries(slice); + } + + return new TimeSeries(); + } + /** * List all yaml configurations as JSON enhanced with detection config id, isActive and createBy information. * --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org