This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 983c5d8 [TE] Update TE-Raptor API to support multiple metrics (#4140)
983c5d8 is described below
commit 983c5d879e0cc9ccf02e0428bd48fa2da51d2970
Author: Akshay Rai <[email protected]>
AuthorDate: Fri Apr 19 15:12:11 2019 -0700
[TE] Update TE-Raptor API to support multiple metrics (#4140)
* Leverage JAX-RS automatic parameter conversion
---
.../api/user/dashboard/UserDashboardResource.java | 153 ++++++++++++++-------
.../user/dashboard}/UserDashboardResourceTest.java | 58 ++++++--
2 files changed, 149 insertions(+), 62 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResource.java
index 48612f2..dab31bb 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResource.java
@@ -25,10 +25,13 @@ import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
@@ -38,6 +41,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.commons.lang.StringUtils;
import org.apache.pinot.thirdeye.api.Constants;
import org.apache.pinot.thirdeye.constant.AnomalyFeedbackType;
@@ -85,6 +89,65 @@ public class UserDashboardResource {
this.detectionAlertDAO = detectionAlertDAO;
}
+ List<AnomalySummary> queryAnomalies(Long start, Long end, String
application, String group, String metric,
+ String dataset, List<MetricDatasetPair> metricDatasetPairs, boolean
fetchTrueAnomaly, Integer limit) {
+ if (limit == null) {
+ LOG.warn("No upper limit specified while fetching anomalies. Defaulting
to " + ANOMALIES_LIMIT_DEFAULT);
+ limit = ANOMALIES_LIMIT_DEFAULT;
+ }
+ Preconditions.checkNotNull(start, "Please specify the start time of the
anomaly retrieval window");
+
+ // TODO: Prefer to have intersection of anomalies rather than union
+ List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
+ // Fetch anomalies by group
+ anomalies.addAll(fetchAnomaliesBySubsGroup(start, end, group));
+ // Fetch anomalies by application
+ anomalies.addAll(fetchAnomaliesByApplication(start, end, application));
+ // Fetch anomalies by metric and/or dataset
+ anomalies.addAll(fetchAnomaliesByMetricDataset(start, end, metric,
dataset));
+ // Fetch anomalies by metric dataset pairs
+ anomalies.addAll(fetchAnomaliesByMetricDatasetPairs(start, end,
metricDatasetPairs));
+
+ // sort descending by start time
+ Collections.sort(anomalies, (o1, o2) -> -1 *
Long.compare(o1.getStartTime(), o2.getStartTime()));
+
+ if (fetchTrueAnomaly) {
+ // Filter and retain only true anomalies
+ List<MergedAnomalyResultDTO> trueAnomalies = new ArrayList<>();
+ for (MergedAnomalyResultDTO anomaly : anomalies) {
+ if (anomaly.getFeedback() != null &&
anomaly.getFeedback().getFeedbackType().isAnomaly()) {
+ trueAnomalies.add(anomaly);
+ }
+ }
+ anomalies = trueAnomalies;
+ }
+ // filter child anomalies
+ anomalies = anomalies.stream().filter(anomaly ->
!anomaly.isChild()).collect(Collectors.toList());
+ // limit result size
+ anomalies = anomalies.subList(0, Math.min(anomalies.size(), limit));
+
+ return getAnomalyFormattedOutput(anomalies);
+ }
+
+ protected static class MetricDatasetPair {
+ String datasetName;
+ String metricName;
+
+ MetricDatasetPair(String dataset, String metric) {
+ this.datasetName = dataset;
+ this.metricName = metric;
+ }
+
+ public static MetricDatasetPair fromString(String metricDatasetPair){
+ String[] metricDataset = metricDatasetPair.trim().split("::");
+ if (metricDataset.length != 2) {
+ throw new RuntimeException("Unable to parse dataset::metric pair " +
metricDatasetPair);
+ }
+
+ return new MetricDatasetPair(metricDataset[0], metricDataset[1]);
+ }
+ }
+
/**
* Returns a list of AnomalySummary for a set of query parameters. Anomalies
are
* sorted by start time (descending).
@@ -122,7 +185,7 @@ public class UserDashboardResource {
@GET
@Path("/anomalies")
@ApiOperation(value = "Query anomalies")
- public List<AnomalySummary> queryAnomalies(
+ public Response fetchAnomalies(
@ApiParam(value = "start time of anomaly retrieval window")
@QueryParam("start") Long start,
@ApiParam(value = "end time of anomaly retrieval window")
@@ -135,57 +198,25 @@ public class UserDashboardResource {
@QueryParam("metric") String metric,
@ApiParam(value = "The name of the pinot table to which this metric
belongs")
@QueryParam("dataset") String dataset,
+ @ApiParam(value = "Specify multiple dataset::metric pairs")
+ @QueryParam("metricAlias") List<MetricDatasetPair> metricDatasetPairs,
@ApiParam(value = "Specify if you want to only fetch true anomalies")
@QueryParam("fetchTrueAnomaly") @DefaultValue("false") boolean
fetchTrueAnomaly,
@ApiParam(value = "max number of results")
- @QueryParam("limit") Integer limit) throws Exception {
- LOG.info("[USER DASHBOARD] Fetching anomalies with filters. Start: " +
start + " end: " + end + " metric: "
- + metric + " dataset: " + dataset + " application: " + application + "
group: " + group
- + " fetchTrueAnomaly: " + fetchTrueAnomaly + " limit: " + limit);
-
- // Safety conditions
- if (limit == null) {
- LOG.warn("No upper limit specified while fetching anomalies. Defaulting
to " + ANOMALIES_LIMIT_DEFAULT);
- limit = ANOMALIES_LIMIT_DEFAULT;
- }
- Preconditions.checkNotNull(start, "Please specify the start time of the
anomaly retrieval window");
-
- // TODO support index select on user-reported anomalies
-// predicates.add(Predicate.OR(
-// Predicate.EQ("notified", true),
-// Predicate.EQ("anomalyResultSource",
AnomalyResultSource.USER_LABELED_ANOMALY)));
-
- // TODO: Prefer to have intersection of anomalies rather than union
- List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
- // Fetch anomalies by group
- anomalies.addAll(fetchAnomaliesBySubsGroup(start, end, group));
- // Fetch anomalies by application
- anomalies.addAll(fetchAnomaliesByApplication(start, end, application));
- // Fetch anomalies by metric and/or dataset
- anomalies.addAll(fetchAnomaliesByMetricDataset(start, end, metric,
dataset));
-
- // sort descending by start time
- Collections.sort(anomalies, (o1, o2) -> -1 *
Long.compare(o1.getStartTime(), o2.getStartTime()));
-
- if (fetchTrueAnomaly) {
- // Filter and retain only true anomalies
- List<MergedAnomalyResultDTO> trueAnomalies = new ArrayList<>();
- for (MergedAnomalyResultDTO anomaly : anomalies) {
- if (anomaly.getFeedback() != null &&
anomaly.getFeedback().getFeedbackType().isAnomaly()) {
- trueAnomalies.add(anomaly);
- }
- }
- anomalies = trueAnomalies;
+ @QueryParam("limit") Integer limit) {
+ Map<String, String> responseMessage = new HashMap<>();
+ List<AnomalySummary> output;
+ try {
+ output = queryAnomalies(start, end, application, group, metric, dataset,
metricDatasetPairs, fetchTrueAnomaly, limit);
+ } catch (Exception e) {
+ LOG.warn("Error while fetching anomalies.", e.getMessage());
+ responseMessage.put("message", "Failed to fetch all the anomalies.");
+ responseMessage.put("more-info", "Error = " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
}
- // filter child anomalies
- anomalies = anomalies.stream().filter(anomaly ->
!anomaly.isChild()).collect(Collectors.toList());
- // limit result size
- anomalies = anomalies.subList(0, Math.min(anomalies.size(), limit));
-
- List<AnomalySummary> output = getAnomalyFormattedOutput(anomalies);
LOG.info("Successfully returned " + output.size() + " anomalies.");
- return output;
+ return Response.ok(output).build();
}
private List<AnomalySummary>
getAnomalyFormattedOutput(List<MergedAnomalyResultDTO> anomalies) {
@@ -238,6 +269,30 @@ public class UserDashboardResource {
return output;
}
+ private Collection<MergedAnomalyResultDTO>
fetchAnomaliesByMetricDatasetPairs(Long start, Long end,
List<MetricDatasetPair> metricsRef) {
+ List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
+
+ if (metricsRef == null) {
+ return Collections.emptyList();
+ }
+
+ List<Predicate> predicates = new ArrayList<>();
+ predicates.add(Predicate.GE("endTime", start));
+ if (end != null) {
+ predicates.add(Predicate.LT("startTime", end));
+ }
+
+ for (MetricDatasetPair metricDatasetPair : metricsRef) {
+ List<Predicate> metricDatasetPred = new ArrayList<>(predicates);
+
+ metricDatasetPred.add(Predicate.EQ("collection",
metricDatasetPair.datasetName));
+ metricDatasetPred.add(Predicate.EQ("metric",
metricDatasetPair.metricName));
+
anomalies.addAll(this.anomalyDAO.findByPredicate(Predicate.AND(metricDatasetPred.toArray(new
Predicate[0]))));
+ }
+
+ return anomalies;
+ }
+
private Collection<MergedAnomalyResultDTO>
fetchAnomaliesByMetricDataset(Long start, Long end, String metric, String
dataset) {
if (StringUtils.isBlank(metric) && StringUtils.isBlank(dataset)) {
return Collections.emptyList();
@@ -260,7 +315,7 @@ public class UserDashboardResource {
return
this.anomalyDAO.findByPredicate(Predicate.AND(predicates.toArray(new
Predicate[predicates.size()])));
}
- private Collection<MergedAnomalyResultDTO> fetchAnomaliesByApplication(Long
start, Long end, String application) throws Exception {
+ private Collection<MergedAnomalyResultDTO> fetchAnomaliesByApplication(Long
start, Long end, String application) {
if (StringUtils.isBlank(application)) {
return Collections.emptyList();
}
@@ -276,7 +331,7 @@ public class UserDashboardResource {
return fetchAnomaliesByConfigIds(start, end, detectionConfigIds);
}
- private Collection<MergedAnomalyResultDTO> fetchAnomaliesBySubsGroup(Long
start, Long end, String group) throws Exception {
+ private Collection<MergedAnomalyResultDTO> fetchAnomaliesBySubsGroup(Long
start, Long end, String group) {
if (StringUtils.isBlank(group)) {
return Collections.emptyList();
}
@@ -292,7 +347,7 @@ public class UserDashboardResource {
return fetchAnomaliesByConfigIds(start, end, detectionConfigIds);
}
- private Collection<MergedAnomalyResultDTO> fetchAnomaliesByConfigIds(Long
start, Long end, Set<Long> detectionConfigIds) throws Exception {
+ private Collection<MergedAnomalyResultDTO> fetchAnomaliesByConfigIds(Long
start, Long end, Set<Long> detectionConfigIds) {
if (detectionConfigIds.isEmpty()) {
return Collections.emptyList();
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resource/v2/UserDashboardResourceTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResourceTest.java
similarity index 76%
rename from
thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resource/v2/UserDashboardResourceTest.java
rename to
thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResourceTest.java
index 40a3efc..1d06469 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resource/v2/UserDashboardResourceTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/user/dashboard/UserDashboardResourceTest.java
@@ -1,23 +1,38 @@
-package org.apache.pinot.thirdeye.dashboard.resource.v2;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.api.user.dashboard;
import java.util.HashMap;
import java.util.Map;
-import org.apache.pinot.thirdeye.api.user.dashboard.UserDashboardResource;
import org.apache.pinot.thirdeye.dashboard.resources.v2.pojo.AnomalySummary;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import java.util.ArrayList;
import java.util.Arrays;
@@ -72,7 +87,6 @@ public class UserDashboardResourceTest {
// anomalies
this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
- this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
this.anomalyIds = new ArrayList<>();
this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(100, 500,
this.detectionIds.get(0), "test_metric", "test_dataset"))); // myDetectionA
this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(800, 1200,
this.detectionIds.get(0), "test_metric", "test_dataset"))); // myDetectionA
@@ -110,47 +124,65 @@ public class UserDashboardResourceTest {
@Test
public void testAnomaliesByApplication() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
"myApplicationA", null, null, null, false, null);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
"myApplicationA", null, null, null, null, false, null);
Assert.assertEquals(anomalies.size(), 2);
Assert.assertEquals(extractIds(anomalies), makeSet(this.anomalyIds.get(1),
this.anomalyIds.get(2)));
}
@Test
public void testAnomaliesByApplicationInvalid() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
"Invalid", null, null, null, false, null);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
"Invalid", null, null, null, null, false, null);
Assert.assertEquals(anomalies.size(), 0);
}
@Test
public void testAnomaliesByGroup() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, "myAlertB", null, null, false, null);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, "myAlertB", null, null, null, false, null);
Assert.assertEquals(anomalies.size(), 2);
Assert.assertEquals(extractIds(anomalies), makeSet(this.anomalyIds.get(3),
this.anomalyIds.get(4)));
}
@Test
public void testAnomaliesByGroupInvalid() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, "Invalid", null, null, false, null);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, "Invalid", null, null, null, false, null);
Assert.assertEquals(anomalies.size(), 0);
}
@Test
public void testAnomaliesLimit() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
"myApplicationA", null, null, null, false, 1);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
"myApplicationA", null, null, null, null, false, 1);
Assert.assertEquals(anomalies.size(), 1);
Assert.assertEquals(extractIds(anomalies),
makeSet(this.anomalyIds.get(1)));
}
@Test
public void testAnomaliesByMetric() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, null, "test_metric", "test_dataset", false, null);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, null, "test_metric", "test_dataset", null, false, null);
Assert.assertEquals(anomalies.size(), 3);
Assert.assertEquals(extractIds(anomalies), makeSet(this.anomalyIds.get(1),
this.anomalyIds.get(2), this.anomalyIds.get(3)));
}
@Test
public void testAnomaliesByDataset() throws Exception {
- List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, null, null, "test_dataset", false, null);
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, null, null, "test_dataset", null, false, null);
+ Assert.assertEquals(anomalies.size(), 4);
+ Assert.assertEquals(extractIds(anomalies), makeSet(this.anomalyIds.get(1),
this.anomalyIds.get(2), this.anomalyIds.get(3), this.anomalyIds.get(4)));
+ }
+
+ @Test
+ public void testAnomaliesByMetricDatasetPairs() throws Exception {
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, null, null, null,
+ Collections.singletonList(new
UserDashboardResource.MetricDatasetPair("test_dataset", "test_metric")), false,
null);
+ Assert.assertEquals(anomalies.size(), 3);
+ Assert.assertEquals(extractIds(anomalies), makeSet(this.anomalyIds.get(1),
this.anomalyIds.get(2), this.anomalyIds.get(3)));
+ }
+
+ @Test
+ public void testAnomaliesByMultipleMetricDatasetPairs() throws Exception {
+ List<UserDashboardResource.MetricDatasetPair> metricPairs = new
ArrayList<>();
+ metricPairs.add(new
UserDashboardResource.MetricDatasetPair("test_dataset", "test_metric"));
+ metricPairs.add(new
UserDashboardResource.MetricDatasetPair("test_dataset", "test_metric_2"));
+ List<AnomalySummary> anomalies = this.resource.queryAnomalies(1000L, null,
null, null, null, null, metricPairs, false, null);
Assert.assertEquals(anomalies.size(), 4);
Assert.assertEquals(extractIds(anomalies), makeSet(this.anomalyIds.get(1),
this.anomalyIds.get(2), this.anomalyIds.get(3), this.anomalyIds.get(4)));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
