This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4f66513 [TE] Flatten Anomaly Results to a list of Map (#4491)
4f66513 is described below
commit 4f6651392bde5ea106f0ff94a8df5955667d4c41
Author: Yungyu Chung <[email protected]>
AuthorDate: Mon Aug 5 11:09:51 2019 -0700
[TE] Flatten Anomaly Results to a list of Map (#4491)
* Flatten anomaly results to a Map for UI presentation
* Add findByStartTimeInRangeAndDetectionConfigId to
MergedAnomalyResultManager and add test cases
---
.../resources/AnomalyFlattenResource.java | 143 +++++++++++++++++++++
.../datalayer/bao/MergedAnomalyResultManager.java | 2 +
.../bao/jdbc/MergedAnomalyResultManagerImpl.java | 9 ++
.../bao/TestMergedAnomalyResultManager.java | 66 ++++++++++
.../resources/TestAnomalyFlattenResource.java | 124 ++++++++++++++++++
5 files changed, 344 insertions(+)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyFlattenResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyFlattenResource.java
new file mode 100644
index 0000000..2238e81
--- /dev/null
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyFlattenResource.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.dashboard.resources;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+import io.swagger.annotations.ApiOperation;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyFeedback;
+import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+
+
+/**
+ * Flatten the anomaly results for UI purpose.
+ * Convert a list of anomalies to rows of columns so that UI can directly
convert the anomalies to table
+ */
+@Path("thirdeye/table")
+public class AnomalyFlattenResource {
+ private static final DAORegistry DAO_REGISTRY = DAORegistry.getInstance();
+ private MergedAnomalyResultManager mergedAnomalyResultDAO;
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static final String ANOMALY_ID = "anomalyId";
+ public static final String ANOMALY_COMMENT = "comment";
+ public static final String FUNCTION_ID = "functionId";
+ public static final String WINDOW_START = "start";
+ public static final String WINDOW_END = "end";
+ private static final String DIMENSION_KEYS = "keys";
+
+ private static final List<String> REQUIRED_JSON_KEYS
=Arrays.asList(FUNCTION_ID, WINDOW_START, WINDOW_END);
+ private static final String DEFAULT_DELIMINATOR = ",";
+
+ public AnomalyFlattenResource() {
+ this.mergedAnomalyResultDAO = DAO_REGISTRY.getMergedAnomalyResultDAO();
+ }
+
+ public AnomalyFlattenResource(MergedAnomalyResultManager
mergedAnomalyResultDAO) {
+ this.mergedAnomalyResultDAO = mergedAnomalyResultDAO;
+ }
+
+ /**
+ * Flatten a list of anomaly results to a list of map
+ * @param jsonPayload a json string; the jsonPayload should include keys:
[functionId, start, end].
+ * start and end are in the format of epoch time
+ * @return a list of map
+ * @throws IOException
+ */
+ @GET
+ @ApiOperation(value = "View a flatted merged anomalies for collection")
+ public List<Map<String, String>> flatAnomalyResults(String jsonPayload)
throws IOException {
+ if (StringUtils.isBlank(jsonPayload)) {
+ throw new IllegalArgumentException("jsonPayload cannot be null or
empty");
+ }
+ Map<String, Object> inputMap = OBJECT_MAPPER.readValue(jsonPayload,
Map.class);
+
+ // Assert if reuired keys are in the map
+ for (String requiredKey : REQUIRED_JSON_KEYS) {
+ if (!inputMap.containsKey(requiredKey)) {
+ throw new IllegalArgumentException(String.format("Miss %s in input
json String; %s are required", requiredKey,
+ REQUIRED_JSON_KEYS.toString()));
+ }
+ }
+
+ // Retrieve anomalies
+ long functionId = Long.valueOf(inputMap.get(FUNCTION_ID).toString());
+ long start = Long.valueOf(inputMap.get(WINDOW_START).toString());
+ long end = Long.valueOf(inputMap.get(WINDOW_END).toString());
+ List<MergedAnomalyResultDTO> anomalies = mergedAnomalyResultDAO.
+ findByStartTimeInRangeAndDetectionConfigId(start, end, functionId);
+
+ // flatten anomaly result information
+ List<String> tableKeys = null;
+ if (inputMap.containsKey(DIMENSION_KEYS)){
+ Object dimensionKeysObj = inputMap.get(DIMENSION_KEYS);
+ if (dimensionKeysObj instanceof List) {
+ tableKeys = (List<String>) dimensionKeysObj;
+ } else {
+ tableKeys =
Arrays.asList(dimensionKeysObj.toString().split(DEFAULT_DELIMINATOR));
+ }
+ }
+ List<Map<String, String>> resultList = new ArrayList<>();
+ for (MergedAnomalyResultDTO result : anomalies) {
+ resultList.add(flatAnomalyResult(result, tableKeys));
+ }
+ return resultList;
+ }
+
+ /**
+ * Flat an anomaly to a flat map structure
+ * @param anomalyResult an instance of MergedAnomalyResultDTO
+ * @param tableKeys a list of keys in dimensions; if null, use all keys in
dimension
+ * @return a map of information in the anomaly result with the required keys
+ */
+ public static Map<String, String> flatAnomalyResult(MergedAnomalyResultDTO
anomalyResult,
+ List<String> tableKeys) {
+ Preconditions.checkNotNull(anomalyResult);
+ Map<String, String> flatMap = new HashMap<>();
+ flatMap.put(ANOMALY_ID, Long.toString(anomalyResult.getId()));
+ DimensionMap dimension = anomalyResult.getDimensions();
+ if (tableKeys == null) {
+ tableKeys = new ArrayList<>(dimension.keySet());
+ }
+ for (String key : tableKeys) {
+ flatMap.put(key, dimension.containsKey(key)? dimension.get(key) : "");
+ }
+ AnomalyFeedback feedback = anomalyResult.getFeedback();
+ if (feedback != null) {
+ flatMap.put(ANOMALY_COMMENT, feedback.getComment());
+ } else {
+ flatMap.put(ANOMALY_COMMENT, "");
+ }
+ flatMap.put(anomalyResult.getMetric(),
Double.toString(anomalyResult.getAvgCurrentVal()));
+ return flatMap;
+ }
+}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java
index b147193..dad46e2 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java
@@ -60,6 +60,8 @@ public interface MergedAnomalyResultManager extends
AbstractManager<MergedAnomal
List<MergedAnomalyResultDTO> findByStartTimeInRangeAndFunctionId(long
startTime, long endTime, long functionId);
+ List<MergedAnomalyResultDTO> findByStartTimeInRangeAndDetectionConfigId(long
startTime, long endTime, long detectionConfigId);
+
List<MergedAnomalyResultDTO> findByTime(long startTime, long endTime);
List<MergedAnomalyResultDTO>
findUnNotifiedByFunctionIdAndIdLesserThanAndEndTimeGreaterThanLastOneDay(long
functionId,
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
index c66721f..1960c39 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
@@ -253,6 +253,15 @@ public class MergedAnomalyResultManagerImpl extends
AbstractManagerImpl<MergedAn
Predicate predicate =
Predicate.AND(Predicate.LT("startTime", endTime),
Predicate.GT("endTime", startTime),
Predicate.EQ("functionId", functionId));
+ return findByPredicate(predicate);
+ }
+
+ @Override
+ public List<MergedAnomalyResultDTO>
findByStartTimeInRangeAndDetectionConfigId(long startTime, long endTime,
+ long detectionConfigId) {
+ Predicate predicate =
+ Predicate.AND(Predicate.LT("startTime", endTime),
Predicate.GT("endTime", startTime),
+ Predicate.EQ("detectionConfigId", detectionConfigId));
List<MergedAnomalyResultBean> list = genericPojoDao.get(predicate,
MergedAnomalyResultBean.class);
return convertMergedAnomalyBean2DTO(list);
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/bao/TestMergedAnomalyResultManager.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/bao/TestMergedAnomalyResultManager.java
index fe85c0b..0f55842 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/bao/TestMergedAnomalyResultManager.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/bao/TestMergedAnomalyResultManager.java
@@ -17,7 +17,9 @@
package org.apache.pinot.thirdeye.datalayer.bao;
import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
+import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.datalayer.DaoTestUtils;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,6 +28,7 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
+import org.joda.time.DateTime;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -44,6 +47,7 @@ public class TestMergedAnomalyResultManager{
private DAOTestBase testDAOProvider;
private AnomalyFunctionManager anomalyFunctionDAO;
+ private DetectionConfigManager detectionConfigDAO;
private MergedAnomalyResultManager mergedAnomalyResultDAO;
@BeforeClass
@@ -51,6 +55,7 @@ public class TestMergedAnomalyResultManager{
testDAOProvider = DAOTestBase.getInstance();
DAORegistry daoRegistry = DAORegistry.getInstance();
anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
+ detectionConfigDAO = daoRegistry.getDetectionConfigManager();
mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
}
@@ -137,6 +142,33 @@ public class TestMergedAnomalyResultManager{
}
@Test
+ public void testFindByStartTimeInRangeAndDetectionConfigId() {
+ long detectionConfigId = detectionConfigDAO.save(mockDetectionConfig());
+ List<MergedAnomalyResultDTO> anomalies = mockAnomalies(detectionConfigId);
+ for (MergedAnomalyResultDTO anomaly : anomalies) {
+ this.mergedAnomalyResultDAO.save(anomaly);
+ }
+ List<MergedAnomalyResultDTO> fetchedAnomalies = mergedAnomalyResultDAO
+ .findByStartTimeInRangeAndDetectionConfigId(
+ new DateTime(2019, 1, 1, 0, 0).getMillis(),
+ new DateTime(2019, 1, 3, 0, 0).getMillis(),
+ detectionConfigId);
+ Assert.assertEquals(fetchedAnomalies.size(), anomalies.size());
+ for (int i = 0; i < anomalies.size(); i ++) {
+ MergedAnomalyResultDTO actual = fetchedAnomalies.get(i);
+ MergedAnomalyResultDTO expected = anomalies.get(i);
+ Assert.assertNotNull(actual.getId());
+ Assert.assertEquals(actual.getDetectionConfigId(),
expected.getDetectionConfigId());
+ Assert.assertEquals(actual.getDimensions(), expected.getDimensions());
+ }
+ // Clean up
+ for (int i = 0; i < anomalies.size(); i++) {
+ this.mergedAnomalyResultDAO.delete(fetchedAnomalies.get(i));
+ }
+ this.detectionConfigDAO.deleteById(detectionConfigId);
+ }
+
+ @Test
public void testSaveChildrenIndependently() {
MergedAnomalyResultDTO parent = new MergedAnomalyResultDTO();
parent.setStartTime(1000);
@@ -248,4 +280,38 @@ public class TestMergedAnomalyResultManager{
Assert.assertFalse(read.getChildren().iterator().next().getChildren().iterator().next().getChildren().isEmpty());
Assert.assertEquals(read.getChildren().iterator().next().getChildren().iterator().next().getChildren().iterator().next().getStartTime(),
1600);
}
+
+
+ public static DetectionConfigDTO mockDetectionConfig() {
+ DetectionConfigDTO detectionConfig = new DetectionConfigDTO();
+ detectionConfig.setName("Only For Test");
+ return detectionConfig;
+ }
+
+ public static List<MergedAnomalyResultDTO> mockAnomalies(long
detectionConfigId) {
+ MergedAnomalyResultDTO anomaly1 = new MergedAnomalyResultDTO();
+ anomaly1.setMetric("metric");
+ anomaly1.setDetectionConfigId(detectionConfigId);
+ anomaly1.setStartTime(new DateTime(2019, 1, 1, 0, 0).getMillis());
+ anomaly1.setEndTime(new DateTime(2019, 1, 1, 12, 0).getMillis());
+ DimensionMap dimension1 = new DimensionMap();
+ dimension1.put("what", "a");
+ dimension1.put("where", "b");
+ dimension1.put("when", "c");
+ dimension1.put("how", "d");
+ anomaly1.setDimensions(dimension1);
+ MergedAnomalyResultDTO anomaly2 = new MergedAnomalyResultDTO();
+ anomaly2.setMetric("metric");
+ anomaly2.setDetectionConfigId(detectionConfigId);
+ anomaly2.setStartTime(new DateTime(2019, 1, 2, 10, 0).getMillis());
+ anomaly2.setEndTime(new DateTime(2019, 1, 2, 20, 0).getMillis());
+ DimensionMap dimension2 = new DimensionMap();
+ dimension2.put("what", "e");
+ dimension2.put("where", "f");
+ dimension2.put("when", "g");
+ dimension2.put("how", "h");
+ anomaly2.setDimensions(dimension2);
+
+ return Arrays.asList(anomaly1, anomaly2);
+ }
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/resources/TestAnomalyFlattenResource.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/resources/TestAnomalyFlattenResource.java
new file mode 100644
index 0000000..a0317e2
--- /dev/null
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/resources/TestAnomalyFlattenResource.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.datasource.resources;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.thirdeye.dashboard.resources.AnomalyFlattenResource;
+import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.bao.TestMergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.joda.time.DateTime;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static
org.apache.pinot.thirdeye.dashboard.resources.AnomalyFlattenResource.*;
+
+
+public class TestAnomalyFlattenResource {
+ private DAOTestBase testDAOProvider;
+ public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private DetectionConfigManager detectionConfigDAO;
+ private MergedAnomalyResultManager mergedAnomalyResultDAO;
+ private long detectionConfigId;
+ @BeforeClass
+ void beforeClass() {
+ testDAOProvider = DAOTestBase.getInstance();
+ DAORegistry daoRegistry = DAORegistry.getInstance();
+ this.detectionConfigDAO = daoRegistry.getDetectionConfigManager();
+ this.mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
+ this.detectionConfigId =
detectionConfigDAO.save(TestMergedAnomalyResultManager.mockDetectionConfig());
+ for (MergedAnomalyResultDTO anomaly :
TestMergedAnomalyResultManager.mockAnomalies(detectionConfigId)) {
+ mergedAnomalyResultDAO.save(anomaly);
+ }
+ }
+
+ @AfterClass(alwaysRun = true)
+ void afterClass() {
+ testDAOProvider.cleanup();
+ }
+
+ @Test
+ public void testFlatAnomalyResults() throws Exception {
+ Map<String, Object> jsonMap = new HashMap<String, Object>() {
+ {
+ put(FUNCTION_ID, detectionConfigId);
+ put(WINDOW_START, new DateTime(2019, 1, 1, 0, 0).getMillis());
+ put(WINDOW_END, new DateTime(2019, 1, 3, 0, 0).getMillis());
+ }
+ };
+ AnomalyFlattenResource resource = new AnomalyFlattenResource();
+ List<Map<String, String>> actualResults =
resource.flatAnomalyResults(OBJECT_MAPPER.writeValueAsString(jsonMap));
+ List<Map<String, String>> expectedResults = Arrays.asList(
+ new HashMap<String, String>() {
+ {
+ put(ANOMALY_ID, Long.toString(2));
+ put("what", "a");
+ put("where", "b");
+ put("when", "c");
+ put("how", "d");
+ put("metric", Double.toString(0d));
+ put(ANOMALY_COMMENT, "");
+ }
+ },
+ new HashMap<String, String>() {
+ {
+ put(ANOMALY_ID, Long.toString(3));
+ put("what", "e");
+ put("where", "f");
+ put("when", "g");
+ put("how", "h");
+ put("metric", Double.toString(0d));
+ put(ANOMALY_COMMENT, "");
+ }
+ }
+ );
+ Assert.assertEquals(actualResults, expectedResults);
+ }
+
+ @Test
+ public void testFlatAnomalyResult() {
+ List<MergedAnomalyResultDTO> anomalies =
TestMergedAnomalyResultManager.mockAnomalies(detectionConfigId);
+ MergedAnomalyResultDTO anomaly = anomalies.get(0);
+ anomaly.setId(1l);
+ Map<String, String> actualMap =
AnomalyFlattenResource.flatAnomalyResult(anomaly, null);
+ Map<String, String> expectMap = new HashMap<String, String>() {
+ {
+ put(ANOMALY_ID, Long.toString(anomaly.getId()));
+ put("what", "a");
+ put("where", "b");
+ put("when", "c");
+ put("how", "d");
+ put("metric", Double.toString(0d));
+ put(ANOMALY_COMMENT, "");
+ }
+ };
+ Assert.assertEquals(actualMap, expectMap);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
