This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c4e1ac9 [TE] Auto-Resolving Create-Update API for alerts (#4295)
c4e1ac9 is described below
commit c4e1ac91a68971105ee9f61e4bfc14381eb53a4d
Author: Akshay Rai <[email protected]>
AuthorDate: Tue Jun 11 10:18:41 2019 -0700
[TE] Auto-Resolving Create-Update API for alerts (#4295)
* Autoresolve api
* Addressed Xiaohui's comments
* Fixed tests
---
.../validators/DetectionConfigValidator.java | 20 +++---
.../thirdeye/detection/yaml/YamlResource.java | 76 +++++++++++++++++++++-
.../thirdeye/detection/yaml/YamlResourceTest.java | 57 +++++++++++++++-
.../yaml/detection/detection-config-1.yaml | 22 +++++++
.../yaml/detection/detection-config-2.yaml | 22 +++++++
5 files changed, 186 insertions(+), 11 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
index 77c1cdf..42dc0d9 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
@@ -146,9 +146,11 @@ public class DetectionConfigValidator implements
ConfigValidator<DetectionConfig
String alertName = MapUtils.getString(detectionYaml, PROP_NAME);
// Validate all compulsory fields
- Preconditions.checkArgument(detectionYaml.containsKey(PROP_METRIC),
+ String metric = MapUtils.getString(detectionYaml, PROP_METRIC);
+ Preconditions.checkArgument(StringUtils.isNotEmpty(metric),
"Missing property (" + PROP_METRIC + ") in sub-alert " + alertName);
- Preconditions.checkArgument(detectionYaml.containsKey(PROP_DATASET),
+ String dataset = MapUtils.getString(detectionYaml, PROP_DATASET);
+ Preconditions.checkArgument(StringUtils.isNotEmpty(dataset),
"Missing property (" + PROP_DATASET + ") in sub-alert " + alertName);
Preconditions.checkArgument(detectionYaml.containsKey(PROP_RULES),
"Missing property (" + PROP_RULES + ") in sub-alert " + alertName);
@@ -160,10 +162,15 @@ public class DetectionConfigValidator implements
ConfigValidator<DetectionConfig
+ " indentation level of detection yaml it applies to.");
// Check if the metric defined in the config exists
- MetricConfigDTO metricConfig = provider
- .fetchMetric(MapUtils.getString(detectionYaml, PROP_METRIC),
MapUtils.getString(detectionYaml, PROP_DATASET));
+ MetricConfigDTO metricConfig = provider.fetchMetric(metric, dataset);
Preconditions.checkArgument(metricConfig != null,
- "Invalid metric (not found) in sub-alert " + alertName);
+ "Metric doesn't exist in our records. Metric " + metric + " in
sub-alert " + alertName);
+
+ // Check if the dataset defined in the config exists
+ DatasetConfigDTO datasetConfig = provider
+
.fetchDatasets(Collections.singletonList(metricConfig.getDataset())).get(metricConfig.getDataset());
+ Preconditions.checkArgument(datasetConfig != null,
+ "Dataset doesn't exist in our records. Dataset " + dataset + " in
sub-alert " + alertName);
// We support only one grouper per metric
Preconditions.checkArgument(ConfigUtils.getList(detectionYaml.get(PROP_GROUPER)).size()
<= 1,
@@ -197,9 +204,6 @@ public class DetectionConfigValidator implements
ConfigValidator<DetectionConfig
// Safety condition: Validate if maxDuration is greater than 15 minutes
Map<String, Object> mergerProperties = MapUtils.getMap(detectionYaml,
PROP_MERGER, new HashMap());
if (mergerProperties.get(PROP_MAX_DURATION) != null) {
- DatasetConfigDTO datasetConfig = provider
- .fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
- .get(metricConfig.getDataset());
Preconditions.checkArgument(
MapUtils.getLong(mergerProperties, PROP_MAX_DURATION) >=
datasetConfig.bucketTimeGranularity().toMillis(),
"The maxDuration field set is not acceptable. Please check the the
document and set it correctly.");
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 6b5dde0..e3806ea 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -27,6 +27,7 @@ import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -98,6 +99,7 @@ public class YamlResource {
private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String PROP_SUBS_GROUP_NAME = "subscriptionGroupName";
+ private static final String PROP_DETECTION_NAME = "detectionName";
// default onboarding replay period
private static final long ONBOARDING_REPLAY_LOOKBACK =
TimeUnit.DAYS.toMillis(30);
@@ -287,6 +289,10 @@ public class YamlResource {
).build();
}
+ long createDetectionPipeline(String yamlDetectionConfig) {
+ return createDetectionPipeline(yamlDetectionConfig, 0, 0);
+ }
+
long createDetectionPipeline(String yamlDetectionConfig, long startTime,
long endTime)
throws IllegalArgumentException {
Preconditions.checkArgument(StringUtils.isNotBlank(yamlDetectionConfig),
"The Yaml Payload in the request is empty.");
@@ -342,11 +348,15 @@ public class YamlResource {
}
LOG.info("Detection Pipeline created with id " + detectionConfigId + "
using payload " + payload);
- responseMessage.put("message", "The subscription group was created
successfully.");
+ responseMessage.put("message", "Alert was created successfully.");
responseMessage.put("more-info", "Record saved with id " +
detectionConfigId);
return Response.ok().entity(responseMessage).build();
}
+ void updateDetectionPipeline(long detectionID, String yamlDetectionConfig) {
+ updateDetectionPipeline(detectionID, yamlDetectionConfig, 0, 0);
+ }
+
void updateDetectionPipeline(long detectionID, String yamlDetectionConfig,
long startTime, long endTime)
throws IllegalArgumentException {
DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(detectionID);
@@ -414,6 +424,70 @@ public class YamlResource {
return Response.ok().entity(responseMessage).build();
}
+ private DetectionConfigDTO fetchExistingDetection(String payload) {
+ DetectionConfigDTO existingDetectionConfig = null;
+
+ // Extract the detectionName from payload
+ Map<String, Object> detectionConfigMap = new HashMap<>();
+ detectionConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(payload)));
+ String detectionName = MapUtils.getString(detectionConfigMap,
PROP_DETECTION_NAME);
+ Preconditions.checkNotNull(detectionName, "Missing property detectionName
in the detection config.");
+
+ // Check if detection already existing
+ Collection<DetectionConfigDTO> detectionConfigs = this.detectionConfigDAO
+ .findByPredicate(Predicate.EQ("name", detectionName));
+ if (detectionConfigs != null && !detectionConfigs.isEmpty()) {
+ existingDetectionConfig = detectionConfigs.iterator().next();
+ }
+
+ return existingDetectionConfig;
+ }
+
+ long createOrUpdateDetectionPipeline(String payload) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "The Yaml
Payload in the request is empty.");
+ long detectionId;
+ DetectionConfigDTO existingDetection = fetchExistingDetection(payload);
+ if (existingDetection != null) {
+ detectionId = existingDetection.getId();
+ updateDetectionPipeline(detectionId, payload);
+ } else {
+ detectionId = createDetectionPipeline(payload);
+ }
+
+ return detectionId;
+ }
+
+ /**
+ Set up a detection pipeline using a YAML config - create new or update
existing
+ @param payload YAML config string
+ @return a message contains the saved detection config id
+ */
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.TEXT_PLAIN)
+ @ApiOperation("Create a new detection pipeline or update existing if one
already exists")
+ public Response createOrUpdateDetectionPipelineApi(@ApiParam("yaml config")
String payload) {
+ Map<String, String> responseMessage = new HashMap<>();
+ long detectionConfigId;
+ try {
+ detectionConfigId = createOrUpdateDetectionPipeline(payload);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Validation error while creating/updating detection pipeline
with payload " + payload, e);
+ responseMessage.put("message", "Validation Error! " + e.getMessage());
+ return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
+ } catch (Exception e) {
+ LOG.error("Error creating/updating detection pipeline with payload " +
payload, e);
+ responseMessage.put("message", "Failed to create the detection pipeline.
Reach out to the ThirdEye team.");
+ responseMessage.put("more-info", "Error = " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
+ }
+
+ LOG.info("Detection Pipeline created/updated id " + detectionConfigId + "
using payload " + payload);
+ responseMessage.put("message", "The alert was created/updated
successfully.");
+ responseMessage.put("more-info", "Record saved/updated with id " +
detectionConfigId);
+ return Response.ok().entity(responseMessage).build();
+ }
+
long createSubscriptionGroup(String yamlAlertConfig) throws
IllegalArgumentException {
Preconditions.checkArgument(StringUtils.isNotBlank(yamlAlertConfig),
"The Yaml Payload in the request is empty.");
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
index da07db9..f72110c 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
@@ -1,14 +1,20 @@
package org.apache.pinot.thirdeye.detection.yaml;
+import java.util.concurrent.TimeUnit;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import
org.apache.pinot.thirdeye.detection.annotation.registry.DetectionAlertRegistry;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
+import
org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
+import org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector;
+import
org.apache.pinot.thirdeye.detection.yaml.translator.CompositePipelineConfigTranslator;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -16,7 +22,6 @@ import org.testng.annotations.Test;
public class YamlResourceTest {
-
private DAOTestBase testDAOProvider;
private YamlResource yamlResource;
private DAORegistry daoRegistry;
@@ -36,6 +41,8 @@ public class YamlResourceTest {
config2.setName("test_detection_2");
alertId2 = detectionDAO.save(config2);
+
DetectionRegistry.getInstance().registerYamlConvertor(CompositePipelineConfigTranslator.class.getName(),
"COMPOSITE");
+ DetectionRegistry.registerComponent(ThresholdRuleDetector.class.getName(),
"THRESHOLD");
DetectionAlertRegistry.getInstance().registerAlertScheme("EMAIL",
"EmailClass");
DetectionAlertRegistry.getInstance().registerAlertScheme("IRIS",
"IrisClass");
DetectionAlertRegistry.getInstance().registerAlertSuppressor("TIME_WINDOW",
"TimeWindowClass");
@@ -48,7 +55,53 @@ public class YamlResourceTest {
}
@Test
- public void testCreateDetectionAlertConfig() throws IOException {
+ public void testCreateOrUpdateDetectionConfig() throws IOException {
+ String blankYaml = "";
+ try {
+ this.yamlResource.createOrUpdateDetectionPipeline(blankYaml);
+ Assert.fail("Exception not thrown on empty yaml");
+ } catch (Exception e) {
+ Assert.assertEquals(e.getMessage(), "The Yaml Payload in the request is
empty.");
+ }
+
+ MetricConfigDTO metricConfig = new MetricConfigDTO();
+ metricConfig.setAlias("test_alias");
+ metricConfig.setName("test_metric");
+ metricConfig.setDataset("test_dataset");
+ daoRegistry.getMetricConfigDAO().save(metricConfig);
+
+ DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO();
+ datasetConfigDTO.setDataset("test_dataset");
+ datasetConfigDTO.setTimeUnit(TimeUnit.DAYS);
+ datasetConfigDTO.setTimeDuration(1);
+ daoRegistry.getDatasetConfigDAO().save(datasetConfigDTO);
+
+ // Create a new detection
+ String validYaml =
IOUtils.toString(this.getClass().getResourceAsStream("detection/detection-config-1.yaml"));
+ try {
+ long id = this.yamlResource.createOrUpdateDetectionPipeline(validYaml);
+ DetectionConfigDTO detection =
daoRegistry.getDetectionConfigManager().findById(id);
+ Assert.assertNotNull(detection);
+ Assert.assertEquals(detection.getName(), "testPipeline");
+ } catch (Exception e) {
+ Assert.fail("Exception should not be thrown for valid yaml. Message: " +
e + " Cause: " + e.getCause());
+ }
+
+ // Update above created detection
+ String updatedYaml =
IOUtils.toString(this.getClass().getResourceAsStream("detection/detection-config-2.yaml"));
+ try {
+ long id = this.yamlResource.createOrUpdateDetectionPipeline(updatedYaml);
+ DetectionConfigDTO detection =
daoRegistry.getDetectionConfigManager().findById(id);
+ Assert.assertNotNull(detection);
+ Assert.assertEquals(detection.getName(), "testPipeline");
+ Assert.assertEquals(detection.getDescription(), "My modified pipeline");
+ } catch (Exception e) {
+ Assert.fail("Exception should not be thrown if detection already exists.
Message: " + e + " Cause: " + e.getCause());
+ }
+ }
+
+ @Test
+ public void testCreateOrDetectionAlertConfig() throws IOException {
String blankYaml = "";
try {
this.yamlResource.createSubscriptionGroup(blankYaml);
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/detection-config-1.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/detection-config-1.yaml
new file mode 100644
index 0000000..9c19a76
--- /dev/null
+++
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/detection-config-1.yaml
@@ -0,0 +1,22 @@
+detectionName: testPipeline
+description: My test pipeline
+type: METRIC_ALERT
+metric: test_metric
+dataset: test_dataset
+filters:
+ D1:
+ - v1
+ - v2
+ D2:
+ - v3
+dimensionExploration:
+ dimensions:
+ - D1
+ - D2
+ minContribution: 0.05
+rules:
+ - detection:
+ - type: THRESHOLD
+ name: maxThreshold_1
+ params:
+ max: 100
diff --git
a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/detection-config-2.yaml
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/detection-config-2.yaml
new file mode 100644
index 0000000..1330155
--- /dev/null
+++
b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/detection-config-2.yaml
@@ -0,0 +1,22 @@
+detectionName: testPipeline
+description: My modified pipeline
+type: METRIC_ALERT
+metric: test_metric
+dataset: test_dataset
+filters:
+ D1:
+ - v1
+ - v2
+ D2:
+ - v3
+dimensionExploration:
+ dimensions:
+ - D1
+ - D2
+ minContribution: 0.05
+rules:
+ - detection:
+ - type: THRESHOLD
+ name: maxThreshold_1
+ params:
+ max: 100
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
