This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new ac058c5 [TE] Refactor detection & preview API for better debugging
(#3812)
ac058c5 is described below
commit ac058c5c9b5caf489d751ce04920e53072d7e386
Author: Akshay Rai <[email protected]>
AuthorDate: Fri Feb 8 16:59:59 2019 -0800
[TE] Refactor detection & preview API for better debugging (#3812)
---
.../detection/DetectionMigrationResource.java | 13 +-
.../detection/validators/ConfigValidator.java | 2 +-
.../validators/DetectionConfigValidator.java | 111 ++++++
...dator.java => SubscriptionConfigValidator.java} | 15 +-
.../yaml/YamlDetectionAlertConfigTranslator.java | 1 -
.../thirdeye/detection/yaml/YamlResource.java | 371 ++++++++++-----------
.../thirdeye/detection/yaml/YamlResourceTest.java | 4 +-
7 files changed, 307 insertions(+), 210 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
index 795e917..1e041cc 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
@@ -19,6 +19,7 @@
package org.apache.pinot.thirdeye.detection;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.util.ArrayList;
@@ -319,12 +320,12 @@ public class DetectionMigrationResource {
}
// Migrate anomaly function config to the detection config by converting
to YAML and then to Detection Config
- Map<String, String> responseMessage = new HashMap<>();
- Map<String, Object> detectionYAMLMap =
translateAnomalyFunctionToYaml(anomalyFunctionDTO);
- detectionConfig = new
YamlResource().translateToDetectionConfig(detectionYAMLMap, responseMessage);
-
- if (detectionConfig == null) {
- throw new RuntimeException("Couldn't translate yaml to detection config
due to " + responseMessage.get("message"));
+ try {
+ Map<String, Object> detectionYAMLMap =
translateAnomalyFunctionToYaml(anomalyFunctionDTO);
+ detectionConfig = new
YamlResource().translateToDetectionConfig(detectionYAMLMap);
+ Preconditions.checkNotNull(detectionConfig);
+ } catch (Exception e) {
+ throw new RuntimeException("Error translating anomaly function config to
the detection config" + e.getMessage());
}
// Save the migrated anomaly function
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ConfigValidator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ConfigValidator.java
index 4879674..aa5be6f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ConfigValidator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ConfigValidator.java
@@ -55,7 +55,7 @@ public abstract class ConfigValidator {
try {
Map<String, Object> yamlConfigMap = (Map<String, Object>)
YAML.load(yamlConfig);
} catch (Exception e) {
- throw new ValidationException("Error parsing the Yaml input. Check for
syntax issues.");
+ throw new ValidationException("Error parsing the Yaml payload. Check for
syntax issues.");
}
return true;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
new file mode 100644
index 0000000..bcd7586
--- /dev/null
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.validators;
+
+import javax.xml.bind.ValidationException;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
+import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
+import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
+import org.apache.pinot.thirdeye.detection.DataProvider;
+import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
+import
org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator;
+import org.apache.pinot.thirdeye.detection.yaml.YamlDetectionTranslatorLoader;
+import org.yaml.snakeyaml.Yaml;
+
+
+public class DetectionConfigValidator extends ConfigValidator {
+
+ private static DetectionConfigValidator INSTANCE;
+
+ private final DataProvider provider;
+ private final DetectionPipelineLoader loader;
+
+ public static DetectionConfigValidator getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE = new DetectionConfigValidator();
+ }
+ return INSTANCE;
+ }
+
+ DetectionConfigValidator() {
+ MetricConfigManager metricDAO =
DAORegistry.getInstance().getMetricConfigDAO();
+ DatasetConfigManager datasetDAO =
DAORegistry.getInstance().getDatasetConfigDAO();
+
+ TimeSeriesLoader timeseriesLoader =
+ new DefaultTimeSeriesLoader(metricDAO, datasetDAO,
ThirdEyeCacheRegistry.getInstance().getQueryCache());
+
+ AggregationLoader aggregationLoader =
+ new DefaultAggregationLoader(metricDAO, datasetDAO,
ThirdEyeCacheRegistry.getInstance().getQueryCache(),
+ ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache());
+
+ this.loader = new DetectionPipelineLoader();
+
+ this.provider = new DefaultDataProvider(metricDAO, datasetDAO,
+ DAORegistry.getInstance().getEventDAO(),
+ DAORegistry.getInstance().getMergedAnomalyResultDAO(),
+ timeseriesLoader, aggregationLoader, loader);
+ }
+
+ /**
+ * Validate the pipeline by loading and initializing components
+ */
+ private void semanticValidation(DetectionConfigDTO detectionConfig) throws
ValidationException {
+ try {
+ // backup and swap out id
+ Long id = detectionConfig.getId();
+ detectionConfig.setId(-1L);
+
+ // try to load the detection pipeline and init all the components
+ this.loader.from(provider, detectionConfig, 0, 0);
+
+ // set id back
+ detectionConfig.setId(id);
+ } catch (Exception e) {
+ throw new ValidationException("Semantic error while initializing the
detection pipeline.");
+ }
+ }
+
+ /**
+ * Perform validation on the detection config like verifying if all the
required fields are set
+ */
+ public void validateConfig(DetectionConfigDTO detectionConfig) throws
ValidationException {
+ semanticValidation(detectionConfig);
+
+ // TODO: Add more static validations here
+ }
+
+ /**
+ * Perform validation on the updated detection config. Check for fields
which shouldn't be
+ * updated by the user.
+ */
+ public void validateUpdatedConfig(DetectionConfigDTO updatedConfig,
DetectionConfigDTO oldConfig)
+ throws ValidationException {
+ validateConfig(updatedConfig);
+
+ // TODO: Add more checks here
+ }
+}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java
similarity index 89%
rename from
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java
rename to
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java
index 07e83d2..34bdba9 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java
@@ -26,23 +26,23 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.pinot.thirdeye.datalayer.util.Predicate;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.detection.ConfigUtils;
import static
org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
-public class DetectionAlertConfigValidator extends ConfigValidator {
+public class SubscriptionConfigValidator extends ConfigValidator {
- private static final DetectionAlertConfigValidator INSTANCE = new
DetectionAlertConfigValidator();
+ private static final SubscriptionConfigValidator INSTANCE = new
SubscriptionConfigValidator();
private static final String PROP_CLASS_NAME = "className";
- public static DetectionAlertConfigValidator getInstance() {
+ public static SubscriptionConfigValidator getInstance() {
return INSTANCE;
}
/**
* Perform validation on the alert config like verifying if all the required
fields are set
*/
- @SuppressWarnings("unchecked")
public void validateConfig(DetectionAlertConfigDTO alertConfig) throws
ValidationException {
// Check for all the required fields in the alert
if (StringUtils.isEmpty(alertConfig.getName())) {
@@ -68,14 +68,14 @@ public class DetectionAlertConfigValidator extends
ConfigValidator {
+ " subscribed detections, and type.");
}
// detectionConfigIds cannot be empty
- List<Long> detectionIds = (List<Long>)
alertConfig.getProperties().get(PROP_DETECTION_CONFIG_IDS);
+ List<Long> detectionIds =
ConfigUtils.getLongs(alertConfig.getProperties().get(PROP_DETECTION_CONFIG_IDS));
if (detectionIds == null || detectionIds.isEmpty()) {
throw new ValidationException("A notification group should subscribe to
at least one alert. If you wish to"
+ " unsubscribe, set active to false.");
}
// At least one recipient must be specified
- Map<String, Object> recipients = (Map<String, Object>)
alertConfig.getProperties().get(PROP_RECIPIENTS);
- if (recipients == null || recipients.isEmpty() || ((List<String>)
recipients.get("to")).isEmpty()) {
+ Map<String, Object> recipients =
ConfigUtils.getMap(alertConfig.getProperties().get(PROP_RECIPIENTS));
+ if (recipients.isEmpty() ||
ConfigUtils.getList(recipients.get("to")).isEmpty()) {
throw new ValidationException("Please specify at least one recipient in
the notification group. If you wish to"
+ " unsubscribe, set active to false.");
}
@@ -93,7 +93,6 @@ public class DetectionAlertConfigValidator extends
ConfigValidator {
* Perform validation on the updated alert config. Check for fields which
shouldn't be
* updated by the user.
*/
- @SuppressWarnings("unchecked")
public void validateUpdatedConfig(DetectionAlertConfigDTO
updatedAlertConfig, DetectionAlertConfigDTO oldAlertConfig)
throws ValidationException {
validateConfig(updatedAlertConfig);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
index 2ed1a9a..ba3eede 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
@@ -161,7 +161,6 @@ public class YamlDetectionAlertConfigTranslator {
/**
* Generates the {@link DetectionAlertConfigDTO} from the YAML Alert Map
*/
- @SuppressWarnings("unchecked")
public DetectionAlertConfigDTO translate(Map<String,Object> yamlAlertConfig)
{
DetectionAlertConfigDTO alertConfigDTO = new DetectionAlertConfigDTO();
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index e5dd378..41c84d6 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableMap;
import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
-import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -68,7 +67,8 @@ import
org.apache.pinot.thirdeye.detection.DefaultDataProvider;
import org.apache.pinot.thirdeye.detection.DetectionPipeline;
import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
-import
org.apache.pinot.thirdeye.detection.validators.DetectionAlertConfigValidator;
+import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
+import
org.apache.pinot.thirdeye.detection.validators.SubscriptionConfigValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -80,14 +80,14 @@ public class YamlResource {
protected static final Logger LOG =
LoggerFactory.getLogger(YamlResource.class);
private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final String PROP_DETECTION_NAME = "detectionName";
private static final String PROP_SUBS_GROUP_NAME = "subscriptionGroupName";
private final DetectionConfigManager detectionConfigDAO;
private final DetectionAlertConfigManager detectionAlertConfigDAO;
private final YamlDetectionTranslatorLoader translatorLoader;
private final YamlDetectionAlertConfigTranslator alertConfigTranslator;
- private final DetectionAlertConfigValidator notificationValidator;
+ private final DetectionConfigValidator detectionValidator;
+ private final SubscriptionConfigValidator subscriptionValidator;
private final DataProvider provider;
private final MetricConfigManager metricDAO;
private final DatasetConfigManager datasetDAO;
@@ -100,7 +100,8 @@ public class YamlResource {
this.detectionConfigDAO =
DAORegistry.getInstance().getDetectionConfigManager();
this.detectionAlertConfigDAO =
DAORegistry.getInstance().getDetectionAlertConfigManager();
this.translatorLoader = new YamlDetectionTranslatorLoader();
- this.notificationValidator = DetectionAlertConfigValidator.getInstance();
+ this.detectionValidator = DetectionConfigValidator.getInstance();
+ this.subscriptionValidator = SubscriptionConfigValidator.getInstance();
this.alertConfigTranslator = new
YamlDetectionAlertConfigTranslator(this.detectionConfigDAO);
this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
@@ -120,59 +121,35 @@ public class YamlResource {
this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO,
anomalyDAO, timeseriesLoader, aggregationLoader, loader);
}
- public DetectionConfigDTO translateToDetectionConfig(Map<String, Object>
yamlConfig, Map<String, String> responseMessage) {
- return buildDetectionConfigFromYaml(0, 0, yamlConfig, null,
responseMessage);
+ public DetectionConfigDTO translateToDetectionConfig(Map<String, Object>
yamlConfig) throws Exception {
+ return buildDetectionConfigFromYaml(0, 0, yamlConfig, null);
}
/*
* Build the detection config from a yaml.
- * Returns null if building or validation failed. Error messages stored in
responseMessage.
*/
private DetectionConfigDTO buildDetectionConfigFromYaml(long startTime, long
endTime, Map<String, Object> yamlConfig,
- DetectionConfigDTO existingDetectionConfig, Map<String, String>
responseMessage) {
+ DetectionConfigDTO existingDetectionConfig) throws Exception {
+
+ // Configure the tuning window
if (startTime == 0L && endTime == 0L) {
// default tuning window 28 days
endTime = System.currentTimeMillis();
startTime = endTime - TimeUnit.DAYS.toMillis(28);
}
- try{
- YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
- DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
- .withExistingDetectionConfig(existingDetectionConfig)
- .generateDetectionConfig();
- validatePipeline(detectionConfig);
- return detectionConfig;
- } catch (InvocationTargetException e){
- // exception thrown in validate pipeline via reflection
- LOG.error("Validate pipeline error", e);
- responseMessage.put("message", e.getCause().getMessage());
- } catch (Exception e) {
- LOG.error("yaml translation error", e);
- responseMessage.put("message", e.getMessage());
- }
- return null;
- }
-
- /*
- * Init the pipeline to check if detection pipeline property is valid
semantically.
- */
- private void validatePipeline(DetectionConfigDTO detectionConfig) throws
Exception {
- Long id = detectionConfig.getId();
- // swap out id
- detectionConfig.setId(-1L);
- // try to load the detection pipeline and init all the components
- this.loader.from(provider, detectionConfig, 0, 0);
- // set id back
- detectionConfig.setId(id);
+ YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
+ return translator.withTrainingWindow(startTime, endTime)
+ .withExistingDetectionConfig(existingDetectionConfig)
+ .generateDetectionConfig();
}
@POST
@Path("/create-alert")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
- @ApiOperation("Use yaml to create both notification and detection yaml. ")
- public Response createYamlAlert(@ApiParam(value = "a json contains both
notification and detection yaml as string") String payload,
+ @ApiOperation("Use yaml to create both subscription and detection yaml. ")
+ public Response createYamlAlert(@ApiParam(value = "a json contains both
subscription and detection yaml as string") String payload,
@ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) throws Exception{
Map<String, String> yamls = OBJECT_MAPPER.readValue(payload, Map.class);
@@ -180,42 +157,15 @@ public class YamlResource {
return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"Empty payload")).build();
}
if (!yamls.containsKey("detection")) {
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml is missing")).build();
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection pipeline yaml is missing")).build();
}
if (!yamls.containsKey("notification")){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"notification yaml is missing")).build();
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"subscription group yaml is missing")).build();
}
// Detection
String detectionYaml = yamls.get("detection");
-
- Map<String, Object> detectionYamlConfig;
- try {
- detectionYamlConfig = ConfigUtils.getMap(this.yaml.load(detectionYaml));
- } catch (Exception e){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
- }
-
- // check if detection config already exists
- String name = MapUtils.getString(detectionYamlConfig, PROP_DETECTION_NAME);
- List<DetectionConfigDTO> detectionConfigDTOs =
this.detectionConfigDAO.findByPredicate(
- Predicate.EQ("name", name));
- if (!detectionConfigDTOs.isEmpty()){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection name already exist: " + name )).build();
- }
-
- HashMap<String, String> responseMessage = new HashMap<>();
- DetectionConfigDTO detectionConfig =
- buildDetectionConfigFromYaml(startTime, endTime, detectionYamlConfig,
null, responseMessage);
- if (detectionConfig == null) {
- return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- detectionConfig.setYaml(detectionYaml);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- if (detectionConfigId == null){
- return Response.serverError().entity(ImmutableMap.of("message", "Save
detection config failed")).build();
- }
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+ long detectionConfigId = createDetectionPipeline(detectionYaml, startTime,
endTime);
// Notification
String notificationYaml = yamls.get("notification");
@@ -246,7 +196,34 @@ public class YamlResource {
}
long alertId =
Long.parseLong(ConfigUtils.getMap(response.getEntity()).get("detectionAlertConfigId").toString());
- return Response.ok().entity(ImmutableMap.of("detectionConfigId",
detectionConfig.getId(), "detectionAlertConfigId", alertId)).build();
+ return Response.ok().entity(ImmutableMap.of("detectionConfigId",
detectionConfigId, "detectionAlertConfigId", alertId)).build();
+ }
+
+ long createDetectionPipeline(String yamlDetectionConfig, long startTime,
long endTime) throws Exception {
+ detectionValidator.validateYAMLConfig(yamlDetectionConfig);
+
+ // Translate config from YAML to detection config (JSON)
+ TreeMap<String, Object> newDetectionConfigMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
newDetectionConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlDetectionConfig)));
+ DetectionConfigDTO detectionConfig =
buildDetectionConfigFromYaml(startTime, endTime, newDetectionConfigMap, null);
+ detectionConfig.setYaml(yamlDetectionConfig);
+
+ // Check for duplicates
+ List<DetectionConfigDTO> detectionConfigDTOS = detectionConfigDAO
+ .findByPredicate(Predicate.EQ("name", detectionConfig.getName()));
+ if (!detectionConfigDTOS.isEmpty()) {
+ throw new ValidationException("Detection name is already taken. Please
use a different detectionName.");
+ }
+
+ // Validate the config before saving it
+ detectionValidator.validateConfig(detectionConfig);
+
+ // Save the detection config
+ Preconditions.checkNotNull(detectionConfig);
+ Long id = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(id, "Error while saving the detection
pipeline");
+
+ return detectionConfig.getId();
}
/**
@@ -260,37 +237,51 @@ public class YamlResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.TEXT_PLAIN)
@ApiOperation("Set up a detection pipeline using a YAML config")
- public Response setUpDetectionPipeline(
+ public Response createDetectionPipelineApi(
@ApiParam("yaml config") String payload,
@ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
- if (StringUtils.isBlank(payload)){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"empty payload")).build();
- }
- Map<String, Object> yamlConfig;
+ Map<String, String> responseMessage = new HashMap<>();
+ long detectionConfigId;
try {
- yamlConfig = (Map<String, Object>) this.yaml.load(payload);
- } catch (Exception e){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
+ detectionConfigId = createDetectionPipeline(payload, startTime, endTime);
+ } catch (ValidationException e) {
+ LOG.warn("Validation error while creating detection pipeline with
payload " + payload, e);
+ responseMessage.put("message", "Validation Error! " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
+ } catch (Exception e) {
+ LOG.error("Error creating subscription group with payload " + payload,
e);
+ responseMessage.put("message", "Failed to create the subscription group.
Reach out to the ThirdEye team.");
+ responseMessage.put("more-info", "Error = " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
}
- // check if detection config already exists
- String name = MapUtils.getString(yamlConfig, PROP_DETECTION_NAME);
- List<DetectionConfigDTO> detectionConfigDTOs =
this.detectionConfigDAO.findByPredicate(
- Predicate.EQ("name", name));
- if (!detectionConfigDTOs.isEmpty()){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection name already exist: " + name )).build();
- }
- Map<String, String> responseMessage = new HashMap<>();
- DetectionConfigDTO detectionConfig =
- buildDetectionConfigFromYaml(startTime, endTime, yamlConfig, null,
responseMessage);
- if (detectionConfig == null) {
- return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
+ LOG.info("Detection Pipeline created with id " + detectionConfigId + "
using payload " + payload);
+ responseMessage.put("message", "The subscription group was created
successfully.");
+ responseMessage.put("more-info", "Record saved with id " +
detectionConfigId);
+ return Response.ok().entity(responseMessage).build();
+ }
+
+ void updateDetectionPipeline(long detectionID, String yamlDetectionConfig,
long startTime, long endTime) throws Exception {
+ DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(detectionID);
+ if (existingDetectionConfig == null) {
+ throw new RuntimeException("Cannot find detection pipeline " +
detectionID);
}
- detectionConfig.setYaml(payload);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
- return Response.ok(detectionConfig).build();
+ detectionValidator.validateYAMLConfig(yamlDetectionConfig);
+
+ // Translate config from YAML to detection config (JSON)
+ TreeMap<String, Object> newDetectionConfigMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
newDetectionConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlDetectionConfig)));
+ DetectionConfigDTO detectionConfig =
buildDetectionConfigFromYaml(startTime, endTime, newDetectionConfigMap,
existingDetectionConfig);
+ detectionConfig.setYaml(yamlDetectionConfig);
+
+ // Validate updated config before saving it
+ detectionValidator.validateUpdatedConfig(detectionConfig,
existingDetectionConfig);
+
+ // Save the detection config
+ Preconditions.checkNotNull(detectionConfig);
+ Long id = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(id, "Error while saving the detection
pipeline");
}
/**
@@ -306,45 +297,37 @@ public class YamlResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.TEXT_PLAIN)
@ApiOperation("Edit a detection pipeline using a YAML config")
- public Response editDetectionPipeline(
+ public Response updateDetectionPipelineApi(
@ApiParam("yaml config") String payload,
@ApiParam("the detection config id to edit") @PathParam("id") long id,
@ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
- if (StringUtils.isBlank(payload)){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"empty payload")).build();
- }
- Map<String, Object> yamlConfig;
+ Map<String, String> responseMessage = new HashMap<>();
try {
- yamlConfig = (Map<String, Object>) this.yaml.load(payload);
- } catch (Exception e){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
+ updateDetectionPipeline(id, payload, startTime, endTime);
+ } catch (ValidationException e) {
+ LOG.warn("Validation error while creating detection pipeline with
payload " + payload, e);
+ responseMessage.put("message", "Validation Error! " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
+ } catch (Exception e) {
+ LOG.error("Error creating subscription group with payload " + payload,
e);
+ responseMessage.put("message", "Failed to create the subscription group.
Reach out to the ThirdEye team.");
+ responseMessage.put("more-info", "Error = " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
}
- Map<String, String> responseMessage = new HashMap<>();
- // retrieve id if detection config already exists
- DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(id);
- if (existingDetectionConfig == null) {
- return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- DetectionConfigDTO detectionConfig =
- buildDetectionConfigFromYaml(startTime, endTime, yamlConfig,
existingDetectionConfig, responseMessage);
- if (detectionConfig == null) {
- return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- detectionConfig.setYaml(payload);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
- return Response.ok(detectionConfig).build();
+ LOG.info("Detection Pipeline " + id + " updated successfully with payload
" + payload);
+ responseMessage.put("message", "The detection Pipeline was created
successfully.");
+ responseMessage.put("detectionConfigId", String.valueOf(id));
+ return Response.ok().entity(responseMessage).build();
}
- @SuppressWarnings("unchecked")
long createSubscriptionGroup(String yamlAlertConfig) throws
ValidationException {
- notificationValidator.validateYAMLConfig(yamlAlertConfig);
+ subscriptionValidator.validateYAMLConfig(yamlAlertConfig);
// Translate config from YAML to detection alert config (JSON)
TreeMap<String, Object> newAlertConfigMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- newAlertConfigMap.putAll((Map<String, Object>)
this.yaml.load(yamlAlertConfig));
+
newAlertConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlAlertConfig)));
DetectionAlertConfigDTO alertConfig =
this.alertConfigTranslator.translate(newAlertConfigMap);
alertConfig.setYaml(yamlAlertConfig);
@@ -355,42 +338,44 @@ public class YamlResource {
throw new ValidationException("Subscription group name is already taken.
Please use a different name.");
}
- // Validate the config
- notificationValidator.validateConfig(alertConfig);
+ // Validate the config before saving it
+ subscriptionValidator.validateConfig(alertConfig);
// Save the detection alert config
Preconditions.checkNotNull(alertConfig);
Long id = this.detectionAlertConfigDAO.save(alertConfig);
- Preconditions.checkNotNull(id);
+ Preconditions.checkNotNull(id, "Error while saving the subscription
group");
return alertConfig.getId();
}
+ @POST
+ @Path("/notification")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.TEXT_PLAIN)
+ @ApiOperation("Create a notification group using a YAML config")
@SuppressWarnings("unchecked")
- void updateSubscriptionGroup(long oldAlertConfigID, String yamlAlertConfig)
throws ValidationException {
- DetectionAlertConfigDTO oldAlertConfig =
this.detectionAlertConfigDAO.findById(oldAlertConfigID);
- if (oldAlertConfig == null) {
- throw new RuntimeException("Cannot find subscription group " +
oldAlertConfigID);
+ public Response createSubscriptionGroupApi(
+ @ApiParam("payload") String yamlAlertConfig) {
+ Map<String, String> responseMessage = new HashMap<>();
+ long detectionAlertConfigId;
+ try {
+ detectionAlertConfigId = createSubscriptionGroup(yamlAlertConfig);
+ } catch (ValidationException e) {
+ LOG.warn("Validation error while creating subscription group with
payload " + yamlAlertConfig, e);
+ responseMessage.put("message", "Validation Error! " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
+ } catch (Exception e) {
+ LOG.error("Error creating subscription group with payload " +
yamlAlertConfig, e);
+ responseMessage.put("message", "Failed to create the subscription group.
Reach out to the ThirdEye team.");
+ responseMessage.put("more-info", "Error = " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
}
- notificationValidator.validateYAMLConfig(yamlAlertConfig);
-
- // Translate payload to detection alert config
- TreeMap<String, Object> newAlertConfigMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- newAlertConfigMap.putAll((Map<String, Object>)
this.yaml.load(yamlAlertConfig));
- DetectionAlertConfigDTO newAlertConfig =
this.alertConfigTranslator.translate(newAlertConfigMap);
- // Update existing alert config with the newly supplied config.
- DetectionAlertConfigDTO updatedAlertConfig =
updateDetectionAlertConfig(oldAlertConfig, newAlertConfig);
- updatedAlertConfig.setYaml(yamlAlertConfig);
-
- // Validate before updating the config
- notificationValidator.validateUpdatedConfig(updatedAlertConfig,
oldAlertConfig);
-
- // Save the updated notification config
- int detectionAlertConfigId =
this.detectionAlertConfigDAO.update(updatedAlertConfig);
- if (detectionAlertConfigId <= 0) {
- throw new RuntimeException("Failed to update the detection alert
config.");
- }
+ LOG.info("Notification group created with id " + detectionAlertConfigId +
" using payload " + yamlAlertConfig);
+ responseMessage.put("message", "The subscription group was created
successfully.");
+ responseMessage.put("detectionAlertConfigId",
String.valueOf(detectionAlertConfigId));
+ return Response.ok().entity(responseMessage).build();
}
/**
@@ -399,7 +384,7 @@ public class YamlResource {
* Update all the fields except the vector clocks and high watermark. The
clocks and watermarks
* are managed by the platform. They shouldn't be reset by the user.
*/
- DetectionAlertConfigDTO updateDetectionAlertConfig(DetectionAlertConfigDTO
oldAlertConfig,
+ private DetectionAlertConfigDTO
updateDetectionAlertConfig(DetectionAlertConfigDTO oldAlertConfig,
DetectionAlertConfigDTO newAlertConfig) {
oldAlertConfig.setName(newAlertConfig.getName());
oldAlertConfig.setCronExpression(newAlertConfig.getCronExpression());
@@ -416,33 +401,30 @@ public class YamlResource {
return oldAlertConfig;
}
- @POST
- @Path("/notification")
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.TEXT_PLAIN)
- @ApiOperation("Create a notification group using a YAML config")
- @SuppressWarnings("unchecked")
- public Response createSubscriptionGroupApi(
- @ApiParam("payload") String yamlAlertConfig) {
- Map<String, String> responseMessage = new HashMap<>();
- long detectionAlertConfigId;
- try {
- detectionAlertConfigId = createSubscriptionGroup(yamlAlertConfig);
- } catch (ValidationException e) {
- LOG.warn("Validation error while creating notification group with
payload " + yamlAlertConfig, e);
- responseMessage.put("message", "Validation Error! " + e.getMessage());
- return Response.serverError().entity(responseMessage).build();
- } catch (Exception e) {
- LOG.error("Error creating notification group with payload " +
yamlAlertConfig, e);
- responseMessage.put("message", "Failed to create the notification group.
Reach out to the ThirdEye team.");
- responseMessage.put("more-info", "Error = " + e.getMessage());
- return Response.serverError().entity(responseMessage).build();
+ void updateSubscriptionGroup(long oldAlertConfigID, String yamlAlertConfig)
throws ValidationException {
+ DetectionAlertConfigDTO oldAlertConfig =
this.detectionAlertConfigDAO.findById(oldAlertConfigID);
+ if (oldAlertConfig == null) {
+ throw new RuntimeException("Cannot find subscription group " +
oldAlertConfigID);
}
+ subscriptionValidator.validateYAMLConfig(yamlAlertConfig);
- LOG.info("Notification group created with id " + detectionAlertConfigId +
" using payload " + yamlAlertConfig);
- responseMessage.put("message", "The notification group was created
successfully.");
- responseMessage.put("detectionAlertConfigId",
String.valueOf(detectionAlertConfigId));
- return Response.ok().entity(responseMessage).build();
+ // Translate payload to detection alert config
+ TreeMap<String, Object> newAlertConfigMap = new
TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
newAlertConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlAlertConfig)));
+ DetectionAlertConfigDTO newAlertConfig =
this.alertConfigTranslator.translate(newAlertConfigMap);
+
+ // Update existing alert config with the newly supplied config.
+ DetectionAlertConfigDTO updatedAlertConfig =
updateDetectionAlertConfig(oldAlertConfig, newAlertConfig);
+ updatedAlertConfig.setYaml(yamlAlertConfig);
+
+ // Validate before updating the config
+ subscriptionValidator.validateUpdatedConfig(updatedAlertConfig,
oldAlertConfig);
+
+ // Save the updated subscription config
+ int detectionAlertConfigId =
this.detectionAlertConfigDAO.update(updatedAlertConfig);
+ if (detectionAlertConfigId <= 0) {
+ throw new RuntimeException("Failed to update the detection alert
config.");
+ }
}
@PUT
@@ -457,12 +439,12 @@ public class YamlResource {
try {
updateSubscriptionGroup(id, yamlAlertConfig);
} catch (ValidationException e) {
- LOG.warn("Validation error while updating notification group " + id + "
with payload " + yamlAlertConfig, e);
+ LOG.warn("Validation error while updating subscription group " + id + "
with payload " + yamlAlertConfig, e);
responseMessage.put("message", "Validation Error! " + e.getMessage());
return Response.serverError().entity(responseMessage).build();
} catch (Exception e) {
- LOG.error("Error updating notification group " + id + " with payload " +
yamlAlertConfig, e);
- responseMessage.put("message", "Failed to update the notification group.
Reach out to the ThirdEye team.");
+ LOG.error("Error updating subscription group " + id + " with payload " +
yamlAlertConfig, e);
+ responseMessage.put("message", "Failed to update the subscription group.
Reach out to the ThirdEye team.");
responseMessage.put("more-info", "Error = " + e.getMessage());
return Response.serverError().entity(responseMessage).build();
}
@@ -478,32 +460,37 @@ public class YamlResource {
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.TEXT_PLAIN)
@ApiOperation("Preview the anomaly detection result of a YAML configuration")
- public Response yamlPreview(
+ public Response yamlPreviewApi(
@QueryParam("start") long start,
@QueryParam("end") long end,
@QueryParam("tuningStart") long tuningStart,
@QueryParam("tuningEnd") long tuningEnd,
@ApiParam("jsonPayload") String payload) throws Exception {
- if (StringUtils.isBlank(payload)){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"empty payload")).build();
- }
- Map<String, Object> yamlConfig;
+ Map<String, String> responseMessage = new HashMap<>();
+ DetectionPipelineResult result;
try {
- yamlConfig = (Map<String, Object>) this.yaml.load(payload);
- } catch (Exception e){
- return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
- }
+ detectionValidator.validateYAMLConfig(payload);
- Map<String, String> responseMessage = new HashMap<>();
- DetectionConfigDTO detectionConfig =
- buildDetectionConfigFromYaml(tuningStart, tuningEnd, yamlConfig, null,
responseMessage);
- if (detectionConfig == null) {
- return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
+ // Translate config from YAML to detection config (JSON)
+ Map<String, Object> newDetectionConfigMap = new
HashMap<>(ConfigUtils.getMap(this.yaml.load(payload)));
+ DetectionConfigDTO detectionConfig =
buildDetectionConfigFromYaml(tuningStart, tuningEnd, newDetectionConfigMap,
null);
+ Preconditions.checkNotNull(detectionConfig);
+ detectionConfig.setId(Long.MAX_VALUE);
+
+ DetectionPipeline pipeline = this.loader.from(this.provider,
detectionConfig, start, end);
+ result = pipeline.run();
+
+ } catch (ValidationException e) {
+ LOG.warn("Validation error while running preview with payload " +
payload, e);
+ responseMessage.put("message", "Validation Error! " + e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
+ } catch (Exception e) {
+ LOG.error("Error running preview with payload " + payload, e);
+ responseMessage.put("message", "Failed to run the preview due to " +
e.getMessage());
+ return Response.serverError().entity(responseMessage).build();
}
- detectionConfig.setId(Long.MAX_VALUE);
- DetectionPipeline pipeline = this.loader.from(this.provider,
detectionConfig, start, end);
- DetectionPipelineResult result = pipeline.run();
+ LOG.info("Preview successful using payload " + payload);
return Response.ok(result).build();
}
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
index 279340b..a181b5e 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
@@ -61,7 +61,7 @@ public class YamlResourceTest {
this.yamlResource.createSubscriptionGroup(inValidYaml);
Assert.fail("Exception not thrown on empty yaml");
} catch (Exception e) {
- Assert.assertEquals(e.getMessage(), "Error parsing the Yaml input. Check
for syntax issues.");
+ Assert.assertEquals(e.getMessage(), "Error parsing the Yaml payload.
Check for syntax issues.");
}
String noSubscriptGroupYaml = "application: test_application";
@@ -147,7 +147,7 @@ public class YamlResourceTest {
this.yamlResource.updateSubscriptionGroup(oldId, inValidYaml);
Assert.fail("Exception not thrown on empty yaml");
} catch (Exception e) {
- Assert.assertEquals(e.getMessage(), "Error parsing the Yaml input. Check
for syntax issues.");
+ Assert.assertEquals(e.getMessage(), "Error parsing the Yaml payload.
Check for syntax issues.");
}
String noSubscriptGroupYaml = "application: test_application";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
