jihaozh closed pull request #3609: [TE] yaml - create alert endpoint
URL: https://github.com/apache/incubator-pinot/pull/3609
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
index b19913ff85..170fdbb0c7 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/api/Constants.java
@@ -27,4 +27,5 @@
public static final String DASHBOARD_TAG = "Dashboard";
public static final String ONBOARD_TAG = "Onboard";
public static final String YAML_TAG = "Yaml";
+ public static final String DETECTION_TAG = "detection";
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionResource.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionResource.java
index dd63d06db5..2647e9fd7f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DetectionResource.java
@@ -18,12 +18,15 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.linkedin.thirdeye.anomaly.detection.DetectionJobSchedulerUtils;
+import com.linkedin.thirdeye.api.Constants;
import com.linkedin.thirdeye.constant.AnomalyResultSource;
import com.linkedin.thirdeye.datalayer.bao.DatasetConfigManager;
+import com.linkedin.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import com.linkedin.thirdeye.datalayer.bao.DetectionConfigManager;
import com.linkedin.thirdeye.datalayer.bao.EventManager;
import com.linkedin.thirdeye.datalayer.bao.MergedAnomalyResultManager;
import com.linkedin.thirdeye.datalayer.bao.MetricConfigManager;
+import com.linkedin.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import com.linkedin.thirdeye.datalayer.dto.DetectionConfigDTO;
import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.datasource.DAORegistry;
@@ -35,6 +38,8 @@
import com.linkedin.thirdeye.detection.finetune.GridSearchTuningAlgorithm;
import com.linkedin.thirdeye.detection.finetune.TuningAlgorithm;
import com.linkedin.thirdeye.detection.spi.model.AnomalySlice;
+import com.wordnik.swagger.annotations.Api;
+import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
import java.text.ParseException;
import java.util.ArrayList;
@@ -46,6 +51,7 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -63,6 +69,7 @@
@Path("/detection")
@Produces(MediaType.APPLICATION_JSON)
+@Api(tags = {Constants.DETECTION_TAG})
public class DetectionResource {
private static final Logger LOG =
LoggerFactory.getLogger(DetectionResource.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -77,6 +84,7 @@
private final DetectionPipelineLoader loader;
private final DataProvider provider;
private final DetectionConfigManager configDAO;
+ private final DetectionAlertConfigManager detectionAlertConfigDAO;
public DetectionResource() {
this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
@@ -84,6 +92,7 @@ public DetectionResource() {
this.eventDAO = DAORegistry.getInstance().getEventDAO();
this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
this.configDAO = DAORegistry.getInstance().getDetectionConfigManager();
+ this.detectionAlertConfigDAO =
DAORegistry.getInstance().getDetectionAlertConfigManager();
TimeSeriesLoader timeseriesLoader =
new DefaultTimeSeriesLoader(metricDAO, datasetDAO,
ThirdEyeCacheRegistry.getInstance().getQueryCache());
@@ -97,6 +106,22 @@ public DetectionResource() {
this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO,
anomalyDAO, timeseriesLoader, aggregationLoader, loader);
}
+ @Path("/{id}")
+ @GET
+ @ApiOperation("get a detection config with yaml")
+ public Response getDetectionConfig(@ApiParam("the detection config id")
@PathParam("id") long id){
+ DetectionConfigDTO config = this.configDAO.findById(id);
+ return Response.ok(config).build();
+ }
+
+ @Path("/notification/{id}")
+ @GET
+ @ApiOperation("get a detection alert config with yaml")
+ public Response getDetectionAlertConfig(@ApiParam("the detection alert
config id") @PathParam("id") long id){
+ DetectionAlertConfigDTO config = this.detectionAlertConfigDAO.findById(id);
+ return Response.ok(config).build();
+ }
+
@POST
@Path("/preview")
public Response detectionPreview(
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
index 109da58c42..9db98a967c 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/yaml/YamlResource.java
@@ -1,5 +1,6 @@
package com.linkedin.thirdeye.detection.yaml;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.linkedin.thirdeye.api.Constants;
@@ -22,6 +23,8 @@
import com.linkedin.thirdeye.detection.DefaultDataProvider;
import com.linkedin.thirdeye.detection.DetectionPipelineLoader;
import com.wordnik.swagger.annotations.Api;
+import com.wordnik.swagger.annotations.ApiImplicitParam;
+import com.wordnik.swagger.annotations.ApiImplicitParams;
import com.wordnik.swagger.annotations.ApiOperation;
import
com.linkedin.thirdeye.detection.validators.DetectionAlertConfigValidator;
import com.wordnik.swagger.annotations.ApiParam;
@@ -53,11 +56,11 @@
@Api(tags = {Constants.YAML_TAG})
public class YamlResource {
protected static final Logger LOG =
LoggerFactory.getLogger(YamlResource.class);
+ private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static final String PROP_SUBS_GROUP_NAME = "subscriptionGroupName";
public static final String PROP_DETECTION_NAME = "detectionName";
-
private final DetectionConfigManager detectionConfigDAO;
private final DetectionAlertConfigManager detectionAlertConfigDAO;
private final YamlDetectionTranslatorLoader translatorLoader;
@@ -95,6 +98,110 @@ public YamlResource() {
this.provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO,
anomalyDAO, timeseriesLoader, aggregationLoader, loader);
}
+ @POST
+ @Path("/create-alert")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @ApiOperation("Use yaml to create both notification and detection yaml. ")
+ public Response createYamlAlert(@ApiParam(value = "a json contains both
notification and detection yaml as string") String payload,
+ @ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
+ @ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) throws Exception{
+ Map<String, String> yamls = OBJECT_MAPPER.readValue(payload, Map.class);
+
+ if (StringUtils.isBlank(payload)){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"Empty payload")).build();
+ }
+ if (!yamls.containsKey("detection")) {
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml is missing")).build();
+ }
+ if (!yamls.containsKey("notification")){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"notification yaml is missing")).build();
+ }
+
+ // get detection yaml
+ String detectionYaml = yamls.get("detection");
+
+ Map<String, Object> detectionYamlConfig;
+ try {
+ detectionYamlConfig = (Map<String, Object>)
this.yaml.load(detectionYaml);
+ } catch (Exception e){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
+ }
+
+ // check if detection config already exists
+ String name = MapUtils.getString(detectionYamlConfig, PROP_DETECTION_NAME);
+ List<DetectionConfigDTO> detectionConfigDTOs =
this.detectionConfigDAO.findByPredicate(
+ Predicate.EQ("name", name));
+ if (!detectionConfigDTOs.isEmpty()){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection name already exist: " + name )).build();
+ }
+
+ HashMap<String, String> responseMessage = new HashMap<>();
+ DetectionConfigDTO detectionConfig =
+ buildDetectionConfigFromYaml(startTime, endTime, detectionYamlConfig,
null, responseMessage);
+ if (detectionConfig == null) {
+ return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
+ }
+ detectionConfig.setYaml(detectionYaml);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+
+ // notification
+ // TODO: Inject detectionConfigId into detection alert config
+ DetectionAlertConfigDTO alertConfig =
createDetectionAlertConfig(yamls.get("notification"), responseMessage);
+ if (alertConfig == null) {
+ // revert
+ this.detectionAlertConfigDAO.deleteById(detectionConfigId);
+ return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
+ }
+ Long detectionAlertConfigId =
this.detectionAlertConfigDAO.save(alertConfig);
+ if (detectionAlertConfigId == null){
+ // revert
+ this.detectionAlertConfigDAO.deleteById(detectionConfigId);
+ return Response.serverError().entity(ImmutableMap.of("message", "Save
detection alert config failed")).build();
+ }
+ LOG.info("saved detection alert config id {}", detectionAlertConfigId);
+
+ return Response.ok().entity(ImmutableMap.of("detectionConfigId",
detectionConfig.getId(), "detectionAlertConfigId",
alertConfig.getId())).build();
+ }
+
+ /*
+ * Build the detection config from a yaml.
+ * Returns null if building or validation failed. Error messages stored in
responseMessage.
+ */
+ private DetectionConfigDTO buildDetectionConfigFromYaml(long startTime, long
endTime, Map<String, Object> yamlConfig,
+ DetectionConfigDTO existingDetectionConfig, Map<String, String>
responseMessage) {
+ try{
+ YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
+ DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
+ .withExistingDetectionConfig(existingDetectionConfig)
+ .generateDetectionConfig();
+ validatePipeline(detectionConfig);
+ return detectionConfig;
+ } catch (InvocationTargetException e){
+ // exception thrown in validate pipeline via reflection
+ LOG.error("Validate pipeline error", e);
+ responseMessage.put("message", e.getCause().getMessage());
+ } catch (Exception e) {
+ LOG.error("yaml translation error", e);
+ responseMessage.put("message", e.getMessage());
+ }
+ return null;
+ }
+
+ /*
+ * Init the pipeline to check if detection pipeline property is valid
semantically.
+ */
+ private void validatePipeline(DetectionConfigDTO detectionConfig) throws
Exception {
+ Long id = detectionConfig.getId();
+ // swap out id
+ detectionConfig.setId(-1L);
+ // try to load the detection pipeline and init all the components
+ this.loader.from(provider, detectionConfig, 0, 0);
+ // set id back
+ detectionConfig.setId(id);
+ }
+
/**
Set up a detection pipeline using a YAML config
@param payload YAML config string
@@ -110,38 +217,33 @@ public Response setUpDetectionPipeline(
@ApiParam("yaml config") String payload,
@ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
- String errorMessage;
+ if (StringUtils.isBlank(payload)){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"empty payload")).build();
+ }
+ Map<String, Object> yamlConfig;
try {
- Preconditions.checkArgument(StringUtils.isNotBlank(payload), "Empty
payload");
- Map<String, Object> yamlConfig = (Map<String, Object>)
this.yaml.load(payload);
-
- // retrieve id if detection config already exists
- List<DetectionConfigDTO> detectionConfigDTOs =
- this.detectionConfigDAO.findByPredicate(Predicate.EQ("name",
MapUtils.getString(yamlConfig, PROP_DETECTION_NAME)));
- DetectionConfigDTO existingDetectionConfig = null;
- if (!detectionConfigDTOs.isEmpty()) {
- existingDetectionConfig = detectionConfigDTOs.get(0);
- }
-
- YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
- DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
- .withExistingDetectionConfig(existingDetectionConfig)
- .generateDetectionConfig();
- detectionConfig.setYaml(payload);
- validatePipeline(detectionConfig);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+ yamlConfig = (Map<String, Object>) this.yaml.load(payload);
+ } catch (Exception e){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
+ }
- return Response.ok(detectionConfig).build();
- } catch (InvocationTargetException e){
- // exception thrown in validate pipeline via reflection
- LOG.error("Validate pipeline error", e);
- errorMessage = e.getCause().getMessage();
- } catch (Exception e) {
- LOG.error("yaml translation error", e);
- errorMessage = e.getMessage();
+ // check if detection config already exists
+ String name = MapUtils.getString(yamlConfig, PROP_DETECTION_NAME);
+ List<DetectionConfigDTO> detectionConfigDTOs =
this.detectionConfigDAO.findByPredicate(
+ Predicate.EQ("name", name));
+ if (!detectionConfigDTOs.isEmpty()){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection name already exist: " + name )).build();
+ }
+ Map<String, String> responseMessage = new HashMap<>();
+ DetectionConfigDTO detectionConfig =
+ buildDetectionConfigFromYaml(startTime, endTime, yamlConfig, null,
responseMessage);
+ if (detectionConfig == null) {
+ return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
}
- return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", errorMessage)).build();
+ detectionConfig.setYaml(payload);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+ return Response.ok(detectionConfig).build();
}
/**
@@ -162,33 +264,31 @@ public Response editDetectionPipeline(
@ApiParam("the detection config id to edit") @PathParam("id") long id,
@ApiParam("tuning window start time for tunable components")
@QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components")
@QueryParam("endTime") long endTime) {
- String errorMessage;
+ if (StringUtils.isBlank(payload)){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"empty payload")).build();
+ }
+ Map<String, Object> yamlConfig;
try {
- Preconditions.checkArgument(StringUtils.isNotBlank(payload), "Empty
payload");
- Map<String, Object> yamlConfig = (Map<String, Object>)
this.yaml.load(payload);
-
- DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(id);
- Preconditions.checkArgument(existingDetectionConfig != null, "Existing
detection config " + id + " not found");
-
- YamlDetectionConfigTranslator translator =
this.translatorLoader.from(yamlConfig, this.provider);
- DetectionConfigDTO detectionConfig =
translator.withTrainingWindow(startTime, endTime)
- .withExistingDetectionConfig(existingDetectionConfig)
- .generateDetectionConfig();
- detectionConfig.setYaml(payload);
- validatePipeline(detectionConfig);
- Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
- Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+ yamlConfig = (Map<String, Object>) this.yaml.load(payload);
+ } catch (Exception e){
+ return
Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message",
"detection yaml parsing error, " + e.getMessage())).build();
+ }
- return Response.ok(detectionConfig).build();
- } catch (InvocationTargetException e){
- // exception thrown in validate pipeline via reflection
- LOG.error("Validate pipeline error", e);
- errorMessage = e.getCause().getMessage();
- } catch (Exception e) {
- LOG.error("yaml translation error", e);
- errorMessage = e.getMessage();
+ Map<String, String> responseMessage = new HashMap<>();
+ // retrieve id if detection config already exists
+ DetectionConfigDTO existingDetectionConfig =
this.detectionConfigDAO.findById(id);
+ if (existingDetectionConfig == null) {
+ return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
+ }
+ DetectionConfigDTO detectionConfig =
+ buildDetectionConfigFromYaml(startTime, endTime, yamlConfig,
existingDetectionConfig, responseMessage);
+ if (detectionConfig == null) {
+ return
Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
}
- return Response.status(400).entity(ImmutableMap.of("status", "400",
"message", errorMessage)).build();
+ detectionConfig.setYaml(payload);
+ Long detectionConfigId = this.detectionConfigDAO.save(detectionConfig);
+ Preconditions.checkNotNull(detectionConfigId, "Save detection config
failed");
+ return Response.ok(detectionConfig).build();
}
@POST
@@ -224,18 +324,6 @@ public Response createDetectionAlertConfig(
return Response.ok().entity(responseMessage).build();
}
- /*
- * Init the pipeline to check if detection pipeline property is valid
semantically.
- */
- private void validatePipeline(DetectionConfigDTO detectionConfig) throws
Exception {
- Long id = detectionConfig.getId();
- // swap out id
- detectionConfig.setId(-1L);
- // try to load the detection pipeline and init all the components
- this.loader.from(provider, detectionConfig, 0, 0);
- // set id back
- detectionConfig.setId(id);
- }
@SuppressWarnings("unchecked")
public DetectionAlertConfigDTO createDetectionAlertConfig(String
yamlAlertConfig, Map<String, String> responseMessage ) {
@@ -339,7 +427,7 @@ public DetectionAlertConfigDTO
updateDetectionAlertConfig(DetectionAlertConfigDT
}
/**
- List all yaml configurations enhanced with detection config id, isActive
and createBy information.
+ List all yaml configurations as JSON. enhanced with detection config id,
isActive and createBy information.
@param id id of a specific detection config yaml to list (optional)
@return the yaml configuration converted in to JSON, with enhanced
information from detection config DTO.
*/
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
