This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push: new 516a168 [TE] Add support for configuring additional custom fields in Jira Alert (#5386) 516a168 is described below commit 516a16820d7be247c42ff0385654bcecbbaf560b Author: Akshay Rai <ak...@linkedin.com> AuthorDate: Fri May 15 09:48:00 2020 -0700 [TE] Add support for configuring additional custom fields in Jira Alert (#5386) --- .../dashboard/ThirdEyeDashboardApplication.java | 3 +- .../alert/StatefulDetectionAlertFilter.java | 4 +- .../alert/scheme/DetectionJiraAlerter.java | 7 ++- .../thirdeye/detection/yaml/YamlResource.java | 51 ++++++++++++++++++++-- .../thirdeye/notification/commons/JiraEntity.java | 15 ++++++- .../notification/commons/ThirdEyeJiraClient.java | 42 ++++++++++++------ .../formatter/channels/JiraContentFormatter.java | 2 + .../thirdeye/detection/yaml/YamlResourceTest.java | 2 +- .../commons/TestThirdEyeJiraClient.java | 12 +++-- 9 files changed, 110 insertions(+), 28 deletions(-) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java index 99d8064..1ac0973 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java @@ -191,7 +191,8 @@ public class ThirdEyeDashboardApplication DAO_REGISTRY.getDetectionConfigManager(), DAO_REGISTRY.getDetectionAlertConfigManager())); env.jersey().register(new DetectionResource()); env.jersey().register(new DetectionAlertResource(DAO_REGISTRY.getDetectionAlertConfigManager())); - env.jersey().register(new YamlResource(config.getDetectionPreviewConfig(), config.getAlertOnboardingPermitPerSecond())); + env.jersey().register(new YamlResource(config.getAlerterConfiguration(), config.getDetectionPreviewConfig(), + config.getAlertOnboardingPermitPerSecond())); env.jersey().register(new SqlDataSourceResource()); TimeSeriesLoader timeSeriesLoader = new DefaultTimeSeriesLoader( diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java index 56c0613..b099677 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java @@ -48,13 +48,11 @@ public abstract class StatefulDetectionAlertFilter extends DetectionAlertFilter public static final String PROP_BCC = "bcc"; public static final String PROP_RECIPIENTS = "recipients"; - private static final String PROP_SEND_ONCE = "sendOnce"; - // Time beyond which we do not want to notify anomalies private static final long ANOMALY_NOTIFICATION_LOOKBACK_TIME = TimeUnit.DAYS.toMillis(14); public StatefulDetectionAlertFilter(DataProvider provider, DetectionAlertConfigDTO config, long endTime) { - super(provider, config, endTime);//prepareStatement.setObject(parameterIndex++, pair.getValue(), info.sqlType); + super(provider, config, endTime); } protected final Set<MergedAnomalyResultDTO> filter(Map<Long, Long> vectorClocks) { diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java index 70d6fc2..4d2c7bc 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java @@ -63,6 +63,9 @@ import static org.apache.pinot.thirdeye.notification.commons.JiraConfiguration.* * labels: # optional, default - thirdeye label is always appended * - test-label-1 * - test-label-2 + * custom: + * test1: value1 + * test2: value2 */ @AlertScheme(type = "JIRA") public class DetectionJiraAlerter extends DetectionAlertScheme { @@ -92,7 +95,7 @@ public class DetectionJiraAlerter extends DetectionAlertScheme { } private void updateJiraAlert(Issue issue, JiraEntity jiraEntity) { - // Append labels + // Append labels - do not remove existing labels jiraEntity.getLabels().addAll(issue.getLabels()); jiraEntity.setLabels(jiraEntity.getLabels().stream().distinct().collect(Collectors.toList())); @@ -152,7 +155,7 @@ public class DetectionJiraAlerter extends DetectionAlertScheme { try { JiraEntity jiraEntity = buildJiraEntity(result.getKey(), result.getValue()); - // Fetch the most recent reported issue + // Fetch the most recent reported issues within mergeGap by jira service account under the project List<Issue> issues = jiraClient.getIssues(jiraEntity.getJiraProject(), jiraEntity.getLabels(), this.jiraAdminConfig.getJiraUser(), jiraEntity.getMergeGap()); Optional<Issue> latestJiraIssue = issues.stream().max( diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java index 0d8446d..c8601fe 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java @@ -56,7 +56,10 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration; import org.apache.pinot.thirdeye.anomaly.task.TaskConstants; +import org.apache.pinot.thirdeye.anomaly.task.TaskContext; +import org.apache.pinot.thirdeye.anomaly.task.TaskRunner; import org.apache.pinot.thirdeye.api.Constants; import org.apache.pinot.thirdeye.auth.ThirdEyePrincipal; import org.apache.pinot.thirdeye.dashboard.DetectionPreviewConfiguration; @@ -88,6 +91,8 @@ import org.apache.pinot.thirdeye.detection.DetectionPipeline; import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader; import org.apache.pinot.thirdeye.detection.DetectionPipelineResult; import org.apache.pinot.thirdeye.detection.TaskUtils; +import org.apache.pinot.thirdeye.detection.alert.DetectionAlertTaskInfo; +import org.apache.pinot.thirdeye.detection.alert.DetectionAlertTaskRunner; import org.apache.pinot.thirdeye.detection.cache.builder.AnomaliesCacheBuilder; import org.apache.pinot.thirdeye.detection.cache.builder.TimeSeriesCacheBuilder; import org.apache.pinot.thirdeye.detection.onboard.YamlOnboardingTaskInfo; @@ -148,8 +153,10 @@ public class YamlResource { private final long previewTimeout; private final DetectionConfigFormatter detectionConfigFormatter; private final RateLimiter onboardingRateLimiter; + private final Map<String, Map<String, Object>> alerterConfig; - public YamlResource(DetectionPreviewConfiguration previewConfig, double alertOnboardingPermitPerSecond) { + public YamlResource(Map<String, Map<String, Object>> alerterConfig, DetectionPreviewConfiguration previewConfig, + double alertOnboardingPermitPerSecond) { this.detectionConfigDAO = DAORegistry.getInstance().getDetectionConfigManager(); this.subscriptionConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager(); this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO(); @@ -162,6 +169,7 @@ public class YamlResource { this.yaml = new Yaml(); this.executor = Executors.newFixedThreadPool(previewConfig.getParallelism()); this.previewTimeout = previewConfig.getTimeout(); + this.alerterConfig = alerterConfig; this.onboardingRateLimiter = RateLimiter.create(alertOnboardingPermitPerSecond); TimeSeriesLoader timeseriesLoader = @@ -182,8 +190,8 @@ public class YamlResource { this.detectionConfigFormatter = new DetectionConfigFormatter(metricDAO, datasetDAO); } - public YamlResource(DetectionPreviewConfiguration previewConfig) { - this(previewConfig, Double.MAX_VALUE); + public YamlResource(Map<String, Map<String, Object>> alerterConfig, DetectionPreviewConfiguration previewConfig) { + this(alerterConfig, previewConfig, Double.MAX_VALUE); } /* @@ -1107,4 +1115,41 @@ public class YamlResource { LOG.info("Successfully returned " + yamls.size() + " detection configs."); return Response.ok(yamls).build(); } + + /** + * Api to trigger a notification alert. Mostly used for ad-hoc testing. + * Alert will be sent only if there are new anomalies since the last watermark. + * Watermarks will be updated after notifying anomalies if any. + */ + @PUT + @Path("/notify/{id}") + @ApiOperation("Send notification email for detection alert config") + public Response triggerNotification( + @ApiParam("Subscription configuration id for the alert") @NotNull @PathParam("id") long subscriptionId) { + LOG.info("Triggering subscription task with id " + subscriptionId); + Map<String, String> responseMessage = new HashMap<>(); + try { + // Build the task context + ThirdEyeAnomalyConfiguration config = new ThirdEyeAnomalyConfiguration(); + config.setAlerterConfiguration(alerterConfig); + TaskContext taskContext = new TaskContext(); + taskContext.setThirdEyeAnomalyConfiguration(config); + + // Run the notification task. This will update the subscription watermark as well. + DetectionAlertTaskInfo taskInfo = new DetectionAlertTaskInfo(subscriptionId); + TaskRunner taskRunner = new DetectionAlertTaskRunner(); + taskRunner.execute(taskInfo, taskContext); + } catch (Exception e) { + LOG.error("Exception while triggering the notification task with id " + subscriptionId, e); + responseMessage.put("message", "Failed to trigger the notification"); + responseMessage.put("more-info", "Triggered subscription id " + subscriptionId + ". Error = " + e.getMessage()); + return Response.serverError().entity(responseMessage).build(); + } + + LOG.info("Subscription with id " + subscriptionId + " triggered successfully"); + responseMessage.put("message", "Subscription was triggered successfully."); + responseMessage.put("more-info", "Triggered subscription id " + subscriptionId); + responseMessage.put("detectionAlertConfigId", String.valueOf(subscriptionId)); + return Response.ok().entity(responseMessage).build(); + } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/JiraEntity.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/JiraEntity.java index 8863ce6..6bb43ff 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/JiraEntity.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/JiraEntity.java @@ -21,6 +21,7 @@ package org.apache.pinot.thirdeye.notification.commons; import java.io.File; import java.util.List; +import java.util.Map; import java.util.Objects; import org.apache.commons.lang3.StringUtils; @@ -38,6 +39,9 @@ public class JiraEntity { private List<String> labels; private List<String> components; + // Ability to configure non-standard customized jira fields + private Map<String, Object> customFieldsMap; + // Report new anomalies by reopening jira tickets created within // the last mergeGap milliseconds private long mergeGap; @@ -120,6 +124,14 @@ public class JiraEntity { this.components = components; } + public Map<String, Object> getCustomFieldsMap() { + return customFieldsMap; + } + + public void setCustomFieldsMap(Map<String, Object> customFieldsMap) { + this.customFieldsMap = customFieldsMap; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("Jira{"); @@ -129,6 +141,7 @@ public class JiraEntity { sb.append(", summary='").append(summary).append('\''); sb.append(", labels='").append(labels).append('\''); sb.append(", components='").append(components).append('\''); + sb.append(", custom='").append(customFieldsMap).append('\''); sb.append(", mergeGap='").append(mergeGap).append('\''); sb.append(", snapshot='").append(snapshot == null ? "" : snapshot.getName()).append('\''); sb.append('}'); @@ -137,6 +150,6 @@ public class JiraEntity { @Override public int hashCode() { - return Objects.hash(jiraProject, jiraIssueTypeId, assignee, summary, labels, mergeGap, snapshot, components); + return Objects.hash(jiraProject, jiraIssueTypeId, assignee, summary, labels, customFieldsMap, mergeGap, snapshot, components); } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java index 8ce9097..10e00d0 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java @@ -28,7 +28,7 @@ import com.atlassian.jira.rest.client.api.domain.CimProject; import com.atlassian.jira.rest.client.api.domain.Comment; import com.atlassian.jira.rest.client.api.domain.Issue; import com.atlassian.jira.rest.client.api.domain.Transition; -import com.atlassian.jira.rest.client.api.domain.input.FieldInput; +import com.atlassian.jira.rest.client.api.domain.input.ComplexIssueInputFieldValue; import com.atlassian.jira.rest.client.api.domain.input.IssueInput; import com.atlassian.jira.rest.client.api.domain.input.IssueInputBuilder; import com.atlassian.jira.rest.client.api.domain.input.TransitionInput; @@ -62,6 +62,7 @@ public class ThirdEyeJiraClient { public static final String PROP_ISSUE_TYPE = "issuetype"; public static final String PROP_PROJECT = "project"; public static final String PROP_ASSIGNEE = "assignee"; + public static final String PROP_CUSTOM = "custom"; public static final String PROP_MERGE_GAP = "mergeGap"; public static final String PROP_LABELS = "labels"; public static final String PROP_COMPONENTS = "components"; @@ -160,14 +161,11 @@ public class ThirdEyeJiraClient { * Updates existing issue with assignee and labels */ public void updateIssue(Issue issue, JiraEntity jiraEntity) { - IssueInput issueInput = new IssueInputBuilder() - .setAssigneeName(jiraEntity.getAssignee()) - .setFieldInput(new FieldInput(PROP_LABELS, jiraEntity.getLabels())) - .build(); - - String prevAssignee = issue.getAssignee() == null ? "unassigned" : issue.getAssignee().getName(); - LOG.info("Updating Jira {} with [assignee={}, labels={}]. Previous state [assignee={}, labels={}]", issue.getKey(), - jiraEntity.getAssignee(), jiraEntity.getLabels(), prevAssignee, issue.getLabels()); + IssueInputBuilder issueBuilder = new IssueInputBuilder(); + setJiraAlertUpdatableFields(issueBuilder, jiraEntity); + IssueInput issueInput = issueBuilder.build(); + + LOG.info("Updating Jira {} with {}", issue.getKey(), issueInput.toString()); restClient.getIssueClient().updateIssue(issue.getKey(), issueInput).claim(); if (jiraEntity.getSnapshot() != null && jiraEntity.getSnapshot().exists()) { restClient.getIssueClient().addAttachments(issue.getAttachmentsUri(), jiraEntity.getSnapshot()).claim(); @@ -219,6 +217,26 @@ public class ThirdEyeJiraClient { return basicIssue.getKey(); } + /** + * Set jira fields which should be updated when a new alert is fired + */ + private void setJiraAlertUpdatableFields(IssueInputBuilder issueBuilder, JiraEntity jiraEntity) { + issueBuilder.setAssigneeName(jiraEntity.getAssignee()); + issueBuilder.setFieldValue(PROP_LABELS, jiraEntity.getLabels()); + + if (jiraEntity.getComponents() != null && !jiraEntity.getComponents().isEmpty()) { + issueBuilder.setComponentsNames(jiraEntity.getComponents()); + } + + if (jiraEntity.getCustomFieldsMap() != null) { + for (Map.Entry<String, Object> customFieldEntry : jiraEntity.getCustomFieldsMap().entrySet()) { + issueBuilder.setFieldValue( + customFieldEntry.getKey(), + ComplexIssueInputFieldValue.with("name", customFieldEntry.getValue().toString())); + } + } + } + IssueInput buildIssue(JiraEntity jiraEntity) { IssueInputBuilder issueBuilder = new IssueInputBuilder(); @@ -239,13 +257,9 @@ public class ThirdEyeJiraClient { issueBuilder.setProjectKey(jiraEntity.getJiraProject()); issueBuilder.setSummary(jiraEntity.getSummary()); issueBuilder.setIssueTypeId(jiraEntity.getJiraIssueTypeId()); - issueBuilder.setAssigneeName(jiraEntity.getAssignee()); issueBuilder.setDescription(jiraEntity.getDescription()); - issueBuilder.setFieldInput(new FieldInput(PROP_LABELS, jiraEntity.getLabels())); - if (jiraEntity.getComponents() != null && !jiraEntity.getComponents().isEmpty()) { - issueBuilder.setComponentsNames(jiraEntity.getComponents()); - } + setJiraAlertUpdatableFields(issueBuilder, jiraEntity); return issueBuilder.build(); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/JiraContentFormatter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/JiraContentFormatter.java index 79e89df..88df2bd 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/JiraContentFormatter.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/JiraContentFormatter.java @@ -171,6 +171,8 @@ public class JiraContentFormatter extends AlertContentFormatter { jiraEntity.setDescription(buildDescription(jiraTemplate, templateValues)); jiraEntity.setComponents(ConfigUtils.getList(alertClientConfig.get(PROP_COMPONENTS))); jiraEntity.setSnapshot(buildSnapshot()); + Map<String, Object> customFieldsMap = ConfigUtils.getMap(alertClientConfig.get(PROP_CUSTOM)); + jiraEntity.setCustomFieldsMap(customFieldsMap); return jiraEntity; } diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java index 696b061..3bffc18 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java @@ -39,7 +39,7 @@ public class YamlResourceTest { public void beforeClass() { testDAOProvider = DAOTestBase.getInstance(); this.user = new ThirdEyePrincipal("test", "test"); - this.yamlResource = new YamlResource(new DetectionPreviewConfiguration()); + this.yamlResource = new YamlResource(null, new DetectionPreviewConfiguration()); this.daoRegistry = DAORegistry.getInstance(); DetectionConfigManager detectionDAO = this.daoRegistry.getDetectionConfigManager(); DetectionConfigDTO config1 = new DetectionConfigDTO(); diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/commons/TestThirdEyeJiraClient.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/commons/TestThirdEyeJiraClient.java index 3bdf231..68377d9 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/commons/TestThirdEyeJiraClient.java +++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/commons/TestThirdEyeJiraClient.java @@ -27,6 +27,10 @@ public class TestThirdEyeJiraClient { jiraEntity.setAssignee("test_assignee"); jiraEntity.setLabels(Arrays.asList("test_1", "test_2")); jiraEntity.setDescription("test_description"); + Map<String, Object> custom = new HashMap<>(); + custom.put("test1", "value1"); + custom.put("test2", "value2"); + jiraEntity.setCustomFieldsMap(custom); JiraConfiguration jiraConfig = new JiraConfiguration(); jiraConfig.setJiraHost("host"); @@ -47,14 +51,16 @@ public class TestThirdEyeJiraClient { // Assert if all the parameters are set Assert.assertEquals(((ComplexIssueInputFieldValue) issueInput.getField("assignee").getValue()) .getValuesMap().values().toString(), "[test_assignee]"); + Assert.assertEquals(((ComplexIssueInputFieldValue) issueInput.getField("project").getValue()) + .getValuesMap().values().toString(), "[test_project]"); Assert.assertEquals(((List) issueInput.getField("labels").getValue()), Arrays.asList("test_1", "test_2")); Assert.assertEquals(issueInput.getField("summary").getValue(), "test_summary"); Assert.assertEquals(issueInput.getField("description").getValue(), "test_description"); - Assert.assertEquals(((ComplexIssueInputFieldValue) issueInput.getField("project").getValue()) - .getValuesMap().values().toString(), "[test_project]"); + Assert.assertEquals(issueInput.getField("test1").getValue().toString(), "ComplexIssueInputFieldValue{valuesMap={name=value1}}"); + Assert.assertEquals(issueInput.getField("test2").getValue().toString(), "ComplexIssueInputFieldValue{valuesMap={name=value2}}"); // Assert if all the required fields are sets - Assert.assertEquals(issueInput.getFields().size(), 7); + Assert.assertEquals(issueInput.getFields().size(), 9); Assert.assertTrue(issueInput.getFields().keySet().contains("anotherrequiredfield")); Assert.assertFalse(issueInput.getFields().keySet().contains("notrequiredfield")); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org