[incubator-pinot] branch master updated (fd78e6a -> 9e757ef)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from fd78e6a Make transform functions support underscore in the query functions (#6125) add 9e757ef [TE] add owners field into the subscription validation (#6128) No new revisions were added by this update. Summary of changes: .../validators/detection/detection-config-schema.json | 11 +++ .../validators/subscription/subscription-config-schema.json | 10 ++ 2 files changed, 21 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] entity anomaly logging for ad-hoc debugging (#6001)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 143f398 [TE] entity anomaly logging for ad-hoc debugging (#6001)
143f398 is described below
commit 143f398fc1a3e23df991c43820049ad3cd1dc730
Author: Akshay Rai
AuthorDate: Sun Sep 13 14:55:08 2020 -0700
[TE] entity anomaly logging for ad-hoc debugging (#6001)
---
.../tools/RunAdhocDatabaseQueriesTool.java | 87 --
1 file changed, 80 insertions(+), 7 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
index c2da5df..f8d576e 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
@@ -18,6 +18,7 @@ package org.apache.pinot.thirdeye.tools;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Multimap;
import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
import org.apache.pinot.thirdeye.constant.AnomalyResultSource;
import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
@@ -80,9 +81,22 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.collections4.CollectionUtils;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
+import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
+import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
+import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import
org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import org.jfree.util.Log;
+import org.apache.pinot.thirdeye.detection.cache.builder.AnomaliesCacheBuilder;
+import
org.apache.pinot.thirdeye.detection.cache.builder.TimeSeriesCacheBuilder;
+import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,6 +125,11 @@ public class RunAdhocDatabaseQueriesTool {
private ClassificationConfigManager classificationConfigDAO;
private ApplicationManager applicationDAO;
+ private static final DateTimeZone TIMEZONE =
DateTimeZone.forID("America/Los_Angeles");
+ private static final DateTimeFormatter DATE_FORMAT =
DateTimeFormat.forPattern("-MM-dd");
+ private static final String ENTITY_STATS_TEMPLATE = "#children = {},
properties = {}";
+ private static final String ENTITY_TIME_TEMPLATE = "[create {}, start {},
end {}]";
+
public RunAdhocDatabaseQueriesTool(File persistenceFile)
throws Exception {
init(persistenceFile);
@@ -692,8 +711,7 @@ public class RunAdhocDatabaseQueriesTool {
// find all recent anomalies (last 14 days) with end time <=
watermark;
long lookback = TimeUnit.DAYS.toMillis(14);
- Predicate predicate = Predicate.AND(
- Predicate.LE("createTime", clockEntry.getValue()),
+ Predicate predicate = Predicate.AND(Predicate.LE("createTime",
clockEntry.getValue()),
Predicate.GT("createTime", clockEntry.getValue() - lookback),
Predicate.EQ("detectionConfigId", detectionConfigDTO.getId()));
List anomalies =
mergedResultDAO.findByPredicate(predicate);
@@ -723,9 +741,65 @@ public class RunAdhocDatabaseQueriesTool {
}
}
- public static void main(String[] args) throws Exception {
+ private void printEntityAnomalyDetails(MergedAnomalyResultDTO anomaly,
String indent, int index) {
+LOG.info("");
+LOG.info("Exploring Entity Anomaly {} with id {}", index, anomaly.getId());
+LOG.info(ENTITY_STATS_TEMPLATE, anomaly.getChildren().size(),
anomaly.getProperties());
+LOG.info(ENTITY_TIME_TEMPLATE,
+new DateTime(anomaly.getCreatedTime(), TIMEZONE),
+DATE_FORMAT.print(new DateTime(anomaly.getStartTime(), TIMEZONE)),
+DATE_FORMAT.print(new DateTime(anomaly.getEndTime(), TIMEZONE)));
+ }
-File persistenceFile = new File("/Users/akrai/persistence-linux.yml");
+ /**
+ * Visualizes the en
[incubator-pinot] branch master updated: [TE] add lastTimestamp into the detection schema (#5964)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1e17f6d [TE] add lastTimestamp into the detection schema (#5964)
1e17f6d is described below
commit 1e17f6d64103cef2bb33899bad09e0d37c628710
Author: Akshay Rai
AuthorDate: Thu Sep 3 10:48:45 2020 -0700
[TE] add lastTimestamp into the detection schema (#5964)
---
.../main/resources/validators/detection/detection-config-schema.json | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
index 8a07d24..59ee067 100644
---
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
+++
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
@@ -51,6 +51,9 @@
},
"grouper": {
"$ref": "#/definitions/grouper"
+},
+"lastTimestamp": {
+ "type": "number"
}
},
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] only merge same trend in ChildKeepingMerge (#5920)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2dbc94c [TE] only merge same trend in ChildKeepingMerge (#5920)
2dbc94c is described below
commit 2dbc94c201557e3bce5a3a944abac46c644c3cbd
Author: Vincent Chen
AuthorDate: Thu Sep 3 10:18:33 2020 -0700
[TE] only merge same trend in ChildKeepingMerge (#5920)
This PR is to fix the issue that ChildKeepMergeWrapper merges anomalies
with different trend. After this PR, the wrapper will only merge anomalies with
same trend, e.g anomalies with UP trend, meaning current value higher than
predicted value.
---
.../wrapper/ChildKeepingMergeWrapper.java | 13 --
.../thirdeye/detection/DetectionTestUtils.java | 30 ++
.../wrapper/ChildKeepingMergeWrapperTest.java | 48 ++
3 files changed, 80 insertions(+), 11 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index d2a1320..064e4f1 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -21,6 +21,7 @@ package org.apache.pinot.thirdeye.detection.wrapper;
import com.google.common.collect.Collections2;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -45,6 +46,7 @@ import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
*/
public class ChildKeepingMergeWrapper extends BaselineFillingMergeWrapper {
private static final String PROP_GROUP_KEY = "groupKey";
+ private static final String PROP_PATTERN_KEY = "pattern";
public ChildKeepingMergeWrapper(DataProvider provider, DetectionConfigDTO
config, long startTime, long endTime) {
super(provider, config, startTime, endTime);
@@ -91,10 +93,15 @@ public class ChildKeepingMergeWrapper extends
BaselineFillingMergeWrapper {
if (anomaly.getProperties().containsKey(PROP_GROUP_KEY)) {
groupKey = anomaly.getProperties().get(PROP_GROUP_KEY);
}
-
+ String patternKey = "";
+ if (anomaly.getProperties().containsKey(PROP_PATTERN_KEY)) {
+patternKey = anomaly.getProperties().get(PROP_PATTERN_KEY);
+ } else if (!Double.isNaN(anomaly.getAvgBaselineVal()) &&
!Double.isNaN(anomaly.getAvgCurrentVal())) {
+patternKey = (anomaly.getAvgCurrentVal() >
anomaly.getAvgBaselineVal()) ? "UP" : "DOWN";
+ }
MergeWrapper.AnomalyKey key =
- new MergeWrapper.AnomalyKey(anomaly.getMetric(),
anomaly.getCollection(), anomaly.getDimensions(), groupKey, "",
- anomaly.getType());
+ new MergeWrapper.AnomalyKey(anomaly.getMetric(),
anomaly.getCollection(), anomaly.getDimensions(),
+ StringUtils.join(Arrays.asList(groupKey, patternKey), ","), "",
anomaly.getType());
MergedAnomalyResultDTO parent = parents.get(key);
if (parent == null || anomaly.getStartTime() - parent.getEndTime() >
this.maxGap) {
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
index 745ba95..8a9eac6 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionTestUtils.java
@@ -33,7 +33,8 @@ import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
public class DetectionTestUtils {
private static final Long PROP_ID_VALUE = 1000L;
- public static MergedAnomalyResultDTO makeAnomaly(Long configId, Long
legacyFunctionId, long start, long end, String metric, String dataset,
Map dimensions) {
+ public static MergedAnomalyResultDTO makeAnomaly(Long configId, Long
legacyFunctionId, long start, long end,
+ String metric, String dataset, Map dimensions,
Map props) {
MergedAnomalyResultDTO anomaly = new MergedAnomalyResultDTO();
anomaly.setDetectionConfigId(configId);
anomaly.setStartTime(start);
@@ -41,6 +42,7 @@ public class DetectionTestUtils {
anomaly.setMetric(metric);
anomaly.setCollection(dataset);
anomaly.setFunctionId(legacyFunctionId);
+anomaly.setProperties(props);
Multimap filters = HashMultimap.create();
for (Map.Entry dimension : dimensions.entrySet()) {
@@
[incubator-pinot] branch master updated (d444285 -> be27d52)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from d444285 Incorporating embedded exception while trying to fetch stream
offset (#5956)
add be27d52 [TE][bug] Treat ThirdEye alert creation as a transaction
(#5958)
No new revisions were added by this update.
Summary of changes:
thirdeye/thirdeye-pinot/config/dashboard.yml | 1 -
thirdeye/thirdeye-pinot/config/detector.yml| 1 -
.../thirdeye/detection/yaml/YamlResource.java | 104 -
.../dataquality/DataQualityTaskRunnerTest.java | 2 +
.../thirdeye/detection/yaml/YamlResourceTest.java | 95 ---
...ction-config-2.yaml => detection-config-3.yaml} | 2 +-
.../yaml/subscription/subscription-config-6.yaml | 4 +-
...on-config-6.yaml => subscription-config-7.yaml} | 2 +-
8 files changed, 146 insertions(+), 65 deletions(-)
copy
thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/detection/{detection-config-2.yaml
=> detection-config-3.yaml} (91%)
copy
thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/subscription/{subscription-config-6.yaml
=> subscription-config-7.yaml} (89%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] polish the rca template in email (#5939)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c3446ba [TE] polish the rca template in email (#5939)
c3446ba is described below
commit c3446bac9c5c0882e0855a77674fc4eb211dbdf1
Author: Akshay Rai
AuthorDate: Fri Aug 28 16:15:18 2020 -0700
[TE] polish the rca template in email (#5939)
* polished the rca table with line separators and text wrapping
* fixed the rca url in the template
* removed deprecated fields from configuration
---
.../anomaly/ThirdEyeAnomalyApplication.java| 18 +++--
.../anomaly/ThirdEyeAnomalyConfiguration.java | 45
.../thirdeye/common/ThirdEyeConfiguration.java | 45 +++-
.../dashboard/ThirdEyeDashboardConfiguration.java | 1 -
.../dashboard/resources/v2/RootCauseResource.java | 2 +-
.../detector/metric-anomalies-template.ftl | 72 +--
...rca-highlights-cube-algo-response-rendered.html | 84 ++
...st-email-rca-highlights-cube-algo-response.json | 2 +-
8 files changed, 94 insertions(+), 175 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
index 0435848..e1a8ba5 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
@@ -97,7 +97,7 @@ public class ThirdEyeAnomalyApplication
}
@Override
- public void run(final ThirdEyeAnomalyConfiguration config, final Environment
environment)
+ public void run(final ThirdEyeAnomalyConfiguration config, final Environment
env)
throws Exception {
LOG.info("Starting ThirdeyeAnomalyApplication : Scheduler {} Worker {}",
config.isScheduler(), config.isWorker());
super.initDAOs();
@@ -107,10 +107,14 @@ public class ThirdEyeAnomalyApplication
LOG.error("Exception while loading caches", e);
}
-environment.getObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
-environment.getObjectMapper().registerModule(makeMapperModule());
+env.getObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+env.getObjectMapper().registerModule(makeMapperModule());
-environment.lifecycle().manage(new Managed() {
+env.lifecycle().manage(lifecycleManager(config, env));
+ }
+
+ private Managed lifecycleManager(ThirdEyeAnomalyConfiguration config,
Environment env) {
+return new Managed() {
@Override
public void start() throws Exception {
@@ -140,14 +144,14 @@ public class ThirdEyeAnomalyApplication
new
HolidayEventsLoader(config.getHolidayEventsLoaderConfiguration(),
config.getCalendarApiKeyPath(),
DAORegistry.getInstance().getEventDAO());
holidayEventsLoader.start();
- environment.jersey().register(new
HolidayEventResource(holidayEventsLoader));
+ env.jersey().register(new HolidayEventResource(holidayEventsLoader));
}
if (config.isMockEventsLoader()) {
mockEventsLoader = new
MockEventsLoader(config.getMockEventsLoaderConfiguration(),
DAORegistry.getInstance().getEventDAO());
mockEventsLoader.run();
}
if (config.isPinotProxy()) {
- environment.jersey().register(new PinotDataSourceResource());
+ env.jersey().register(new PinotDataSourceResource());
}
if (config.isDetectionPipeline()) {
detectionScheduler = new
DetectionCronScheduler(DAORegistry.getInstance().getDetectionConfigManager());
@@ -206,7 +210,7 @@ public class ThirdEyeAnomalyApplication
modelDownloaderManager.shutdown();
}
}
-});
+};
}
private void updateAdminSession(String adminUser, String sessionKey) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
index c46dea7..8a08bb0 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
@@ -32,8 +32,6 @@ public class ThirdEyeAnomalyConfiguration extends
ThirdEyeConfiguration {
private boolean alert = false;
private boolean autoload = false;
- private boolean classifier = false;
- private boolean dataCompleteness = false;
private boolean holidayEventsLoader = false;
private boolean mockEventsLoader
[incubator-pinot] branch master updated: [TE] update schema to support window parameters (#5933)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8f0145e [TE] update schema to support window parameters (#5933)
8f0145e is described below
commit 8f0145eb39ed84b53c9c4b858b5d9662a4386941
Author: Akshay Rai
AuthorDate: Thu Aug 27 16:16:31 2020 -0700
[TE] update schema to support window parameters (#5933)
---
.../detection/wrapper/AnomalyDetectorWrapper.java | 5 ++-
.../detection/detection-config-schema.json | 51 +++---
.../subscription/subscription-config-schema.json | 3 ++
.../composite-detection-config-good-1.yaml | 4 ++
4 files changed, 56 insertions(+), 7 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
index cf8621a..52cc63a 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
@@ -69,8 +69,9 @@ import static
org.apache.pinot.thirdeye.detection.yaml.translator.DetectionConfi
*/
public class AnomalyDetectorWrapper extends DetectionPipeline {
private static final String PROP_METRIC_URN = "metricUrn";
+ private static final String PROP_DETECTOR_COMPONENT_NAME =
"detectorComponentName";
- // moving window detection properties
+ // moving window detection properties (configurable in yaml)
private static final String PROP_MOVING_WINDOW_DETECTION =
"isMovingWindowDetection";
private static final String PROP_WINDOW_DELAY = "windowDelay";
private static final String PROP_WINDOW_DELAY_UNIT = "windowDelayUnit";
@@ -78,10 +79,10 @@ public class AnomalyDetectorWrapper extends
DetectionPipeline {
private static final String PROP_WINDOW_UNIT = "windowUnit";
private static final String PROP_FREQUENCY = "frequency";
private static final String PROP_DETECTOR = "detector";
- private static final String PROP_DETECTOR_COMPONENT_NAME =
"detectorComponentName";
private static final String PROP_TIMEZONE = "timezone";
private static final String PROP_BUCKET_PERIOD = "bucketPeriod";
private static final String PROP_CACHE_PERIOD_LOOKBACK =
"cachingPeriodLookback";
+
// fail detection job if it failed successively for the first 5 windows
private static final long EARLY_TERMINATE_WINDOW = 5;
// expression to consolidate the time series
diff --git
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
index ced3f61..8a07d24 100644
---
a/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
+++
b/thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
@@ -146,7 +146,48 @@
}
},
-"eachComponent": {
+"anomalyDetectorComponent": {
+ "type": "array",
+ "items": {
+"type": "object",
+"required": [ "type", "name" ],
+"properties": {
+ "type": {
+"type": "string"
+ },
+ "name": {
+"type": "string"
+ },
+ "params": {
+"type": "object"
+ },
+ "windowDelay": {
+"type": "number"
+ },
+ "windowDelayUnit": {
+"type": "string"
+ },
+ "windowSize": {
+"type": "number"
+ },
+ "windowUnit": {
+"type": "string"
+ },
+ "bucketPeriod": {
+"type": "string"
+ },
+ "timeZone": {
+"type": "string"
+ },
+ "cachingPeriodLookback": {
+"type": "string"
+ }
+},
+"additionalProperties": false
+ }
+},
+
+"genericComponent": {
"type": "array",
"items": {
"type": "object",
@@ -174,13 +215,13 @@
"required": [ "detection" ],
"
[incubator-pinot] branch master updated (ced3a70 -> 1b38909)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from ced3a70 Fix CSV and JSON converter on BYTES column (#5931)
add 1b38909 [TE] ThirdEye User Config (Yaml) Validation (#5930)
No new revisions were added by this update.
Summary of changes:
thirdeye/pom.xml | 1 +
thirdeye/thirdeye-pinot/pom.xml| 7 +
.../trigger/DataAvailabilityTaskScheduler.java | 3 +
.../api/detection/AnomalyDetectionResource.java| 6 +-
.../dashboard/resources/v2/AnomaliesResource.java | 1 -
.../thirdeye/detection/DefaultDataProvider.java| 1 -
.../registry/DetectionAlertRegistry.java | 1 -
.../validators/ConfigValidationException.java | 57
.../validators/ConfigValidationUtils.java | 57
.../detection/validators/ConfigValidator.java | 25 +-
.../validators/DetectionConfigValidator.java | 226 ++--
.../validators/SubscriptionConfigValidator.java| 117 +++--
.../validators/ThirdEyeUserConfigValidator.java| 75 ++
.../thirdeye/detection/yaml/YamlResource.java | 286 ++---
.../yaml/translator/ConfigTranslator.java | 26 +-
.../yaml/translator/DetectionConfigTranslator.java | 17 +-
.../translator/DetectionMetricAttributeHolder.java | 2 +-
.../translator/SubscriptionConfigTranslator.java | 23 +-
.../builder/DetectionConfigPropertiesBuilder.java | 4 +-
.../detection/detection-config-schema.json | 214 +++
.../subscription/subscription-config-schema.json | 110
.../detection/AnomalyDetectionResourceTest.java| 3 +-
.../pinot/thirdeye/datalayer/DaoTestUtils.java | 14 +-
.../pinot/thirdeye/detection/DataProviderTest.java | 2 -
.../pinot/thirdeye/detection/MockDataProvider.java | 11 +-
.../dataquality/DataQualityTaskRunnerTest.java | 8 +-
.../validators/DetectionConfigValidatorTest.java | 172 +
.../SubscriptionConfigValidatorTest.java | 272
.../thirdeye/detection/yaml/YamlResourceTest.java | 52 +++-
.../translator/DetectionConfigTranslatorTest.java | 5 +-
.../YamlDetectionAlertConfigTranslatorTest.java| 7 +-
.../integration/NotificationTaskSchedulerTest.java | 12 +-
.../detection/migrated-detection-config-1.json | 2 +-
.../detection/migrated-detection-config-2.json | 2 +-
yaml => composite-detection-config-bad-1.yaml} | 8 +-
...yaml => composite-detection-config-good-1.yaml} | 4 +-
...e-config-1.yaml => detection-config-bad-1.yaml} | 43 +++-
.../validators/detection-config-good-1.yaml| 82 ++
.../validators/entity-pipeline-config-2.yaml | 4 +-
.../yaml/subscription/subscription-config-1.yaml | 12 +-
.../yaml/subscription/subscription-config-2.yaml | 8 +-
.../yaml/subscription/subscription-config-3.yaml | 8 +-
.../yaml/subscription/subscription-config-4.yaml | 8 +-
.../yaml/subscription/subscription-config-5.yaml | 8 +-
.../compositePipelineTranslatorTestResult-5.json | 24 +-
.../yaml/translator/pipeline-config-1.yaml | 5 +-
.../yaml/translator/pipeline-config-2.yaml | 2 +-
.../yaml/translator/pipeline-config-3.yaml | 2 +-
.../yaml/translator/pipeline-config-5.yaml | 4 +-
.../detection/yaml/translator/sla-config-5.yaml| 8 +-
.../yaml/translator/sla-config-translated-5.json | 12 +-
.../src/test/resources/sample-alert-config.yml | 2 +-
52 files changed, 1454 insertions(+), 611 deletions(-)
create mode 100644
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ConfigValidationException.java
create mode 100644
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ConfigValidationUtils.java
create mode 100644
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/ThirdEyeUserConfigValidator.java
create mode 100644
thirdeye/thirdeye-pinot/src/main/resources/validators/detection/detection-config-schema.json
create mode 100644
thirdeye/thirdeye-pinot/src/main/resources/validators/subscription/subscription-config-schema.json
create mode 100644
thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidatorTest.java
copy
thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/validators/{entity-pipeline-config-2.yaml
=> composite-detection-config-bad-1.yaml} (93%)
copy
thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/validators/{entity-pipeline-config-2.yaml
=> composite-detection-config-good-1.yaml} (95%)
copy
thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/validators/{entity-pipelin
[incubator-pinot] branch master updated: [TE] cube api bug fix to fetch by dataset name (#5921)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9e65bdf [TE] cube api bug fix to fetch by dataset name (#5921)
9e65bdf is described below
commit 9e65bdf641ddb054ce1a4d874e76c1c5dbf383fa
Author: Akshay Rai
AuthorDate: Tue Aug 25 13:14:06 2020 -0700
[TE] cube api bug fix to fetch by dataset name (#5921)
---
.../org/apache/pinot/thirdeye/dashboard/resources/SummaryResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/SummaryResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/SummaryResource.java
index 7965d9b..6caf3bc 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/SummaryResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/SummaryResource.java
@@ -118,7 +118,7 @@ public class SummaryResource {
Dimensions dimensions;
if (StringUtils.isBlank(groupByDimensions) ||
JAVASCRIPT_NULL_STRING.equals(groupByDimensions)) {
-DatasetConfigDTO datasetConfigDTO =
datasetConfigDAO.findByDataset(dataset);
+DatasetConfigDTO datasetConfigDTO =
datasetConfigDAO.findByDataset(datasetName);
List dimensionNames = new ArrayList<>();
if (datasetConfigDTO != null) {
dimensionNames = datasetConfigDTO.getDimensions();
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] add anomaly detection as a service - new table for storing ad-hoc data (#5866)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c425b0f [TE] add anomaly detection as a service - new table for
storing ad-hoc data (#5866)
c425b0f is described below
commit c425b0feabf674bd5e8de3b034c126bb8ffdd9fc
Author: Yanwen(Jason) Lin <34975811+jasonyanw...@users.noreply.github.com>
AuthorDate: Tue Aug 25 15:04:00 2020 -0400
[TE] add anomaly detection as a service - new table for storing ad-hoc data
(#5866)
Phase 2 is separated into two parts. This PR is for the 1st part. The main
change is that a new table is created for storing online ad-hoc data. The PR
for Phase 1 can be found in #5769.
Here are some other minor changes:
* Remove the cleanExistingState method in the AnomalyDetectionResource.
This is not necessary because each request is distinct now and the monitor job
will periodically clean them (dataset/metric/detection configs, ad-hoc data).
* Remove the suffix appended to metric as the metric name is allowed to be
duplicate.
The 2nd part of phase 2 is about creating a new endpoint to provide
registering and CRUD operations on ad-hoc data.
---
.../apache/pinot/thirdeye/api/detection/AnomalyDetectionResource.java | 2 ++
.../pinot/thirdeye/api/detection/AnomalyDetectionResourceTest.java | 1 +
2 files changed, 3 insertions(+)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResource.java
index 16b476c..ac3d863 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResource.java
@@ -184,6 +184,8 @@ public class AnomalyDetectionResource {
// TODO: refactor code to resolve request configurations in one place
(e.g. default/customized config names)
+// TODO: refactor code to resolve request configurations in one place
(e.g. default/customized config names)
+
try {
Preconditions.checkNotNull(start, "Detection start time is not
provided");
Preconditions.checkNotNull(end, "Detection end time is not provided");
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResourceTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResourceTest.java
index d6bfd98..7b52d98 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResourceTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/api/detection/AnomalyDetectionResourceTest.java
@@ -48,6 +48,7 @@ import java.io.IOException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+
public class AnomalyDetectionResourceTest {
private static final String DEFAULT_DETECTION_NAME = "online_detection";
private static final String DEFAULT_DATASET_NAME = "online_dataset";
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] Fix corner case in SLA alert - round up datasetLastRefreshTime (#5909)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new afc7539 [TE] Fix corner case in SLA alert - round up
datasetLastRefreshTime (#5909)
afc7539 is described below
commit afc75394d127e8a870206eb4300f8c773154a513
Author: Akshay Rai
AuthorDate: Tue Aug 25 12:00:30 2020 -0700
[TE] Fix corner case in SLA alert - round up datasetLastRefreshTime (#5909)
---
.../components/DataSlaQualityChecker.java | 40 ++-
.../spec/DataSlaQualityCheckerSpec.java| 7 +-
.../dataquality/DataQualityTaskRunnerTest.java | 80 +++---
3 files changed, 69 insertions(+), 58 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/components/DataSlaQualityChecker.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/components/DataSlaQualityChecker.java
index 42d6860..9f4f110 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/components/DataSlaQualityChecker.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/components/DataSlaQualityChecker.java
@@ -77,7 +77,6 @@ public class DataSlaQualityChecker implements
AnomalyDetector delay;
}
/**
- * Align and round off start time to the upper boundary of the granularity
+ * Align and round off timestamp to the upper boundary of the granularity
+ *
+ * Examples:
+ * a. 20th Aug 05:00 pm will be rounded up to 21th Aug 12:00 am
+ * b. 20th Aug 12:00 am will be rounded up to 21th Aug 12:00 am
*/
private static long alignToUpperBoundary(long timestamp, DatasetConfigDTO
datasetConfig) {
Period granularityPeriod =
datasetConfig.bucketTimeGranularity().toPeriod();
DateTimeZone timezone = DateTimeZone.forID(datasetConfig.getTimezone());
-DateTime startTime = new DateTime(timestamp - 1,
timezone).plus(granularityPeriod);
-return (startTime.getMillis() /
granularityPeriod.toStandardDuration().getMillis()) *
granularityPeriod.toStandardDuration().getMillis();
+DateTime startTime = new DateTime(timestamp,
timezone).plus(granularityPeriod);
+return (startTime.getMillis() /
granularityPeriod.toStandardDuration().getMillis())
+* granularityPeriod.toStandardDuration().getMillis();
}
/**
* Creates a DATA_SLA anomaly from ceiling(start) to ceiling(end) for the
detection id.
* If existing DATA_SLA anomalies are present, then it will be merged
accordingly.
+ *
+ * @param datasetLastRefreshTime the timestamp corresponding to the last
record in the data (UTC)
+ * @param datasetLastRefreshTimeRoundUp the timestamp rounded off to the
upper granular bucket. See
+ * @{link #alignToUpperBoundary}
*/
- private MergedAnomalyResultDTO createDataSLAAnomaly(MetricSlice slice,
DatasetConfigDTO datasetConfig) {
-MergedAnomalyResultDTO anomaly = DetectionUtils.makeAnomaly(
-alignToUpperBoundary(slice.getStart(), datasetConfig),
-alignToUpperBoundary(slice.getEnd(), datasetConfig));
+ private MergedAnomalyResultDTO createDataSLAAnomaly(MetricSlice slice,
DatasetConfigDTO datasetConfig,
+ long datasetLastRefreshTime, long datasetLastRefreshTimeRoundUp) {
+MergedAnomalyResultDTO anomaly =
DetectionUtils.makeAnomaly(slice.getStart(), slice.getEnd());
anomaly.setCollection(datasetConfig.getName());
anomaly.setType(AnomalyType.DATA_SLA);
anomaly.setAnomalyResultSource(AnomalyResultSource.DATA_QUALITY_DETECTION);
// Store the metadata in the anomaly
Map properties = new HashMap<>();
-properties.put("datasetLastRefreshTime", String.valueOf(slice.getStart() -
1));
-properties.put("sla", sla);
+properties.put("datasetLastRefreshTime",
String.valueOf(datasetLastRefreshTime));
+properties.put("datasetLastRefreshTimeRoundUp",
String.valueOf(datasetLastRefreshTimeRoundUp));
+properties.put("sla", this.sla);
anomaly.setProperties(properties);
return anomaly;
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/spec/DataSlaQualityCheckerSpec.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/spec/DataSlaQualityCheckerSpec.java
index edfbc19..5a67d01 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/spec/DataSlaQualityCheckerSpec.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/dataquality/spec/DataSlaQualityCheckerSpec.java
@@ -20,16 +20,17 @@
package org.apache.pinot.thirdeye.detection.dataquality.spec;
import com.fa
[incubator-pinot] branch master updated: [TE] Remove deprecated classes under /detection/algorithm (#5908)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1b3fb99 [TE] Remove deprecated classes under /detection/algorithm
(#5908)
1b3fb99 is described below
commit 1b3fb999395923561cae7165b7a491e061b096d2
Author: Akshay Rai
AuthorDate: Tue Aug 25 11:49:25 2020 -0700
[TE] Remove deprecated classes under /detection/algorithm (#5908)
---
.../thirdeye/detection/DefaultDataProvider.java| 2 +-
.../detection/DetectionPipelineLoader.java | 2 +-
.../pinot/thirdeye/detection/DetectionUtils.java | 4 +-
.../detection/algorithm/BaselineAlgorithm.java | 139
.../algorithm/BaselineRuleFilterWrapper.java | 117
.../detection/algorithm/MovingWindowAlgorithm.java | 762 -
.../algorithm/RuleBasedFilterWrapper.java | 91 ---
.../detection/algorithm/ThresholdAlgorithm.java| 93 ---
.../algorithm/ThresholdRuleFilterWrapper.java | 73 --
.../components/AbsoluteChangeRuleDetector.java | 2 +-
.../detection/components/HoltWintersDetector.java | 2 +-
.../components/MeanVarianceRuleDetector.java | 1 -
.../components/PercentageChangeRuleDetector.java | 2 +-
.../components/ThresholdRuleDetector.java | 2 +-
.../detection/algorithm/BaselineAlgorithmTest.java | 139
.../algorithm/MovingWindowAlgorithmTest.java | 414 ---
.../algorithm/ThresholdAlgorithmTest.java | 89 ---
.../MergeDimensionThresholdIntegrationTest.java| 158 -
.../mergeDimensionThresholdProperties.json | 31 -
19 files changed, 8 insertions(+), 2115 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
index 4d82342..c996196 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
@@ -215,7 +215,7 @@ public class DefaultDataProvider implements DataProvider {
}
@Override
- public DetectionPipeline loadPipeline(DetectionConfigDTO config, long start,
long end) throws Exception {
+ public DetectionPipeline loadPipeline(DetectionConfigDTO config, long start,
long end) {
return this.loader.from(this, config, start, end);
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineLoader.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineLoader.java
index dff23c8..7af6681 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineLoader.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineLoader.java
@@ -32,7 +32,7 @@ public class DetectionPipelineLoader {
Constructor constructor =
Class.forName(className).getConstructor(DataProvider.class,
DetectionConfigDTO.class, long.class, long.class);
return (DetectionPipeline) constructor.newInstance(provider, config,
start, end);
} catch (Exception e) {
- throw new IllegalArgumentException("Failed to initialize the detection
pipeline.", e.getCause());
+ throw new IllegalArgumentException("Failed to initialize the detection
pipeline.", e);
}
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
index f0eeb2f..9322bb5 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
@@ -106,12 +106,12 @@ public class DetectionUtils {
* @param slice metric slice
* @param df time series with COL_TIME and at least one boolean value series
* @param seriesName name of the value series
- * @param endTime end time of this detection window
* @param monitoringGranularityPeriod the monitoring granularity period
* @param dataset dataset config for the metric
* @return list of anomalies
*/
- public static List makeAnomalies(MetricSlice slice,
DataFrame df, String seriesName, long endTime, Period
monitoringGranularityPeriod, DatasetConfigDTO dataset) {
+ public static List makeAnomalies(MetricSlice slice,
DataFrame df, String seriesName,
+ Period monitoringGranularityPeriod, DatasetConfigDTO dataset) {
if (df.isEmpty()) {
return Collections.emptyList();
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/a
[incubator-pinot] branch master updated (a8974c7 -> 4c4777d)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from a8974c7 [TE] Updated ThirdEye docker launch script to accept MODE as a second arg (#5914) add 4c4777d [TE] frontend - harleyjj/alert-details - fix bug in forecast dates (#5916) No new revisions were added by this update. Summary of changes: .../app/pods/components/alert-details/component.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] add anomaly detection as a service - endpoints for managing online detection data (#5894)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a9818f5 [TE] add anomaly detection as a service - endpoints for
managing online detection data (#5894)
a9818f5 is described below
commit a9818f554cd0334f5786ad8db6a7c5fc0d0d2fff
Author: Yanwen(Jason) Lin <34975811+jasonyanw...@users.noreply.github.com>
AuthorDate: Mon Aug 24 14:39:41 2020 -0400
[TE] add anomaly detection as a service - endpoints for managing online
detection data (#5894)
Phase 2 is separated into two parts. This PR is for the 2nd part. Please
note that this PR has a dependency. PR #5866 should be merged before this PR
get merged. The PR for Phase 2 1st part can be found in #5866 (this is also the
commit c118b16 in current PR). The PR for Phase 1 can be found in #5769.
The main change is that several CRUD endpoints are created for managing
online detection data. Those endpoints are listed as follows:
* POST /anomaly-detection/data - register data. The data should be provided
in the request. The dataset and metric configurations can optionally be
provided to customize configurations. A data-id will be returned.
* GET /anomaly-detection/data/{data-id} - retrieve registered data with the
corresponding dataset and metric configurations.
* PUT /anomaly-detection/data/{data-id} - update registered data. The data
should be provided in the request. The dataset and metric configurations can
optionally be provided to customize configurations. The data-id will be
returned.
* DELETE /anomaly-detection/data/{data-id} - delete registered data. The
data-id will be returned.
And one extra optional path parameter data-id is provided for online
detection service endpoint: POST /anomaly-detection. If the user has used the
above endpoints to register their online detection data, they do not need to
provide data in the online detection service request (POST /anomaly-detection)
and only need to provide the data-id with optional detection configuration.
Here are some other minor changes:
* Add dataset and metric configurations in response as well (similar to
detection configuration)
* Force user to provide start and end time. If they are not provided,
return with error message
* The parameters used in request/response are all reformated to use hyphen
format. For example, detectionConfiguration is changed to
detection-configuration.
---
.../anomaly/monitor/MonitorTaskRunner.java | 10 +
.../api/detection/AnomalyDetectionResource.java| 576 -
.../datalayer/bao/OnlineDetectionDataManager.java | 9 +
.../bao/jdbc/OnlineDetectionDataManagerImpl.java | 25 +
.../thirdeye/datalayer/dao/GenericPojoDao.java | 4 +
.../datalayer/dto/OnlineDetectionDataDTO.java | 6 +
.../datalayer/entity/OnlineDetectionDataIndex.java | 22 +
.../thirdeye/datalayer/pojo/MetricConfigBean.java | 10 -
.../datalayer/pojo/OnlineDetectionDataBean.java| 51 ++
.../thirdeye/datalayer/util/DaoProviderUtil.java | 3 +
.../pinot/thirdeye/datasource/DAORegistry.java | 5 +
.../online/OnlineThirdEyeDataSource.java | 35 +-
.../src/main/resources/schema/create-schema.sql| 12 +
.../detection/AnomalyDetectionResourceTest.java| 266 +++---
.../bao/TestOnlineDetectionDataManager.java| 63 +++
.../thirdeye/api/detection/payload-bad-custom.json | 6 +-
.../pinot/thirdeye/api/detection/payload-bad.json | 6 +-
.../api/detection/payload-good-custom.json | 6 +-
.../pinot/thirdeye/api/detection/payload-good.json | 6 +-
.../api/detection/payload-update-test1.json| 2 +
.../api/detection/payload-update-test2.json| 2 +
.../api/detection/payload-update-test3.json| 3 +
22 files changed, 876 insertions(+), 252 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java
index 763e75b..a5c1ca3 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java
@@ -281,6 +281,16 @@ public class MonitorTaskRunner implements TaskRunner {
} catch (Exception e) {
LOG.error("Exception when deleting old evaluations.", e);
}
+
+// Delete expired online detection data
+try {
+ int deletedOnlineDetectionDatas =
DAO_REGISTRY.getOnlineDetectionDataManager()
+
.deleteRecordsOlderThanDays(monitorTaskInfo.getDefaultRetentionDays());
+ LOG.info("Deleted {} online detection dat
[incubator-pinot] branch master updated (6dd54f8 -> 2b58bfb)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 6dd54f8 Fix the variable names for off-heap alloc configs (#5852) add 2b58bfb [TE] clean up legacy code (#5842) No new revisions were added by this update. Summary of changes: .../anomaly/alert/util/DataReportHelper.java | 276 - .../thirdeye/anomaly/alert/util/EmailHelper.java | 130 .../dashboard/resources/SummaryResource.java | 24 +- .../views/contributor/ContributionCell.java| 183 .../views/contributor/ContributionViewTable.java | 48 --- .../contributor/ContributionViewTableBuilder.java | 142 - .../views/contributor/ContributorViewHandler.java | 331 - .../views/contributor/ContributorViewRequest.java | 27 -- .../views/contributor/ContributorViewResponse.java | 109 --- .../thirdeye/detection/yaml/YamlResource.java | 8 +- .../content/BaseNotificationContent.java | 2 - .../pinot/thirdeye/detector/anomaly-report-v2.ftl | 136 - .../pinot/thirdeye/detector/anomaly-report.ftl | 150 -- .../thirdeye/detector/custom-anomaly-report.ftl| 96 -- .../detector/data-report-by-metric-dimension.ftl | 105 --- .../dashboard/handler/ContributorTest.java | 70 - 16 files changed, 20 insertions(+), 1817 deletions(-) delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/alert/util/DataReportHelper.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/views/contributor/ContributionCell.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/views/contributor/ContributionViewTable.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/views/contributor/ContributionViewTableBuilder.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/views/contributor/ContributorViewHandler.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/views/contributor/ContributorViewRequest.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/views/contributor/ContributorViewResponse.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/resources/org/apache/pinot/thirdeye/detector/anomaly-report-v2.ftl delete mode 100644 thirdeye/thirdeye-pinot/src/main/resources/org/apache/pinot/thirdeye/detector/anomaly-report.ftl delete mode 100644 thirdeye/thirdeye-pinot/src/main/resources/org/apache/pinot/thirdeye/detector/custom-anomaly-report.ftl delete mode 100644 thirdeye/thirdeye-pinot/src/main/resources/org/apache/pinot/thirdeye/detector/data-report-by-metric-dimension.ftl delete mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/handler/ContributorTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (db48107 -> f4949e9)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from db48107 Adding controller healthcheck endpoint: /health (#5846) add f4949e9 [TE] Added reset application API +refactor (#5847) No new revisions were added by this update. Summary of changes: .../api/application/ApplicationResource.java | 132 + 1 file changed, 85 insertions(+), 47 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (cc7a344 -> 7f8e65c)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from cc7a344 [TE] UI changes for anomalies page v3(#5824) add 7f8e65c [TE] Fix template rendering errors (#5839) No new revisions were added by this update. Summary of changes: .../content/templates/MetricAnomaliesContent.java | 4 ++-- .../detector/metric-anomalies-template.ftl | 9 +++- .../templates/TestMetricAnomaliesContent.java | 26 +- 3 files changed, 15 insertions(+), 24 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (999f567 -> 7cc75ad)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 999f567 [TE] Guice Injection for Dashboard Server (#5792) add 7cc75ad [TE] make rca insights in email resilient to issues in rca response (#5794) No new revisions were added by this update. Summary of changes: thirdeye/thirdeye-pinot/config/h2db.mv.db | Bin 2949120 -> 2490368 bytes .../common/restclient/ThirdEyeRcaRestClient.java | 2 +- .../dashboard/resources/SummaryResource.java | 75 +++-- .../dashboard/resources/v2/RootCauseResource.java | 2 +- .../content/templates/MetricAnomaliesContent.java | 14 ++-- .../detector/metric-anomalies-template.ftl | 55 --- .../restclient/TestThirdEyeRcaRestClient.java | 2 +- .../alert/scheme/DetectionJiraAlerterTest.java | 2 +- .../templates/TestMetricAnomaliesContent.java | 67 -- ...rca-highlights-cube-algo-response-rendered.html | 44 ++-- ...st-email-rca-highlights-cube-algo-response.json | 63 - 11 files changed, 211 insertions(+), 115 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (0e95a67 -> 47623c0)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 0e95a67 Do not apply shade plugin on parent modules (#5761) add 47623c0 [TE] Error handling refactor of EntityManagerResource (#5788) No new revisions were added by this update. Summary of changes: .../resources/BadRequestWebException.java | 55 .../dashboard/resources/EntityManagerResource.java | 43 .../dashboard/resources/ResourceUtils.java | 60 ++ 3 files changed, 137 insertions(+), 21 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/BadRequestWebException.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/ResourceUtils.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] remove linkedin bintray from pom and add override (#5773) (#5775)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 32348be [TE] remove linkedin bintray from pom and add override
(#5773) (#5775)
32348be is described below
commit 32348be61b69e11fb3fd902dfbfe24ddb7214f7f
Author: Vincent Chen
AuthorDate: Fri Jul 31 12:16:54 2020 -0700
[TE] remove linkedin bintray from pom and add override (#5773) (#5775)
---
.travis/.travis_te_nightly_build.sh | 2 +-
thirdeye/pom.xml| 7 ---
2 files changed, 1 insertion(+), 8 deletions(-)
diff --git a/.travis/.travis_te_nightly_build.sh
b/.travis/.travis_te_nightly_build.sh
index 07007ba..790509d 100755
--- a/.travis/.travis_te_nightly_build.sh
+++ b/.travis/.travis_te_nightly_build.sh
@@ -27,7 +27,7 @@ if [ -n "${DEPLOY_BUILD_OPTS}" ]; then
mvn versions:set -DnewVersion="$BUILD_VERSION${DEV_VERSION}" -q -B
mvn versions:commit -q -B
# Deploy ThirdEye backend to bintray
- mvn -pl '!thirdeye-frontend' deploy -s ../.travis/.ci.settings.xml
-DskipTests -q
+ mvn -pl '!thirdeye-frontend' deploy -s ../.travis/.ci.settings.xml
-DskipTests -q
-DaltDeploymentRepository=bintray-linkedin-maven::default::'https://api.bintray.com/maven/linkedin/maven/thirdeye/;publish=1;override=1'
# Deploy ThirdEye frontend to NPM
cd thirdeye-frontend/
npm version ${BUILD_VERSION}${DEV_VERSION}
diff --git a/thirdeye/pom.xml b/thirdeye/pom.xml
index 08eee1c..e30c7fa 100644
--- a/thirdeye/pom.xml
+++ b/thirdeye/pom.xml
@@ -86,13 +86,6 @@
20170516
2.12.0
-
-
-
- bintray-linkedin-maven
-
https://api.bintray.com/maven/linkedin/maven/thirdeye/;publish=1
-
-
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch thirdeye-elr updated (74ca8b6 -> 6a244e2)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch thirdeye-elr in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 74ca8b6 [TE] minor fix (#5777) add 6a244e2 revert comment-out for merging (#5779) No new revisions were added by this update. Summary of changes: .travis/.travis_set_deploy_build_opts.sh | 8 1 file changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch thirdeye-elr updated (323d1b1 -> 74ca8b6)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch thirdeye-elr in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 323d1b1 [TE] minor fix to trigger ELR (#5776) add 74ca8b6 [TE] minor fix (#5777) No new revisions were added by this update. Summary of changes: .travis/.travis_set_deploy_build_opts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch thirdeye-elr updated (6cd415f -> 323d1b1)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch thirdeye-elr in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 6cd415f [TE] remove linkedin bintray from pom and add override (#5773) add 323d1b1 [TE] minor fix to trigger ELR (#5776) No new revisions were added by this update. Summary of changes: .travis/.travis_set_deploy_build_opts.sh | 4 ++-- .travis/.travis_te_nightly_build.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (10117ae -> cb7de23)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 10117ae Add untar failure server meter (#5768) add cb7de23 [TE] Added a backfill start date for Anomaly Detection (#5740) No new revisions were added by this update. Summary of changes: .../thirdeye/detection/yaml/YamlResource.java | 27 ++ .../yaml/translator/DetectionConfigTranslator.java | 16 - 2 files changed, 32 insertions(+), 11 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch thirdeye-elr updated: [TE] remove linkedin bintray from pom and add override (#5773)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch thirdeye-elr
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/thirdeye-elr by this push:
new 6cd415f [TE] remove linkedin bintray from pom and add override (#5773)
6cd415f is described below
commit 6cd415f5efd7e9dba3bc24579001d177ae1d1001
Author: Vincent Chen
AuthorDate: Thu Jul 30 11:37:09 2020 -0700
[TE] remove linkedin bintray from pom and add override (#5773)
---
.travis/.travis_te_nightly_build.sh | 6 +++---
thirdeye/pom.xml| 7 ---
2 files changed, 3 insertions(+), 10 deletions(-)
diff --git a/.travis/.travis_te_nightly_build.sh
b/.travis/.travis_te_nightly_build.sh
index 07007ba..418e4c3 100755
--- a/.travis/.travis_te_nightly_build.sh
+++ b/.travis/.travis_te_nightly_build.sh
@@ -18,7 +18,7 @@
# under the License.
#
-if [ -n "${DEPLOY_BUILD_OPTS}" ]; then
+# if [ -n "${DEPLOY_BUILD_OPTS}" ]; then
echo "Deploying ThirdEye to bintray"
# Generate new version number
cd thirdeye/
@@ -27,10 +27,10 @@ if [ -n "${DEPLOY_BUILD_OPTS}" ]; then
mvn versions:set -DnewVersion="$BUILD_VERSION${DEV_VERSION}" -q -B
mvn versions:commit -q -B
# Deploy ThirdEye backend to bintray
- mvn -pl '!thirdeye-frontend' deploy -s ../.travis/.ci.settings.xml
-DskipTests -q
+ mvn -pl '!thirdeye-frontend' deploy -s ../.travis/.ci.settings.xml
-DskipTests -q
-DaltDeploymentRepository=bintray-linkedin-maven::default::'https://api.bintray.com/maven/linkedin/maven/thirdeye/;publish=1;override=1'
# Deploy ThirdEye frontend to NPM
cd thirdeye-frontend/
npm version ${BUILD_VERSION}${DEV_VERSION}
npm-login-noninteractive
npm publish
-fi
+# fi
diff --git a/thirdeye/pom.xml b/thirdeye/pom.xml
index 08eee1c..e30c7fa 100644
--- a/thirdeye/pom.xml
+++ b/thirdeye/pom.xml
@@ -86,13 +86,6 @@
20170516
2.12.0
-
-
-
- bintray-linkedin-maven
-
https://api.bintray.com/maven/linkedin/maven/thirdeye/;publish=1
-
-
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch thirdeye-elr created (now 10117ae)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch thirdeye-elr in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at 10117ae Add untar failure server meter (#5768) No new revisions were added by this update. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (1f69041 -> b1ab891)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 1f69041 [Part 2] Add geo support - add a geo aggregate function st_union (#5744) add b1ab891 [TE] Embed cube algorithm results in email (#5770) No new revisions were added by this update. Summary of changes: .../content/templates/MetricAnomaliesContent.java | 2 +- .../detector/metric-anomalies-template.ftl | 47 +++ .../restclient/MockThirdEyeRcaRestClient.java | 1 - .../alert/scheme/DetectionEmailAlerterTest.java| 1 - .../templates/TestMetricAnomaliesContent.java | 48 ++- ...rca-highlights-cube-algo-response-rendered.html | 115 ...st-email-rca-highlights-cube-algo-response.json | 149 + .../resources/test-metric-anomalies-template.html | 1 + 8 files changed, 356 insertions(+), 8 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/test/resources/test-email-rca-highlights-cube-algo-response-rendered.html create mode 100644 thirdeye/thirdeye-pinot/src/test/resources/test-email-rca-highlights-cube-algo-response.json - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] Thirdeye docker log crash fix (#5749)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 405b4ef [TE] Thirdeye docker log crash fix (#5749)
405b4ef is described below
commit 405b4ef7f529647095740db04798404559a7b77a
Author: Alexander Pucher
AuthorDate: Fri Jul 24 15:16:20 2020 -0700
[TE] Thirdeye docker log crash fix (#5749)
Recent changes to ThirdEye logging aren't reflected in the config files
shipped with the docker image and break the launch of apachepinot/thirdeye.
This PR modifies the configuration to launch the container successfully.
---
docker/images/pinot-thirdeye/bin/start-thirdeye.sh | 4 ++--
docker/images/pinot-thirdeye/config/ephemeral/dashboard.yml| 4
docker/images/pinot-thirdeye/config/ephemeral/detector.yml | 6 +++---
docker/images/pinot-thirdeye/config/pinot-quickstart/dashboard.yml | 4
docker/images/pinot-thirdeye/config/pinot-quickstart/detector.yml | 6 +++---
5 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
b/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
index 92b8d87..56498a8 100755
--- a/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
+++ b/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
@@ -39,11 +39,11 @@ fi
echo "Running thirdeye backend config: ${CONFIG_DIR}"
[ -f "${CONFIG_DIR}/data-sources/data-sources-config-backend.yml" ] && cp
"${CONFIG_DIR}/data-sources/data-sources-config-backend.yml"
"${CONFIG_DIR}/data-sources/data-sources-config.yml"
-java -cp "./bin/thirdeye-pinot.jar"
org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyApplication "${CONFIG_DIR}" &
+java -Dlog4j.configurationFile=log4j2.xml -cp "./bin/thirdeye-pinot.jar"
org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyApplication "${CONFIG_DIR}" &
sleep 10
echo "Running thirdeye frontend config: ${CONFIG_DIR}"
[ -f "${CONFIG_DIR}/data-sources/data-sources-config-frontend.yml" ] && cp
"${CONFIG_DIR}/data-sources/data-sources-config-frontend.yml"
"${CONFIG_DIR}/data-sources/data-sources-config.yml"
-java -cp "./bin/thirdeye-pinot.jar"
org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardApplication
"${CONFIG_DIR}" &
+java -Dlog4j.configurationFile=log4j2.xml -cp "./bin/thirdeye-pinot.jar"
org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardApplication
"${CONFIG_DIR}" &
wait
diff --git a/docker/images/pinot-thirdeye/config/ephemeral/dashboard.yml
b/docker/images/pinot-thirdeye/config/ephemeral/dashboard.yml
index 09c4e86..c5298b1 100644
--- a/docker/images/pinot-thirdeye/config/ephemeral/dashboard.yml
+++ b/docker/images/pinot-thirdeye/config/ephemeral/dashboard.yml
@@ -1,3 +1,5 @@
+logging:
+ type: external
authConfig:
authEnabled: false
authKey: ""
@@ -24,6 +26,8 @@ alerterConfiguration:
smtpHost: localhost
smtpPort: 25
server:
+ requestLog:
+type: external
type: default
applicationConnectors:
- type: http
diff --git a/docker/images/pinot-thirdeye/config/ephemeral/detector.yml
b/docker/images/pinot-thirdeye/config/ephemeral/detector.yml
index da08eb1..a8516ec 100644
--- a/docker/images/pinot-thirdeye/config/ephemeral/detector.yml
+++ b/docker/images/pinot-thirdeye/config/ephemeral/detector.yml
@@ -1,8 +1,8 @@
logging:
- level: INFO
- loggers:
-org.hibernate.engine.internal: WARN
+ type: external
server:
+ requestLog:
+type: external
type: default
rootPath: '/api/*'
applicationContextPath: /
diff --git a/docker/images/pinot-thirdeye/config/pinot-quickstart/dashboard.yml
b/docker/images/pinot-thirdeye/config/pinot-quickstart/dashboard.yml
index b7b4b47..b039b8d 100644
--- a/docker/images/pinot-thirdeye/config/pinot-quickstart/dashboard.yml
+++ b/docker/images/pinot-thirdeye/config/pinot-quickstart/dashboard.yml
@@ -1,3 +1,5 @@
+logging:
+ type: external
authConfig:
authEnabled: false
authKey: ""
@@ -24,6 +26,8 @@ alerterConfiguration:
smtpHost: localhost
smtpPort: 25
server:
+ requestLog:
+type: external
type: default
applicationConnectors:
- type: http
diff --git a/docker/images/pinot-thirdeye/config/pinot-quickstart/detector.yml
b/docker/images/pinot-thirdeye/config/pinot-quickstart/detector.yml
index 6bca237..1ad3312 100644
--- a/docker/images/pinot-thirdeye/config/pinot-quickstart/detector.yml
+++ b/docker/images/pinot-thirdeye/config/pinot-quickstart/detector.yml
@@ -1,8 +1,8 @@
logging:
- level: INFO
- loggers:
-org.hibernate.engine.internal: WARN
+ type: external
server:
+ requestLog:
+type: external
type: default
rootPath: '/api/*'
applicationContextPath: /
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] pinot - harleyjj/detectionutils - return current when predicted baseline cannot be trained (#5736)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2af8d8e [TE] pinot - harleyjj/detectionutils - return current when
predicted baseline cannot be trained (#5736)
2af8d8e is described below
commit 2af8d8ee8b6a7587d988d29453379ecc375e5997
Author: Harley Jackson
AuthorDate: Fri Jul 24 15:06:10 2020 -0700
[TE] pinot - harleyjj/detectionutils - return current when predicted
baseline cannot be trained (#5736)
*When detection/predicted-baseline/{anomalyId} endpoint is called and there
is insufficient data to generate the baseline,
an error is currently returned.
*This fix returns the current time series instead, so that data will still
be displayed in the graph.
---
.../apache/pinot/thirdeye/detection/DetectionUtils.java | 16 +++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
index 4352d1a..f0eeb2f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
@@ -252,6 +252,7 @@ public class DetectionUtils {
long start, long end, DetectionPipelineLoader loader, DataProvider
provider) throws Exception {
String baselineProviderComponentName =
anomaly.getProperties().get(PROP_BASELINE_PROVIDER_COMPONENT_NAME);
BaselineProvider baselineProvider = new RuleBaselineProvider();
+TimeSeries returnTimeSeries;
if (baselineProviderComponentName != null && config != null &&
config.getComponentSpecs().containsKey(baselineProviderComponentName))
{
@@ -265,7 +266,20 @@ public class DetectionUtils {
InputDataFetcher dataFetcher = new DefaultInputDataFetcher(provider,
config.getId());
baselineProvider.init(spec, dataFetcher);
}
-return
baselineProvider.computePredictedTimeSeries(MetricSlice.from(metricId, start,
end, filters));
+
+try {
+ returnTimeSeries =
baselineProvider.computePredictedTimeSeries(MetricSlice.from(metricId, start,
end, filters));
+} catch (Exception e) {
+ // send current if the predicted baseline can't be trained
+ BaselineProvider alternateProvider = new RuleBaselineProvider();
+ RuleBaselineProviderSpec spec = new RuleBaselineProviderSpec();
+ spec.setOffset("current");
+ InputDataFetcher dataFetcher = new DefaultInputDataFetcher(provider,
config.getId());
+ alternateProvider.init(spec, dataFetcher);
+ returnTimeSeries =
alternateProvider.computePredictedTimeSeries(MetricSlice.from(metricId, start,
end, filters));
+}
+
+return returnTimeSeries;
}
/**
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] Setup a rest client pipeline for comms between ThirdEye services; added RCA highlights API as an example (#5713)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new f5bb36b [TE] Setup a rest client pipeline for comms between ThirdEye services; added RCA highlights API as an example (#5713) f5bb36b is described below commit f5bb36b8e3b3a0ac01cfc09c9027098e3aa65552 Author: Akshay Rai AuthorDate: Fri Jul 24 14:36:22 2020 -0700 [TE] Setup a rest client pipeline for comms between ThirdEye services; added RCA highlights API as an example (#5713) Given an anomaly, added an RCA highlights API which returns the cube results (and more). The PR also enables the notification pipeline to retrieve these results via a rest client. The results will be embedded into the alert email in a subsequent PR. Changes: * Created a generic Jersey client API (AbstractRestClient) for GET and POST requests and implemented an RCA rest client (ThirdEyeRcaRestClient) used for communicating with the RCA services. * Packaged the rest client under the commons package so that it can be leveraged by all the modules. * The Thirdeye rest client (admin service) account is configured in the detector.yaml configuration file. Ex: ``` thirdeyeRestClient: adminUser: admin sessionKey: 1234 ``` * Introduced a constants package under commons and moved some of the RCA constants there. * Added unit tests for testing compose url and RCA rest client. * Tested the entire pipeline by deploying locally. --- .../anomaly/ThirdEyeAnomalyApplication.java| 19 +++ .../anomaly/ThirdEyeAnomalyConfiguration.java | 10 ++ .../rca/MultiDimensionalSummaryConstants.java | 34 + .../constants/rca/RootCauseResourceConstants.java | 29 .../common/restclient/AbstractRestClient.java | 166 + .../common/restclient/ParseResponseFunction.java | 40 + .../pinot/thirdeye/common/restclient/Protocol.java | 35 + .../common/restclient/ThirdEyeRcaRestClient.java | 109 ++ .../ThirdEyeRestClientConfiguration.java | 46 ++ .../pinot/thirdeye/common/utils/SessionUtils.java | 16 ++ .../cube/entry/MultiDimensionalSummaryCLITool.java | 45 +++--- .../dashboard/ThirdEyeDashboardApplication.java| 2 +- .../dashboard/resources/SummaryResource.java | 51 --- .../dashboard/resources/v2/RootCauseResource.java | 54 ++- .../alert/scheme/DetectionAlertScheme.java | 4 + .../alert/scheme/DetectionEmailAlerter.java| 2 +- .../alert/scheme/DetectionJiraAlerter.java | 3 +- .../content/BaseNotificationContent.java | 12 +- .../content/templates/EntityGroupKeyContent.java | 2 +- .../templates/HierarchicalAnomaliesContent.java| 4 +- .../content/templates/MetricAnomaliesContent.java | 59 ++-- .../common/restclient/MockAbstractRestClient.java | 52 +++ .../restclient/MockThirdEyeRcaRestClient.java | 56 +++ .../common/restclient/TestAbstractRestClient.java | 61 .../restclient/TestThirdEyeRcaRestClient.java | 65 .../alert/scheme/DetectionEmailAlerterTest.java| 15 ++ .../alert/scheme/DetectionJiraAlerterTest.java | 18 ++- .../templates/TestMetricAnomaliesContent.java | 12 +- .../channels/TestJiraContentFormatter.java | 17 ++- 29 files changed, 956 insertions(+), 82 deletions(-) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java index 2a3686d..439b057 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java @@ -27,10 +27,13 @@ import org.apache.pinot.thirdeye.anomaly.events.HolidayEventsLoader; import org.apache.pinot.thirdeye.anomaly.events.MockEventsLoader; import org.apache.pinot.thirdeye.anomaly.monitor.MonitorJobScheduler; import org.apache.pinot.thirdeye.anomaly.task.TaskDriver; +import org.apache.pinot.thirdeye.common.restclient.ThirdEyeRestClientConfiguration; import org.apache.pinot.thirdeye.common.time.TimeGranularity; import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardService; import org.apache.pinot.thirdeye.common.BaseThirdEyeApplication; import org.apache.pinot.thirdeye.common.ThirdEyeSwaggerBundle; +import org.apache.pinot.thirdeye.common.utils.SessionUtils; +import org.apache.pinot.thirdeye.datalayer.dto.SessionDTO; import org.apache.pinot.thirdeye.datasource.DAORegistry; import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry; import
[incubator-pinot] branch master updated: [TE] Make few internal documentationi and mocks publicwq (#5743)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cb149be [TE] Make few internal documentationi and mocks publicwq
(#5743)
cb149be is described below
commit cb149bed54b926b5841e7cc1919c5c3fc8dd8199
Author: Akshay Rai
AuthorDate: Fri Jul 24 09:22:17 2020 -0700
[TE] Make few internal documentationi and mocks publicwq (#5743)
---
thirdeye/docs/alert_setup.rst | 2 +-
thirdeye/docs/detection_pipeline_architecture.rst | 165 ++
.../docs/detection_pipeline_execution_flow.rst | 192 +
thirdeye/docs/index.rst| 2 +
.../{alert_setup.rst => thirdeye_architecture.rst} | 13 +-
.../{alert_setup.rst => thirdeye_ui_mocks.rst} | 15 +-
6 files changed, 372 insertions(+), 17 deletions(-)
diff --git a/thirdeye/docs/alert_setup.rst b/thirdeye/docs/alert_setup.rst
index bda73f0..be00ebe 100644
--- a/thirdeye/docs/alert_setup.rst
+++ b/thirdeye/docs/alert_setup.rst
@@ -29,4 +29,4 @@ Alert Setup
advanced_config
templates
appendix
-contribute_detection
\ No newline at end of file
+contribute_detection
diff --git a/thirdeye/docs/detection_pipeline_architecture.rst
b/thirdeye/docs/detection_pipeline_architecture.rst
new file mode 100644
index 000..0419b6c
--- /dev/null
+++ b/thirdeye/docs/detection_pipeline_architecture.rst
@@ -0,0 +1,165 @@
+..
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+..
+
+.. _detection-pipeline-architecture:
+
+##
+Detection Pipeline Architecture
+##
+
+This document summarizes the motivation and rationale behind the 2018
refactoring of ThirdEye's anomaly framework library. We describe critical user,
dev, and ops pain points, define a list of requirements and discuss our
approach to building a scalable and robust detection framework for ThirdEye. We
also discuss conscious design trade-offs made in order to facilitate future
modification and maintenance of the system.
+
+Motivation
+
+ThirdEye has been adopted by numerous teams for production monitoring of
business and system metrics, and ThirdEye's user base grows consistently. With
this come challenges of scale and a demand for excellence in development and
operation. ThirdEye has outgrown the assumptions and limits of its existing
detection framework in multiple dimensions and we need to address technical
debt in order to enable the on-boarding of new use-cases and continue to
satisfy our existing customers. We exp [...]
+
+Lack of support for business rules and custom work flows
+***
+Anomaly detection workflows have many common components such as the monitoring
of different metrics and the drill-down into multiple dimensions of time
series. However, each individual detection use-case usually comes with an
additional set of business rules to integrate smoothly with established
processes at LinkedIn. These typically include cut-off thresholds and fixed
lists of sub-dimensions to monitor or ignore, but may extend to custom alerting
workflows and the grouping of detected [...]
+
+Difficult debugging and modification
+**
+The existing framework has grown over multiple generations of developers with
different preferences, designs, and goals. This lead to an inconsistent and
undocumented approach to architecture and code design. Worse, existing design
documentation is outdated and misleading. This is exacerbated by a lack of
testing infrastructure, integration tests, and unit tests. Many assumptions
about the behavior of the framework are implicit and only exist in the heads of
past maintainers of platform [...]
+
+Tight coupling prevents testing
+
+Another property of the existing detection code base is tight coupling of
individual components and the leaking of state in unexpecte
[incubator-pinot] branch master updated (31fdc5d -> b8c1c85)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 31fdc5d UI Enhancement: Pinot UI expandable leftmost column(Page
Links) (#5723)
add b8c1c85 [TE] Added DataSource.getMinDateTime() API for Pinot (#5727)
No new revisions were added by this update.
Summary of changes:
thirdeye/.gitignore| 1 +
.../thirdeye/datasource/ThirdEyeDataSource.java| 10 +
...eMaxTime.java => PinotDataSourceTimeQuery.java} | 43 +-
.../datasource/pinot/PinotThirdEyeDataSource.java | 19 +-
4 files changed, 47 insertions(+), 26 deletions(-)
rename
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/{PinotDataSourceMaxTime.java
=> PinotDataSourceTimeQuery.java} (79%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (7708341 -> 2aad02b)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 7708341 Re-implement TarGzCompressionUtils (#5665)
add 2aad02b [TE] Created a separate doc for Application. +cosmetic
refactor (#5720)
No new revisions were added by this update.
Summary of changes:
thirdeye/docs/{intro.rst => about.rst} | 0
thirdeye/docs/application.rst | 96 ++
thirdeye/docs/configuration.rst| 38 --
thirdeye/docs/getting_started.rst | 21 +---
thirdeye/docs/introduction.rst | 5 +-
5 files changed, 100 insertions(+), 60 deletions(-)
rename thirdeye/docs/{intro.rst => about.rst} (100%)
create mode 100644 thirdeye/docs/application.rst
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] frontend - harleyjj/alert-details - Align time picker on Alert Overview and Preview with RCA (#5607)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7405f38 [TE] frontend - harleyjj/alert-details - Align time picker on
Alert Overview and Preview with RCA (#5607)
7405f38 is described below
commit 7405f38651f892991531f60790777097bd6196e6
Author: Harley Jackson
AuthorDate: Wed Jul 1 10:32:19 2020 -0700
[TE] frontend - harleyjj/alert-details - Align time picker on Alert
Overview and Preview with RCA (#5607)
---
.../app/pods/components/alert-details/component.js | 57 --
.../app/pods/components/alert-details/template.hbs | 118 ++---
.../app/styles/components/alert-details.scss | 26 +
.../app/styles/shared/_styles.scss | 4 +
4 files changed, 132 insertions(+), 73 deletions(-)
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
index 9149500..eb82429 100644
--- a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
+++ b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
@@ -30,17 +30,14 @@ import { getValueFromYaml } from
'thirdeye-frontend/utils/yaml-tools';
import { inject as service } from '@ember/service';
import { task } from 'ember-concurrency';
import floatToPercent from 'thirdeye-frontend/utils/float-to-percent';
-import { setUpTimeRangeOptions } from
'thirdeye-frontend/utils/manage-alert-utils';
import moment from 'moment';
import _ from 'lodash';
import config from 'thirdeye-frontend/config/environment';
const TABLE_DATE_FORMAT = 'MMM DD, hh:mm A'; // format for anomaly table
const TIME_PICKER_INCREMENT = 5; // tells date picker hours field how
granularly to display time
-const DEFAULT_ACTIVE_DURATION = '1m'; // setting this date range selection as
default (Last 24 Hours)
const UI_DATE_FORMAT = 'MMM D, hh:mm a'; // format for date picker to use
(usually varies by route or metric)
const DISPLAY_DATE_FORMAT = '-MM-DD HH:mm'; // format used consistently
across app to display custom date range
-const TIME_RANGE_OPTIONS = ['48h', '1w', '1m', '3m'];
const ANOMALY_LEGEND_THRESHOLD = 20; // If number of anomalies is larger than
this threshold, don't show the legend
export default Component.extend({
@@ -602,7 +599,7 @@ export default Component.extend({
} = this.getProperties('anomaliesOld', 'anomaliesCurrent',
'analysisRange', 'stateOfAnomaliesAndTimeSeries');
let tableData = [];
const humanizedObject = {
-queryDuration: (get(this, 'duration') || DEFAULT_ACTIVE_DURATION),
+queryDuration: analysisRange[1] - analysisRange[0],
queryStart: analysisRange[0],
queryEnd: analysisRange[1]
};
@@ -746,7 +743,7 @@ export default Component.extend({
component: 'custom/anomalies-table/investigation-link',
title: 'RCA',
propertyName: 'id'
- }]
+ }];
return [...settingsColumn, ...startColumn, ...dimensionColumn,
...middleColumns, ...rightmostColumns, ...rcaColumn];
}
@@ -831,24 +828,29 @@ export default Component.extend({
* @type {Object[]} - array of objects, each of which represents each date
pill
*/
pill: computed(
-'analysisRange', 'startDate', 'endDate', 'duration',
+'analysisRange', 'startDate', 'endDate',
function() {
const analysisRange = get(this, 'analysisRange');
const startDate = Number(analysisRange[0]);
const endDate = Number(analysisRange[1]);
- const duration = get(this, 'duration') || DEFAULT_ACTIVE_DURATION;
const predefinedRanges = {
-'Today': [moment().startOf('day'), moment().startOf('day').add(1,
'days')],
-'Last 24 hours': [moment().subtract(1, 'day'), moment()],
-'Yesterday': [moment().subtract(1, 'day').startOf('day'),
moment().startOf('day')],
-'Last Week': [moment().subtract(1, 'week').startOf('day'),
moment().startOf('day')]
+'Last 48 Hours': [moment().subtract(48, 'hour').startOf('hour'),
moment().startOf('hour')],
+'Last Week': [moment().subtract(1, 'week').startOf('day'),
moment().startOf('day')],
+'Last 30 Days': [moment().subtract(1, 'month').startOf('day'),
moment().startOf('day')],
+'Last 3 Months': [moment().subtract(3, 'month').startOf('day'),
moment().startOf('day')]
};
-
+ if (!this.get('isPreviewMode')) {
+const futureRanges = {
+ 'Next 48 Hours': [moment().add(48, 'hour').startOf('hour'),
moment().startOf('hour')],
+ 'Next Week': [moment().add(1, 'week').startOf('day'),
moment().startOf('day')],
+ 'Next 30 Days': [moment().add(1, 'month').startOf('day'),
moment().startOf('day')]
+};
+Object.assign
[incubator-pinot] branch master updated: [TE] fix run-backend.sh with log4j2 (#5629)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new 33f5dd7 [TE] fix run-backend.sh with log4j2 (#5629) 33f5dd7 is described below commit 33f5dd78ade1b082282eb64add226217b94bb230 Author: Vincent Chen AuthorDate: Fri Jun 26 14:23:02 2020 -0700 [TE] fix run-backend.sh with log4j2 (#5629) --- thirdeye/run-backend.sh | 2 +- thirdeye/thirdeye-pinot/config/detector.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/thirdeye/run-backend.sh b/thirdeye/run-backend.sh index 8e09c51..d70d7e4 100755 --- a/thirdeye/run-backend.sh +++ b/thirdeye/run-backend.sh @@ -4,5 +4,5 @@ echo "Launching ThirdEye backend worker in demo mode" echo "***" cd thirdeye-pinot -java -cp "./target/thirdeye-pinot-1.0.0-SNAPSHOT.jar" org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyApplication "./config" +java -Dlog4j.configurationFile=log4j2.xml -cp "./target/thirdeye-pinot-1.0.0-SNAPSHOT.jar" org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyApplication "./config" cd .. diff --git a/thirdeye/thirdeye-pinot/config/detector.yml b/thirdeye/thirdeye-pinot/config/detector.yml index dd1d9d9..f9f15f1 100644 --- a/thirdeye/thirdeye-pinot/config/detector.yml +++ b/thirdeye/thirdeye-pinot/config/detector.yml @@ -11,6 +11,8 @@ server: adminConnectors: - type: http port: 1868 + requestLog: +type: external alert: false autoload: false classifier: false - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] logging config change for demo mode (#5621)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new a5de96c [TE] logging config change for demo mode (#5621) a5de96c is described below commit a5de96cc585359c2684591079709f05740e41c0a Author: Vincent Chen AuthorDate: Thu Jun 25 16:20:58 2020 -0700 [TE] logging config change for demo mode (#5621) --- thirdeye/run-frontend.sh | 2 +- thirdeye/thirdeye-pinot/config/dashboard.yml | 4 thirdeye/thirdeye-pinot/config/detector.yml | 4 +--- .../thirdeye-pinot/src/main/resources/log4j2-test.xml| 13 - thirdeye/thirdeye-pinot/src/main/resources/log4j2.xml| 16 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/thirdeye/run-frontend.sh b/thirdeye/run-frontend.sh index 072a590..694476f 100755 --- a/thirdeye/run-frontend.sh +++ b/thirdeye/run-frontend.sh @@ -4,5 +4,5 @@ echo "Launching ThirdEye Dashboard in demo mode" echo "***" cd thirdeye-pinot -java -cp "./target/thirdeye-pinot-1.0.0-SNAPSHOT.jar" org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardApplication "./config" +java -Dlog4j.configurationFile=log4j2.xml -cp "./target/thirdeye-pinot-1.0.0-SNAPSHOT.jar" org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardApplication "./config" cd .. diff --git a/thirdeye/thirdeye-pinot/config/dashboard.yml b/thirdeye/thirdeye-pinot/config/dashboard.yml index b7b4b47..84e8967 100644 --- a/thirdeye/thirdeye-pinot/config/dashboard.yml +++ b/thirdeye/thirdeye-pinot/config/dashboard.yml @@ -23,6 +23,8 @@ alerterConfiguration: smtpConfiguration: smtpHost: localhost smtpPort: 25 +logging: + type: external server: type: default applicationConnectors: @@ -31,6 +33,8 @@ server: adminConnectors: - type: http port: 1427 + requestLog: +type: external whitelistDatasets: [] swagger: resourcePackage: "org.apache.pinot.thirdeye.dashboard.resources,org.apache.pinot.thirdeye.dashboard.resources.v2,org.apache.pinot.thirdeye.anomaly.onboard,org.apache.pinot.thirdeye.detection,org.apache.pinot.thirdeye.detection.yaml" diff --git a/thirdeye/thirdeye-pinot/config/detector.yml b/thirdeye/thirdeye-pinot/config/detector.yml index b5c1951..dd1d9d9 100644 --- a/thirdeye/thirdeye-pinot/config/detector.yml +++ b/thirdeye/thirdeye-pinot/config/detector.yml @@ -1,7 +1,5 @@ logging: - level: INFO - loggers: -org.hibernate.engine.internal: WARN + type: external server: type: default rootPath: '/api/*' diff --git a/thirdeye/thirdeye-pinot/src/main/resources/log4j2-test.xml b/thirdeye/thirdeye-pinot/src/main/resources/log4j2-test.xml deleted file mode 100644 index 1733507..000 --- a/thirdeye/thirdeye-pinot/src/main/resources/log4j2-test.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/thirdeye/thirdeye-pinot/src/main/resources/log4j2.xml b/thirdeye/thirdeye-pinot/src/main/resources/log4j2.xml new file mode 100644 index 000..43f3853 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (ab59c19 -> f8347f6)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from ab59c19 [TE] add the applications to the alert search endpoint response (#5614) add f8347f6 [TE] migrate logging from logback to log4j2 (#5606) No new revisions were added by this update. Summary of changes: thirdeye/pom.xml | 18 ++ thirdeye/thirdeye-pinot/pom.xml| 26 +++ .../thirdeye/common/BaseThirdEyeApplication.java | 9 + .../rootcause/impl/RCAFrameworkRunner.java | 39 +- .../src/main/resources/log4j2-test.xml | 13 .../thirdeye-pinot/src/main/resources/logback.xml | 16 - .../test/resources/sample-config-dir/detector.yml | 4 +-- 7 files changed, 68 insertions(+), 57 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/main/resources/log4j2-test.xml delete mode 100644 thirdeye/thirdeye-pinot/src/main/resources/logback.xml - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] Enable creating new application in the Admin page (#5601)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 83c6a2d [TE] Enable creating new application in the Admin page (#5601)
83c6a2d is described below
commit 83c6a2d7ccb2b51624e17f34df5fddae663bb4e2
Author: Xiaohui Sun
AuthorDate: Mon Jun 22 09:13:05 2020 -0700
[TE] Enable creating new application in the Admin page (#5601)
Co-authored-by: Xiaohui Sun
Add a "Create New" in the entity editor drop down.
This could be used to create a new application.
---
.../thirdeye-pinot/src/main/resources/assets/js/lib/entity-editor.js | 1 +
1 file changed, 1 insertion(+)
diff --git
a/thirdeye/thirdeye-pinot/src/main/resources/assets/js/lib/entity-editor.js
b/thirdeye/thirdeye-pinot/src/main/resources/assets/js/lib/entity-editor.js
index adb2f12..675353e 100644
--- a/thirdeye/thirdeye-pinot/src/main/resources/assets/js/lib/entity-editor.js
+++ b/thirdeye/thirdeye-pinot/src/main/resources/assets/js/lib/entity-editor.js
@@ -28,6 +28,7 @@ function renderEntitySelector() {
getData("/thirdeye/entity/" + entityType, "admin").done(function (data) {
entityData = data;
var select = "Select";
+ select += " Create New ";
for (var i in data) {
select += "" +
buildNameForEntity(data[i], entityType)
+ "";
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] Add maxValueDaily and maxValueHourly data filters (#5528)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a969ede [TE] Add maxValueDaily and maxValueHourly data filters (#5528)
a969ede is described below
commit a969ede4f2fd5194ba615f1c7782dce462ef722e
Author: Akshay Rai
AuthorDate: Mon Jun 22 00:37:50 2020 -0700
[TE] Add maxValueDaily and maxValueHourly data filters (#5528)
---
.../pinot/thirdeye/common/utils/MetricUtils.java | 40 ++
.../detection/algorithm/DimensionWrapper.java | 27 ---
.../components/ThresholdRuleAnomalyFilter.java | 16 -
.../detection/algorithm/DimensionWrapperTest.java | 40 ++
.../MergeDimensionThresholdIntegrationTest.java| 13 +--
5 files changed, 97 insertions(+), 39 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/utils/MetricUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/utils/MetricUtils.java
new file mode 100644
index 000..bf632e5
--- /dev/null
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/utils/MetricUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.common.utils;
+
+import org.apache.pinot.thirdeye.constant.MetricAggFunction;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
+
+
+/**
+ * Utility class for ThirdEye metrics
+ */
+public class MetricUtils {
+
+ private MetricUtils() {}
+
+ /**
+ * check if the metric aggregation is cumulative
+ */
+ public static boolean isAggCumulative(MetricConfigDTO metric) {
+MetricAggFunction aggFunction = metric.getDefaultAggFunction();
+return aggFunction.equals(MetricAggFunction.SUM) ||
aggFunction.equals(MetricAggFunction.COUNT);
+ }
+}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/DimensionWrapper.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/DimensionWrapper.java
index d9d034d..4e3f0b5 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/DimensionWrapper.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/DimensionWrapper.java
@@ -34,6 +34,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.collections4.MapUtils;
+import org.apache.pinot.thirdeye.common.utils.MetricUtils;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
@@ -52,7 +53,6 @@ import org.apache.pinot.thirdeye.detection.PredictionResult;
import org.apache.pinot.thirdeye.detection.cache.CacheConfig;
import
org.apache.pinot.thirdeye.detection.spi.exception.DetectorDataInsufficientException;
import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
-import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
import org.joda.time.DateTime;
@@ -101,7 +101,9 @@ public class DimensionWrapper extends DetectionPipeline {
private final double minContribution;
private final double minValue;
private final double minValueHourly;
+ private final double maxValueHourly;
private final double minValueDaily;
+ private final double maxValueDaily;
private final double minLiveZone;
private final double liveBucketPercentageThreshold;
private final Period lookback;
@@ -122,10 +124,13 @@ public class DimensionWrapper extends DetectionPipeline {
// the metric used in dimension exploration
this.metricUrn = MapUtils.getString(config.getProperties(), "metricUrn",
null);
+
this.minContribution = MapUtils.getDoubleValue(config.getProperties(),
"minContributio
[incubator-pinot] branch master updated: [TE] recover deleted api enpoints (#5556)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 33578ac [TE] recover deleted api enpoints (#5556)
33578ac is described below
commit 33578ac4edbafbfca484c0555adca235231cb489
Author: Vincent Chen
AuthorDate: Fri Jun 12 18:08:35 2020 -0700
[TE] recover deleted api enpoints (#5556)
---
.../dashboard/resources/v2/AnomaliesResource.java | 31 ++
1 file changed, 31 insertions(+)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java
index f3f1c95..51e1b43 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/AnomaliesResource.java
@@ -196,6 +196,37 @@ public class AnomaliesResource {
return anomaliesWrapper;
}
+ /**
+ * Find anomalies by anomaly ids
+ * @param startTime
+ * @param endTime
+ * @param anomalyIdsString
+ * @return
+ * @throws Exception
+ */
+ @GET
+ @Path("search/anomalyIds/{startTime}/{endTime}/{pageNumber}")
+ public AnomaliesWrapper getAnomaliesByAnomalyIds(
+ @PathParam("startTime") Long startTime,
+ @PathParam("endTime") Long endTime,
+ @PathParam("pageNumber") int pageNumber,
+ @QueryParam("anomalyIds") String anomalyIdsString,
+ @QueryParam("searchFilters") String searchFiltersJSON,
+ @QueryParam("filterOnly") @DefaultValue("false") boolean filterOnly)
throws Exception {
+
+String[] anomalyIds = anomalyIdsString.split(",");
+List mergedAnomalies = new ArrayList<>();
+for (String id : anomalyIds) {
+ Long anomalyId = Long.valueOf(id);
+ MergedAnomalyResultDTO anomaly =
mergedAnomalyResultDAO.findById(anomalyId);
+ if (anomaly != null) {
+mergedAnomalies.add(anomaly);
+ }
+}
+AnomaliesWrapper
+anomaliesWrapper =
constructAnomaliesWrapperFromMergedAnomalies(mergedAnomalies,
searchFiltersJSON, pageNumber, filterOnly);
+return anomaliesWrapper;
+ }
// --- HELPER FUNCTIONS
/**
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (cd774c7 -> 91f8844)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from cd774c7 Return Select only Literal Expression at Broker (#5513) add 91f8844 [TE][subscription] Bug fix to fix the notification pipeline when detections are disabled (#5545) No new revisions were added by this update. Summary of changes: .../pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (04e12bd -> bea55a0)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 04e12bd Config for raw index writer version (#5503) add bea55a0 [TE] Add data quality/sla rule to the detection template (#5515) No new revisions were added by this update. Summary of changes: thirdeye/thirdeye-frontend/app/utils/yaml-tools.js | 38 -- 1 file changed, 21 insertions(+), 17 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (4988d72 -> b6cb44c)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 4988d72 [TE] add back frontend as submodule (#5457) add b6cb44c [TE] clean up somce code and disable sla alerts till we have the ramp feature (#5458) No new revisions were added by this update. Summary of changes: .../trigger/DataAvailabilityTaskScheduler.java | 38 +- .../apache/pinot/thirdeye/detection/TaskUtils.java | 31 +- .../dataquality/DataQualityPipelineJob.java| 14 +--- 3 files changed, 40 insertions(+), 43 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (d8ea89c -> 4988d72)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from d8ea89c Create PULL_REQUEST_TEMPLATE.md (#5345) add 4988d72 [TE] add back frontend as submodule (#5457) No new revisions were added by this update. Summary of changes: .travis/.travis_te_nightly_build.sh | 4 ++-- thirdeye/pom.xml| 1 + thirdeye/thirdeye-frontend/package.json | 2 +- thirdeye/thirdeye-frontend/pom.xml | 22 -- 4 files changed, 24 insertions(+), 5 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] Add support for configuring additional custom fields in Jira Alert (#5386)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 516a168 [TE] Add support for configuring additional custom fields in
Jira Alert (#5386)
516a168 is described below
commit 516a16820d7be247c42ff0385654bcecbbaf560b
Author: Akshay Rai
AuthorDate: Fri May 15 09:48:00 2020 -0700
[TE] Add support for configuring additional custom fields in Jira Alert
(#5386)
---
.../dashboard/ThirdEyeDashboardApplication.java| 3 +-
.../alert/StatefulDetectionAlertFilter.java| 4 +-
.../alert/scheme/DetectionJiraAlerter.java | 7 ++-
.../thirdeye/detection/yaml/YamlResource.java | 51 --
.../thirdeye/notification/commons/JiraEntity.java | 15 ++-
.../notification/commons/ThirdEyeJiraClient.java | 42 --
.../formatter/channels/JiraContentFormatter.java | 2 +
.../thirdeye/detection/yaml/YamlResourceTest.java | 2 +-
.../commons/TestThirdEyeJiraClient.java| 12 +++--
9 files changed, 110 insertions(+), 28 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
index 99d8064..1ac0973 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
@@ -191,7 +191,8 @@ public class ThirdEyeDashboardApplication
DAO_REGISTRY.getDetectionConfigManager(),
DAO_REGISTRY.getDetectionAlertConfigManager()));
env.jersey().register(new DetectionResource());
env.jersey().register(new
DetectionAlertResource(DAO_REGISTRY.getDetectionAlertConfigManager()));
-env.jersey().register(new YamlResource(config.getDetectionPreviewConfig(),
config.getAlertOnboardingPermitPerSecond()));
+env.jersey().register(new YamlResource(config.getAlerterConfiguration(),
config.getDetectionPreviewConfig(),
+config.getAlertOnboardingPermitPerSecond()));
env.jersey().register(new SqlDataSourceResource());
TimeSeriesLoader timeSeriesLoader = new DefaultTimeSeriesLoader(
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
index 56c0613..b099677 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
@@ -48,13 +48,11 @@ public abstract class StatefulDetectionAlertFilter extends
DetectionAlertFilter
public static final String PROP_BCC = "bcc";
public static final String PROP_RECIPIENTS = "recipients";
- private static final String PROP_SEND_ONCE = "sendOnce";
-
// Time beyond which we do not want to notify anomalies
private static final long ANOMALY_NOTIFICATION_LOOKBACK_TIME =
TimeUnit.DAYS.toMillis(14);
public StatefulDetectionAlertFilter(DataProvider provider,
DetectionAlertConfigDTO config, long endTime) {
-super(provider, config,
endTime);//prepareStatement.setObject(parameterIndex++, pair.getValue(),
info.sqlType);
+super(provider, config, endTime);
}
protected final Set filter(Map
vectorClocks) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
index 70d6fc2..4d2c7bc 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
@@ -63,6 +63,9 @@ import static
org.apache.pinot.thirdeye.notification.commons.JiraConfiguration.*
* labels: # optional, default - thirdeye label is always
appended
* - test-label-1
* - test-label-2
+ * custom:
+ * test1: value1
+ * test2: value2
*/
@AlertScheme(type = "JIRA")
public class DetectionJiraAlerter extends DetectionAlertScheme {
@@ -92,7 +95,7 @@ public class DetectionJiraAlerter extends
DetectionAlertScheme {
}
private void updateJiraAlert(Issue issue, JiraEntity jiraEntity) {
-// Append labels
+// Append labels - do not remove existing labels
jiraEntity.getLabels().addAll(issue.getLabels());
j
[incubator-pinot] branch master updated: Make timezone configurable on frontend (#5374)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 32cb705 Make timezone configurable on frontend (#5374)
32cb705 is described below
commit 32cb705bbad1debed68363773238448fc87f2e85
Author: Tamas Nemeth
AuthorDate: Thu May 14 02:10:57 2020 +0200
Make timezone configurable on frontend (#5374)
---
thirdeye/thirdeye-frontend/app/pods/application/route.js | 2 +-
.../app/pods/components/alert-details/component.js | 5 +++--
.../app/pods/components/anomaly-summary/component.js | 7 +++
.../app/pods/components/anomaly-summary/template.hbs | 2 +-
.../app/pods/components/share-custom-template/component.js | 5 -
thirdeye/thirdeye-frontend/app/pods/home/share-dashboard/route.js | 4 +++-
.../app/pods/services/rootcause-aggregates-cache/service.js| 4 +++-
.../app/pods/services/rootcause-breakdowns-cache/service.js| 4 +++-
.../app/pods/services/rootcause-timeseries-cache/service.js| 3 ++-
thirdeye/thirdeye-frontend/app/utils/build-tooltip.js | 4 +++-
thirdeye/thirdeye-frontend/app/utils/rca-utils.js | 4 +++-
thirdeye/thirdeye-frontend/config/environment.js | 2 ++
12 files changed, 35 insertions(+), 11 deletions(-)
diff --git a/thirdeye/thirdeye-frontend/app/pods/application/route.js
b/thirdeye/thirdeye-frontend/app/pods/application/route.js
index 647c613..5901792 100644
--- a/thirdeye/thirdeye-frontend/app/pods/application/route.js
+++ b/thirdeye/thirdeye-frontend/app/pods/application/route.js
@@ -39,7 +39,7 @@ export default Route.extend(ApplicationRouteMixin, {
});
}
-this.get('moment').setTimeZone('America/Los_Angeles');
+this.get('moment').setTimeZone(config.timeZone);
},
model(params, transition) {
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
index 7bd40f0..6e4bd76 100644
--- a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
+++ b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
@@ -33,6 +33,7 @@ import floatToPercent from
'thirdeye-frontend/utils/float-to-percent';
import { setUpTimeRangeOptions } from
'thirdeye-frontend/utils/manage-alert-utils';
import moment from 'moment';
import _ from 'lodash';
+import config from 'thirdeye-frontend/config/environment';
const TABLE_DATE_FORMAT = 'MMM DD, hh:mm A'; // format for anomaly table
const TIME_PICKER_INCREMENT = 5; // tells date picker hours field how
granularly to display time
@@ -695,7 +696,7 @@ export default Component.extend({
}] : [];
const startColumn = [{
template: 'custom/anomalies-table/start-duration',
-title: 'Start / Duration (PDT)',
+title: `Start / Duration
(${moment().tz(config.timeZone).format('z')})`,
propertyName: 'startDateStr',
sortedBy: 'start',
sortDirection: 'desc',
@@ -1154,7 +1155,7 @@ export default Component.extend({
selectedRule,
uniqueTimeSeries
} = this.getProperties('metricUrn', 'analysisRange', 'selectedBaseline',
'showRules', 'selectedRule', 'uniqueTimeSeries');
-const timeZone = 'America/Los_Angeles';
+const timeZone = config.timeZone;
this.setProperties({
errorTimeseries: null,
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/component.js
index b11a381..0be588a 100644
---
a/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/component.js
+++
b/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/component.js
@@ -24,6 +24,7 @@ import { checkStatus, humanizeFloat, buildBounds } from
'thirdeye-frontend/utils
import columns from 'thirdeye-frontend/shared/anomaliesTableColumns';
import moment from 'moment';
import _ from 'lodash';
+import config from 'thirdeye-frontend/config/environment';
const TABLE_DATE_FORMAT = 'MMM DD, hh:mm A'; // format for anomaly table and
legend
@@ -52,6 +53,12 @@ export default Component.extend({
* predicted time series
*/
predicted: null,
+
+ /**
+ * timezone id
+ */
+ timezoneId: moment().tz(config.timeZone).format('z'),
+
/**
* imported color mapping for graph
*/
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/template.hbs
b/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/template.hbs
index 201e6f5..3303c1e 100644
---
a/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/template.hbs
+++
b/thirdeye/thirdeye-frontend/app/pods/components/anomaly-summary/template.hbs
@@ -34,7 +34,7
[incubator-pinot] branch master updated (bcb8342 -> 3aead87)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from bcb8342 [TE] frontend - harleyjj/anomalies-table - add modified time and delete function to anomalies table (#5367) add 3aead87 [TE] frontend - harley/share-dashboard - make share-dashboard template support comparing two metrics (#5289) No new revisions were added by this update. Summary of changes: .../app/pods/components/share-custom-template/component.js | 12 +--- 1 file changed, 9 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (7562d4d -> bcb8342)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 7562d4d Schema method to fetch field spec of time column (#5370) add bcb8342 [TE] frontend - harleyjj/anomalies-table - add modified time and delete function to anomalies table (#5367) No new revisions were added by this update. Summary of changes: .../app/pods/components/alert-details/component.js | 30 +++-- .../anomalies-table/modify-time/component.js | 12 ++ .../anomalies-table/modify-time/template.hbs | 7 ++ .../custom/anomalies-table/resolution/component.js | 84 ++ .../custom/anomalies-table/resolution/template.hbs | 128 + .../pods/custom/anomalies-table/rule/component.js | 90 +++ .../pods/custom/anomalies-table/rule/template.hbs | 31 + .../app/pods/services/api/anomalies/service.js | 4 + thirdeye/thirdeye-frontend/app/utils/utils.js | 14 ++- 9 files changed, 339 insertions(+), 61 deletions(-) create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/anomalies-table/modify-time/component.js create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/anomalies-table/modify-time/template.hbs create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/anomalies-table/rule/component.js create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/anomalies-table/rule/template.hbs - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE][subscription] Subscription pipeline should use anomaly create ti… (#5233)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git The following commit(s) were added to refs/heads/master by this push: new c577088 [TE][subscription] Subscription pipeline should use anomaly create ti… (#5233) c577088 is described below commit c577088783518034e27d624e84b0df0629d22b1f Author: Akshay Rai AuthorDate: Fri Apr 17 17:09:42 2020 -0700 [TE][subscription] Subscription pipeline should use anomaly create ti… (#5233) This PR merges the changes from #5152 addressing a bug, adding safety checks and improving the unit testing coverage. * [Bug] Fetch anomalies based on the create time (UTC) rather than the anomaly end time (could essentially be in any timezone depending on the dataset) * [Safety] Make sure anomalies older than 14 days are never notified. * [Safety] Do not scan anomalies from detections which have been disabled. * [Tests] Fixed and improved unit tests across all the alert filters. PS: This change will be followed by another PR with a script to update the watermarks of existing subscription groups. --- .../datalayer/bao/MergedAnomalyResultManager.java | 2 +- .../bao/jdbc/MergedAnomalyResultManagerImpl.java | 9 +- .../datalayer/pojo/DetectionAlertConfigBean.java | 21 +- .../thirdeye/detection/DefaultDataProvider.java| 3 + .../pinot/thirdeye/detection/alert/AlertUtils.java | 10 +- .../detection/alert/DetectionAlertJob.java | 17 +- .../detection/alert/DetectionAlertTaskRunner.java | 24 +- .../alert/StatefulDetectionAlertFilter.java| 50 ++-- .../filter/DimensionsRecipientAlertFilter.java | 17 +- .../alert/filter/PerUserDimensionAlertFilter.java | 18 +- .../detection/alert/filter/SubscriptionUtils.java | 1 - .../ToAllRecipientsDetectionAlertFilter.java | 23 +- .../components/ThresholdRuleDetector.java | 2 +- .../validators/SubscriptionConfigValidator.java| 5 - .../translator/SubscriptionConfigTranslator.java | 1 - .../thirdeye/detection/alert/SendAlertTest.java| 8 +- .../detection/alert/filter/AlertFilterUtils.java | 43 .../filter/DimensionsRecipientAlertFilterTest.java | 149 +++- .../filter/PerUserDimensionAlertFilterTest.java| 74 -- .../ToAllRecipientsDetectionAlertFilterTest.java | 262 - 20 files changed, 401 insertions(+), 338 deletions(-) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java index 081b39d..5e2a187 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/MergedAnomalyResultManager.java @@ -63,7 +63,7 @@ public interface MergedAnomalyResultManager extends AbstractManager findByStartEndTimeInRangeAndDetectionConfigId(long startTime, long endTime, long detectionConfigId); - List findByStartTimeInRangeAndDetectionConfigId(long startTime, long endTime, long detectionConfigId); + List findByCreatedTimeInRangeAndDetectionConfigId(long startTime, long endTime, long detectionConfigId); List findAnomaliesWithinBoundary(long startTime, long endTime, long detectionConfigId); diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java index 26a4a96..1d5a7a7 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java @@ -21,12 +21,11 @@ package org.apache.pinot.thirdeye.datalayer.bao.jdbc; import com.google.common.base.Preconditions; import com.google.inject.Singleton; +import java.sql.Timestamp; import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFeedbackDTO; import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO; -import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO; import org.apache.pinot.thirdeye.datalayer.pojo.AnomalyFeedbackBean; import org.apache.pinot.thirdeye.datalayer.pojo.AnomalyFunctionBean; -import org.apache.pinot.thirdeye.datalayer.pojo.DetectionConfigBean; import org.apache.pinot.thirdeye.datalayer.pojo.MetricConfigBean; import java.util.ArrayList; import java.util.Collections; @@ -269,9 +268,11 @@ public class MergedAnomalyResultManagerImpl extends AbstractManagerImpl findByStartTimeInRangeAndDetectionConfigId(long startTime, long endTime, long
[incubator-pinot] branch master updated (881469d -> 159e4bb)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 881469d [TE] script to migrate subscription group watermarks from anomaly endtime to create time. (#5244) add 159e4bb [TE] Update notificaion template to support Data SLA anomalies; Don't show NA for predicted values (#5242) No new revisions were added by this update. Summary of changes: .../apache/pinot/thirdeye/anomaly/AnomalyType.java | 16 ++- .../content/BaseNotificationContent.java | 133 +++-- .../content/templates/EntityGroupKeyContent.java | 14 ++- .../templates/HierarchicalAnomaliesContent.java| 16 ++- .../content/templates/MetricAnomaliesContent.java | 15 ++- .../detector/jira-metric-anomalies-template.ftl| 4 +- .../detector/metric-anomalies-template.ftl | 6 +- .../templates/TestMetricAnomaliesContent.java | 58 +++-- .../channels/TestJiraContentFormatter.java | 33 - .../resources/test-jira-anomalies-template.ftl | 7 +- .../resources/test-metric-anomalies-template.html | 69 +-- 11 files changed, 318 insertions(+), 53 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (7f04bdf -> 881469d)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 7f04bdf [TE] Runner to generate SLA based metric data missing alerts (#5200) add 881469d [TE] script to migrate subscription group watermarks from anomaly endtime to create time. (#5244) No new revisions were added by this update. Summary of changes: .../tools/RunAdhocDatabaseQueriesTool.java | 104 + 1 file changed, 104 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (79c0838 -> 7f04bdf)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 79c0838 [TE] frontend - harleyjj/alert-details - fix buggy flag for showing forecast message (#5243) add 7f04bdf [TE] Runner to generate SLA based metric data missing alerts (#5200) No new revisions were added by this update. Summary of changes: .../apache/pinot/thirdeye/anomaly/AnomalyType.java | 2 +- .../thirdeye/anomaly/task/TaskRunnerFactory.java | 4 +- .../datalayer/bao/MergedAnomalyResultManager.java | 2 + .../bao/jdbc/MergedAnomalyResultManagerImpl.java | 9 + .../datalayer/pojo/DetectionConfigBean.java| 14 +- .../pinot/thirdeye/detection/DataProvider.java | 2 + .../pinot/thirdeye/detection/DetectionUtils.java | 13 +- .../apache/pinot/thirdeye/detection/TaskUtils.java | 1 - .../availability/DataAvailabilityTaskRunner.java | 85 .../detection/datasla/DatasetSlaTaskRunner.java| 320 ++ .../thirdeye/scheduler/DetectionCronScheduler.java | 2 +- .../scheduler/SubscriptionCronScheduler.java | 1 + .../trigger/DataAvailabilityTaskSchedulerTest.java | 5 +- .../pinot/thirdeye/detection/MockDataProvider.java | 3 + .../datasla/DatasetSlaTaskRunnerTest.java | 478 + .../wrapper/EntityAnomalyMergeWrapperTest.java | 13 +- 16 files changed, 847 insertions(+), 107 deletions(-) delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/availability/DataAvailabilityTaskRunner.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/datasla/DatasetSlaTaskRunner.java create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/datasla/DatasetSlaTaskRunnerTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (0880f10 -> 79c0838)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 0880f10 fix generator.sh after pinot version bump (#5237) add 79c0838 [TE] frontend - harleyjj/alert-details - fix buggy flag for showing forecast message (#5243) No new revisions were added by this update. Summary of changes: .../app/pods/components/alert-details/component.js | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (06bd2c6 -> 60bc83a)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 06bd2c6 [TE] endpoint - harleyjj/metricBreakdownPipeline - add flag for ignoring score when adding dimensions to response (#5212) add 60bc83a Revert "[TE][subscription] update subscription watermarks to use anomaly create time instead of end time (#5152)" (#5227) No new revisions were added by this update. Summary of changes: .../datalayer/bao/MergedAnomalyResultManager.java | 2 +- .../bao/jdbc/MergedAnomalyResultManagerImpl.java | 7 ++- .../datalayer/pojo/DetectionAlertConfigBean.java | 21 +-- .../thirdeye/detection/DefaultDataProvider.java| 3 --- .../pinot/thirdeye/detection/alert/AlertUtils.java | 9 +++- .../detection/alert/DetectionAlertJob.java | 17 --- .../detection/alert/DetectionAlertTaskRunner.java | 24 +- .../alert/StatefulDetectionAlertFilter.java| 18 +--- .../filter/DimensionsRecipientAlertFilter.java | 17 +-- .../alert/filter/PerUserDimensionAlertFilter.java | 18 ++-- .../detection/alert/filter/SubscriptionUtils.java | 1 + .../ToAllRecipientsDetectionAlertFilter.java | 22 ++-- .../components/ThresholdRuleDetector.java | 2 +- .../validators/SubscriptionConfigValidator.java| 5 + .../translator/SubscriptionConfigTranslator.java | 1 + .../thirdeye/detection/alert/SendAlertTest.java| 6 ++ .../ToAllRecipientsDetectionAlertFilterTest.java | 19 + 17 files changed, 130 insertions(+), 62 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (a1401de -> 06bd2c6)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from a1401de [TE] frontend - harleyjj/home - use duration param to set date range (#5198) add 06bd2c6 [TE] endpoint - harleyjj/metricBreakdownPipeline - add flag for ignoring score when adding dimensions to response (#5212) No new revisions were added by this update. Summary of changes: .../rootcause/impl/MetricBreakdownPipeline.java | 19 --- 1 file changed, 16 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (df3b904 -> 7ca6f33)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from df3b904 Fix travis cache (#5225)
add 7ca6f33 [TE] frontend - harleyjj/rca - reformat dimension-algorithm
table (#5206)
No new revisions were added by this update.
Summary of changes:
.../components/contribution-table/component.js | 211 -
.../components/contribution-table/template.hbs | 249 -
.../rootcause-dimensions-algorithm/component.js| 24 +-
.../dimensions-table/change-bars/template.hbs | 2 +-
.../pods/custom/dimensions-table/cost/template.hbs | 1 -
.../custom/dimensions-table/dimension/template.hbs | 4 +-
.../{cost => percent-change}/component.js | 0
.../dimensions-table/percent-change/template.hbs | 1 +
.../app/shared/dimensionAnalysisTableConfig.js | 23 +-
thirdeye/thirdeye-frontend/app/styles/app.scss | 1 -
.../app/styles/components/contribution-table.scss | 132 ---
.../app/styles/pods/custom/dimensions-table.scss | 29 ++-
.../contribution-table/component-test.js | 26 ---
13 files changed, 58 insertions(+), 645 deletions(-)
delete mode 100644
thirdeye/thirdeye-frontend/app/pods/components/contribution-table/component.js
delete mode 100644
thirdeye/thirdeye-frontend/app/pods/components/contribution-table/template.hbs
delete mode 100644
thirdeye/thirdeye-frontend/app/pods/custom/dimensions-table/cost/template.hbs
rename thirdeye/thirdeye-frontend/app/pods/custom/dimensions-table/{cost =>
percent-change}/component.js (100%)
create mode 100644
thirdeye/thirdeye-frontend/app/pods/custom/dimensions-table/percent-change/template.hbs
delete mode 100644
thirdeye/thirdeye-frontend/app/styles/components/contribution-table.scss
delete mode 100644
thirdeye/thirdeye-frontend/tests/integration/pods/components/contribution-table/component-test.js
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] frontend - harleyjj/validation - surface errors in dom for create and edit alert (#5187)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f1e2086 [TE] frontend - harleyjj/validation - surface errors in dom
for create and edit alert (#5187)
f1e2086 is described below
commit f1e20863c8e24300ffca3b37cbf0ca100cc11c2a
Author: Harley Jackson
AuthorDate: Fri Mar 27 13:42:32 2020 -0700
[TE] frontend - harleyjj/validation - surface errors in dom for create and
edit alert (#5187)
* Surfaces errors in configuration and preview to the DOM
* Includes modal for 'more details' if response includes 'more-info' field
* Styling updates to Create Alert and Edit Alert
* On successful alert creation, Create Alert transitions to Alert Overview
for detection
* Fixes bug where successful Create Alert shown as error
---
.../app/pods/components/alert-details/component.js | 35 ++-
.../pods/components/detection-yaml/component.js| 31 +-
.../pods/components/detection-yaml/template.hbs| 108 +++--
.../components/modals/error-modal/component.js | 19
.../components/modals/error-modal/template.hbs | 15 +++
.../pods/components/subscription-yaml/component.js | 9 ++
.../pods/components/subscription-yaml/template.hbs | 57 ---
.../app/pods/components/te-modal/component.js | 1 +
.../app/pods/components/te-modal/template.hbs | 8 +-
.../app/pods/manage/yaml/controller.js | 76 +++
.../app/pods/manage/yaml/template.hbs | 7 ++
.../app/pods/self-serve/create-alert/controller.js | 33 ++-
.../app/pods/self-serve/create-alert/template.hbs | 13 ++-
.../app/styles/components/button.scss | 14 +++
.../app/styles/components/detection-yaml.scss | 41
.../app/styles/components/links.scss | 25 +
.../app/styles/components/subscription-yaml.scss | 14 +++
.../app/styles/pods/self-serve/create-alert.scss | 4 +
.../app/styles/wrapper/styles.scss | 2 +-
thirdeye/thirdeye-frontend/package.json| 1 +
.../acceptance/self-serve-import-metric-test.js| 4 +-
.../thirdeye-frontend/tests/utils/constants.js | 1 +
thirdeye/thirdeye-frontend/yarn.lock | 7 ++
23 files changed, 451 insertions(+), 74 deletions(-)
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
index 4941e66..1989da3 100644
--- a/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
+++ b/thirdeye/thirdeye-frontend/app/pods/components/alert-details/component.js
@@ -855,9 +855,17 @@ export default Component.extend({
}
set(this, 'cachedMetric', getValueFromYaml('metric', alertYaml,
'string'));
} catch (error) {
- const message = (error.body && typeof error.body === 'object') ?
error.body.message : error.message;
- notifications.error(`_getAnomalies failed: ${message}`, 'Error',
toastOptions);
+ const previewErrorMsg = (error.body && typeof error.body === 'object') ?
error.body.message : error.message;
+ const previewErrorInfo = (error.body && typeof error.body === 'object')
? error.body['more-info'] : error['more-info'];
+ notifications.error('Failed to get anomalies, please check warning above
detection configuration', 'Error', toastOptions);
this.set('getAnomaliesError', true);
+ if (this.get('isPreviewMode')) {
+this.get('sendPreviewError')({
+ previewError: true,
+ previewErrorMsg,
+ previewErrorInfo
+});
+ }
}
return {
@@ -1029,6 +1037,13 @@ export default Component.extend({
_fetchAnomalies() {
set(this, 'getAnomaliesError', false);
+if (this.get('isPreviewMode')) {
+ this.get('sendPreviewError')({
+previewError: false,
+previewErrorMsg: null,
+previewErrorInfo: null
+ });
+}
// If the user is running the detection with a new metric, we should reset
the state of time series and anomalies for comparison
if (this._checkMetricIfCreateAlertPreview()) {
@@ -1049,18 +1064,32 @@ export default Component.extend({
if (get(this, 'metricUrn')) {
this._fetchTimeseries();
} else {
-throw new Error('There was no anomaly data returned for the
detection');
+throw new Error('There was no metric or anomaly data returned for
the detection');
}
})
.catch(error => {
if (error.name !== 'TaskCancelation') {
this.get('notifications').error(error, 'Error', toastOptions);
set(this, 'getAnomaliesError', true);
+if (this.get('isPreviewMode')
[incubator-pinot] branch master updated: [TE] Remove hard-coded server data datetimeformatter (#5171)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 562180d [TE] Remove hard-coded server data datetimeformatter (#5171)
562180d is described below
commit 562180dde5a8385eb651c01a480a24bda957f4c8
Author: Xiaohui Sun
AuthorDate: Mon Mar 23 09:26:27 2020 -0700
[TE] Remove hard-coded server data datetimeformatter (#5171)
We should not hard-code the server datetime as UTC.
The datetime information should come from dataset configuration.
---
.../datasource/pinot/resultset/ThirdEyeResultSetUtils.java| 8
1 file changed, 8 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
index 1bb0ffe..9bd622d 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
@@ -90,15 +90,11 @@ public class ThirdEyeResultSetUtils {
dataGranularity = dataTimeSpec.getDataGranularity();
boolean isISOFormat = false;
DateTimeFormatter inputDataDateTimeFormatter = null;
- DateTimeFormatter serverDataDateTimeFormatter = null;
String timeFormat = dataTimeSpec.getFormat();
if (timeFormat != null &&
!timeFormat.equals(TimeSpec.SINCE_EPOCH_FORMAT)) {
isISOFormat = true;
inputDataDateTimeFormatter =
DateTimeFormat.forPattern(timeFormat).withZone(dateTimeZone);
}
- if (isISOFormat && (sourceName.equals(MYSQL) || sourceName.equals(H2))) {
-serverDataDateTimeFormatter =
DateTimeFormat.forPattern(timeFormat).withZone(DateTimeZone.getDefault());
- }
List resultSets = entry.getValue();
for (int i = 0; i < resultSets.size(); i++) {
@@ -123,11 +119,7 @@ public class ThirdEyeResultSetUtils {
if (!isISOFormat) {
millis =
dataGranularity.toMillis(Double.valueOf(groupKeyVal).longValue());
} else {
- if (sourceName.equals(MYSQL)) {
-millis = DateTime.parse(groupKeyVal,
serverDataDateTimeFormatter).getMillis();
- } else {
millis = DateTime.parse(groupKeyVal,
inputDataDateTimeFormatter).getMillis();
- }
}
if (millis < startTime) {
LOG.error("Data point earlier than requested start time {}:
{}", new Date(startTime), new Date(millis));
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (977c79c -> 2cfbe7e)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 977c79c [TE] frontend - harleyjj/rca - cube table updates (#5159) add 2cfbe7e [TE][subscription] update subscription watermarks to use anomaly create time instead of end time (#5152) No new revisions were added by this update. Summary of changes: .../datalayer/bao/MergedAnomalyResultManager.java | 2 +- .../bao/jdbc/MergedAnomalyResultManagerImpl.java | 7 +-- .../datalayer/pojo/DetectionAlertConfigBean.java | 21 ++- .../thirdeye/detection/DefaultDataProvider.java| 3 +++ .../pinot/thirdeye/detection/alert/AlertUtils.java | 9 +--- .../detection/alert/DetectionAlertJob.java | 17 +++ .../detection/alert/DetectionAlertTaskRunner.java | 24 +- .../alert/StatefulDetectionAlertFilter.java| 18 +++- .../filter/DimensionsRecipientAlertFilter.java | 17 ++- .../alert/filter/PerUserDimensionAlertFilter.java | 18 ++-- .../detection/alert/filter/SubscriptionUtils.java | 1 - .../ToAllRecipientsDetectionAlertFilter.java | 22 ++-- .../components/ThresholdRuleDetector.java | 2 +- .../validators/SubscriptionConfigValidator.java| 5 - .../translator/SubscriptionConfigTranslator.java | 1 - .../thirdeye/detection/alert/SendAlertTest.java| 6 -- .../ToAllRecipientsDetectionAlertFilterTest.java | 19 - 17 files changed, 62 insertions(+), 130 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (4be032a -> 1376331)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 4be032a [TE] frontend - harleyjj/alert-details - stop showing bounds for minutely granularity on Alert Overview (#5157) add 1376331 [TE] Setup a test Github workflow for ThirdEye (#5158) No new revisions were added by this update. Summary of changes: .../workflows/maven.yml| 30 +++--- 1 file changed, 21 insertions(+), 9 deletions(-) copy kubernetes/helm/pinot-presto-cli.sh => .github/workflows/maven.yml (59%) mode change 100755 => 100644 - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch akshayrai-patch-1 updated (97a1f8a -> 20d9bca)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch akshayrai-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 97a1f8a Fix the cron expression add 20d9bca add license header No new revisions were added by this update. Summary of changes: .github/workflows/maven.yml | 19 +++ 1 file changed, 19 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (9933870 -> 4be032a)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 9933870 Move SegmentMetadata from pinot-common to pinot-core (#5156) add 4be032a [TE] frontend - harleyjj/alert-details - stop showing bounds for minutely granularity on Alert Overview (#5157) No new revisions were added by this update. Summary of changes: .../thirdeye-frontend/app/pods/components/alert-details/component.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch akshayrai-patch-1 updated (cfb51e2 -> 97a1f8a)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch akshayrai-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from cfb51e2 [TE] Setup a test Github workflow for ThirdEye add 97a1f8a Fix the cron expression No new revisions were added by this update. Summary of changes: .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] 01/01: [TE] Setup a test Github workflow for ThirdEye
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a commit to branch akshayrai-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git commit cfb51e2e5a5130cbc564377acd2dfb42d9dbfb54 Author: Akshay Rai AuthorDate: Mon Mar 16 16:19:23 2020 -0700 [TE] Setup a test Github workflow for ThirdEye --- .github/workflows/maven.yml | 24 1 file changed, 24 insertions(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 000..ebf0ab9 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,24 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Artifact Publish Test Workflow + +on: + schedule: + - cron: "0 0 17 1/1 * ? *" + +jobs: + build: + +runs-on: ubuntu-latest + +steps: +- uses: actions/checkout@v2 +- name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: +java-version: 1.8 +- name: Build with Maven + run: | +cd thirdeye +mvn -T 1C clean compile package - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch akshayrai-patch-1 created (now cfb51e2)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch akshayrai-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. at cfb51e2 [TE] Setup a test Github workflow for ThirdEye This branch includes the following new commits: new cfb51e2 [TE] Setup a test Github workflow for ThirdEye The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (d989427 -> 494874d)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from d989427 Fix the SQL group-by for empty data table (#5151) add 494874d [TE] Detection creation endpoints should trigger Replay & Tuning (#5142) No new revisions were added by this update. Summary of changes: .../thirdeye/detection/yaml/YamlResource.java | 48 -- 1 file changed, 26 insertions(+), 22 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (6b6fa56a -> 888700d)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 6b6fa56a Update license and notice (#5145) add 888700d [TE] Remove validation to make recipients backwards compatible (#5149) No new revisions were added by this update. Summary of changes: .../thirdeye/detection/validators/SubscriptionConfigValidator.java| 4 1 file changed, 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (2ec6ed8 -> 91850cd)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 2ec6ed8 Synthetic Time Series Generator for pinot-tools (#5118) add 91850cd [TE] Cleaned up default detection & subscription yaml (#5121) No new revisions were added by this update. Summary of changes: thirdeye/thirdeye-frontend/app/utils/yaml-tools.js | 61 +++--- .../components/detection-yaml/component-test.js| 2 +- .../components/subscription-yaml/component-test.js | 2 +- 3 files changed, 33 insertions(+), 32 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE] added a couple of validations to check if detection is subscribed & valid; cleaned up other validations (#5113)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8ec8499 [TE] added a couple of validations to check if detection is
subscribed & valid; cleaned up other validations (#5113)
8ec8499 is described below
commit 8ec849968f6025ca471929b6861c0ae1b5f993d0
Author: Akshay Rai
AuthorDate: Fri Mar 6 09:46:48 2020 -0800
[TE] added a couple of validations to check if detection is subscribed &
valid; cleaned up other validations (#5113)
Validations added:
* Make sure subscription group subscribes to the detectionName while
creating an alert
* Make sure all the subscribed detections are valid
* recipients should be configured under the EMAIL scheme params and not at
the root level
Validations removed:
* type is not compulsory in subscription group
* type is not compulsory for sub-entity alerts.
Added unit tests for newly added validations
---
.../alert/scheme/DetectionEmailAlerter.java| 2 +-
.../validators/DetectionConfigValidator.java | 39
.../validators/SubscriptionConfigValidator.java| 52 +++---
.../thirdeye/detection/yaml/YamlResource.java | 32 -
.../yaml/translator/DetectionConfigTranslator.java | 4 +-
.../translator/SubscriptionConfigTranslator.java | 3 +-
.../validators/DetectionConfigValidatorTest.java | 9
.../thirdeye/detection/yaml/YamlResourceTest.java | 48 +---
.../yaml/detection/detection-config-1.yaml | 2 +-
.../yaml/detection/detection-config-2.yaml | 2 +-
.../yaml/subscription/subscription-config-1.yaml | 15 ---
.../yaml/subscription/subscription-config-2.yaml | 14 +++---
.../yaml/subscription/subscription-config-3.yaml | 14 +++---
.../yaml/subscription/subscription-config-4.yaml | 14 +++---
.../yaml/subscription/subscription-config-5.yaml | 13 +++---
.../src/test/resources/sample-alert-config.yml | 14 +++---
16 files changed, 171 insertions(+), 106 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
index 24177b2..d429125 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
@@ -63,7 +63,7 @@ import static
org.apache.pinot.thirdeye.notification.commons.SmtpConfiguration.S
public class DetectionEmailAlerter extends DetectionAlertScheme {
private static final Logger LOG =
LoggerFactory.getLogger(DetectionEmailAlerter.class);
- private static final String PROP_RECIPIENTS = "recipients";
+ public static final String PROP_RECIPIENTS = "recipients";
private static final String PROP_TO = "to";
private static final String PROP_CC = "cc";
private static final String PROP_BCC = "bcc";
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
index 8af3462..fbefa5f 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionConfigValidator.java
@@ -21,8 +21,6 @@ package org.apache.pinot.thirdeye.detection.validators;
import com.google.common.base.Preconditions;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -38,6 +36,7 @@ import
org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import org.quartz.CronExpression;
import static org.apache.pinot.thirdeye.detection.ConfigUtils.*;
+import static
org.apache.pinot.thirdeye.detection.alert.scheme.DetectionEmailAlerter.*;
public class DetectionConfigValidator implements
ConfigValidator {
@@ -58,6 +57,7 @@ public class DetectionConfigValidator implements
ConfigValidator detectionYaml,
String parentAlertName)
@@ -227,33 +226,31 @@ public class DetectionConfigValidator implements
ConfigValidator detectionYaml) throws
IllegalArgumentException {
-// Validate detectionName
-
Preconditions.checkArgument(detectionYaml.containsKey(PROP_DETECTION_NAME),
"Property missing: " + PROP_DETECTION_NAME);
-String alertName = MapUtils.getString(detectionYaml, PROP_DETECTION_NAME);
+/
[incubator-pinot] branch master updated (b880c47 -> 9f33abe)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from b880c47 Fix the duplicate broker refresh message (#5102) add 9f33abe [TE] Add additional safeguard around jira description length (#5100) No new revisions were added by this update. Summary of changes: .../detection/alert/scheme/DetectionJiraAlerter.java| 13 ++--- 1 file changed, 10 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE][subscription] destroy jira connections after use (#5097)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 00fd911 [TE][subscription] destroy jira connections after use (#5097)
00fd911 is described below
commit 00fd911936f10420c612cbe7200b9b6a89ddcafa
Author: Akshay Rai
AuthorDate: Thu Feb 27 10:54:57 2020 -0800
[TE][subscription] destroy jira connections after use (#5097)
---
.../thirdeye/detection/alert/DetectionAlertTaskRunner.java | 1 +
.../thirdeye/detection/alert/scheme/DetectionAlertScheme.java | 4
.../thirdeye/detection/alert/scheme/DetectionJiraAlerter.java | 6 ++
.../thirdeye/notification/commons/ThirdEyeJiraClient.java | 10 +-
4 files changed, 20 insertions(+), 1 deletion(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskRunner.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskRunner.java
index 7291756..cfa27a1 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskRunner.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskRunner.java
@@ -112,6 +112,7 @@ public class DetectionAlertTaskRunner implements TaskRunner
{
detAlertTaskFactory.loadAlertSchemes(alertConfig,
taskContext.getThirdEyeAnomalyConfiguration(), result);
for (DetectionAlertScheme alertScheme : alertSchemes) {
alertScheme.run();
+alertScheme.destroy();
}
updateAlertConfigWatermarks(result, alertConfig);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
index 8aaf6d7..c89fc2b 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
@@ -54,6 +54,10 @@ public abstract class DetectionAlertScheme {
public abstract void run() throws Exception;
+ public void destroy() {
+// do nothing
+ }
+
/**
* Plug the appropriate template based on configuration.
*/
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
index 49f978c..df807df 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionJiraAlerter.java
@@ -21,6 +21,7 @@ package org.apache.pinot.thirdeye.detection.alert.scheme;
import com.atlassian.jira.rest.client.api.domain.Issue;
import com.google.common.base.Preconditions;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -171,6 +172,11 @@ public class DetectionJiraAlerter extends
DetectionAlertScheme {
}
@Override
+ public void destroy() {
+this.jiraClient.close();
+ }
+
+ @Override
public void run() throws Exception {
Preconditions.checkNotNull(result);
if (result.getAllAnomalies().size() == 0) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
index 2820905..8ce9097 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java
@@ -35,6 +35,7 @@ import
com.atlassian.jira.rest.client.api.domain.input.TransitionInput;
import
com.atlassian.jira.rest.client.internal.async.AsynchronousJiraRestClientFactory;
import com.google.common.base.Joiner;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
@@ -60,7 +61,6 @@ public class ThirdEyeJiraClient {
private static final String JIRA_REOPEN_TRANSITION = "Reopen";
public static final String PROP_ISSUE_TYPE = "issuetype";
public static final String PROP_PROJECT = "project";
- public static final String PROP_SUMMARY = "summary";
public static final String PROP_ASSIGNEE = "assignee";
public static final String PROP_MERGE_GAP = "mergeGap";
public s
[incubator-pinot] branch master updated: [TE][subscription] Dimension Recipient Alerter should be agnostic to individual errors; Emit metrics to track emails and jiras (#5091)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fdfb97e [TE][subscription] Dimension Recipient Alerter should be
agnostic to individual errors; Emit metrics to track emails and jiras (#5091)
fdfb97e is described below
commit fdfb97e3da41149d3605b3931e22e8542704e36c
Author: Akshay Rai
AuthorDate: Wed Feb 26 08:38:55 2020 -0800
[TE][subscription] Dimension Recipient Alerter should be agnostic to
individual errors; Emit metrics to track emails and jiras (#5091)
---
.../thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java| 18 ++
.../detection/alert/scheme/DetectionAlertScheme.java | 5 +++--
.../detection/alert/scheme/DetectionEmailAlerter.java | 5 -
.../detection/alert/scheme/DetectionJiraAlerter.java | 10 --
4 files changed, 33 insertions(+), 5 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
index 7e40d29..d6de6c2 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
@@ -173,6 +173,24 @@ public class ThirdeyeMetricsUtil {
public static final Counter eventScheduledTaskFallbackCounter =
metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"eventScheduledTaskFallbackCounter");
+ public static final Counter emailAlertsSucesssCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"emailAlertsSucesssCounter");
+
+ public static final Counter emailAlertsFailedCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"emailAlertsFailedCounter");
+
+ public static final Counter jiraAlertsSuccessCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"jiraAlertsSuccessCounter");
+
+ public static final Counter jiraAlertsFailedCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"jiraAlertsFailedCounter");
+
+ public static final Counter jiraAlertsNumTicketsCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"jiraAlertsNumTicketsCounter");
+
+ public static final Counter jiraAlertsNumCommentsCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class,
"jiraAlertsNumCommentsCounter");
+
public static MetricsRegistry getMetricsRegistry() {
return metricsRegistry;
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
index 0a4ab18..8aaf6d7 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionAlertScheme.java
@@ -85,12 +85,13 @@ public abstract class DetectionAlertScheme {
* Fail the alert task if unable to notify owner. However, in case of
dimensions recipient alerter,
* do not fail the alert if a subset of recipients are invalid.
*/
- void handleAlertFailure(int size, IllegalArgumentException e) {
+ void handleAlertFailure(int numOfAnomalies, Exception e) throws Exception {
+// Dimension recipients not enabled
if (this.result.getResult().size() == 1) {
throw e;
} else {
LOG.warn("Skipping! Found illegal arguments while sending {} anomalies
for alert {}." + " Exception message: ",
- size, this.subsConfig.getId(), e);
+ numOfAnomalies, this.subsConfig.getId(), e);
}
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
index d4e4ab9..24177b2 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
@@ -35,6 +35,7 @@ import org.apache.commons.collections4.MapUtils;
import org.apache.commons.mail.DefaultAuthenticator;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
+import org.apache.pinot.thirdeye.anomaly.utils.ThirdeyeMetricsUtil;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertCo
[incubator-pinot] branch master updated (ef6509e -> b2608ad)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from ef6509e Minor improvement for RoutingManager (#5092) add b2608ad [TE] Upgrade mysql version to 5.1.41 to fix compatibility issues connectin with 8.* (#5093) No new revisions were added by this update. Summary of changes: thirdeye/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (ef69b75 -> 35f2117)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from ef69b75 [TE][subscription] Fix duplicate anomaly report issue and clean up subscription pipelines (#5081) add 35f2117 [TE][subscription] Minor bug fix in the empty email check (#5084) No new revisions were added by this update. Summary of changes: .../ToAllRecipientsDetectionAlertFilter.java | 4 ++- .../detection/alert/filter/AlertFilterUtils.java | 12 .../ToAllRecipientsDetectionAlertFilterTest.java | 33 ++ 3 files changed, 48 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (7dfc5c1 -> ef69b75)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 7dfc5c1 Adding quickstart into Pinot admin command (#5080) add ef69b75 [TE][subscription] Fix duplicate anomaly report issue and clean up subscription pipelines (#5081) No new revisions were added by this update. Summary of changes: .../alert/StatefulDetectionAlertFilter.java| 9 +- .../filter/DimensionDetectionAlertFilter.java | 121 - .../filter/DimensionsRecipientAlertFilter.java | 4 +- .../alert/filter/PerUserDimensionAlertFilter.java | 51 +++--- .../detection/alert/filter/SubscriptionUtils.java | 17 +- .../ToAllRecipientsDetectionAlertFilter.java | 16 +- .../filter/DimensionDetectionAlertFilterTest.java | 197 - .../filter/PerUserDimensionAlertFilterTest.java| 17 +- .../ToAllRecipientsDetectionAlertFilterTest.java | 13 +- 9 files changed, 84 insertions(+), 361 deletions(-) delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/DimensionDetectionAlertFilter.java delete mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/DimensionDetectionAlertFilterTest.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (94122e8 -> 884af43)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 94122e8 [TE] frontend - harleyjj/alerts - update application filter for many-to-many relationship (#5062) add 884af43 [TE] Remove fromAddress as a compulsory field from subscription group (#5078) No new revisions were added by this update. Summary of changes: .../thirdeye/detection/alert/DetectionAlertTaskRunner.java| 11 --- .../detection/alert/scheme/DetectionEmailAlerter.java | 10 -- .../detection/validators/SubscriptionConfigValidator.java | 1 - 3 files changed, 8 insertions(+), 14 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (bfa176c -> 7cca144)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from bfa176c Update Selection Query Limit with pre-configured value (#5040)
add 7cca144 [TE] [data-availability] Scheduling of data availability
tasks & some clean up (#5056)
No new revisions were added by this update.
Summary of changes:
.../anomaly/ThirdEyeAnomalyApplication.java| 20 ++--
.../trigger/DataAvailabilityTaskScheduler.java | 70 +
.../pinot/thirdeye/anomaly/task/TaskConstants.java | 1 +
.../thirdeye/anomaly/task/TaskInfoFactory.java | 3 +
.../thirdeye/anomaly/task/TaskRunnerFactory.java | 4 +
.../anomaly/utils/ThirdeyeMetricsUtil.java | 3 +
.../datalayer/pojo/DetectionConfigBean.java| 15 ++-
.../thirdeye/detection/DetectionDataSLAJob.java| 68 +
.../thirdeye/detection/DetectionPipelineJob.java | 60 ++-
.../pinot/thirdeye/detection/DetectionUtils.java | 8 ++
.../{DetectionPipelineJob.java => TaskUtils.java} | 89 ++--
.../detection/alert/DetectionAlertJob.java | 18 +---
.../availability/DataAvailabilityTaskRunner.java | 85
.../thirdeye/detection/yaml/YamlResource.java | 12 +--
.../DetectionCronScheduler.java} | 112 -
.../SubscriptionCronScheduler.java}| 82 ---
.../ThirdEyeCronScheduler.java}| 46 +
.../trigger/DataAvailabilityTaskSchedulerTest.java | 46 +++--
.../integration/NotificationTaskSchedulerTest.java | 14 +--
19 files changed, 469 insertions(+), 287 deletions(-)
create mode 100644
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionDataSLAJob.java
copy
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/{DetectionPipelineJob.java
=> TaskUtils.java} (58%)
create mode 100644
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/availability/DataAvailabilityTaskRunner.java
rename
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{detection/DetectionPipelineScheduler.java
=> scheduler/DetectionCronScheduler.java} (66%)
rename
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{detection/alert/DetectionAlertScheduler.java
=> scheduler/SubscriptionCronScheduler.java} (82%)
copy
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{detection/Evaluation.java
=> scheduler/ThirdEyeCronScheduler.java} (50%)
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (b9e4d7d -> 0a1c17a)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from b9e4d7d [TE] frontend - harleyjj/alert-details - bug fix and consolidation of anomaly filtering to a method (#5048) add 0a1c17a [TE] weekly detection - holt winter algorithm prediction window alignment (#5052) No new revisions were added by this update. Summary of changes: .../detection/components/HoltWintersDetector.java | 23 ++ 1 file changed, 15 insertions(+), 8 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (b4b1512 -> b9e4d7d)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from b4b1512 Add forceDelete option for task queue deletion (#5051) add b9e4d7d [TE] frontend - harleyjj/alert-details - bug fix and consolidation of anomaly filtering to a method (#5048) No new revisions were added by this update. Summary of changes: .../app/pods/components/alert-details/component.js | 62 ++ 1 file changed, 28 insertions(+), 34 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (89fec1e -> 0b3a60d)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 89fec1e Change SqlConformanceLevel from LENIENT to BABEL (#5031) add 0b3a60d [TE] Add reference link to labelling anomalies in alert notifications (#5037) No new revisions were added by this update. Summary of changes: .../detection/yaml/translator/SubscriptionConfigTranslator.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (73b93da -> 76d6934)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 73b93da [TE] Allow fallback for out-of-window cases (#5025) add 76d6934 [TE] frontend - harleyjj/alert-details - fix anomaly chart mapping to be robust against imprecision (#5012) No new revisions were added by this update. Summary of changes: .../app/pods/components/alert-details/component.js | 214 ++--- 1 file changed, 101 insertions(+), 113 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (01a9187 -> 73b93da)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 01a9187 [TE][bug] Bring back the cleanCode method accidentally removed in #4900 (#5024) add 73b93da [TE] Allow fallback for out-of-window cases (#5025) No new revisions were added by this update. Summary of changes: .../trigger/DataAvailabilityTaskScheduler.java | 5 +++-- .../trigger/DataAvailabilityTaskSchedulerTest.java | 20 +++- 2 files changed, 22 insertions(+), 3 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (0c1be2f -> 01a9187)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 0c1be2f [TE] auto enable data-availaibity trigger for PINOT detections without cron (#5009) add 01a9187 [TE][bug] Bring back the cleanCode method accidentally removed in #4900 (#5024) No new revisions were added by this update. Summary of changes: .../apache/pinot/thirdeye/detection/DefaultDataProvider.java | 10 ++ .../detection/cache/builder/AnomaliesCacheBuilder.java | 6 ++ .../detection/cache/builder/TimeSeriesCacheBuilder.java| 6 ++ 3 files changed, 22 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (6a97127 -> 0c1be2f)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 6a97127 Decouple server instance id with hostname/port config. (#4995) add 0c1be2f [TE] auto enable data-availaibity trigger for PINOT detections without cron (#5009) No new revisions were added by this update. Summary of changes: .../utils/DataAvailabilitySchedulingConfiguration.java| 2 +- .../pinot/thirdeye/detection/DetectionPipelineScheduler.java | 4 ++-- .../detection/yaml/translator/DetectionConfigTranslator.java | 11 +++ .../pinot/thirdeye/detection/yaml/YamlResourceTest.java | 4 +++- .../yaml/translator/DetectionConfigTranslatorTest.java| 6 ++ .../src/test/resources/sample-detection-config.yml| 3 +++ 6 files changed, 26 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (f6d33be -> 96a55cb)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from f6d33be For RANGE predicate queries touching offline segments, use sorted inverted index (#5013) add 96a55cb [TE][detection] Implement anomalies cache to speed up detection (#4900) No new revisions were added by this update. Summary of changes: thirdeye/pom.xml | 2 +- .../dashboard/resources/v2/AnomaliesResource.java | 8 +- .../pinot/thirdeye/datalayer/dto/AbstractDTO.java | 3 +- .../datalayer/dto/MergedAnomalyResultDTO.java | 6 +- .../thirdeye/datalayer/pojo/AbstractBean.java | 3 +- .../datalayer/pojo/MergedAnomalyResultBean.java| 3 +- .../thirdeye/datasource/ThirdEyeCacheRegistry.java | 2 +- .../thirdeye/detection/DefaultDataProvider.java| 162 .../detection/DetectionPipelineTaskRunner.java | 5 +- .../thirdeye/detection/DetectionResource.java | 13 +- .../pinot/thirdeye/detection/DetectionUtils.java | 22 +++ .../pinot/thirdeye/detection/alert/AlertUtils.java | 8 +- .../detection/alert/DetectionAlertTaskFactory.java | 6 +- .../detection/alert/DetectionAlertTaskRunner.java | 1 - .../detection/algorithm/DimensionWrapper.java | 29 .../detection/algorithm/MovingWindowAlgorithm.java | 10 +- .../cache/builder/AnomaliesCacheBuilder.java | 170 + .../cache/builder/TimeSeriesCacheBuilder.java | 168 .../finetune/GridSearchTuningAlgorithm.java| 6 +- .../onboard/YamlOnboardingTaskRunner.java | 5 +- .../thirdeye/detection/spi/model/AnomalySlice.java | 38 - .../detection/wrapper/AnomalyDetectorWrapper.java | 2 - .../thirdeye/detection/yaml/YamlResource.java | 6 +- .../src/main/resources/schema/drop-tables.sql | 16 +- .../csv/CSVThirdEyeDataSourceIntegrationTest.java | 1 - .../MockThirdEyeDataSourceIntegrationTest.java | 1 - .../pinot/thirdeye/detection/DataProviderTest.java | 33 +--- .../pinot/thirdeye/detection/MockDataProvider.java | 21 +-- .../alert/DetectionAlertTaskFactoryTest.java | 2 +- .../thirdeye/detection/alert/SendAlertTest.java| 35 + .../alert/scheme/DetectionEmailAlerterTest.java| 6 +- .../alert/scheme/DetectionJiraAlerterTest.java | 4 +- .../detection/algorithm/DimensionWrapperTest.java | 40 + .../integration/NotificationTaskSchedulerTest.java | 5 +- .../templates/TestEntityGroupKeyContent.java | 6 +- .../templates/TestMetricAnomaliesContent.java | 17 ++- .../channels/TestJiraContentFormatter.java | 9 +- .../test/resources/data-sources/cache-config.yml | 2 +- 38 files changed, 612 insertions(+), 264 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/builder/AnomaliesCacheBuilder.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/builder/TimeSeriesCacheBuilder.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (08c0c2b -> a9b7b52)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 08c0c2b Modifying segment fetcher doc to v2 endpoint (#4923)
add a9b7b52 [TE] add documentation for centralized caching in ThirdEye
(#4917)
No new revisions were added by this update.
Summary of changes:
thirdeye/docs/appendix.rst | 2 +-
thirdeye/docs/cache_intro.rst | 35 +++
thirdeye/docs/{introduction.rst => caches.rst} | 10 +-
thirdeye/docs/configuration.rst| 41 +
thirdeye/docs/couchbase.rst| 123 +
thirdeye/docs/index.rst| 3 +-
thirdeye/docs/setup_cache_datastore.rst| 43 +
7 files changed, 251 insertions(+), 6 deletions(-)
create mode 100644 thirdeye/docs/cache_intro.rst
copy thirdeye/docs/{introduction.rst => caches.rst} (87%)
create mode 100644 thirdeye/docs/couchbase.rst
create mode 100644 thirdeye/docs/setup_cache_datastore.rst
-
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (29d88fb -> 58ff866)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 29d88fb Replacing usage of Class.forName with PluginManager.createInstance (#4907) add 58ff866 [TE] clean up and fix bugs in notification pipeline (#4908) No new revisions were added by this update. Summary of changes: .../alert/DetectionAlertFilterNotification.java | 4 +--- .../detection/alert/StatefulDetectionAlertFilter.java | 4 +++- .../alert/filter/DimensionDetectionAlertFilter.java | 4 +++- .../alert/filter/PerUserDimensionAlertFilter.java | 4 +++- .../detection/alert/filter/SubscriptionUtils.java | 8 .../filter/ToAllRecipientsDetectionAlertFilter.java | 16 ++-- .../detection/alert/filter/AlertFilterUtils.java| 17 + .../alert/filter/DimensionDetectionAlertFilterTest.java | 13 +++-- .../alert/filter/PerUserDimensionAlertFilterTest.java | 6 +++--- .../filter/ToAllRecipientsDetectionAlertFilterTest.java | 13 - .../suppress/DetectionTimeWindowSuppressorTest.java | 6 -- 11 files changed, 47 insertions(+), 48 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (e4c3b56 -> 6bf7f64)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e4c3b56 [TE] Stop producing so many logs -- don't check for holes in the middle of timeseries for complete cache misses (#4890) add 6bf7f64 [TE] Prefetch based on data granularity in dimension exploration jobs (#4894) No new revisions were added by this update. Summary of changes: .../detection/algorithm/DimensionWrapper.java | 33 +- .../apache/pinot/thirdeye/util/ThirdEyeUtils.java | 14 ++--- 2 files changed, 36 insertions(+), 11 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (e01ccc2 -> e4c3b56)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e01ccc2 Initial code dump for dynamically loading pinot plugins (#4899) add e4c3b56 [TE] Stop producing so many logs -- don't check for holes in the middle of timeseries for complete cache misses (#4890) No new revisions were added by this update. Summary of changes: .../detection/cache/ThirdEyeCacheResponse.java | 42 -- .../apache/pinot/thirdeye/util/IntervalUtils.java | 36 +++ 2 files changed, 60 insertions(+), 18 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated: [TE][notification] Support configuring reference links per dimensional recipients (#4893)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7b4a4e7 [TE][notification] Support configuring reference links per
dimensional recipients (#4893)
7b4a4e7 is described below
commit 7b4a4e7ff28a39a96c6b2ddc7183692f60b57377
Author: Akshay Rai
AuthorDate: Fri Dec 6 08:45:54 2019 -0800
[TE][notification] Support configuring reference links per dimensional
recipients (#4893)
This PR enables configuration of referenceLinks and potential other
subscription related settings in the DIMENSIONS_ALERTER_PIPELINE. For example,
this can enable users to configure different oncall-runbooks per anomalous
dimension.
---
.../alert/DetectionAlertFilterNotification.java| 23
.../alert/StatefulDetectionAlertFilter.java| 4 +-
.../filter/DimensionDetectionAlertFilter.java | 14 ++---
.../filter/DimensionsRecipientAlertFilter.java | 15 +++---
.../alert/filter/PerUserDimensionAlertFilter.java | 14 ++---
.../detection/alert/filter/SubscriptionUtils.java | 58
.../ToAllRecipientsDetectionAlertFilter.java | 15 +++---
.../alert/scheme/DetectionEmailAlerter.java| 17 +++---
.../alert/scheme/DetectionJiraAlerter.java | 10 ++--
.../detection/alert/filter/AlertFilterUtils.java | 19 +--
...ava => DimensionsRecipientAlertFilterTest.java} | 61 +++---
.../alert/scheme/DetectionEmailAlerterTest.java| 12 +++--
.../alert/scheme/DetectionJiraAlerterTest.java | 12 +++--
.../DetectionTimeWindowSuppressorTest.java | 20 ---
.../tools/RunAdhocDatabaseQueriesTool.java | 14 +++--
15 files changed, 216 insertions(+), 92 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertFilterNotification.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertFilterNotification.java
index b5ba6f9..bbb3f88 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertFilterNotification.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertFilterNotification.java
@@ -24,6 +24,7 @@ import com.google.common.collect.Multimap;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
/**
@@ -31,24 +32,24 @@ import java.util.Objects;
*/
public class DetectionAlertFilterNotification {
- Map notificationSchemeProps;
+ DetectionAlertConfigDTO subsConfig;
Multimap dimensionFilters;
- public DetectionAlertFilterNotification(Map
notificationSchemeProps) {
-this(notificationSchemeProps, ArrayListMultimap.create());
+ public DetectionAlertFilterNotification(DetectionAlertConfigDTO subsConfig) {
+this(subsConfig, ArrayListMultimap.create());
}
- public DetectionAlertFilterNotification(Map
notificationSchemeProps, Multimap dimensionFilters) {
-this.notificationSchemeProps = notificationSchemeProps;
+ public DetectionAlertFilterNotification(DetectionAlertConfigDTO subsConfig,
Multimap dimensionFilters) {
+this.subsConfig = subsConfig;
this.dimensionFilters = dimensionFilters;
}
- public Map getNotificationSchemeProps() {
-return notificationSchemeProps;
+ public DetectionAlertConfigDTO getSubscriptionConfig() {
+return subsConfig;
}
- public void setNotificationSchemeProps(Map
notificationSchemeProps) {
-this.notificationSchemeProps = notificationSchemeProps;
+ public void setSubscriptionConfig(DetectionAlertConfigDTO subsConfig) {
+this.subsConfig = subsConfig;
}
public Multimap getDimensionFilters() {
@@ -68,13 +69,13 @@ public class DetectionAlertFilterNotification {
return false;
}
DetectionAlertFilterNotification that = (DetectionAlertFilterNotification)
o;
-return Objects.equals(notificationSchemeProps,
that.notificationSchemeProps) &&
+return Objects.equals(subsConfig, that.subsConfig) &&
Objects.equals(dimensionFilters, that.dimensionFilters);
}
@Override
public int hashCode() {
-return Objects.hash(notificationSchemeProps, dimensionFilters);
+return Objects.hash(subsConfig, dimensionFilters);
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
index 3a15235..41b3df4 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/ap
[incubator-pinot] branch master updated (82359d0 -> d15b33a)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 82359d0 Use Files.createDirectories instead of File.mkdirs in (#4878) add d15b33a [TE] Bug fix in the rca dashboard/summary endpoint (#4880) No new revisions were added by this update. Summary of changes: .../thirdeye/dashboard/resources/SummaryResource.java | 17 + 1 file changed, 13 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (f3558b6 -> 2201a22)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from f3558b6 Capture the exact UPDATE operation in DefaultColumnHandler (#4876) add 2201a22 [TE] Generify centralized cache DAO and add certificate-based authentication support for Couchbase (#4862) No new revisions were added by this update. Summary of changes: .../config/data-sources/cache-config.yml | 32 + .../thirdeye/datasource/ThirdEyeCacheRegistry.java | 17 +++ .../thirdeye/detection/cache/CacheConfig.java | 26 ++ .../detection/cache/CacheConfigLoader.java | 7 +++ .../pinot/thirdeye/detection/cache/CacheDAO.java | 41 .../thirdeye/detection/cache/CacheDataSource.java | 28 ++- .../detection/cache/CentralizedCacheConfig.java| 2 +- .../detection/cache/CouchbaseCacheDAO.java | 56 ++ .../detection/cache/DefaultTimeSeriesCache.java| 4 +- .../org/apache/pinot/thirdeye/util/CacheUtils.java | 49 ++- 10 files changed, 195 insertions(+), 67 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/CacheDAO.java - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (db91339 -> 5d0bcca)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from db91339 [TE] clean up some legacy code (#4837) add 5d0bcca [TE] frontend - harleyjj/bugs - Dont filter zeroes from bounds and display error message on create alert (#4868) No new revisions were added by this update. Summary of changes: .../app/pods/self-serve/create-alert/controller.js| 8 ++-- thirdeye/thirdeye-frontend/app/utils/utils.js | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (64312f4 -> db91339)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 64312f4 Adding pinotFS.open(URI) method to pinotFS (#4873) add db91339 [TE] clean up some legacy code (#4837) No new revisions were added by this update. Summary of changes: .../anomaly/merge/AnomalyTimeBasedSummarizer.java | 1 + .../context/AnomalyDetectionContext.java | 17 - .../anomalydetection/context/AnomalyResult.java| 2 + .../AbstractModularizedAnomalyFunction.java| 369 .../function/MinMaxThresholdFunction.java | 104 - .../ModularizedAnomalyFunctionModelProvider.java | 66 --- .../function/WeekOverWeekRuleFunction.java | 176 .../model/data/AbstractDataModel.java | 36 -- .../anomalydetection/model/data/DataModel.java | 57 --- .../anomalydetection/model/data/NoopDataModel.java | 40 -- .../model/data/SeasonalDataModel.java | 82 .../model/detection/AbstractDetectionModel.java| 36 -- .../model/detection/DetectionModel.java| 49 --- .../detection/MinMaxThresholdDetectionModel.java | 99 - .../model/detection/NoopDetectionModel.java| 32 -- .../detection/SimpleThresholdDetectionModel.java | 124 -- .../model/merge/AbstractMergeModel.java| 36 -- .../anomalydetection/model/merge/MergeModel.java | 48 --- .../model/merge/MinMaxThresholdMergeModel.java | 93 .../model/merge/NoPredictionMergeModel.java| 27 -- .../model/merge/NoopMergeModel.java| 31 -- .../model/merge/SimplePercentageMergeModel.java| 106 - .../model/prediction/AbstractPredictionModel.java | 36 -- .../ExpectedTimeSeriesPredictionModel.java | 31 -- .../model/prediction/NoopPredictionModel.java | 31 -- .../model/prediction/PredictionModel.java | 47 --- .../prediction/SeasonalAveragePredictionModel.java | 109 - .../transform/AbstractTransformationFunction.java | 36 -- .../model/transform/AnomalyRemovalFunction.java| 206 - .../transform/MovingAverageSmoothingFunction.java | 98 - .../TotalCountThresholdRemovalFunction.java| 70 --- .../model/transform/TransformationFunction.java| 54 --- .../model/transform/ZeroRemovalFunction.java | 53 --- .../api/user/dashboard/UserDashboardResource.java | 2 - .../datalayer/dto/MergedAnomalyResultDTO.java | 1 - .../thirdeye/detection/spi/model/AnomalySlice.java | 23 +- .../merge/TestAnomalyTimeBasedSummarizer.java | 101 - .../function/TestAnomalyFunctionFactory.java | 67 --- .../function/TestMinMaxThresholdFunction.java | 206 - .../function/TestWeekOverWeekRuleFunction.java | 468 - .../transform/testAnomalyRemovalFunction.java | 51 --- .../pinot/thirdeye/datalayer/DaoTestUtils.java | 15 - .../bao/TestMergedAnomalyResultManager.java| 61 +-- .../pinot/thirdeye/detection/DataProviderTest.java | 29 +- .../algorithm/LegacyAlertFilterWrapperTest.java| 109 - .../LegacyAnomalyFunctionAlgorithmTest.java| 98 - .../algorithm/LegacyDimensionWrapperTest.java | 154 --- .../algorithm/LegacyMergeWrapperTest.java | 137 -- .../LegacyEmulationIntegrationTest.java| 148 --- .../AnomalyApplicationEndToEndTest.java| 435 --- 50 files changed, 49 insertions(+), 4458 deletions(-) delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/function/AbstractModularizedAnomalyFunction.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/function/MinMaxThresholdFunction.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/function/ModularizedAnomalyFunctionModelProvider.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/function/WeekOverWeekRuleFunction.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/model/data/AbstractDataModel.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/model/data/DataModel.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/model/data/NoopDataModel.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/model/data/SeasonalDataModel.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/model/detection/AbstractDetectionModel.java delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/model/detection
[incubator-pinot] branch master updated (05c89fc -> 03ae23c)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 05c89fc Enhance TagNameUtils to handle the TagOverrideConfig (#4859) add 03ae23c [TE] Check if snapshot exists before attaching file (#4867) No new revisions were added by this update. Summary of changes: .../pinot/thirdeye/notification/commons/ThirdEyeJiraClient.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (54fb020 -> b558e81)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 54fb020 [TE] Enable local debugging of CSV data (#4826) add b558e81 [TE] add default constructor for detection health status (#4861) No new revisions were added by this update. Summary of changes: .../pinot/thirdeye/detection/health/AnomalyCoverageStatus.java | 5 + .../pinot/thirdeye/detection/health/DetectionTaskStatus.java | 9 + .../apache/pinot/thirdeye/detection/health/RegressionStatus.java | 8 3 files changed, 22 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (5c02b14 -> 54fb020)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 5c02b14 Split BrokerReduceService code into several DataTableReducers (#4851) add 54fb020 [TE] Enable local debugging of CSV data (#4826) No new revisions were added by this update. Summary of changes: .../config/data-sources/data-sources-config.yml| 15 thirdeye/thirdeye-pinot/config/data/pageviews.csv | 67 thirdeye/thirdeye-pinot/config/h2db.mv.db | Bin 4947968 -> 2949120 bytes .../datasource/mock/MockThirdEyeDataSource.java| 89 - .../datasource/sql/SqlResponseCacheLoader.java | 63 --- .../pinot/thirdeye/datasource/sql/SqlUtils.java| 15 ++-- 6 files changed, 192 insertions(+), 57 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/config/data/pageviews.csv - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (e6e7bf9 -> 1cc296b)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e6e7bf9 [TE] Use default centralized cache settings if exception occurs during init (#4845) add 1cc296b [TE] Fix duplicate entries in entity report (#4840) No new revisions were added by this update. Summary of changes: .../notification/content/templates/EntityGroupKeyContent.java | 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (e1b3512 -> e6e7bf9)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from e1b3512 Prepare for pinot release 0.3.0 (#4849) add e6e7bf9 [TE] Use default centralized cache settings if exception occurs during init (#4845) No new revisions were added by this update. Summary of changes: .../thirdeye/datasource/ThirdEyeCacheRegistry.java | 23 -- .../pinot/resultset/ThirdEyeResultSetUtils.java| 5 +++-- 2 files changed, 16 insertions(+), 12 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
[incubator-pinot] branch master updated (81703be -> c720b99)
This is an automated email from the ASF dual-hosted git repository. akshayrai09 pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git. from 81703be [TE] Centralized cache proof-of-concept for anomaly detection via Couchbase (#4761) add c720b99 [TE] quick fix for error in retriving legacy anomaly (#4843) No new revisions were added by this update. Summary of changes: .../thirdeye/api/user/dashboard/UserDashboardResource.java| 11 +-- 1 file changed, 9 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
