This is an automated email from the ASF dual-hosted git repository. xhsun pushed a commit to branch disable_long_fail_alerts in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 81583d6152c03be7519c9a8ff3f3d1452e38bb2d Author: Xiaohui Sun <[email protected]> AuthorDate: Thu Apr 2 15:42:51 2020 -0700 [TE] Disable alerts if it has no success run within 30 days --- .../anomaly/monitor/MonitorTaskRunner.java | 28 ++++++++++++++++++++++ .../thirdeye/datalayer/dao/GenericPojoDao.java | 4 ++++ .../pinot/thirdeye/datalayer/dto/AbstractDTO.java | 10 ++++++++ 3 files changed, 42 insertions(+) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java index eb1a81f..0b37ac1 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/monitor/MonitorTaskRunner.java @@ -48,6 +48,7 @@ public class MonitorTaskRunner implements TaskRunner { private static final Logger LOG = LoggerFactory.getLogger(MonitorJobRunner.class); private static final long MAX_TASK_TIME = TimeUnit.HOURS.toMillis(6); + private static final long MAX_FAILED_DISABLE_DAYS = 30; private DAORegistry DAO_REGISTRY = DAORegistry.getInstance(); @@ -120,11 +121,38 @@ public class MonitorTaskRunner implements TaskRunner { // update detection health updateDetectionHealth(); + + // disable alerts that failed consecutively for a long time + disableLongFailedAlerts(); + } catch (Exception e) { LOG.error("Exception in monitor update task", e); } } + /** + * Disable the alert if it was updated before {@MAX_TASK_FAIL_DAYS} but there is no success run since then. + */ + private void disableLongFailedAlerts() { + DetectionConfigManager detectionDAO = DAO_REGISTRY.getDetectionConfigManager(); + List<DetectionConfigDTO> detectionConfigs = detectionDAO.findAllActive(); + long currentTimeMills = System.currentTimeMillis(); + long maxTaskFailMills = TimeUnit.DAYS.toMillis(MAX_FAILED_DISABLE_DAYS); + for (DetectionConfigDTO config : detectionConfigs) { + if (config.getHealth() != null && config.getHealth().getDetectionTaskStatus() != null) { + long lastTaskExecutionTime = config.getHealth().getDetectionTaskStatus().getLastTaskExecutionTime(); + if (config.getUpdateTime() <= currentTimeMills - maxTaskFailMills && + (lastTaskExecutionTime == -1L || lastTaskExecutionTime <= currentTimeMills - maxTaskFailMills)) { + config.setActive(false); + detectionDAO.update(config); + LOG.info("Disable alert " + config.getId() + " since it failed more than " + MAX_FAILED_DISABLE_DAYS + " days"); + LOG.info("Task last update time: " + config.getUpdateTime()); + LOG.info("Last success task execution time: " + lastTaskExecutionTime); + } + } + } + } + private void updateDetectionHealth() { DetectionConfigManager detectionDAO = DAO_REGISTRY.getDetectionConfigManager(); List<DetectionConfigDTO> detectionConfigs = detectionDAO.findAllActive(); diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java index 493ef91..8e983c5 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java @@ -429,6 +429,7 @@ public class GenericPojoDao { E e = OBJECT_MAPPER.readValue(entity.getJsonVal(), beanClass); e.setId(entity.getId()); + e.setUpdateTime(entity.getUpdateTime()); ret.add(e); } } @@ -461,6 +462,7 @@ public class GenericPojoDao { e = OBJECT_MAPPER.readValue(genericJsonEntity.getJsonVal(), pojoClass); e.setId(genericJsonEntity.getId()); e.setVersion(genericJsonEntity.getVersion()); + e.setUpdateTime(genericJsonEntity.getUpdateTime()); } return e; } @@ -492,6 +494,7 @@ public class GenericPojoDao { E e = OBJECT_MAPPER.readValue(genericJsonEntity.getJsonVal(), pojoClass); e.setId(genericJsonEntity.getId()); e.setVersion(genericJsonEntity.getVersion()); + e.setUpdateTime(genericJsonEntity.getUpdateTime()); result.add(e); } } @@ -599,6 +602,7 @@ public class GenericPojoDao { E bean = OBJECT_MAPPER.readValue(entity.getJsonVal(), pojoClass); bean.setId(entity.getId()); bean.setVersion(entity.getVersion()); + bean.setUpdateTime(entity.getUpdateTime()); ret.add(bean); } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/AbstractDTO.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/AbstractDTO.java index fcfe833..c440779 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/AbstractDTO.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/AbstractDTO.java @@ -20,15 +20,25 @@ package org.apache.pinot.thirdeye.datalayer.dto; import java.io.Serializable; +import java.sql.Timestamp; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; public abstract class AbstractDTO implements Serializable { private Long id; private int version; + protected Long updateTime; protected String createdBy; protected String updatedBy; + public Long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Timestamp updateTime) { + this.updateTime = updateTime.getTime(); + } + public Long getId() { return id; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
