vincentchenjl commented on a change in pull request #4724: [TE] Add event
trigger listener for event driven scheduling
URL: https://github.com/apache/incubator-pinot/pull/4724#discussion_r338845153
##########
File path:
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/detection/trigger/TriggerEventListener.java
##########
@@ -0,0 +1,85 @@
+package org.apache.pinot.thirdeye.anomaly.detection.trigger;
+
+import java.util.List;
+import
org.apache.pinot.thirdeye.anomaly.detection.trigger.filter.TriggerEventFilter;
+import
org.apache.pinot.thirdeye.anomaly.detection.trigger.utils.DatasetTriggerInfoRepo;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TriggerEventListener implements Runnable {
+ private static final Logger LOG =
LoggerFactory.getLogger(TriggerEventListener.class);
+ private KafkaTriggerEventConsumer _consumer;
+ private final List<TriggerEventFilter> _filters;
+ private static final DAORegistry DAO_REGISTRY = DAORegistry.getInstance();
+ private DatasetTriggerInfoRepo _datasetTriggerInfoRepo;
+ private DatasetConfigManager _datasetConfigManager;
+ private boolean closed;
+ private long totalEventCount;
+ private long processedEventCount;
+ private long sleepTimeMilli;
+
+ public TriggerEventListener(KafkaTriggerEventConsumer consumer,
List<TriggerEventFilter> filters, long sleepTime) {
+ _consumer = consumer;
+ _filters = filters;
+ _datasetConfigManager = DAO_REGISTRY.getDatasetConfigDAO();
+ _datasetTriggerInfoRepo = DatasetTriggerInfoRepo.getInstance();
+ closed = false;
+ totalEventCount = 0;
+ processedEventCount = 0;
+ sleepTimeMilli = sleepTime;
+ }
+
+ @Override
+ public void run() {
+ while (!(Thread.interrupted() || closed)) {
+ try {
+ processOneBatch();
+ } catch (InterruptedException e) {
+ LOG.error("Caught Interrupted Exception", e);
+ break;
+ }
+ }
+ _consumer.close();
+ LOG.info("TriggerEventListener is closed.");
+ }
+
+ public void close() {
+ _datasetTriggerInfoRepo.close();
+ closed = true;
+ }
+
+ void processOneBatch() throws InterruptedException {
+ List<TriggerEvent> events = _consumer.poll(5000);
+ LOG.info("Received " + events.size() + " events...");
+ totalEventCount += events.size();
+ for (TriggerEvent event : events) {
+ if (checkIfAllPassed(event)) {
+ LOG.info("Processing event: " + event.getDatasetName() + " with
watermark " + event.getHighWatermark());
+ String dataset = event.getDatasetName();
+ _datasetTriggerInfoRepo.setLastUpdateTimestamp(dataset,
event.getHighWatermark());
Review comment:
Yes, the dataset names have the suffix
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
