vincentchenjl commented on a change in pull request #4724: [TE] Add event
trigger listener for event driven scheduling
URL: https://github.com/apache/incubator-pinot/pull/4724#discussion_r338850429
##########
File path:
thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/detection/trigger/DatasetTriggerInfoRepoTest.java
##########
@@ -0,0 +1,141 @@
+package org.apache.pinot.thirdeye.anomaly.detection.trigger;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import
org.apache.pinot.thirdeye.anomaly.detection.trigger.utils.DatasetTriggerInfoRepo;
+import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class DatasetTriggerInfoRepoTest {
+ private static String TEST_DATA_SOURCE = "TestSource";
+ private static String TEST_DATASET_PREFIX = "test_dataset_";
+ private static String TEST_METRIC_PREFIX = "test_metric_";
+ private DAOTestBase testDAOProvider;
+
+ @BeforeClass
+ public void BeforeClass() {
+ testDAOProvider = DAOTestBase.getInstance();
+ DetectionConfigManager detectionConfigManager =
DAORegistry.getInstance().getDetectionConfigManager();
+ MetricConfigManager metricConfigManager =
DAORegistry.getInstance().getMetricConfigDAO();
+ DatasetConfigManager datasetConfigDAO =
DAORegistry.getInstance().getDatasetConfigDAO();
+
+ MetricConfigDTO metric1 = new MetricConfigDTO();
+ metric1.setDataset(TEST_DATASET_PREFIX + 1);
+ metric1.setName(TEST_METRIC_PREFIX + 1);
+ metric1.setActive(true);
+ metric1.setAlias("");
+ long metricId1 = metricConfigManager.save(metric1);
+
+ MetricConfigDTO metric2 = new MetricConfigDTO();
+ metric2.setDataset(TEST_DATASET_PREFIX + 2);
+ metric2.setName(TEST_METRIC_PREFIX + 2);
+ metric2.setActive(true);
+ metric2.setAlias("");
+ long metricId2 = metricConfigManager.save(metric2);
+
+ DetectionConfigDTO detect1 = new DetectionConfigDTO();
+ detect1.setName("test_detection_1");
+ detect1.setActive(true);
+ Map<String, Object> props = new HashMap<>();
+ List<String> metricUrns = new ArrayList<>();
+ metricUrns.add("thirdeye:metric:" + metricId1);
+ metricUrns.add("thirdeye:metric:" + metricId2);
+ props.put("nestedMetricUrns", metricUrns);
+ detect1.setProperties(props);
+ detectionConfigManager.save(detect1);
+
+ DatasetConfigDTO ds1 = new DatasetConfigDTO();
+ ds1.setDataset(TEST_DATASET_PREFIX + 1);
+ ds1.setDataSource(TEST_DATA_SOURCE);
+ ds1.setLastRefreshTime(1000);
+ datasetConfigDAO.save(ds1);
+
+ DatasetConfigDTO ds2 = new DatasetConfigDTO();
+ ds2.setDataset(TEST_DATASET_PREFIX + 2);
+ ds2.setDataSource(TEST_DATA_SOURCE);
+ ds2.setLastRefreshTime(2000);
+ datasetConfigDAO.save(ds2);
+ }
+
+ @Test
+ public void testInitAndGetInstance () {
+ DatasetTriggerInfoRepo datasetTriggerInfoRepo =
DatasetTriggerInfoRepo.initAndGetInstance(
+ 1, Collections.singletonList(TEST_DATA_SOURCE));
+ Assert.assertSame(datasetTriggerInfoRepo,
+ DatasetTriggerInfoRepo.initAndGetInstance(1,
Collections.singletonList(TEST_DATA_SOURCE)));
+ Assert.assertSame(datasetTriggerInfoRepo,
DatasetTriggerInfoRepo.getInstance());
+
Assert.assertTrue(datasetTriggerInfoRepo.isDatasetActive(TEST_DATASET_PREFIX +
1));
+
Assert.assertTrue(datasetTriggerInfoRepo.isDatasetActive(TEST_DATASET_PREFIX +
2));
+
Assert.assertFalse(datasetTriggerInfoRepo.isDatasetActive(TEST_DATASET_PREFIX +
3));
+
Assert.assertEquals(datasetTriggerInfoRepo.getLastUpdateTimestamp(TEST_DATASET_PREFIX
+ 1), 1000);
+
Assert.assertEquals(datasetTriggerInfoRepo.getLastUpdateTimestamp(TEST_DATASET_PREFIX
+ 2), 2000);
+
Assert.assertEquals(datasetTriggerInfoRepo.getLastUpdateTimestamp(TEST_DATASET_PREFIX
+ 3), 0);
+ }
+
+ @Test
+ public void testSetLastUpdateTimestamp() {
+ DatasetTriggerInfoRepo datasetTriggerInfoRepo =
DatasetTriggerInfoRepo.initAndGetInstance(
+ 1, Collections.singletonList(TEST_DATA_SOURCE));
+ datasetTriggerInfoRepo.setLastUpdateTimestamp(TEST_DATASET_PREFIX + 1,
3000);
+
Assert.assertEquals(datasetTriggerInfoRepo.getLastUpdateTimestamp(TEST_DATASET_PREFIX
+ 1), 3000);
+
Assert.assertEquals(datasetTriggerInfoRepo.getLastUpdateTimestamp(TEST_DATASET_PREFIX
+ 2), 2000);
+
Assert.assertEquals(datasetTriggerInfoRepo.getLastUpdateTimestamp(TEST_DATASET_PREFIX
+ 4), 0);
+ }
+
+ @Test
+ public void testRefresh() throws InterruptedException {
+ DetectionConfigManager detectionConfigManager =
DAORegistry.getInstance().getDetectionConfigManager();
+ MetricConfigManager metricConfigManager =
DAORegistry.getInstance().getMetricConfigDAO();
+ DatasetConfigManager datasetConfigDAO =
DAORegistry.getInstance().getDatasetConfigDAO();
+ DatasetTriggerInfoRepo datasetTriggerInfoRepo =
DatasetTriggerInfoRepo.initAndGetInstance(
+ 1, Collections.singletonList(TEST_DATA_SOURCE));
+
+ MetricConfigDTO metric = new MetricConfigDTO();
+ metric.setDataset(TEST_DATASET_PREFIX + 3);
+ metric.setName(TEST_METRIC_PREFIX + 3);
+ metric.setActive(true);
+ metric.setAlias("");
+ long metricId = metricConfigManager.save(metric);
+
+ DetectionConfigDTO detect2 = new DetectionConfigDTO();
+ detect2.setName("test_detection_2");
+ detect2.setActive(true);
+ Map<String, Object> props = new HashMap<>();
+ List<String> metricUrns = new ArrayList<>();
+ metricUrns.add("thirdeye:metric:" + metricId);
+ props.put("nestedMetricUrns", metricUrns);
+ detect2.setProperties(props);
+ detectionConfigManager.save(detect2);
+
+ DatasetConfigDTO ds = new DatasetConfigDTO();
+ ds.setDataset(TEST_DATASET_PREFIX + 3);
+ ds.setDataSource(TEST_DATA_SOURCE);
+ ds.setLastRefreshTime(3000);
+ datasetConfigDAO.save(ds);
+
+ Thread.sleep(65 * 1000); // wait for datasetTriggerInfoRepo to refresh
Review comment:
This issue is that `DatasetTriggerInfoRepo` can only refresh at minute
level. When I run the test suite, it does not add that much time since the
entire suite runs for couple minutes.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
