This is an automated email from the ASF dual-hosted git repository. jihao pushed a commit to branch model-downloaders in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 90a031ae1c7a4a1d3e1f6c732b8d06531fb073a4 Author: Jihao Zhang <[email protected]> AuthorDate: Mon Mar 16 11:20:19 2020 -0700 [TE] anomaly detection model downloaders --- .../anomaly/ThirdEyeAnomalyApplication.java | 9 +++ .../thirdeye/common/ThirdEyeConfiguration.java | 11 ++++ .../dashboard/ThirdEyeDashboardApplication.java | 10 ++++ .../thirdeye/model/download/ModelDownloader.java | 28 +++++++++ .../download/ModelDownloaderConfiguration.java | 45 ++++++++++++++ .../model/download/ModelDownloaderManager.java | 68 ++++++++++++++++++++++ 6 files changed, 171 insertions(+) diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java index 240cdc5..08126dd 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java @@ -41,6 +41,7 @@ import org.apache.pinot.thirdeye.dashboard.resources.EmailResource; import org.apache.pinot.thirdeye.datasource.DAORegistry; import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry; import org.apache.pinot.thirdeye.datasource.pinot.resources.PinotDataSourceResource; +import org.apache.pinot.thirdeye.model.download.ModelDownloaderManager; import org.apache.pinot.thirdeye.scheduler.DetectionCronScheduler; import org.apache.pinot.thirdeye.scheduler.SubscriptionCronScheduler; import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory; @@ -77,6 +78,7 @@ public class ThirdEyeAnomalyApplication private DataAvailabilityTaskScheduler dataAvailabilityTaskScheduler = null; private DetectionCronScheduler detectionScheduler = null; private SubscriptionCronScheduler subscriptionScheduler = null; + private ModelDownloaderManager modelDownloaderManager = null; public static void main(final String[] args) throws Exception { List<String> argList = new ArrayList<>(Arrays.asList(args)); @@ -199,6 +201,10 @@ public class ThirdEyeAnomalyApplication config.getDataAvailabilitySchedulingConfiguration().getScheduleDelayInSec()); dataAvailabilityTaskScheduler.start(); } + if (config.getModelDownloaderConfig() != null) { + modelDownloaderManager = new ModelDownloaderManager(config.getModelDownloaderConfig()); + modelDownloaderManager.start(); + } } @Override @@ -236,6 +242,9 @@ public class ThirdEyeAnomalyApplication if (dataAvailabilityEventListenerDriver != null) { dataAvailabilityEventListenerDriver.shutdown(); } + if (modelDownloaderManager != null) { + modelDownloaderManager.shutdown(); + } } }); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java index bbfa37d..83b0dd8 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java @@ -28,6 +28,7 @@ import java.util.List; import io.dropwizard.Configuration; import java.util.Map; +import org.apache.pinot.thirdeye.model.download.ModelDownloaderConfiguration; public class ThirdEyeConfiguration extends Configuration { @@ -51,6 +52,8 @@ public class ThirdEyeConfiguration extends Configuration { private String failureFromAddress; private String failureToAddress; + private List<ModelDownloaderConfiguration> modelDownloaderConfig; + /** * allow cross request for local development */ @@ -190,4 +193,12 @@ public class ThirdEyeConfiguration extends Configuration { public void setAlerterConfiguration(Map<String, Map<String, Object>> alerterConfigurations) { this.alerterConfigurations = alerterConfigurations; } + + public List<ModelDownloaderConfiguration> getModelDownloaderConfig() { + return modelDownloaderConfig; + } + + public void setModelDownloaderConfig(List<ModelDownloaderConfiguration> modelDownloaderConfig) { + this.modelDownloaderConfig = modelDownloaderConfig; + } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java index bf6a6c7..427173e 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java @@ -78,6 +78,7 @@ import org.apache.pinot.thirdeye.detection.annotation.DetectionConfigurationReso import org.apache.pinot.thirdeye.detection.yaml.YamlResource; import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory; import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory; +import org.apache.pinot.thirdeye.model.download.ModelDownloaderManager; import org.apache.pinot.thirdeye.rootcause.RCAFramework; import org.apache.pinot.thirdeye.rootcause.impl.RCAFrameworkLoader; import org.apache.pinot.thirdeye.tracking.RequestStatisticsLogger; @@ -113,6 +114,7 @@ public class ThirdEyeDashboardApplication private static final Logger LOG = LoggerFactory.getLogger(ThirdEyeDashboardApplication.class); private RequestStatisticsLogger requestStatisticsLogger; + private ModelDownloaderManager modelDownloaderManager; @Override public String getName() { @@ -253,6 +255,11 @@ public class ThirdEyeDashboardApplication env.jersey().register(new AuthValueFactoryProvider.Binder<>(ThirdEyePrincipal.class)); } + if (config.getModelDownloaderConfig() != null) { + modelDownloaderManager = new ModelDownloaderManager(config.getModelDownloaderConfig()); + modelDownloaderManager.start(); + } + env.lifecycle().manage(new Managed() { @Override public void start() throws Exception { @@ -265,6 +272,9 @@ public class ThirdEyeDashboardApplication if (requestStatisticsLogger != null) { requestStatisticsLogger.shutdown(); } + if (modelDownloaderManager != null) { + modelDownloaderManager.shutdown(); + } } }); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java new file mode 100644 index 0000000..f76db13 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java @@ -0,0 +1,28 @@ +package org.apache.pinot.thirdeye.model.download; + +import java.nio.file.Path; +import java.util.Map; + + +/** + * The model downloader interface. It downloads model files (e.x., trained tensorflow models), into a local path. + * The implementation of this class can be configured to run at a certain frequency in ThirdEye server, so that the + * models can be kept up-to-date. + */ +public abstract class ModelDownloader { + private final Map<String, Object> properties; + + /** + * Create a model downloader. + * @param properties the properties + */ + public ModelDownloader(Map<String, Object> properties) { + this.properties = properties; + } + + /** + * fetch the models into the local path. + * @param destination the destination path + */ + public abstract void fetchModel(Path destination); +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java new file mode 100644 index 0000000..20b4dde --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java @@ -0,0 +1,45 @@ +package org.apache.pinot.thirdeye.model.download; + +import java.util.HashMap; +import java.util.Map; +import org.apache.pinot.thirdeye.common.time.TimeGranularity; + + +public class ModelDownloaderConfiguration { + private TimeGranularity runFrequency; + private String className; + private String destinationPath; + private Map<String, Object> properties = new HashMap<>(); + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public Map<String, Object> getProperties() { + return properties; + } + + public void setProperties(Map<String, Object> properties) { + this.properties = properties; + } + + public TimeGranularity getRunFrequency() { + return runFrequency; + } + + public void setRunFrequency(TimeGranularity runFrequency) { + this.runFrequency = runFrequency; + } + + public String getDestinationPath() { + return destinationPath; + } + + public void setDestinationPath(String destinationPath) { + this.destinationPath = destinationPath; + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java new file mode 100644 index 0000000..ed6f205 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java @@ -0,0 +1,68 @@ +package org.apache.pinot.thirdeye.model.download; + +import java.lang.reflect.Constructor; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardService; +import org.apache.pinot.thirdeye.common.time.TimeGranularity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * The model downloader manager. This class manages the life cycle of the model downloader. + * It constructs the model downloader, and then schedules the model downloader to run periodically and downloads the + * models into a local destination path. The class names, the run frequency and the download path can be configured. + */ +public class ModelDownloaderManager { + private static final Logger LOG = LoggerFactory.getLogger(AutoOnboardService.class); + + private final List<ModelDownloaderConfiguration> configs; + private final Map<String, ModelDownloader> modelDownloaders; + private ScheduledExecutorService scheduledExecutorService; + + public ModelDownloaderManager(List<ModelDownloaderConfiguration> modelDownloaderConfigs) { + this.configs = modelDownloaderConfigs; + this.modelDownloaders = new HashMap<>(); + this.scheduledExecutorService = Executors.newScheduledThreadPool(5); + + constructModelDownloaders(); + } + + private void constructModelDownloaders() { + for (ModelDownloaderConfiguration config : this.configs) { + try { + Constructor<?> constructor = Class.forName(config.getClassName()).getConstructor(Map.class); + ModelDownloader downloader = (ModelDownloader) constructor.newInstance(config.getProperties()); + this.modelDownloaders.put(config.getClassName(), downloader); + } catch (Exception e) { + LOG.warn("Failed to initialize model downloader {}", config.getClassName(), e); + } + } + } + + /** + * start the model downloader manager + */ + public void start() { + for (ModelDownloaderConfiguration config : this.configs) { + TimeGranularity runFrequency = config.getRunFrequency(); + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + LOG.info("running the model downloader: {}", config.getClassName()); + this.modelDownloaders.get(config.getClassName()).fetchModel(Paths.get(config.getDestinationPath())); + }, 0L, runFrequency.getSize(), runFrequency.getUnit()); + } + } + + /** + * shut down the manager + */ + public void shutdown() { + LOG.info("Shutting down model downloader manager"); + scheduledExecutorService.shutdown(); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
