jihaozh commented on a change in pull request #3830: [TE] Presto Connector
backend and front end
URL: https://github.com/apache/incubator-pinot/pull/3830#discussion_r256574025
##########
File path:
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/presto/PrestoResponseCacheLoader.java
##########
@@ -0,0 +1,186 @@
+package org.apache.pinot.thirdeye.datasource.presto;
+
+import com.google.common.cache.CacheLoader;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pinot.thirdeye.common.time.TimeSpec;
+import org.apache.pinot.thirdeye.dashboard.Utils;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeResultSet;
+import org.apache.pinot.thirdeye.detection.ConfigUtils;
+import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
+import org.apache.tomcat.jdbc.pool.DataSource;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.Period;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static
org.apache.pinot.thirdeye.datasource.pinot.resultset.ThirdEyeDataFrameResultSet.*;
+
+/**
+ * This class is a CacheLoader which issue queries to Presto
+ * It contains connection pools(DataSource) for each Presto database
configured in data-sources-configs
+ */
+public class PrestoResponseCacheLoader extends CacheLoader<PrestoQuery,
ThirdEyeResultSet> {
+ private static final Logger LOG =
LoggerFactory.getLogger(PrestoResponseCacheLoader.class);
+
+ private static final int INIT_CONNECTIONS = 20;
+
+ private static int MAX_CONNECTIONS;
+ static {
+ try {
+ MAX_CONNECTIONS =
Integer.parseInt(System.getProperty("max_presto_connections", "50"));
+ } catch (Exception e) {
+ MAX_CONNECTIONS = 50;
+ }
+ }
+ private final String PRESTO_URL = "prestoURLs";
+ private static final String JDBC_USER = "user";
+ private static final String JDBC_PASSWORD = "password";
+ private static final String VALIDATION_QUERY = "select 1";
+ private static final int ABANDONED_TIMEOUT = 600_000;
+
+ Map<String, DataSource> dbNameToDataSourceMap = new HashMap<>();
+ private String user;
+ private String password;
+
+
+ public PrestoResponseCacheLoader(Map<String, Object> properties) {
+ Map<String, String> dbNameToURLMap =
ConfigUtils.getMap(properties.get(PRESTO_URL));
+ user = (String)properties.get(JDBC_USER);
+ password = (String)properties.get(JDBC_PASSWORD);
+
+ for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
+ DataSource dataSource = new DataSource();
+ dataSource.setInitialSize(INIT_CONNECTIONS);
+ dataSource.setMaxActive(MAX_CONNECTIONS);
+ dataSource.setUsername(user);
+ dataSource.setPassword(password);
+ dataSource.setUrl(entry.getValue());
+
+ dataSource.setValidationQuery(VALIDATION_QUERY);
+ dataSource.setTestWhileIdle(true);
+ dataSource.setTestOnBorrow(true);
+ // when returning connection to pool
+ dataSource.setTestOnReturn(true);
+ dataSource.setRollbackOnReturn(true);
+
+ // Timeout before an abandoned(in use) connection can be removed.
+ dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
+ dataSource.setRemoveAbandoned(true);
+
+ dbNameToDataSourceMap.put(entry.getKey(), dataSource);
+ }
+ }
+
+
+ /**
+ * This method gets the dimension filters for the given dataset from the
presto data source,
+ * and returns them as map of dimension name to values
+ * @param dataset
+ * @return dimension filters map
+ */
+ public Map<String, List<String>> getDimensionFilters(String dataset) throws
Exception {
+ LOG.info("Getting dimension filters for " + dataset);
+ DatasetConfigDTO datasetConfig =
ThirdEyeUtils.getDatasetConfigFromName(dataset);
+
+ String[] tableComponents = dataset.split("\\.");
+ String dbName = tableComponents[0];
+ String tableName = ThirdEyeUtils.computePrestoTableName(dataset);
+ Map<String, List<String>> dimensionFilters = new HashMap<>();
+ Connection conn;
+ DataSource dataSource = dbNameToDataSourceMap.get(dbName);
+
+ try {
+ for (String dimension: datasetConfig.getDimensions()) {
+ dimensionFilters.put(dimension, new ArrayList<>());
+ conn = dataSource.getConnection();
+ Statement stmt = conn.createStatement();
+ ResultSet rs;
+
+ System.out.println(SqlUtils.getDimensionFiltersSQL(dimension,
tableName));
Review comment:
remove println. consider logging them if necessary.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
