This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 52f69d1 [TE] Added support for Vertica as a data source (#4404)
52f69d1 is described below
commit 52f69d16a0e99376348ad060fb4062cffc9886cc
Author: Takahiko Saito <[email protected]>
AuthorDate: Wed Jul 10 11:55:01 2019 -0700
[TE] Added support for Vertica as a data source (#4404)
Added support for Vertica as a data source.
---
.../datasource/sql/SqlResponseCacheLoader.java | 37 ++++++++++++++++++++++
.../pinot/thirdeye/datasource/sql/SqlUtils.java | 3 ++
2 files changed, 40 insertions(+)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
index 5877291..3caa095 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
@@ -60,19 +60,23 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
private static int MAX_CONNECTIONS = 50;
private static final String PRESTO = "Presto";
private static final String MYSQL = "MySQL";
+ private static final String VERTICA = "Vertica";
private static final String DATASETS = "datasets";
private static final String H2 = "H2";
private static final String USER = "user";
private static final String DB = "db";
private static final String PASSWORD = "password";
+ private static final String DRIVER = "driver";
private static final DateTime MIN_DATETIME = DateTime.parse("1970-01-01");
private static final int ABANDONED_TIMEOUT = 60000;
private Map<String, DataSource> prestoDBNameToDataSourceMap = new
HashMap<>();
private Map<String, DataSource> mysqlDBNameToDataSourceMap = new HashMap<>();
+ private Map<String, DataSource> verticaDBNameToDataSourceMap = new
HashMap<>();
private static Map<String, String> prestoDBNameToURLMap = new HashMap<>();
private static Map<String, String> mysqlDBNameToURLMap = new HashMap<>();
+ private static Map<String, String> verticaDBNameToURLMap = new HashMap<>();
private static String h2Url;
DataSource h2DataSource;
@@ -133,6 +137,34 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
}
}
+ // Init Vertica datasources
+ if (properties.containsKey(VERTICA)) {
+ List<Map<String, Object>> verticaMapList =
ConfigUtils.getList(properties.get(VERTICA));
+ for (Map<String, Object> objMap: verticaMapList) {
+ Map<String, String> dbNameToURLMap = (Map)objMap.get(DB);
+ String verticaUser = (String)objMap.get(USER);
+ String verticaPassword = getPassword(objMap);
+ String verticaDriver = (String)objMap.get(DRIVER);
+
+ for (Map.Entry<String, String> entry: dbNameToURLMap.entrySet()) {
+ DataSource dataSource = new DataSource();
+ dataSource.setInitialSize(INIT_CONNECTIONS);
+ dataSource.setMaxActive(MAX_CONNECTIONS);
+ dataSource.setUsername(verticaUser);
+ dataSource.setPassword(verticaPassword);
+ dataSource.setDriverClassName(verticaDriver);
+ dataSource.setUrl(entry.getValue());
+
+ // Timeout before an abandoned(in use) connection can be removed.
+ dataSource.setRemoveAbandonedTimeout(ABANDONED_TIMEOUT);
+ dataSource.setRemoveAbandoned(true);
+
+ verticaDBNameToDataSourceMap.put(entry.getKey(), dataSource);
+ verticaDBNameToURLMap.putAll(dbNameToURLMap);
+ }
+ }
+ }
+
// Init H2 datasource
if (properties.containsKey(H2)) {
h2DataSource = new DataSource();
@@ -293,6 +325,8 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
dataSource = prestoDBNameToDataSourceMap.get(SQLQuery.getDbName());
} else if (sourceName.equals(MYSQL)) {
dataSource = mysqlDBNameToDataSourceMap.get(SQLQuery.getDbName());
+ } else if (sourceName.equals(VERTICA)) {
+ dataSource = verticaDBNameToDataSourceMap.get(SQLQuery.getDbName());
} else {
dataSource = h2DataSource;
}
@@ -323,6 +357,7 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
Map<String, Map<String,String>> dbNameToURLMap = new LinkedHashMap<>();
dbNameToURLMap.put(PRESTO, prestoDBNameToURLMap);
dbNameToURLMap.put(MYSQL, mysqlDBNameToURLMap);
+ dbNameToURLMap.put(VERTICA, verticaDBNameToURLMap);
Map<String, String> h2ToURLMap = new HashMap<>();
h2ToURLMap.put(H2, h2Url);
@@ -348,6 +383,8 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
return prestoDBNameToDataSourceMap.get(dbName);
} else if (sourceName.equals(MYSQL)) {
return mysqlDBNameToDataSourceMap.get(dbName);
+ } else if (sourceName.equals(VERTICA)) {
+ return verticaDBNameToDataSourceMap.get(dbName);
} else {
return h2DataSource;
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
index dedb9f7..298a842 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
@@ -87,6 +87,7 @@ public class SqlUtils {
private static final String PRESTO = "Presto";
private static final String MYSQL = "MySQL";
private static final String H2 = "H2";
+ private static final String VERTICA = "Vertica";
/**
* Insert a table to SQL database, currently only used by H2, that can be
read by ThirdEye
@@ -595,6 +596,8 @@ public class SqlUtils {
return "UNIX_TIMESTAMP(STR_TO_DATE(CAST(" + timeColumn + " AS CHAR), '"
+ timeFormatToMySQLFormat(timeFormat) + "'))";
} else if (sourceName.equals(H2)){
return "TO_UNIXTIME(PARSEDATETIME(CAST(" + timeColumn + " AS VARCHAR),
'" + timeFormat + "'))";
+ } else if (sourceName.equals(VERTICA)) {
+ return "EXTRACT(EPOCH FROM to_timestamp(to_char(" + timeColumn + "), '"
+ timeFormat + "'))";
}
return "";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
