This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9b73770 [TE] Dynamically change H2 demo database timestamps (#4157)
9b73770 is described below
commit 9b737701d7e82ca09ed3d85553229716aa42c3d3
Author: Dian Tang <[email protected]>
AuthorDate: Fri Apr 26 10:05:51 2019 -0700
[TE] Dynamically change H2 demo database timestamps (#4157)
---
.../config/data-sources/data-sources-config.yml | 2 ++
.../pinot/resultset/ThirdEyeResultSetUtils.java | 2 +-
.../datasource/sql/SqlResponseCacheLoader.java | 40 +++++++++++++++++++++-
.../pinot/thirdeye/datasource/sql/SqlUtils.java | 5 ++-
4 files changed, 44 insertions(+), 5 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/config/data-sources/data-sources-config.yml
b/thirdeye/thirdeye-pinot/config/data-sources/data-sources-config.yml
index 3492d61..1c8379d 100644
--- a/thirdeye/thirdeye-pinot/config/data-sources/data-sources-config.yml
+++ b/thirdeye/thirdeye-pinot/config/data-sources/data-sources-config.yml
@@ -10,6 +10,7 @@ dataSourceConfigs:
timeColumn: "date"
granularity: "1DAYS"
timeFormat: "yyyy-MM-dd"
+ timezone: "UTC"
metrics:
value: "SUM"
dataFile: "daily.csv"
@@ -17,6 +18,7 @@ dataSourceConfigs:
timeColumn: "datetime"
granularity: "1HOURS"
timeFormat: "yyyy-MM-dd HH:mm:ss"
+ timezone: "UTC"
metrics:
value: "SUM"
dataFile: "hourly.csv"
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
index e164489..e7d49cc 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
@@ -121,7 +121,7 @@ public class ThirdEyeResultSetUtils {
if (!isISOFormat) {
millis =
dataGranularity.toMillis(Double.valueOf(groupKeyVal).longValue());
} else {
- if (sourceName.equals(MYSQL) || sourceName.equals(H2)) {
+ if (sourceName.equals(MYSQL)) {
millis = DateTime.parse(groupKeyVal,
serverDataDateTimeFormatter).getMillis();
} else {
millis = DateTime.parse(groupKeyVal,
inputDataDateTimeFormatter).getMillis();
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
index 9f1953b..f6ea6b8 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
@@ -31,6 +31,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.dashboard.Utils;
@@ -64,6 +65,7 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
private static final String USER = "user";
private static final String DB = "db";
private static final String PASSWORD = "password";
+ private static final DateTime MIN_DATETIME = DateTime.parse("1970-01-01");
private static final int ABANDONED_TIMEOUT = 60000;
private Map<String, DataSource> prestoDBNameToDataSourceMap = new
HashMap<>();
@@ -164,6 +166,10 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
SqlUtils.createTable(h2DataSource, tableName,
dataset.getTimeColumn(), metrics, dataset.getDimensions());
SqlUtils.onBoardSqlDataset(dataset);
+ List<H2Row> h2Rows = new ArrayList<>();
+ DateTime maxDateTime = MIN_DATETIME;
+ DateTimeFormatter fmt =
DateTimeFormat.forPattern(dataset.getTimeFormat()).withZone(DateTimeZone.forID(dataset.getTimezone()));
+
if (dataset.getDataFile().length() > 0) {
String thirdEyeConfigDir = System.getProperty("dw.rootDir");
String fileURI = thirdEyeConfigDir + "/data/" +
dataset.getDataFile();
@@ -172,7 +178,20 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
String columnNames = scanner.nextLine();
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
- SqlUtils.insertCSVRow(h2DataSource, tableName, columnNames,
line);
+ String[] items = line.split(",");
+ DateTime dateTime = DateTime.parse(items[0], fmt);
+ if (dateTime.isAfter(maxDateTime)) {
+ maxDateTime = dateTime;
+ }
+ h2Rows.add(new H2Row(dateTime, items[1]));
+ }
+ // Calculate the day difference between today and the last day
of data point
+ int days = (int) ((DateTime.now().getMillis() -
maxDateTime.getMillis()) / TimeUnit.DAYS.toMillis(1));
+ for (H2Row h2Row: h2Rows) {
+ String[] items = new String[2];
+ items[0] = fmt.print(h2Row.getDateTime().plusDays(days));
+ items[1] = h2Row.getVal();
+ SqlUtils.insertCSVRow(h2DataSource, tableName, columnNames,
items);
}
}
}
@@ -327,4 +346,23 @@ public class SqlResponseCacheLoader extends
CacheLoader<SqlQuery, ThirdEyeResult
return h2DataSource;
}
}
+
+ // Container class for one row in H2 CSV
+ final static class H2Row {
+ DateTime dateTime;
+ String val;
+
+ H2Row(DateTime dateTime, String val) {
+ this.dateTime = dateTime;
+ this.val = val;
+ }
+
+ public DateTime getDateTime() {
+ return dateTime;
+ }
+
+ public String getVal() {
+ return val;
+ }
+ }
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
index 804d843..dedb9f7 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlUtils.java
@@ -127,12 +127,11 @@ public class SqlUtils {
*
* @param tableName table name
* @param columnNames column names in CSV, separated by ,
- * @param row one raw row in CSV
+ * @param items row items
* @throws SQLException
*/
- public static void insertCSVRow(DataSource ds, String tableName, String
columnNames, String row) throws SQLException {
+ public static void insertCSVRow(DataSource ds, String tableName, String
columnNames, String[] items) throws SQLException {
// Put quotes around values that contains spaces
- String[] items = row.split(",");
StringBuilder sb = new StringBuilder();
String prefix = "";
for (String item: items) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
