This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 71de311 [TE] Fix MySQL and H2 timestamp automatic timezone conversion
issues (#4113)
71de311 is described below
commit 71de311dfb329f94335cb683a2881377c43c140c
Author: Dian Tang <[email protected]>
AuthorDate: Tue Apr 16 12:20:03 2019 -0700
[TE] Fix MySQL and H2 timestamp automatic timezone conversion issues (#4113)
---
.../app/pods/self-serve/import-sql-metric/controller.js | 4 ++--
.../datasource/pinot/PinotThirdEyeDataSource.java | 3 ++-
.../pinot/resultset/ThirdEyeResultSetUtils.java | 17 ++++++++++++++---
.../thirdeye/datasource/sql/SqlThirdEyeDataSource.java | 10 ++++------
4 files changed, 22 insertions(+), 12 deletions(-)
diff --git
a/thirdeye/thirdeye-frontend/app/pods/self-serve/import-sql-metric/controller.js
b/thirdeye/thirdeye-frontend/app/pods/self-serve/import-sql-metric/controller.js
index e5ebb1c..0bbd760 100644
---
a/thirdeye/thirdeye-frontend/app/pods/self-serve/import-sql-metric/controller.js
+++
b/thirdeye/thirdeye-frontend/app/pods/self-serve/import-sql-metric/controller.js
@@ -14,7 +14,7 @@ export default Controller.extend({
timeColumn: '',
selectedTimeFormat: '',
selectedTimeGranularity: '',
- selectedTimezone: '',
+ selectedTimezone: 'UTC',
response: '',
init() {
@@ -22,7 +22,7 @@ export default Controller.extend({
this.aggregationOptions = ['SUM', 'AVG', 'COUNT', 'MAX' ];
this.timeFormatOptions = ['EPOCH', 'yyyyMMdd', 'yyyy-MM-dd',
'yyyy-MM-dd-HH', 'yyyy-MM-dd HH:mm:ss', 'yyyyMMddHHmmss'];
this.timeGranularityOptions = ['1MILLISECONDS', '1SECONDS', '1MINUTES',
'1HOURS', '1DAYS', '1WEEKs', '1MONTHS', '1YEARS'];
- this.timezoneOptions = ["Pacific/Midway", "US/Hawaii", "US/Alaska",
"US/Pacific", "US/Arizona", "US/Mountain", "US/Central",
+ this.timezoneOptions = ["UTC", "Pacific/Midway", "US/Hawaii", "US/Alaska",
"US/Pacific", "US/Arizona", "US/Mountain", "US/Central",
"US/Eastern", "America/Caracas", "America/Manaus", "America/Santiago",
"Canada/Newfoundland", "Brazil/East", "America/Buenos_Aires",
"America/Godthab", "America/Montevideo", "Atlantic/South_Georgia",
"Atlantic/Azores", "Atlantic/Cape_Verde", "Africa/Casablanca",
"Europe/London", "Europe/Berlin", "Europe/Belgrade", "Europe/Brussels",
"Europe/Warsaw", "Africa/Algiers", "Asia/Amman", "Europe/Athens",
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
index c91a21f..6d71bea 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
@@ -198,7 +198,8 @@ public class PinotThirdEyeDataSource implements
ThirdEyeDataSource {
metricFunctionToResultSetList.put(metricFunction,
resultSetGroup.getResultSets());
}
- List<String[]> resultRows =
ThirdEyeResultSetUtils.parseResultSets(request, metricFunctionToResultSetList,
"Pinot");
+ List<String[]> resultRows =
ThirdEyeResultSetUtils.parseResultSets(request, metricFunctionToResultSetList,
+ "Pinot");
return new RelationalThirdEyeResponse(request, resultRows, timeSpec);
} catch (Exception e) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
index 393d45b..eb18e6b 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resultset/ThirdEyeResultSetUtils.java
@@ -45,9 +45,12 @@ import org.slf4j.LoggerFactory;
public class ThirdEyeResultSetUtils {
private static final Logger LOG =
LoggerFactory.getLogger(ThirdEyeResultSetUtils.class);
+ private static final String MYSQL = "MySQL";
+ private static final String H2 = "H2";
public static List<String[]> parseResultSets(ThirdEyeRequest request,
- Map<MetricFunction, List<ThirdEyeResultSet>>
metricFunctionToResultSetList, String sourceName) throws ExecutionException {
+ Map<MetricFunction, List<ThirdEyeResultSet>>
metricFunctionToResultSetList,
+ String sourceName) throws ExecutionException {
int numGroupByKeys = 0;
boolean hasGroupBy = false;
@@ -85,11 +88,15 @@ public class ThirdEyeResultSetUtils {
dataGranularity = dataTimeSpec.getDataGranularity();
boolean isISOFormat = false;
DateTimeFormatter inputDataDateTimeFormatter = null;
+ DateTimeFormatter serverDataDateTimeFormatter = null;
String timeFormat = dataTimeSpec.getFormat();
if (timeFormat != null &&
!timeFormat.equals(TimeSpec.SINCE_EPOCH_FORMAT)) {
isISOFormat = true;
inputDataDateTimeFormatter =
DateTimeFormat.forPattern(timeFormat).withZone(dateTimeZone);
}
+ if (sourceName.equals(MYSQL) || sourceName.equals(H2) ) {
+ serverDataDateTimeFormatter =
DateTimeFormat.forPattern(timeFormat).withZone(DateTimeZone.getDefault());
+ }
List<ThirdEyeResultSet> resultSets = entry.getValue();
for (int i = 0; i < resultSets.size(); i++) {
@@ -116,7 +123,11 @@ public class ThirdEyeResultSetUtils {
if (!isISOFormat) {
millis =
dataGranularity.toMillis(Double.valueOf(groupKeyVal).longValue());
} else {
- millis = DateTime.parse(groupKeyVal,
inputDataDateTimeFormatter).getMillis();
+ if (sourceName.equals(MYSQL) || sourceName.equals(H2)) {
+ millis = DateTime.parse(groupKeyVal,
serverDataDateTimeFormatter).getMillis();
+ } else {
+ millis = DateTime.parse(groupKeyVal,
inputDataDateTimeFormatter).getMillis();
+ }
}
if (millis < startTime) {
LOG.error("Data point earlier than requested start time {}:
{}", new Date(startTime), new Date(millis));
@@ -181,7 +192,7 @@ public class ThirdEyeResultSetUtils {
return Math.max(aggregate, value);
} else if (aggFunction.equals(MetricAggFunction.COUNT) &&
sourceName.equals("Pinot")) {
return aggregate + 1;
- } else if (aggFunction.equals(MetricAggFunction.COUNT) &&
sourceName.equals("SQL")) {
+ } else if (aggFunction.equals(MetricAggFunction.COUNT)) { // For all other
COUNT cases
return aggregate + value;
} else {
throw new IllegalArgumentException(String.format("Unknown aggregation
function '%s'", aggFunction));
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
index 3fd1fed..eb72276 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
@@ -45,9 +45,6 @@ public class SqlThirdEyeDataSource implements
ThirdEyeDataSource {
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE =
ThirdEyeCacheRegistry.getInstance();
protected LoadingCache<RelationalQuery, ThirdEyeResultSetGroup>
sqlResponseCache;
private SqlResponseCacheLoader sqlResponseCacheLoader;
- public static final String DATA_SOURCE_NAME =
SqlThirdEyeDataSource.class.getSimpleName();
-
-
public SqlThirdEyeDataSource(Map<String, Object> properties) throws
Exception {
sqlResponseCacheLoader = new SqlResponseCacheLoader(properties);
@@ -62,8 +59,8 @@ public class SqlThirdEyeDataSource implements
ThirdEyeDataSource {
@Override
public ThirdEyeResponse execute(ThirdEyeRequest request) throws Exception {
LinkedHashMap<MetricFunction, List<ThirdEyeResultSet>>
metricFunctionToResultSetList = new LinkedHashMap<>();
-
TimeSpec timeSpec = null;
+ String sourceName = "";
try {
for (MetricFunction metricFunction : request.getMetricFunctions()) {
String dataset = metricFunction.getDataset();
@@ -75,7 +72,7 @@ public class SqlThirdEyeDataSource implements
ThirdEyeDataSource {
}
String[] tableComponents = dataset.split("\\.");
- String sourceName = tableComponents[0];
+ sourceName = tableComponents[0];
String dbName = tableComponents[1];
String sqlQuery = SqlUtils.getSql(request, metricFunction,
request.getFilterSet(), dataTimeSpec, sourceName);
@@ -85,7 +82,8 @@ public class SqlThirdEyeDataSource implements
ThirdEyeDataSource {
metricFunctionToResultSetList.put(metricFunction,
thirdEyeResultSetGroup.getResultSets());
}
- List<String[]> resultRows =
ThirdEyeResultSetUtils.parseResultSets(request, metricFunctionToResultSetList,
"SQL");
+ List<String[]> resultRows =
ThirdEyeResultSetUtils.parseResultSets(request, metricFunctionToResultSetList,
+ sourceName);
return new RelationalThirdEyeResponse(request, resultRows, timeSpec);
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
