This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 77e394d [TE] datasource - aggregate to one minute level for epoch
timestamp (#3530)
77e394d is described below
commit 77e394dfdc39a15916f957ce4124eec41113218e
Author: Xiaohui Sun <[email protected]>
AuthorDate: Tue Nov 20 14:02:54 2018 -0800
[TE] datasource - aggregate to one minute level for epoch timestamp (#3530)
Pinot uses UTC time which causes metrics shown inconsistently in ThirdEye
which is PST/PDT by default. This aggregates to one minute level instead of the
actual aggregation granularity.
---
.../com/linkedin/thirdeye/datasource/pinot/PqlUtils.java | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
index 39c1cf1..11281a0 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
@@ -348,11 +348,10 @@ public class PqlUtils {
return AND.join(components);
}
- private static String convertEpochToAggGranularity(String timeColumnName,
TimeSpec timeSpec, TimeGranularity aggregationGranularity) {
- String groupByTimeColumnName =
String.format("dateTimeConvert(%s,'%d:%s:%s','%d:%s:%s','%d:%s')",
timeColumnName,
+ private static String convertEpochToMinuteAggGranularity(String
timeColumnName, TimeSpec timeSpec) {
+ String groupByTimeColumnName =
String.format("dateTimeConvert(%s,'%d:%s:%s','%d:%s:%s','1:MINUTES')",
timeColumnName,
timeSpec.getDataGranularity().getSize(),
timeSpec.getDataGranularity().getUnit(), timeSpec.getFormat(),
- timeSpec.getDataGranularity().getSize(),
timeSpec.getDataGranularity().getUnit(), timeSpec.getFormat(),
- aggregationGranularity.getSize(), aggregationGranularity.getUnit());
+ timeSpec.getDataGranularity().getSize(),
timeSpec.getDataGranularity().getUnit(), timeSpec.getFormat());
return groupByTimeColumnName;
}
@@ -361,11 +360,11 @@ public class PqlUtils {
String timeColumnName = timeSpec.getColumnName();
List<String> groups = new LinkedList<>();
if (aggregationGranularity != null && !groups.contains(timeColumnName)) {
- // Convert the time column to aggregation granularity if it is epoch.
- // E.g.,
dateTimeConvert(timestampInEpoch,'1:MILLISECONDS:EPOCH','1:MILLISECONDS:EPOCH','15:MINUTES')
+ // Convert the time column to 1 minute granularity if it is epoch.
+ // E.g.,
dateTimeConvert(timestampInEpoch,'1:MILLISECONDS:EPOCH','1:MILLISECONDS:EPOCH','1:MINUTES')
if
(timeSpec.getFormat().equals(TimeGranularitySpec.TimeFormat.EPOCH.toString())
&& !timeSpec.getDataGranularity().equals(aggregationGranularity)) {
- String groupByTimeColumnName =
convertEpochToAggGranularity(timeColumnName, timeSpec, aggregationGranularity);
+ String groupByTimeColumnName =
convertEpochToMinuteAggGranularity(timeColumnName, timeSpec);
groups.add(groupByTimeColumnName);
} else {
groups.add(timeColumnName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
