HIVE-20105: Druid-Hive: tpcds query on timestamp throws java.lang.IllegalArgumentException: Cannot create timestamp, parsing error (Nishant Bangarwa, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9dea3b0b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9dea3b0b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9dea3b0b Branch: refs/heads/master Commit: 9dea3b0bf25a9889be447454557f0524e2f972b5 Parents: 8e73f4b Author: Nishant Bangarwa <[email protected]> Authored: Mon Jul 23 22:34:34 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Sat Jul 28 08:17:49 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/druid/serde/DruidSerDe.java | 69 ++-- .../queries/clientpositive/druid_timestamptz.q | 40 +++ .../druid/druid_timestamptz.q.out | 360 ++++++++++++++++++- 3 files changed, 430 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9dea3b0b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java ---------------------------------------------------------------------- diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index 015924d..fcde538 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -393,41 +393,22 @@ import static org.joda.time.format.ISODateTimeFormat.dateOptionalTimeParser; continue; } switch (types[i].getPrimitiveCategory()) { - case TIMESTAMP: - final TimestampWritableV2 timestampWritable; - if (value instanceof Number) { - timestampWritable = new TimestampWritableV2( - Timestamp.ofEpochMilli(((Number) value).longValue())); - } else { - timestampWritable = new TimestampWritableV2( - Timestamp.valueOf((String) value)); - } - output.add(timestampWritable); - break; - case TIMESTAMPLOCALTZ: - final long numberOfMillis; - if (value instanceof Number) { - numberOfMillis = ((Number) value).longValue(); - } else { - // it is an extraction fn need to be parsed - numberOfMillis = dateOptionalTimeParser().parseDateTime((String) value).getMillis(); - } - output.add(new TimestampLocalTZWritable(new TimestampTZ(ZonedDateTime - .ofInstant(Instant.ofEpochMilli(numberOfMillis), - ((TimestampLocalTZTypeInfo) types[i]).timeZone() - )))); - break; - case DATE: - final DateWritableV2 dateWritable; - if (value instanceof Number) { - dateWritable = new DateWritableV2( - Date.ofEpochMilli((((Number) value).longValue()))); - } else { - // it is an extraction fn need to be parsed - dateWritable = new DateWritableV2( - Date.ofEpochMilli(dateOptionalTimeParser().parseDateTime((String) value).getMillis())); - } - output.add(dateWritable); + case TIMESTAMP: + output.add(new TimestampWritableV2( + Timestamp.ofEpochMilli(deserializeToMillis(value)))); + break; + case TIMESTAMPLOCALTZ: + output.add(new TimestampLocalTZWritable( + new TimestampTZ( + ZonedDateTime + .ofInstant( + Instant.ofEpochMilli(deserializeToMillis(value)), + ((TimestampLocalTZTypeInfo) types[i]).timeZone() + )))); + break; + case DATE: + output.add(new DateWritableV2( + Date.ofEpochMilli(deserializeToMillis(value)))); break; case BYTE: output.add(new ByteWritable(((Number) value).byteValue())); @@ -478,6 +459,24 @@ import static org.joda.time.format.ISODateTimeFormat.dateOptionalTimeParser; return output; } + private long deserializeToMillis(Object value) + { + long numberOfMillis; + if (value instanceof Number) { + numberOfMillis = ((Number) value).longValue(); + } else { + // it is an extraction fn need to be parsed + try { + numberOfMillis = dateOptionalTimeParser().parseDateTime((String) value).getMillis(); + } catch (IllegalArgumentException e) { + // we may not be able to parse the date if it already comes in Hive format, + // we retry and otherwise fail + numberOfMillis = Timestamp.valueOf((String) value).toEpochMilli(); + } + } + return numberOfMillis; + } + @Override public ObjectInspector getObjectInspector() { return inspector; } http://git-wip-us.apache.org/repos/asf/hive/blob/9dea3b0b/ql/src/test/queries/clientpositive/druid_timestamptz.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz.q b/ql/src/test/queries/clientpositive/druid_timestamptz.q index 605d240..b21ca9d 100644 --- a/ql/src/test/queries/clientpositive/druid_timestamptz.q +++ b/ql/src/test/queries/clientpositive/druid_timestamptz.q @@ -11,6 +11,14 @@ TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); insert into table tstz1_n0 values(cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone), 'Bill', 10); +-- Create table with druid time column as timestamp +create table tstz1_n1(`__time` timestamp, n string, v integer) +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); + +insert into table tstz1_n1 +values(cast('2016-01-03 12:26:34' as timestamp), 'Bill', 10); + EXPLAIN select `__time` from tstz1_n0; select `__time` from tstz1_n0; @@ -26,7 +34,22 @@ SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n0; EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; +EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; +SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; + +EXPLAIN select `__time` from tstz1_n1; +select `__time` from tstz1_n1; + +EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; +SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; + +EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; +SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; + +EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; +SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; +-- Change timezone to UTC and test again set time zone UTC; EXPLAIN select `__time` from tstz1_n0; select `__time` from tstz1_n0; @@ -50,3 +73,20 @@ SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n0; EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n0; + +EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; +SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time`; + +EXPLAIN select `__time` from tstz1_n1; +select `__time` from tstz1_n1; + +EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; +SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1; + +EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; +SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1; + +EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; +SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time`; + + http://git-wip-us.apache.org/repos/asf/hive/blob/9dea3b0b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out index f37e80f..003b4d6 100644 --- a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out +++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out @@ -24,6 +24,28 @@ values(cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local ti POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@tstz1_n0 +PREHOOK: query: create table tstz1_n1(`__time` timestamp, n string, v integer) +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tstz1_n1 +POSTHOOK: query: create table tstz1_n1(`__time` timestamp, n string, v integer) +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tstz1_n1 +PREHOOK: query: insert into table tstz1_n1 +values(cast('2016-01-03 12:26:34' as timestamp), 'Bill', 10) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tstz1_n1 +POSTHOOK: query: insert into table tstz1_n1 +values(cast('2016-01-03 12:26:34' as timestamp), 'Bill', 10) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tstz1_n1 PREHOOK: query: EXPLAIN select `__time` from tstz1_n0 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select `__time` from tstz1_n0 @@ -74,7 +96,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),null,'UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} druid.query.type scan Select Operator expressions: vc (type: timestamp) @@ -107,7 +129,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["2016-01-03T20:26:34.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),null,'UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["2016-01-03T20:26:34.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} druid.query.type scan Select Operator expressions: vc (type: timestamp) @@ -140,7 +162,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes int - druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),null,'UTC'),'HOUR','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'HOUR','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} druid.query.type scan Select Operator expressions: vc (type: int) @@ -173,7 +195,7 @@ STAGE PLANS: properties: druid.fieldNames vc druid.fieldTypes timestamp - druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),null,'UTC'),'PT1H','','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n0","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(timestamp_parse(timestamp_format(\"__time\",'yyyy-MM-dd\\u0027T\\u0027HH:mm:ss.SSS\\u0027Z\\u0027','US/Pacific'),'','UTC'),'PT1H','','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} druid.query.type scan Select Operator expressions: vc (type: timestamp) @@ -189,6 +211,171 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tstz1_n0 POSTHOOK: Output: hdfs://### HDFS PATH ### 2016-01-03 12:00:00 +PREHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n0 + properties: + druid.fieldNames extract,$f1 + druid.fieldTypes timestamp with local time zone,int + druid.query.json {"queryType":"groupBy","dataSource":"default.tstz1_n0","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longMax","name":"$f1","fieldName":"v"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.type groupBy + Select Operator + expressions: extract (type: timestamp with local time zone), $f1 (type: int) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n0 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n0 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:26:34.0 US/Pacific 10 +PREHOOK: query: EXPLAIN select `__time` from tstz1_n1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN select `__time` from tstz1_n1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames vc + druid.fieldTypes timestamp + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: timestamp) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select `__time` from tstz1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select `__time` from tstz1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:26:34 +PREHOOK: query: EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames vc + druid.fieldTypes int + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'HOUR','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +12 +PREHOOK: query: EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames vc + druid.fieldTypes timestamp + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(\"__time\",'PT1H','','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: timestamp) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:00:00 +PREHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames extract,$f1 + druid.fieldTypes timestamp,int + druid.query.json {"queryType":"groupBy","dataSource":"default.tstz1_n1","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longMax","name":"$f1","fieldName":"v"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.type groupBy + Select Operator + expressions: extract (type: timestamp), $f1 (type: int) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:26:34 10 PREHOOK: query: EXPLAIN select `__time` from tstz1_n0 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select `__time` from tstz1_n0 @@ -420,3 +607,168 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@tstz1_n0 POSTHOOK: Output: hdfs://### HDFS PATH ### 2016-01-03 20:00:00 +PREHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n0 + properties: + druid.fieldNames extract,$f1 + druid.fieldTypes timestamp with local time zone,int + druid.query.json {"queryType":"groupBy","dataSource":"default.tstz1_n0","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longMax","name":"$f1","fieldName":"v"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.type groupBy + Select Operator + expressions: extract (type: timestamp with local time zone), $f1 (type: int) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n0 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT `__time`, max(v) FROM tstz1_n0 GROUP BY `__time` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n0 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 20:26:34.0 UTC 10 +PREHOOK: query: EXPLAIN select `__time` from tstz1_n1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN select `__time` from tstz1_n1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames vc + druid.fieldTypes timestamp + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"\"__time\"","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: timestamp) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select `__time` from tstz1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select `__time` from tstz1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:26:34 +PREHOOK: query: EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames vc + druid.fieldTypes int + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(\"__time\",'HOUR','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT EXTRACT(HOUR FROM CAST(`__time` AS timestamp)) FROM tstz1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +12 +PREHOOK: query: EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames vc + druid.fieldTypes timestamp + druid.query.json {"queryType":"scan","dataSource":"default.tstz1_n1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_floor(\"__time\",'PT1H','','UTC')","outputType":"LONG"}],"columns":["vc"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: timestamp) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT FLOOR(CAST(`__time` AS timestamp) to HOUR) FROM tstz1_n1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:00:00 +PREHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: tstz1_n1 + properties: + druid.fieldNames extract,$f1 + druid.fieldTypes timestamp,int + druid.query.json {"queryType":"groupBy","dataSource":"default.tstz1_n1","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longMax","name":"$f1","fieldName":"v"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]} + druid.query.type groupBy + Select Operator + expressions: extract (type: timestamp), $f1 (type: int) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1_n1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT `__time`, max(v) FROM tstz1_n1 GROUP BY `__time` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1_n1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +2016-01-03 12:26:34 10
