Repository: hive Updated Branches: refs/heads/master 664de83d2 -> 53fda73e8
HIVE-19726: ORC date PPD is broken (Prasanth Jayachandran reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/53fda73e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/53fda73e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/53fda73e Branch: refs/heads/master Commit: 53fda73e82d958c39f02ceefed6e07dcd320ba7d Parents: 664de83 Author: Prasanth Jayachandran <[email protected]> Authored: Wed May 30 13:34:26 2018 -0700 Committer: Prasanth Jayachandran <[email protected]> Committed: Wed May 30 13:34:26 2018 -0700 ---------------------------------------------------------------------- .../test/queries/clientpositive/orc_ppd_date.q | 21 ++ .../clientpositive/llap/orc_ppd_date.q.out | 211 +++++++++++++++++++ .../hive/ql/io/sarg/SearchArgumentImpl.java | 6 - 3 files changed, 232 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/53fda73e/ql/src/test/queries/clientpositive/orc_ppd_date.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/orc_ppd_date.q b/ql/src/test/queries/clientpositive/orc_ppd_date.q index edc2a8f..d0351d7 100644 --- a/ql/src/test/queries/clientpositive/orc_ppd_date.q +++ b/ql/src/test/queries/clientpositive/orc_ppd_date.q @@ -102,3 +102,24 @@ select sum(hash(*)) from newtypesorc_n3 where da between '1970-02-18' and '1970- set hive.optimize.index.filter=true; select sum(hash(*)) from newtypesorc_n3 where da between '1970-02-18' and '1970-02-19'; + +create table test_lrl(c date) stored as orc; +insert into test_lrl values ('1900-01-01'); + +explain select count(*) from test_lrl where c='1900-01-01'; +set hive.optimize.index.filter=true; +select count(*) from test_lrl where c='1900-01-01'; +set hive.optimize.index.filter=false; +select count(*) from test_lrl where c='1900-01-01'; + +explain select * from test_lrl where c=cast('1900-01-01' as date); +set hive.optimize.index.filter=true; +select * from test_lrl where c=cast('1900-01-01' as date); +set hive.optimize.index.filter=false; +select * from test_lrl where c=cast('1900-01-01' as date); + +explain select count(*) from test_lrl where c LIKE '1900-01-01%'; +set hive.optimize.index.filter=true; +select count(*) from test_lrl where c LIKE '1900-01-01%'; +set hive.optimize.index.filter=false; +select count(*) from test_lrl where c LIKE '1900-01-01%'; http://git-wip-us.apache.org/repos/asf/hive/blob/53fda73e/ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out index 330d368..84cd53d 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out @@ -297,3 +297,214 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc_n3 #### A masked pattern was here #### NULL +PREHOOK: query: create table test_lrl(c date) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_lrl +POSTHOOK: query: create table test_lrl(c date) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_lrl +PREHOOK: query: insert into test_lrl values ('1900-01-01') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_lrl +POSTHOOK: query: insert into test_lrl values ('1900-01-01') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_lrl +POSTHOOK: Lineage: test_lrl.c SCRIPT [] +PREHOOK: query: explain select count(*) from test_lrl where c='1900-01-01' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from test_lrl where c='1900-01-01' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_lrl + filterExpr: (c = DATE'1900-01-01') (type: boolean) + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (c = DATE'1900-01-01') (type: boolean) + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from test_lrl where c='1900-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_lrl +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_lrl where c='1900-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_lrl +#### A masked pattern was here #### +1 +PREHOOK: query: select count(*) from test_lrl where c='1900-01-01' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_lrl +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_lrl where c='1900-01-01' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_lrl +#### A masked pattern was here #### +1 +PREHOOK: query: explain select * from test_lrl where c=cast('1900-01-01' as date) +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from test_lrl where c=cast('1900-01-01' as date) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: test_lrl + Filter Operator + predicate: (c = DATE'1900-01-01') (type: boolean) + Select Operator + expressions: DATE'1900-01-01' (type: date) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select * from test_lrl where c=cast('1900-01-01' as date) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_lrl +#### A masked pattern was here #### +POSTHOOK: query: select * from test_lrl where c=cast('1900-01-01' as date) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_lrl +#### A masked pattern was here #### +1900-01-01 +PREHOOK: query: select * from test_lrl where c=cast('1900-01-01' as date) +PREHOOK: type: QUERY +PREHOOK: Input: default@test_lrl +#### A masked pattern was here #### +POSTHOOK: query: select * from test_lrl where c=cast('1900-01-01' as date) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_lrl +#### A masked pattern was here #### +1900-01-01 +PREHOOK: query: explain select count(*) from test_lrl where c LIKE '1900-01-01%' +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from test_lrl where c LIKE '1900-01-01%' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_lrl + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (c like '1900-01-01%') (type: boolean) + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 56 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from test_lrl where c LIKE '1900-01-01%' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_lrl +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_lrl where c LIKE '1900-01-01%' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_lrl +#### A masked pattern was here #### +1 +PREHOOK: query: select count(*) from test_lrl where c LIKE '1900-01-01%' +PREHOOK: type: QUERY +PREHOOK: Input: default@test_lrl +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_lrl where c LIKE '1900-01-01%' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_lrl +#### A masked pattern was here #### +1 http://git-wip-us.apache.org/repos/asf/hive/blob/53fda73e/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java index e720ad1..8aa97d9 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java @@ -109,12 +109,6 @@ public final class SearchArgumentImpl implements SearchArgument { return ((LiteralDelegate) literal).getLiteral(); } - // To get around a kryo 2.22 bug while deserialize a Timestamp into Date - // (https://github.com/EsotericSoftware/kryo/issues/88) - // When we see a Date, convert back into Timestamp - if (literal instanceof java.util.Date) { - return new Timestamp(((java.util.Date)literal).getTime()); - } return literal; }
