Repository: hive Updated Branches: refs/heads/branch-3 86361df3a -> 42ac182b6
HIVE-12192: Hive should carry out timestamp computations in UTC (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) (addendum 2) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42ac182b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42ac182b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42ac182b Branch: refs/heads/branch-3 Commit: 42ac182b636bd7a873d2f5a5309c054d558fc04c Parents: 86361df Author: Jesus Camacho Rodriguez <[email protected]> Authored: Tue Jun 26 11:26:43 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Wed Jun 27 00:45:27 2018 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../optimizer/FixedBucketPruningOptimizer.java | 20 ++++- ...allowincompatible_vectorization_false_date.q | 29 ++++++ ...wincompatible_vectorization_false_date.q.out | 94 ++++++++++++++++++++ 4 files changed, 142 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42ac182b/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 12dad9f..01ac0d3 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -480,6 +480,7 @@ minillaplocal.query.files=\ retry_failure_oom.q,\ runtime_stats_hs2.q,\ bucketsortoptimize_insert_2.q,\ + change_allowincompatible_vectorization_false_date.q,\ check_constraint.q,\ cbo_gby.q,\ cbo_join.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/42ac182b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java index 4f7d3c2..2debaca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java @@ -23,9 +23,12 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.Stack; +import java.util.stream.Collectors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.Timestamp; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; @@ -256,13 +259,26 @@ public class FixedBucketPruningOptimizer extends Transform { private boolean addLiteral(List<Object> literals, PredicateLeaf leaf) { switch (leaf.getOperator()) { case EQUALS: - return literals.add(leaf.getLiteral()); + return literals.add( + convertLiteral(leaf.getLiteral())); case IN: - return literals.addAll(leaf.getLiteralList()); + return literals.addAll( + leaf.getLiteralList().stream().map(l -> convertLiteral(l)).collect(Collectors.toList())); default: return false; } } + + private Object convertLiteral(Object o) { + // This is a bit hackish to fix mismatch between SARG and Hive types + // for Timestamp and Date. TODO: Move those types to storage-api. + if (o instanceof java.sql.Date) { + return Date.valueOf(o.toString()); + } else if (o instanceof java.sql.Timestamp) { + return Timestamp.valueOf(o.toString()); + } + return o; + } } public final class FixedBucketPruningOptimizerCtxt implements http://git-wip-us.apache.org/repos/asf/hive/blob/42ac182b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q new file mode 100644 index 0000000..e750c59 --- /dev/null +++ b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q @@ -0,0 +1,29 @@ +--! qt:dataset:alltypesorc + +set hive.vectorized.execution.enabled=false; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.tez.bucket.pruning=true; +set hive.optimize.index.filter=true; +set hive.metastore.disallow.incompatible.col.type.changes=false; + +create table change_allowincompatible_vectorization_false_date (ts date) partitioned by (s string) clustered by (ts) into 32 buckets stored as orc tblproperties ('transactional'='true'); + +alter table change_allowincompatible_vectorization_false_date add partition(s='aaa'); + +alter table change_allowincompatible_vectorization_false_date add partition(s='bbb'); + +insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') select ctimestamp1 from alltypesorc where ctimestamp1 > '2000-01-01' limit 50; + +insert into table change_allowincompatible_vectorization_false_date partition (s='bbb') select ctimestamp1 from alltypesorc where ctimestamp1 < '2000-01-01' limit 50; + +select count(*) from change_allowincompatible_vectorization_false_date; + +alter table change_allowincompatible_vectorization_false_date change column ts ts timestamp; + +select count(*) from change_allowincompatible_vectorization_false_date; + +insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') values ('2038-03-22 07:26:48.0'); + +select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa'; + http://git-wip-us.apache.org/repos/asf/hive/blob/42ac182b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out new file mode 100644 index 0000000..1bba3f3 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out @@ -0,0 +1,94 @@ +PREHOOK: query: create table change_allowincompatible_vectorization_false_date (ts date) partitioned by (s string) clustered by (ts) into 32 buckets stored as orc tblproperties ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date +POSTHOOK: query: create table change_allowincompatible_vectorization_false_date (ts date) partitioned by (s string) clustered by (ts) into 32 buckets stored as orc tblproperties ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date +PREHOOK: query: alter table change_allowincompatible_vectorization_false_date add partition(s='aaa') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date +POSTHOOK: query: alter table change_allowincompatible_vectorization_false_date add partition(s='aaa') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=aaa +PREHOOK: query: alter table change_allowincompatible_vectorization_false_date add partition(s='bbb') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date +POSTHOOK: query: alter table change_allowincompatible_vectorization_false_date add partition(s='bbb') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=bbb +PREHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') select ctimestamp1 from alltypesorc where ctimestamp1 > '2000-01-01' limit 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=aaa +POSTHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') select ctimestamp1 from alltypesorc where ctimestamp1 > '2000-01-01' limit 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=aaa +POSTHOOK: Lineage: change_allowincompatible_vectorization_false_date PARTITION(s=aaa).ts EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +PREHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='bbb') select ctimestamp1 from alltypesorc where ctimestamp1 < '2000-01-01' limit 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=bbb +POSTHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='bbb') select ctimestamp1 from alltypesorc where ctimestamp1 < '2000-01-01' limit 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=bbb +POSTHOOK: Lineage: change_allowincompatible_vectorization_false_date PARTITION(s=bbb).ts EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +PREHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date +PREHOOK: type: QUERY +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb +#### A masked pattern was here #### +50 +PREHOOK: query: alter table change_allowincompatible_vectorization_false_date change column ts ts timestamp +PREHOOK: type: ALTERTABLE_RENAMECOL +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date +POSTHOOK: query: alter table change_allowincompatible_vectorization_false_date change column ts ts timestamp +POSTHOOK: type: ALTERTABLE_RENAMECOL +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date +PREHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date +PREHOOK: type: QUERY +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from change_allowincompatible_vectorization_false_date +POSTHOOK: type: QUERY +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=bbb +#### A masked pattern was here #### +50 +PREHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') values ('2038-03-22 07:26:48.0') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=aaa +POSTHOOK: query: insert into table change_allowincompatible_vectorization_false_date partition (s='aaa') values ('2038-03-22 07:26:48.0') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@change_allowincompatible_vectorization_false_date@s=aaa +POSTHOOK: Lineage: change_allowincompatible_vectorization_false_date PARTITION(s=aaa).ts SCRIPT [] +PREHOOK: query: select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa' +PREHOOK: type: QUERY +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +#### A masked pattern was here #### +POSTHOOK: query: select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +#### A masked pattern was here #### +2038-03-22 07:26:48
