This is an automated email from the ASF dual-hosted git repository. vgarg pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new bc35a37 HIVE-21862: ORC ppd produces wrong result with timestamp (Vineet Garg, reviewed by Jesus Camacho Rodriguez) bc35a37 is described below commit bc35a371610092ba423b739ca171a2e56003da27 Author: Vineet Garg <vg...@apache.org> AuthorDate: Thu Jul 25 12:11:39 2019 -0700 HIVE-21862: ORC ppd produces wrong result with timestamp (Vineet Garg, reviewed by Jesus Camacho Rodriguez) --- .../hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 2 +- .../java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java | 2 +- .../apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java | 3 +++ .../hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java | 3 ++- .../change_allowincompatible_vectorization_false_date.q | 2 ++ .../change_allowincompatible_vectorization_false_date.q.out | 11 +++++++++++ 6 files changed, 20 insertions(+), 3 deletions(-) diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 6a00220..85a42f9 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -782,7 +782,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void> sargApp = new RecordReaderImpl.SargApplier(sarg, rowIndexStride, evolution, OrcFile.WriterVersion.from(OrcFile.WriterImplementation.ORC_JAVA, fileMetadata.getWriterVersionNum()), - false); + true); } boolean hasAnyData = false; // stripeRgs should have been initialized by this time with an empty array. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index ec4994d..f72705f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -2313,7 +2313,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, } PredicateLeaf leaf = predLeaves.get(pred); try { - truthValues[pred] = RecordReaderImpl.evaluatePredicate(stats, leaf, null); + truthValues[pred] = RecordReaderImpl.evaluatePredicate(stats, leaf, null, true); } catch (NoDynamicValuesException dve) { LOG.debug("Dynamic values are not available here {}", dve.getMessage()); boolean hasNulls = stats.hasNull() || leaf.getOperator() != Operator.NULL_SAFE_EQUALS; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java index 27fe828..bfabdce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java @@ -204,6 +204,9 @@ public class ConvertAstToSearchArg { } return fl.doubleValue(); case TIMESTAMP: + if (lit instanceof org.apache.hadoop.hive.common.type.Timestamp) { + return ((org.apache.hadoop.hive.common.type.Timestamp) lit).toSqlTimestamp(); + } return Timestamp.valueOf(lit.toString()); case DATE: return Date.valueOf(lit.toString()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java index 35053e3..3a40b31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java @@ -251,7 +251,8 @@ public class FixedBucketPruningOptimizer extends Transform { if (o instanceof java.sql.Date) { return Date.valueOf(o.toString()); } else if (o instanceof java.sql.Timestamp) { - return Timestamp.valueOf(o.toString()); + java.sql.Timestamp sqlTimestamp = (java.sql.Timestamp)o; + return Timestamp.ofEpochMilli(sqlTimestamp.getTime(), sqlTimestamp.getNanos()); } return o; } diff --git a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q index e750c59..02d923b 100644 --- a/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q +++ b/ql/src/test/queries/clientpositive/change_allowincompatible_vectorization_false_date.q @@ -27,3 +27,5 @@ insert into table change_allowincompatible_vectorization_false_date partition (s select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa'; +set hive.vectorized.execution.enabled=true; +select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa'; diff --git a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out index 4661a03..f3716b1 100644 --- a/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out +++ b/ql/src/test/results/clientpositive/llap/change_allowincompatible_vectorization_false_date.q.out @@ -84,3 +84,14 @@ POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa #### A masked pattern was here #### 2038-03-22 07:26:48 +PREHOOK: query: select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa' +PREHOOK: type: QUERY +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date +PREHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +#### A masked pattern was here #### +POSTHOOK: query: select ts from change_allowincompatible_vectorization_false_date where ts='2038-03-22 07:26:48.0' and s='aaa' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date +POSTHOOK: Input: default@change_allowincompatible_vectorization_false_date@s=aaa +#### A masked pattern was here #### +2038-03-22 07:26:48