Author: gunther
Date: Mon Oct 27 18:07:57 2014
New Revision: 1634630
URL: http://svn.apache.org/r1634630
Log:
HIVE-8566: Vectorized queries output wrong timestamps (Matt McCline reviewed by
Prasanth J)
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/vector_data_types.q
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_data_types.q.out
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vector_data_types.q.out
Modified:
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1634630&r1=1634629&r2=1634630&view=diff
==============================================================================
---
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
(original)
+++
hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
Mon Oct 27 18:07:57 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
@@ -1074,41 +1075,20 @@ class RecordReaderImpl implements Record
result = (LongColumnVector) previousVector;
}
- // Read present/isNull stream
- super.nextVector(result, batchSize);
-
- data.nextVector(result, batchSize);
- nanoVector.isNull = result.isNull;
- nanos.nextVector(nanoVector, batchSize);
-
- if(result.isRepeating && nanoVector.isRepeating) {
- batchSize = 1;
- }
-
- // Non repeating values preset in the vector. Iterate thru the vector
and populate the time
+ result.reset();
+ Object obj = null;
for (int i = 0; i < batchSize; i++) {
- if (!result.isNull[i]) {
- long ms = (result.vector[result.isRepeating ? 0 : i] +
WriterImpl.BASE_TIMESTAMP)
- * WriterImpl.MILLIS_PER_SECOND;
- long ns = parseNanos(nanoVector.vector[nanoVector.isRepeating ? 0 :
i]);
- // the rounding error exists because java always rounds up when
dividing integers
- // -42001/1000 = -42; and -42001 % 1000 = -1 (+ 1000)
- // to get the correct value we need
- // (-42 - 1)*1000 + 999 = -42001
- // (42)*1000 + 1 = 42001
- if(ms < 0 && ns != 0) {
- ms -= 1000;
- }
- // Convert millis into nanos and add the nano vector value to it
- result.vector[i] = (ms * 1000000) + ns;
+ obj = next(obj);
+ if (obj == null) {
+ result.noNulls = false;
+ result.isNull[i] = true;
+ } else {
+ TimestampWritable writable = (TimestampWritable) obj;
+ Timestamp timestamp = writable.getTimestamp();
+ result.vector[i] = TimestampUtils.getTimeNanoSec(timestamp);
}
}
- if(!(result.isRepeating && nanoVector.isRepeating)) {
- // both have to repeat for the result to be repeating
- result.isRepeating = false;
- }
-
return result;
}
Modified:
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/vector_data_types.q
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/vector_data_types.q?rev=1634630&r1=1634629&r2=1634630&view=diff
==============================================================================
---
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/vector_data_types.q
(original)
+++
hive/branches/branch-0.14/ql/src/test/queries/clientpositive/vector_data_types.q
Mon Oct 27 18:07:57 2014
@@ -40,8 +40,14 @@ EXPLAIN SELECT t, si, i, b, f, d, bo, s,
SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT
20;
+SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q;
+
SET hive.vectorized.execution.enabled=true;
EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY
i LIMIT 20;
-SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT
20;
\ No newline at end of file
+SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i LIMIT
20;
+
+SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q;
\ No newline at end of file
Modified:
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_data_types.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_data_types.q.out?rev=1634630&r1=1634629&r2=1634630&view=diff
==============================================================================
---
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_data_types.q.out
(original)
+++
hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vector_data_types.q.out
Mon Oct 27 18:07:57 2014
@@ -177,6 +177,17 @@ NULL 409 65536 4294967490 46.97 25.92
fa
120 331 65539 4294967324 88.02 40.94 true holly nixon
2013-03-01 09:11:58.703262 96.64 yard duty
71 298 65540 4294967510 91.63 9.6 false gabriella
zipper 2013-03-01 09:11:58.70314 68.14 geology
114 385 65541 4294967458 73.48 34.97 true oscar quirinius
2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM
over1korc ORDER BY i LIMIT 20
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM
over1korc ORDER BY i LIMIT 20
@@ -239,23 +250,34 @@ POSTHOOK: query: SELECT t, si, i, b, f,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
-108 301 65536 4294967357 90.05 17.59 true ethan johnson
1860-11-12 20:05:55.011470936 75.70 undecided
-118 497 65536 4294967381 50.32 12.72 false david nixon
1860-11-12 20:05:55.011484936 83.48 values clariffication
-18 280 65536 4294967320 32.92 45.94 false holly white
1860-11-12 20:05:55.011285936 58.86 topology
-69 489 65536 4294967404 33.52 17.99 false oscar ichabod
1860-11-12 20:05:55.011446936 32.68 topology
-27 405 65536 4294967508 82.24 29.41 true oscar ovid
1860-11-12 20:05:55.011365936 16.85 biology
-42 495 65536 4294967431 43.57 46.81 false tom johnson
1860-11-12 20:05:55.011444936 62.25 american history
-NULL 409 65536 4294967490 46.97 25.92 false fred miller
1860-11-12 20:05:55.011315936 33.45 history
-59 431 65537 4294967326 11.34 2.9 true oscar robinson
1860-11-12 20:05:55.011360936 81.04 zync studies
-63 458 65537 4294967463 21.94 49.71 true fred van buren
1860-11-12 20:05:55.011477936 99.34 wind surfing
-121 355 65537 4294967437 85.9 10.99 true calvin brown
1860-11-12 20:05:55.011453936 4.49 yard duty
-72 267 65537 4294967460 55.2 42.89 false oscar carson
1860-11-12 20:05:55.011352936 51.91 topology
-22 264 65537 4294967419 8.07 10.71 false david xylophone
1860-11-12 20:05:55.011335936 11.87 undecided
-101 283 65538 4294967527 84.03 27.29 true yuri brown
1860-11-12 20:05:55.011505936 44.24 biology
-109 376 65538 4294967453 59.61 35.62 true priscilla
xylophone 1860-11-12 20:05:55.011485936 9.53 study skills
-83 353 65538 4294967339 0.08 23.91 true holly thompson
1860-11-12 20:05:55.011356936 30.27 quiet hour
-29 331 65539 4294967420 73.18 28.96 true ethan brown
1860-11-12 20:05:55.011293936 58.85 zync studies
-70 430 65539 4294967534 18.89 43.84 true tom carson
1860-11-12 20:05:55.011381936 21.93 joggying
-120 331 65539 4294967324 88.02 40.94 true holly nixon
1860-11-12 20:05:55.011461936 96.64 yard duty
-71 298 65540 4294967510 91.63 9.6 false gabriella
zipper 1860-11-12 20:05:55.011339936 68.14 geology
-114 385 65541 4294967458 73.48 34.97 true oscar quirinius
1860-11-12 20:05:55.011342936 72.33 xylophone band
+108 301 65536 4294967357 90.05 17.59 true ethan johnson
2013-03-01 09:11:58.703271 75.70 undecided
+118 497 65536 4294967381 50.32 12.72 false david nixon
2013-03-01 09:11:58.703285 83.48 values clariffication
+18 280 65536 4294967320 32.92 45.94 false holly white
2013-03-01 09:11:58.703086 58.86 topology
+69 489 65536 4294967404 33.52 17.99 false oscar ichabod
2013-03-01 09:11:58.703247 32.68 topology
+27 405 65536 4294967508 82.24 29.41 true oscar ovid
2013-03-01 09:11:58.703166 16.85 biology
+42 495 65536 4294967431 43.57 46.81 false tom johnson
2013-03-01 09:11:58.703245 62.25 american history
+NULL 409 65536 4294967490 46.97 25.92 false fred miller
2013-03-01 09:11:58.703116 33.45 history
+59 431 65537 4294967326 11.34 2.9 true oscar robinson
2013-03-01 09:11:58.703161 81.04 zync studies
+63 458 65537 4294967463 21.94 49.71 true fred van buren
2013-03-01 09:11:58.703278 99.34 wind surfing
+121 355 65537 4294967437 85.9 10.99 true calvin brown
2013-03-01 09:11:58.703254 4.49 yard duty
+72 267 65537 4294967460 55.2 42.89 false oscar carson
2013-03-01 09:11:58.703153 51.91 topology
+22 264 65537 4294967419 8.07 10.71 false david xylophone
2013-03-01 09:11:58.703136 11.87 undecided
+101 283 65538 4294967527 84.03 27.29 true yuri brown
2013-03-01 09:11:58.703306 44.24 biology
+109 376 65538 4294967453 59.61 35.62 true priscilla
xylophone 2013-03-01 09:11:58.703286 9.53 study skills
+83 353 65538 4294967339 0.08 23.91 true holly thompson
2013-03-01 09:11:58.703157 30.27 quiet hour
+29 331 65539 4294967420 73.18 28.96 true ethan brown
2013-03-01 09:11:58.703094 58.85 zync studies
+70 430 65539 4294967534 18.89 43.84 true tom carson
2013-03-01 09:11:58.703182 21.93 joggying
+120 331 65539 4294967324 88.02 40.94 true holly nixon
2013-03-01 09:11:58.703262 96.64 yard duty
+71 298 65540 4294967510 91.63 9.6 false gabriella
zipper 2013-03-01 09:11:58.70314 68.14 geology
+114 385 65541 4294967458 73.48 34.97 true oscar quirinius
2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
Modified:
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vector_data_types.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/vector_data_types.q.out?rev=1634630&r1=1634629&r2=1634630&view=diff
==============================================================================
---
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vector_data_types.q.out
(original)
+++
hive/branches/branch-0.14/ql/src/test/results/clientpositive/vector_data_types.q.out
Mon Oct 27 18:07:57 2014
@@ -171,6 +171,17 @@ NULL 409 65536 4294967490 46.97 25.92
fa
120 331 65539 4294967324 88.02 40.94 true holly nixon
2013-03-01 09:11:58.703262 96.64 yard duty
71 298 65540 4294967510 91.63 9.6 false gabriella
zipper 2013-03-01 09:11:58.70314 68.14 geology
114 385 65541 4294967458 73.48 34.97 true oscar quirinius
2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
PREHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM
over1korc ORDER BY i LIMIT 20
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN select t, si, i, b, f, d, bo, s, ts, dec, bin FROM
over1korc ORDER BY i LIMIT 20
@@ -226,23 +237,34 @@ POSTHOOK: query: SELECT t, si, i, b, f,
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over1korc
#### A masked pattern was here ####
-108 301 65536 4294967357 90.05 17.59 true ethan johnson
1860-11-12 20:05:55.011470936 75.70 undecided
-118 497 65536 4294967381 50.32 12.72 false david nixon
1860-11-12 20:05:55.011484936 83.48 values clariffication
-18 280 65536 4294967320 32.92 45.94 false holly white
1860-11-12 20:05:55.011285936 58.86 topology
-69 489 65536 4294967404 33.52 17.99 false oscar ichabod
1860-11-12 20:05:55.011446936 32.68 topology
-27 405 65536 4294967508 82.24 29.41 true oscar ovid
1860-11-12 20:05:55.011365936 16.85 biology
-42 495 65536 4294967431 43.57 46.81 false tom johnson
1860-11-12 20:05:55.011444936 62.25 american history
-NULL 409 65536 4294967490 46.97 25.92 false fred miller
1860-11-12 20:05:55.011315936 33.45 history
-59 431 65537 4294967326 11.34 2.9 true oscar robinson
1860-11-12 20:05:55.011360936 81.04 zync studies
-63 458 65537 4294967463 21.94 49.71 true fred van buren
1860-11-12 20:05:55.011477936 99.34 wind surfing
-121 355 65537 4294967437 85.9 10.99 true calvin brown
1860-11-12 20:05:55.011453936 4.49 yard duty
-72 267 65537 4294967460 55.2 42.89 false oscar carson
1860-11-12 20:05:55.011352936 51.91 topology
-22 264 65537 4294967419 8.07 10.71 false david xylophone
1860-11-12 20:05:55.011335936 11.87 undecided
-101 283 65538 4294967527 84.03 27.29 true yuri brown
1860-11-12 20:05:55.011505936 44.24 biology
-109 376 65538 4294967453 59.61 35.62 true priscilla
xylophone 1860-11-12 20:05:55.011485936 9.53 study skills
-83 353 65538 4294967339 0.08 23.91 true holly thompson
1860-11-12 20:05:55.011356936 30.27 quiet hour
-29 331 65539 4294967420 73.18 28.96 true ethan brown
1860-11-12 20:05:55.011293936 58.85 zync studies
-70 430 65539 4294967534 18.89 43.84 true tom carson
1860-11-12 20:05:55.011381936 21.93 joggying
-120 331 65539 4294967324 88.02 40.94 true holly nixon
1860-11-12 20:05:55.011461936 96.64 yard duty
-71 298 65540 4294967510 91.63 9.6 false gabriella
zipper 1860-11-12 20:05:55.011339936 68.14 geology
-114 385 65541 4294967458 73.48 34.97 true oscar quirinius
1860-11-12 20:05:55.011342936 72.33 xylophone band
+108 301 65536 4294967357 90.05 17.59 true ethan johnson
2013-03-01 09:11:58.703271 75.70 undecided
+118 497 65536 4294967381 50.32 12.72 false david nixon
2013-03-01 09:11:58.703285 83.48 values clariffication
+18 280 65536 4294967320 32.92 45.94 false holly white
2013-03-01 09:11:58.703086 58.86 topology
+69 489 65536 4294967404 33.52 17.99 false oscar ichabod
2013-03-01 09:11:58.703247 32.68 topology
+27 405 65536 4294967508 82.24 29.41 true oscar ovid
2013-03-01 09:11:58.703166 16.85 biology
+42 495 65536 4294967431 43.57 46.81 false tom johnson
2013-03-01 09:11:58.703245 62.25 american history
+NULL 409 65536 4294967490 46.97 25.92 false fred miller
2013-03-01 09:11:58.703116 33.45 history
+59 431 65537 4294967326 11.34 2.9 true oscar robinson
2013-03-01 09:11:58.703161 81.04 zync studies
+63 458 65537 4294967463 21.94 49.71 true fred van buren
2013-03-01 09:11:58.703278 99.34 wind surfing
+121 355 65537 4294967437 85.9 10.99 true calvin brown
2013-03-01 09:11:58.703254 4.49 yard duty
+72 267 65537 4294967460 55.2 42.89 false oscar carson
2013-03-01 09:11:58.703153 51.91 topology
+22 264 65537 4294967419 8.07 10.71 false david xylophone
2013-03-01 09:11:58.703136 11.87 undecided
+101 283 65538 4294967527 84.03 27.29 true yuri brown
2013-03-01 09:11:58.703306 44.24 biology
+109 376 65538 4294967453 59.61 35.62 true priscilla
xylophone 2013-03-01 09:11:58.703286 9.53 study skills
+83 353 65538 4294967339 0.08 23.91 true holly thompson
2013-03-01 09:11:58.703157 30.27 quiet hour
+29 331 65539 4294967420 73.18 28.96 true ethan brown
2013-03-01 09:11:58.703094 58.85 zync studies
+70 430 65539 4294967534 18.89 43.84 true tom carson
2013-03-01 09:11:58.703182 21.93 joggying
+120 331 65539 4294967324 88.02 40.94 true holly nixon
2013-03-01 09:11:58.703262 96.64 yard duty
+71 298 65540 4294967510 91.63 9.6 false gabriella
zipper 2013-03-01 09:11:58.70314 68.14 geology
+114 385 65541 4294967458 73.48 34.97 true oscar quirinius
2013-03-01 09:11:58.703143 72.33 xylophone band
+PREHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(HASH(*))
+FROM (SELECT t, si, i, b, f, d, bo, s, ts, dec, bin FROM over1korc ORDER BY i)
as q
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1korc
+#### A masked pattern was here ####
+-17045922556
