Author: namit
Date: Thu Mar 28 05:52:42 2013
New Revision: 1461959
URL: http://svn.apache.org/r1461959
Log:
HIVE-4149 wrong results big outer joins with array of ints
(Navis via namit)
Added:
hive/trunk/data/files/tiny_a.txt
hive/trunk/data/files/tiny_b.txt
hive/trunk/ql/src/test/queries/clientpositive/join_array.q
hive/trunk/ql/src/test/results/clientpositive/join_array.q.out
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Added: hive/trunk/data/files/tiny_a.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/data/files/tiny_a.txt?rev=1461959&view=auto
==============================================================================
--- hive/trunk/data/files/tiny_a.txt (added)
+++ hive/trunk/data/files/tiny_a.txt Thu Mar 28 05:52:42 2013
@@ -0,0 +1 @@
+103200920268924913312
Added: hive/trunk/data/files/tiny_b.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/data/files/tiny_b.txt?rev=1461959&view=auto
==============================================================================
--- hive/trunk/data/files/tiny_b.txt (added)
+++ hive/trunk/data/files/tiny_b.txt Thu Mar 28 05:52:42 2013
@@ -0,0 +1,2 @@
+10320092002467760023374853555556606672767778808187889090919092971001031041071081081091101131131131131131131141161161161171161171171171151151171171171211201311311311251251241241281281311311321331341341341342626\N\N\N\N116
+1032009202689249102591061311421592443203984174335536167108269179711046105110931112114212151220122612321267136415491646194821702272232524332534285229252992311932073279332334123637364536343450347336383688373637583812386238733868388341184134412741704216\N\N\N\N3139
Added: hive/trunk/ql/src/test/queries/clientpositive/join_array.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/join_array.q?rev=1461959&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/join_array.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/join_array.q Thu Mar 28
05:52:42 2013
@@ -0,0 +1,10 @@
+create table tinyA(a bigint, b bigint) stored as textfile;
+create table tinyB(a bigint, bList array<int>) stored as textfile;
+
+load data local inpath '../data/files/tiny_a.txt' into table tinyA;
+load data local inpath '../data/files/tiny_b.txt' into table tinyB;
+
+select * from tinyA;
+select * from tinyB;
+
+select tinyB.a, tinyB.bList from tinyB full outer join tinyA on tinyB.a =
tinyA.a;
Added: hive/trunk/ql/src/test/results/clientpositive/join_array.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join_array.q.out?rev=1461959&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join_array.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/join_array.q.out Thu Mar 28
05:52:42 2013
@@ -0,0 +1,53 @@
+PREHOOK: query: create table tinyA(a bigint, b bigint) stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tinyA(a bigint, b bigint) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tinyA
+PREHOOK: query: create table tinyB(a bigint, bList array<int>) stored as
textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tinyB(a bigint, bList array<int>) stored as
textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tinyB
+PREHOOK: query: load data local inpath '../data/files/tiny_a.txt' into table
tinyA
+PREHOOK: type: LOAD
+PREHOOK: Output: default@tinya
+POSTHOOK: query: load data local inpath '../data/files/tiny_a.txt' into table
tinyA
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@tinya
+PREHOOK: query: load data local inpath '../data/files/tiny_b.txt' into table
tinyB
+PREHOOK: type: LOAD
+PREHOOK: Output: default@tinyb
+POSTHOOK: query: load data local inpath '../data/files/tiny_b.txt' into table
tinyB
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@tinyb
+PREHOOK: query: select * from tinyA
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tinya
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tinyA
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tinya
+#### A masked pattern was here ####
+10320092026892491 3312
+PREHOOK: query: select * from tinyB
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tinyb
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tinyB
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tinyb
+#### A masked pattern was here ####
+10320092002467760
[0,23,37,48,53,55,55,56,60,66,72,76,77,78,80,81,87,88,90,90,91,90,92,97,100,103,104,107,108,108,109,110,113,113,113,113,113,113,114,116,116,116,117,116,117,117,117,115,115,117,117,117,121,120,131,131,131,125,125,124,124,128,128,131,131,132,133,134,134,134,134,26,26,null,null,null,null,116]
+10320092026892491
[0,2,59,106,131,142,159,244,320,398,417,433,553,616,710,826,917,971,1046,1051,1093,1112,1142,1215,1220,1226,1232,1267,1364,1549,1646,1948,2170,2272,2325,2433,2534,2852,2925,2992,3119,3207,3279,3323,3412,3637,3645,3634,3450,3473,3638,3688,3736,3758,3812,3862,3873,3868,3883,4118,4134,4127,4170,4216,null,null,null,null,3139]
+PREHOOK: query: select tinyB.a, tinyB.bList from tinyB full outer join tinyA
on tinyB.a = tinyA.a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tinya
+PREHOOK: Input: default@tinyb
+#### A masked pattern was here ####
+POSTHOOK: query: select tinyB.a, tinyB.bList from tinyB full outer join tinyA
on tinyB.a = tinyA.a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tinya
+POSTHOOK: Input: default@tinyb
+#### A masked pattern was here ####
+10320092002467760
[0,23,37,48,53,55,55,56,60,66,72,76,77,78,80,81,87,88,90,90,91,90,92,97,100,103,104,107,108,108,109,110,113,113,113,113,113,113,114,116,116,116,117,116,117,117,117,115,115,117,117,117,121,120,131,131,131,125,125,124,124,128,128,131,131,132,133,134,134,134,134,26,26,null,null,null,null,116]
+10320092026892491
[0,2,59,106,131,142,159,244,320,398,417,433,553,616,710,826,917,971,1046,1051,1093,1112,1142,1215,1220,1226,1232,1267,1364,1549,1646,1948,2170,2272,2325,2433,2534,2852,2925,2992,3119,3207,3279,3323,3412,3637,3645,3634,3450,3473,3638,3688,3736,3758,3812,3862,3873,3868,3883,4118,4134,4127,4170,4216,null,null,null,null,3139]
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java?rev=1461959&r1=1461958&r2=1461959&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
Thu Mar 28 05:52:42 2013
@@ -187,7 +187,7 @@ public class LazyArray extends LazyNonPr
&& 0 == LazyUtils
.compare(bytes.getData(), startPosition[index], elementLength,
nullSequence.getBytes(), 0, nullSequence.getLength())) {
- return null;
+ return arrayElements[index] = null;
}
arrayElements[index] = LazyFactory
.createLazyObject(oi.getListElementObjectInspector());
Modified:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1461959&r1=1461958&r2=1461959&view=diff
==============================================================================
---
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
(original)
+++
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Thu Mar 28 05:52:42 2013
@@ -95,6 +95,20 @@ public class TestLazyArrayMapStruct exte
assertNull((b.getListElementObject(5)));
assertEquals(5, b.getList().size());
+ // -- HIVE-4149
+ b = (LazyArray) LazyFactory.createLazyObject(oi);
+
+ data = new byte[] {'a', '\t', '\\', 'N'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+ assertEquals(new Text("a"), ((LazyString)
b.getListElementObject(0)).getWritableObject());
+ assertNull(b.getListElementObject(1));
+
+ data = new byte[] {'\\', 'N', '\t', 'a'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+ assertNull(b.getListElementObject(0));
+ assertNull(b.getListElementObject(0)); // twice (returns not cleaned
cache)
+ assertEquals(new Text("a"), ((LazyString)
b.getListElementObject(1)).getWritableObject());
+
} catch (Throwable e) {
e.printStackTrace();
throw e;
