This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3015ff2 HIVE-23529 : CTAS is broken for uniontype when
row_deserialize (Mustafa Iman via Ashutosh Chauhan)
3015ff2 is described below
commit 3015ff26ad0761c6332c6935ec83436d46bde561
Author: Mustafa Iman <[email protected]>
AuthorDate: Sun May 24 20:54:24 2020 -0700
HIVE-23529 : CTAS is broken for uniontype when row_deserialize (Mustafa
Iman via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <[email protected]>
---
data/files/data_with_union.txt | 1 +
.../test/resources/testconfiguration.properties | 1 +
.../clientpositive/row_deserialize_with_union.q | 17 ++++++++
.../llap/row_deserialize_with_union.q.out | 45 ++++++++++++++++++++++
.../objectinspector/ObjectInspectorUtils.java | 5 ++-
.../TestStandardObjectInspectors.java | 12 +++---
6 files changed, 73 insertions(+), 8 deletions(-)
diff --git a/data/files/data_with_union.txt b/data/files/data_with_union.txt
new file mode 100644
index 0000000..dd1e7f9
--- /dev/null
+++ b/data/files/data_with_union.txt
@@ -0,0 +1 @@
+0102111foo
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 7a91eef..e1ba435 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -745,6 +745,7 @@ spark.query.files=\
rcfile_bigdata.q,\
reduce_deduplicate_exclude_join.q,\
router_join_ppr.q,\
+ row_deserialize_with_union.q,\
runtime_skewjoin_mapjoin_spark.q,\
sample1.q,\
sample10.q,\
diff --git a/ql/src/test/queries/clientpositive/row_deserialize_with_union.q
b/ql/src/test/queries/clientpositive/row_deserialize_with_union.q
new file mode 100644
index 0000000..cb7d2aa
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/row_deserialize_with_union.q
@@ -0,0 +1,17 @@
+SET hive.vectorized.execution.enabled=true;
+set hive.vectorized.use.row.serde.deserialize=true;
+set hive.vectorized.use.vector.serde.deserialize=false;
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/data_with_union/;
+dfs -copyFromLocal ../../data/files/data_with_union.txt
${system:test.tmp.dir}/data_with_union/data_with_union.txt;
+
+CREATE EXTERNAL TABLE data_with_union(
+ unionfield uniontype<int, string>,
+ arrayfield array<int>,
+ mapfield map<int,int>,
+ structfield struct<`sf1`:int, `sf2`:string>)
+stored as textfile
+location '${system:test.tmp.dir}/data_with_union';
+
+create table data_with_union_2 as select * from data_with_union;
+
+select * from data_with_union_2;
\ No newline at end of file
diff --git
a/ql/src/test/results/clientpositive/llap/row_deserialize_with_union.q.out
b/ql/src/test/results/clientpositive/llap/row_deserialize_with_union.q.out
new file mode 100644
index 0000000..cc0ba5c
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/row_deserialize_with_union.q.out
@@ -0,0 +1,45 @@
+PREHOOK: query: CREATE EXTERNAL TABLE data_with_union(
+ unionfield uniontype<int, string>,
+ arrayfield array<int>,
+ mapfield map<int,int>,
+ structfield struct<`sf1`:int, `sf2`:string>)
+stored as textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@data_with_union
+POSTHOOK: query: CREATE EXTERNAL TABLE data_with_union(
+ unionfield uniontype<int, string>,
+ arrayfield array<int>,
+ mapfield map<int,int>,
+ structfield struct<`sf1`:int, `sf2`:string>)
+stored as textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@data_with_union
+PREHOOK: query: create table data_with_union_2 as select * from data_with_union
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@data_with_union
+PREHOOK: Output: database:default
+PREHOOK: Output: default@data_with_union_2
+POSTHOOK: query: create table data_with_union_2 as select * from
data_with_union
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@data_with_union
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@data_with_union_2
+POSTHOOK: Lineage: data_with_union_2.arrayfield SIMPLE
[(data_with_union)data_with_union.FieldSchema(name:arrayfield, type:array<int>,
comment:null), ]
+POSTHOOK: Lineage: data_with_union_2.mapfield SIMPLE
[(data_with_union)data_with_union.FieldSchema(name:mapfield, type:map<int,int>,
comment:null), ]
+POSTHOOK: Lineage: data_with_union_2.structfield SIMPLE
[(data_with_union)data_with_union.FieldSchema(name:structfield,
type:struct<sf1:int,sf2:string>, comment:null), ]
+POSTHOOK: Lineage: data_with_union_2.unionfield SIMPLE
[(data_with_union)data_with_union.FieldSchema(name:unionfield,
type:uniontype<int,string>, comment:null), ]
+PREHOOK: query: select * from data_with_union_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@data_with_union_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from data_with_union_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@data_with_union_2
+#### A masked pattern was here ####
+{0:1} [0,2] {1:1} {"sf1":1,"sf2":"foo"}
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
index ff69b47..bb3de30 100644
---
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
@@ -485,11 +485,12 @@ public final class ObjectInspectorUtils {
case UNION: {
UnionObjectInspector uoi = (UnionObjectInspector)oi;
List<ObjectInspector> objectInspectors = uoi.getObjectInspectors();
+ byte tag = uoi.getTag(o);
Object object = copyToStandardObject(
uoi.getField(o),
- objectInspectors.get(uoi.getTag(o)),
+ objectInspectors.get(tag),
objectInspectorOption);
- result = object;
+ result = new StandardUnionObjectInspector.StandardUnion(tag, object);
break;
}
default: {
diff --git
a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
index 04ffbb8..2709fca 100644
---
a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
+++
b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java
@@ -497,7 +497,7 @@ public class TestStandardObjectInspectors {
assertEquals(0, ObjectInspectorUtils.compare(union, uoi1,
new StandardUnion((byte) 0, 1), uoi2));
assertTrue(ObjectInspectorUtils.copyToStandardObject(
- union, uoi1).equals(1));
+ union, uoi1).equals(union));
union = new StandardUnion((byte) 1, "two");
assertEquals(1, uoi1.getTag(union));
@@ -506,7 +506,7 @@ public class TestStandardObjectInspectors {
assertEquals(0, ObjectInspectorUtils.compare(union, uoi1,
new StandardUnion((byte) 1, "two"), uoi2));
assertTrue(ObjectInspectorUtils.copyToStandardObject(
- union, uoi1).equals("two"));
+ union, uoi1).equals(union));
union = new StandardUnion((byte) 2, true);
assertEquals(2, uoi1.getTag(union));
@@ -515,7 +515,7 @@ public class TestStandardObjectInspectors {
assertEquals(0, ObjectInspectorUtils.compare(union, uoi1,
new StandardUnion((byte) 2, true), uoi2));
assertTrue(ObjectInspectorUtils.copyToStandardObject(
- union, uoi1).equals(true));
+ union, uoi1).equals(union));
ArrayList<Integer> iList = new ArrayList<Integer>();
iList.add(4);
@@ -527,7 +527,7 @@ public class TestStandardObjectInspectors {
assertEquals(0, ObjectInspectorUtils.compare(union, uoi1,
new StandardUnion((byte) 3, iList.clone()), uoi2));
assertTrue(ObjectInspectorUtils.copyToStandardObject(
- union, uoi1).equals(iList));
+ union, uoi1).equals(union));
HashMap<Integer, String> map = new HashMap<Integer, String>();
map.put(6, "six");
@@ -548,7 +548,7 @@ public class TestStandardObjectInspectors {
assertNotNull(th);
assertEquals("Compare on map type not supported!", th.getMessage());
assertTrue(ObjectInspectorUtils.copyToStandardObject(
- union, uoi1).equals(map));
+ union, uoi1).equals(union));
ArrayList<Object> struct = new ArrayList<Object>(2);
@@ -562,7 +562,7 @@ public class TestStandardObjectInspectors {
assertEquals(0, ObjectInspectorUtils.compare(union, uoi1,
new StandardUnion((byte) 5, struct.clone()), uoi2));
assertTrue(ObjectInspectorUtils.copyToStandardObject(
- union, uoi1).equals(struct));
+ union, uoi1).equals(union));
} catch (Throwable e) {
e.printStackTrace();
