HIVE-12287: Lineage for lateral view shows wrong dependencies (Jimmy, reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5fdeed6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5fdeed6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5fdeed6 Branch: refs/heads/master-fixed Commit: d5fdeed64cf36844bc3e26dff798a31e5f2643c4 Parents: 02629e9 Author: Jimmy Xiang <[email protected]> Authored: Wed Oct 28 14:47:45 2015 -0700 Committer: Jimmy Xiang <[email protected]> Committed: Tue Nov 3 09:35:54 2015 -0800 ---------------------------------------------------------------------- .../ql/optimizer/lineage/OpProcFactory.java | 25 +++++--------- ql/src/test/queries/clientpositive/lineage2.q | 18 ++++++++++ .../clientpositive/infer_bucket_sort.q.out | 4 +-- .../test/results/clientpositive/lineage2.q.out | 30 ++++++++++++++++ .../clientpositive/load_dyn_part15.q.out | 6 ++-- .../multi_insert_lateral_view.q.out | 36 ++++++++++---------- .../clientpositive/spark/load_dyn_part15.q.out | 6 ++-- .../spark/multi_insert_lateral_view.q.out | 36 ++++++++++---------- .../spark/union_lateralview.q.out | 4 +-- .../tez/vectorized_distinct_gby.q.out | 4 +-- .../clientpositive/union_lateralview.q.out | 4 +-- .../vectorized_distinct_gby.q.out | 4 +-- 12 files changed, 109 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java index 5c5d0d6..d95b45b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java @@ -324,26 +324,19 @@ public class OpProcFactory { } // Dirty hack!! - // For the select path the columns are the ones at the end of the + // For the select path the columns are the ones at the beginning of the // current operators schema and for the udtf path the columns are - // at the beginning of the operator schema. + // at the end of the operator schema. ArrayList<ColumnInfo> out_cols = op.getSchema().getSignature(); int out_cols_size = out_cols.size(); int cols_size = cols.size(); - if (isUdtfPath) { - int cnt = 0; - while (cnt < cols_size) { - lCtx.getIndex().mergeDependency(op, out_cols.get(cnt), - lCtx.getIndex().getDependency(inpOp, cols.get(cnt))); - cnt++; - } - } - else { - int cnt = cols_size - 1; - while (cnt >= 0) { - lCtx.getIndex().mergeDependency(op, out_cols.get(out_cols_size - cols_size + cnt), - lCtx.getIndex().getDependency(inpOp, cols.get(cnt))); - cnt--; + int outColOffset = isUdtfPath ? out_cols_size - cols_size : 0; + for (int cnt = 0; cnt < cols_size; cnt++) { + ColumnInfo outCol = out_cols.get(outColOffset + cnt); + if (!outCol.isHiddenVirtualCol()) { + ColumnInfo col = cols.get(cnt); + lCtx.getIndex().mergeDependency(op, outCol, + lCtx.getIndex().getDependency(inpOp, col)); } } return null; http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/queries/clientpositive/lineage2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/lineage2.q b/ql/src/test/queries/clientpositive/lineage2.q index 6bcd1d7..d3fe64a 100644 --- a/ql/src/test/queries/clientpositive/lineage2.q +++ b/ql/src/test/queries/clientpositive/lineage2.q @@ -114,3 +114,21 @@ concat(substr(src1.key,1,1),sum(substr(src1.value,5))) from src1 GROUP BY substr(src1.key,1,1); +drop table if exists relations; +create table relations (identity char(32), type string, + ep1_src_type string, ep1_type string, ep2_src_type string, ep2_type string, + ep1_ids array<string>, ep2_ids array<string>); + +drop table if exists rels_exploded; +create table rels_exploded (identity char(32), type string, + ep1_src_type string, ep1_type string, ep2_src_type string, ep2_type string, + ep1_id char(32), ep2_id char(32)); + +select identity, ep1_id from relations + lateral view explode(ep1_ids) nav_rel as ep1_id; + +insert into rels_exploded select identity, type, + ep1_src_type, ep1_type, ep2_src_type, ep2_type, ep1_id, ep2_id +from relations lateral view explode(ep1_ids) rel1 as ep1_id + lateral view explode (ep2_ids) rel2 as ep2_id; + http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/infer_bucket_sort.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/infer_bucket_sort.q.out b/ql/src/test/results/clientpositive/infer_bucket_sort.q.out index 5d69e2f..1e584e0 100644 --- a/ql/src/test/results/clientpositive/infer_bucket_sort.q.out +++ b/ql/src/test/results/clientpositive/infer_bucket_sort.q.out @@ -978,8 +978,8 @@ SELECT key, value FROM (SELECT key FROM src group by key) a lateral view explode POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_table@part=1 -POSTHOOK: Lineage: test_table PARTITION(part=1).key SCRIPT [] -POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_table PARTITION(part=1).value SCRIPT [] PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1') PREHOOK: type: DESCTABLE PREHOOK: Input: default@test_table http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/lineage2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/lineage2.q.out b/ql/src/test/results/clientpositive/lineage2.q.out index 4184a83..0185d43 100644 --- a/ql/src/test/results/clientpositive/lineage2.q.out +++ b/ql/src/test/results/clientpositive/lineage2.q.out @@ -675,3 +675,33 @@ PREHOOK: Input: default@src1 4 2 4807.0 6 1 666.0 9 1 998.0 +PREHOOK: query: drop table if exists relations +PREHOOK: type: DROPTABLE +PREHOOK: query: create table relations (identity char(32), type string, + ep1_src_type string, ep1_type string, ep2_src_type string, ep2_type string, + ep1_ids array<string>, ep2_ids array<string>) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@relations +PREHOOK: query: drop table if exists rels_exploded +PREHOOK: type: DROPTABLE +PREHOOK: query: create table rels_exploded (identity char(32), type string, + ep1_src_type string, ep1_type string, ep2_src_type string, ep2_type string, + ep1_id char(32), ep2_id char(32)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@rels_exploded +PREHOOK: query: select identity, ep1_id from relations + lateral view explode(ep1_ids) nav_rel as ep1_id +PREHOOK: type: QUERY +PREHOOK: Input: default@relations +#### A masked pattern was here #### +{"version":"1.0","engine":"mr","hash":"bb30b94d13d0b35802db85b4e33230b3","queryText":"select identity, ep1_id from relations\n lateral view explode(ep1_ids) nav_rel as ep1_id","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"nav_rel._col11","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"identity"},{"id":1,"vertexType":"COLUMN","vertexId":"ep1_id"},{"id":2,"vertexType":"COLUMN","vertexId":"default.relations.identity"},{"id":3,"vertexType":"COLUMN","vertexId":"default.relations.ep1_ids"}]} +PREHOOK: query: insert into rels_exploded select identity, type, + ep1_src_type, ep1_type, ep2_src_type, ep2_type, ep1_id, ep2_id +from relations lateral view explode(ep1_ids) rel1 as ep1_id + lateral view explode (ep2_ids) rel2 as ep2_id +PREHOOK: type: QUERY +PREHOOK: Input: default@relations +PREHOOK: Output: default@rels_exploded +{"version":"1.0","engine":"mr","hash":"e76d2efade744d1d5cf74fda064ba6c6","queryText":"insert into rels_exploded select identity, type,\n ep1_src_type, ep1_type, ep2_src_type, ep2_type, ep1_id, ep2_id\nfrom relations lateral view explode(ep1_ids) rel1 as ep1_id\n lateral view explode (ep2_ids) rel2 as ep2_id","edges":[{"sources":[8],"targets":[0],"edgeType":"PROJECTION"},{"sources":[9],"targets":[1],"edgeType":"PROJECTION"},{"sources":[10],"targets":[2],"edgeType":"PROJECTION"},{"sources":[11],"targets":[3],"edgeType":"PROJECTION"},{"sources":[12],"targets":[4],"edgeType":"PROJECTION"},{"sources":[13],"targets":[5],"edgeType":"PROJECTION"},{"sources":[14],"targets":[6],"expression":"CAST( rel1._col11 AS CHAR(32)","edgeType":"PROJECTION"},{"sources":[15],"targets":[7],"expression":"CAST( rel2._col12 AS CHAR(32)","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.rels_exploded.identity"},{"id":1,"vertexType":"COLUMN","vertexId":"default.rels_explo ded.type"},{"id":2,"vertexType":"COLUMN","vertexId":"default.rels_exploded.ep1_src_type"},{"id":3,"vertexType":"COLUMN","vertexId":"default.rels_exploded.ep1_type"},{"id":4,"vertexType":"COLUMN","vertexId":"default.rels_exploded.ep2_src_type"},{"id":5,"vertexType":"COLUMN","vertexId":"default.rels_exploded.ep2_type"},{"id":6,"vertexType":"COLUMN","vertexId":"default.rels_exploded.ep1_id"},{"id":7,"vertexType":"COLUMN","vertexId":"default.rels_exploded.ep2_id"},{"id":8,"vertexType":"COLUMN","vertexId":"default.relations.identity"},{"id":9,"vertexType":"COLUMN","vertexId":"default.relations.type"},{"id":10,"vertexType":"COLUMN","vertexId":"default.relations.ep1_src_type"},{"id":11,"vertexType":"COLUMN","vertexId":"default.relations.ep1_type"},{"id":12,"vertexType":"COLUMN","vertexId":"default.relations.ep2_src_type"},{"id":13,"vertexType":"COLUMN","vertexId":"default.relations.ep2_type"},{"id":14,"vertexType":"COLUMN","vertexId":"default.relations.ep1_ids"},{"id":15,"vertexType":"COLU MN","vertexId":"default.relations.ep2_ids"}]} http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/load_dyn_part15.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/load_dyn_part15.q.out b/ql/src/test/results/clientpositive/load_dyn_part15.q.out index b105497..4c9cd6d 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part15.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part15.q.out @@ -26,9 +26,9 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@load_dyn_part15_test@part_key=%7B2 POSTHOOK: Output: default@load_dyn_part15_test@part_key=1 POSTHOOK: Output: default@load_dyn_part15_test@part_key=3%5D -POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=1).key SCRIPT [] -POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=3]).key SCRIPT [] -POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key={2).key SCRIPT [] +POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=3]).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key={2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] PREHOOK: query: show partitions load_dyn_part15_test PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@load_dyn_part15_test http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out index 4723153..07a0f3e 100644 --- a/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out +++ b/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out @@ -285,10 +285,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 @@ -548,10 +548,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 @@ -800,8 +800,8 @@ POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] @@ -1133,10 +1133,10 @@ POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 -POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 @@ -1512,10 +1512,10 @@ POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 POSTHOOK: Output: default@src_lv4 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.null, ] POSTHOOK: Lineage: src_lv4.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/spark/load_dyn_part15.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part15.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part15.q.out index b105497..4c9cd6d 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part15.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part15.q.out @@ -26,9 +26,9 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@load_dyn_part15_test@part_key=%7B2 POSTHOOK: Output: default@load_dyn_part15_test@part_key=1 POSTHOOK: Output: default@load_dyn_part15_test@part_key=3%5D -POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=1).key SCRIPT [] -POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=3]).key SCRIPT [] -POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key={2).key SCRIPT [] +POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key=3]).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: load_dyn_part15_test PARTITION(part_key={2).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] PREHOOK: query: show partitions load_dyn_part15_test PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@load_dyn_part15_test http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out index 6aec979..d000ad7 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out @@ -200,10 +200,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.value SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 @@ -464,10 +464,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 PREHOOK: type: QUERY PREHOOK: Input: default@src_lv1 @@ -717,8 +717,8 @@ POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] @@ -1045,10 +1045,10 @@ POSTHOOK: Input: default@src_10 POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 -POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: select * from src_lv1 @@ -1419,10 +1419,10 @@ POSTHOOK: Output: default@src_lv1 POSTHOOK: Output: default@src_lv2 POSTHOOK: Output: default@src_lv3 POSTHOOK: Output: default@src_lv4 -POSTHOOK: Lineage: src_lv1.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv1.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] -POSTHOOK: Lineage: src_lv2.key SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: src_lv2.value EXPRESSION [(src_10)src_10.FieldSchema(name:ROW__ID, type:struct<transactionId:bigint,bucketId:int,rowId:bigint>, comment:), ] +POSTHOOK: Lineage: src_lv1.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv1.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.key SIMPLE [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: src_lv2.value SCRIPT [(src_10)src_10.FieldSchema(name:key, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] POSTHOOK: Lineage: src_lv3.value EXPRESSION [(src_10)src_10.null, ] POSTHOOK: Lineage: src_lv4.key SIMPLE [(src_10)src_10.FieldSchema(name:value, type:string, comment:null), ] http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/spark/union_lateralview.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/union_lateralview.q.out b/ql/src/test/results/clientpositive/spark/union_lateralview.q.out index eaa312a..c223ac6 100644 --- a/ql/src/test/results/clientpositive/spark/union_lateralview.q.out +++ b/ql/src/test/results/clientpositive/spark/union_lateralview.q.out @@ -242,9 +242,9 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Output: default@test_union_lateral_view -POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION [] +POSTHOOK: Lineage: test_union_lateral_view.arr_ele SCRIPT [] POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@test_union_lateral_view http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out index da5ce13..db604f8 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_distinct_gby.q.out @@ -14,8 +14,8 @@ POSTHOOK: query: insert into table dtest select c,b from (select array(300,300,3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dtest -POSTHOOK: Lineage: dtest.a SIMPLE [] -POSTHOOK: Lineage: dtest.b EXPRESSION [] +POSTHOOK: Lineage: dtest.a SCRIPT [] +POSTHOOK: Lineage: dtest.b SIMPLE [] PREHOOK: query: explain select sum(distinct a), count(distinct a) from dtest PREHOOK: type: QUERY POSTHOOK: query: explain select sum(distinct a), count(distinct a) from dtest http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/union_lateralview.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/union_lateralview.q.out b/ql/src/test/results/clientpositive/union_lateralview.q.out index 734c1f4..7b89e50 100644 --- a/ql/src/test/results/clientpositive/union_lateralview.q.out +++ b/ql/src/test/results/clientpositive/union_lateralview.q.out @@ -236,9 +236,9 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Output: default@test_union_lateral_view -POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION [] +POSTHOOK: Lineage: test_union_lateral_view.arr_ele SCRIPT [] POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), (srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@test_union_lateral_view http://git-wip-us.apache.org/repos/asf/hive/blob/d5fdeed6/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out index b5c667f..9bc25ba 100644 --- a/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out +++ b/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out @@ -14,8 +14,8 @@ POSTHOOK: query: insert into table dtest select c,b from (select array(300,300,3 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dtest -POSTHOOK: Lineage: dtest.a SIMPLE [] -POSTHOOK: Lineage: dtest.b EXPRESSION [] +POSTHOOK: Lineage: dtest.a SCRIPT [] +POSTHOOK: Lineage: dtest.b SIMPLE [] PREHOOK: query: explain select sum(distinct a), count(distinct a) from dtest PREHOOK: type: QUERY POSTHOOK: query: explain select sum(distinct a), count(distinct a) from dtest
