HIVE-14861: Support precedence for set operator using parentheses (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2435e702 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2435e702 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2435e702 Branch: refs/heads/master Commit: 2435e702dab5bd50c6605af15e229f678b8f545d Parents: 15039c0 Author: Pengcheng Xiong <[email protected]> Authored: Fri Oct 7 13:53:36 2016 -0700 Committer: Pengcheng Xiong <[email protected]> Committed: Fri Oct 7 13:53:36 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/FromClauseParser.g | 2 +- .../apache/hadoop/hive/ql/parse/HiveParser.g | 104 +-- .../hadoop/hive/ql/parse/IdentifiersParser.g | 2 +- .../ptf_negative_AmbiguousWindowDefn.q | 6 +- .../clientnegative/ptf_negative_NoWindowDefn.q | 4 +- .../clientnegative/windowing_after_orderby.q | 7 + .../test/queries/clientpositive/union_paren.q | 54 ++ .../create_or_replace_view6.q.out | 2 +- .../results/clientnegative/unionClusterBy.q.out | 2 +- .../clientnegative/unionDistributeBy.q.out | 2 +- .../results/clientnegative/unionLimit.q.out | 2 +- .../results/clientnegative/unionOrderBy.q.out | 2 +- .../results/clientnegative/unionSortBy.q.out | 2 +- .../windowing_after_orderby.q.out | 9 + .../clientpositive/cbo_rp_lineage2.q.out | 2 +- .../results/clientpositive/complex_alias.q.out | 4 +- .../clientpositive/constant_prop_1.q.out | 12 +- .../results/clientpositive/input_part7.q.out | 4 +- .../clientpositive/llap/explainuser_2.q.out | 442 ++++++------ .../clientpositive/optimize_nullscan.q.out | 8 +- .../clientpositive/tez/explainanalyze_2.q.out | 704 ++++++++++--------- .../results/clientpositive/union_offcbo.q.out | 112 +-- .../results/clientpositive/union_paren.q.out | 260 +++++++ .../test/results/clientpositive/union_ppr.q.out | 4 +- 24 files changed, 1083 insertions(+), 669 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index ad9abce..bf35d60 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -222,7 +222,7 @@ subQuerySource @init { gParent.pushMsg("subquery source", state); } @after { gParent.popMsg(state); } : - LPAREN queryStatementExpression[false] RPAREN KW_AS? identifier -> ^(TOK_SUBQUERY queryStatementExpression identifier) + LPAREN queryStatementExpression RPAREN KW_AS? identifier -> ^(TOK_SUBQUERY queryStatementExpression identifier) ; //---------------------- Rules for parsing PTF clauses ----------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index bf78545..bef3acf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -717,7 +717,7 @@ explainStatement : KW_EXPLAIN ( explainOption* execStatement -> ^(TOK_EXPLAIN execStatement explainOption*) | - KW_REWRITE queryStatementExpression[true] -> ^(TOK_EXPLAIN_SQ_REWRITE queryStatementExpression)) + KW_REWRITE queryStatementExpression -> ^(TOK_EXPLAIN_SQ_REWRITE queryStatementExpression)) ; explainOption @@ -729,7 +729,7 @@ explainOption execStatement @init { pushMsg("statement", state); } @after { popMsg(state); } - : queryStatementExpression[true] + : queryStatementExpression | loadStatement | exportStatement | importStatement @@ -2310,14 +2310,14 @@ setOperator | KW_UNION KW_DISTINCT? -> ^(TOK_UNIONDISTINCT) ; -queryStatementExpression[boolean topLevel] +queryStatementExpression : /* Would be nice to do this as a gated semantic perdicate But the predicate gets pushed as a lookahead decision. Calling rule doesnot know about topLevel */ - (w=withClause {topLevel}?)? - queryStatementExpressionBody[topLevel] { + (w=withClause)? + queryStatementExpressionBody { if ($w.tree != null) { $queryStatementExpressionBody.tree.insertChild(0, $w.tree); } @@ -2325,10 +2325,10 @@ queryStatementExpression[boolean topLevel] -> queryStatementExpressionBody ; -queryStatementExpressionBody[boolean topLevel] +queryStatementExpressionBody : - fromStatement[topLevel] - | regularBody[topLevel] + fromStatement + | regularBody ; withClause @@ -2338,16 +2338,16 @@ withClause cteStatement : - identifier KW_AS LPAREN queryStatementExpression[false] RPAREN + identifier KW_AS LPAREN queryStatementExpression RPAREN -> ^(TOK_SUBQUERY queryStatementExpression identifier) ; -fromStatement[boolean topLevel] +fromStatement : (singleFromStatement -> singleFromStatement) (u=setOperator r=singleFromStatement -> ^($u {$fromStatement.tree} $r) )* - -> {u != null && topLevel}? ^(TOK_QUERY + -> {u != null}? ^(TOK_QUERY ^(TOK_FROM ^(TOK_SUBQUERY {$fromStatement.tree} @@ -2376,11 +2376,11 @@ The valuesClause rule below ensures that the parse tree for very similar to the tree for "insert into table FOO select a,b from BAR". Since virtual table name is implicit, it's represented as TOK_ANONYMOUS. */ -regularBody[boolean topLevel] +regularBody : i=insertClause ( - s=selectStatement[topLevel] + s=selectStatement {$s.tree.getFirstChildWithType(TOK_INSERT).replaceChildren(0, 0, $i.tree);} -> {$s.tree} | valuesClause @@ -2392,38 +2392,63 @@ regularBody[boolean topLevel] ) ) | - selectStatement[topLevel] + selectStatement ; -selectStatement[boolean topLevel] +atomSelectStatement : - ( s=selectClause f=fromClause? w=whereClause? g=groupByClause? h=havingClause? + win=window_clause? + -> ^(TOK_QUERY $f? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + $s $w? $g? $h? $win?)) + | + LPAREN! selectStatement RPAREN! + ; + +selectStatement + : + a=atomSelectStatement + set=setOpSelectStatement[$atomSelectStatement.tree]? o=orderByClause? c=clusterByClause? d=distributeByClause? sort=sortByClause? - win=window_clause? l=limitClause? - -> ^(TOK_QUERY $f? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) - $s $w? $g? $h? $o? $c? - $d? $sort? $win? $l?)) - ) - (set=setOpSelectStatement[$selectStatement.tree, topLevel])? + { + if(set == null){ + $a.tree.getFirstChildWithType(TOK_INSERT).addChild($o.tree); + $a.tree.getFirstChildWithType(TOK_INSERT).addChild($c.tree); + $a.tree.getFirstChildWithType(TOK_INSERT).addChild($d.tree); + $a.tree.getFirstChildWithType(TOK_INSERT).addChild($sort.tree); + $a.tree.getFirstChildWithType(TOK_INSERT).addChild($l.tree); + } + } -> {set == null}? - {$selectStatement.tree} + {$a.tree} -> {o==null && c==null && d==null && sort==null && l==null}? {$set.tree} - -> {throwSetOpException()} + -> ^(TOK_QUERY + ^(TOK_FROM + ^(TOK_SUBQUERY + {$set.tree} + {adaptor.create(Identifier, generateUnionAlias())} + ) + ) + ^(TOK_INSERT + ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) + ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF)) + $o? $c? $d? $sort? $l? + ) + ) ; -setOpSelectStatement[CommonTree t, boolean topLevel] +setOpSelectStatement[CommonTree t] : - (u=setOperator b=simpleSelectStatement + (u=setOperator b=atomSelectStatement -> {$setOpSelectStatement.tree != null && u.tree.getType()==HiveParser.TOK_UNIONDISTINCT}? ^(TOK_QUERY ^(TOK_FROM @@ -2454,15 +2479,8 @@ setOpSelectStatement[CommonTree t, boolean topLevel] ) -> ^(TOK_UNIONALL {$t} $b) )+ - o=orderByClause? - c=clusterByClause? - d=distributeByClause? - sort=sortByClause? - win=window_clause? - l=limitClause? - -> {o==null && c==null && d==null && sort==null && win==null && l==null && !topLevel}? - {$setOpSelectStatement.tree} - -> ^(TOK_QUERY + -> {$setOpSelectStatement.tree.getChild(0).getType()==HiveParser.TOK_UNIONALL}? + ^(TOK_QUERY ^(TOK_FROM ^(TOK_SUBQUERY {$setOpSelectStatement.tree} @@ -2472,27 +2490,15 @@ setOpSelectStatement[CommonTree t, boolean topLevel] ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF)) - $o? $c? $d? $sort? $win? $l? ) ) - ; - -simpleSelectStatement - : - selectClause - fromClause? - whereClause? - groupByClause? - havingClause? - ((window_clause) => window_clause)? - -> ^(TOK_QUERY fromClause? ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE)) - selectClause whereClause? groupByClause? havingClause? window_clause?)) + -> {$setOpSelectStatement.tree} ; selectStatementWithCTE : (w=withClause)? - selectStatement[true] { + selectStatement { if ($w.tree != null) { $selectStatement.tree.insertChild(0, $w.tree); } http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 4a44173..6ae731f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -476,7 +476,7 @@ precedenceEqualOperator subQueryExpression : - LPAREN! selectStatement[true] RPAREN! + LPAREN! selectStatement RPAREN! ; precedenceEqualExpression http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q b/ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q index b2465fd..5128576 100644 --- a/ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q +++ b/ql/src/test/queries/clientnegative/ptf_negative_AmbiguousWindowDefn.q @@ -4,9 +4,9 @@ sum(p_size) over (w1) as s1, sum(p_size) over (w2) as s2, sum(p_size) over (w3) as s3 from part -distribute by p_mfgr -sort by p_mfgr window w1 as (rows between 2 preceding and 2 following), w2 as (rows between unbounded preceding and current row), - w3 as w3; + w3 as w3 +distribute by p_mfgr +sort by p_mfgr; http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q b/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q index 8defb3a..99932cd 100644 --- a/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q +++ b/ql/src/test/queries/clientnegative/ptf_negative_NoWindowDefn.q @@ -3,7 +3,7 @@ select p_mfgr, p_name, p_size, sum(p_size) over (w1) as s1, sum(p_size) over (w2) as s2 from part +window w1 as (rows between 2 preceding and 2 following) distribute by p_mfgr -sort by p_mfgr -window w1 as (rows between 2 preceding and 2 following); +sort by p_mfgr; http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/queries/clientnegative/windowing_after_orderby.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/windowing_after_orderby.q b/ql/src/test/queries/clientnegative/windowing_after_orderby.q new file mode 100644 index 0000000..b9acb34 --- /dev/null +++ b/ql/src/test/queries/clientnegative/windowing_after_orderby.q @@ -0,0 +1,7 @@ +create table empsalary (depname varchar(10), salary int); + +SELECT sum(salary) OVER w as s , avg(salary) OVER w as a + FROM empsalary + order by s + WINDOW w AS (PARTITION BY depname ORDER BY salary DESC); + http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/queries/clientpositive/union_paren.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/union_paren.q b/ql/src/test/queries/clientpositive/union_paren.q new file mode 100644 index 0000000..0b38b68 --- /dev/null +++ b/ql/src/test/queries/clientpositive/union_paren.q @@ -0,0 +1,54 @@ +set hive.mapred.mode=nonstrict; + +explain select * from src union all select * from src; + +create table t1(c int); + +insert into t1 values (1),(1),(2); + +create table t2(c int); + +insert into t2 values (2),(1),(2); + +create table t3(c int); + +insert into t3 values (2),(3),(2); + +(select * from t1) union all select * from t2 union select * from t3 order by c; + +(select * from t1) union all (select * from t2 union select * from t3) order by c; + +(select * from src order by key limit 1); + +(select * from src) union all select * from src order by key limit 1; + +(select * from src limit 1) union all select * from src order by key limit 1; + +((select * from src)) union all select * from src order by key limit 1; + +select * from src union all ((select * from src)) order by key limit 1; + +select * from src union all ((select * from src limit 1)) order by key limit 1; + +select * from src union all (select * from src) order by key limit 1; + +(select * from src order by key) union all (select * from src) order by key limit 1; + +(select * from src order by key) union all (select * from src limit 1) order by key limit 1; + +select count(*) from (select key from src union select key from src)cool_cust; + +--similar tpcds q14 + +with cross_items as + (select key, k + from src, + (select iss.key k + from src iss + union all + select ics.key k + from src ics + ) x + where key = k +) +select * from cross_items order by key limit 1; http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/create_or_replace_view6.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/create_or_replace_view6.q.out b/ql/src/test/results/clientnegative/create_or_replace_view6.q.out index b919e72..6f571dd 100644 --- a/ql/src/test/results/clientnegative/create_or_replace_view6.q.out +++ b/ql/src/test/results/clientnegative/create_or_replace_view6.q.out @@ -16,4 +16,4 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@v -FAILED: ParseException line 2:52 cannot recognize input near 'blah' '<EOF>' '<EOF>' in select clause +FAILED: ParseException line 2:52 cannot recognize input near 'blah' '<EOF>' '<EOF>' in create view statement http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/unionClusterBy.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/unionClusterBy.q.out b/ql/src/test/results/clientnegative/unionClusterBy.q.out index 4705f3b..7f01413 100644 --- a/ql/src/test/results/clientnegative/unionClusterBy.q.out +++ b/ql/src/test/results/clientnegative/unionClusterBy.q.out @@ -1 +1 @@ -FAILED: ParseException line 6:19 Failed to recognize predicate '<EOF>'. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement +FAILED: ParseException line 5:0 missing EOF at 'union' near 'key' http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/unionDistributeBy.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/unionDistributeBy.q.out b/ql/src/test/results/clientnegative/unionDistributeBy.q.out index 4705f3b..7f01413 100644 --- a/ql/src/test/results/clientnegative/unionDistributeBy.q.out +++ b/ql/src/test/results/clientnegative/unionDistributeBy.q.out @@ -1 +1 @@ -FAILED: ParseException line 6:19 Failed to recognize predicate '<EOF>'. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement +FAILED: ParseException line 5:0 missing EOF at 'union' near 'key' http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/unionLimit.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/unionLimit.q.out b/ql/src/test/results/clientnegative/unionLimit.q.out index 4705f3b..e8463b5 100644 --- a/ql/src/test/results/clientnegative/unionLimit.q.out +++ b/ql/src/test/results/clientnegative/unionLimit.q.out @@ -1 +1 @@ -FAILED: ParseException line 6:19 Failed to recognize predicate '<EOF>'. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement +FAILED: ParseException line 5:0 missing EOF at 'union' near '1' http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/unionOrderBy.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/unionOrderBy.q.out b/ql/src/test/results/clientnegative/unionOrderBy.q.out index 4705f3b..7f01413 100644 --- a/ql/src/test/results/clientnegative/unionOrderBy.q.out +++ b/ql/src/test/results/clientnegative/unionOrderBy.q.out @@ -1 +1 @@ -FAILED: ParseException line 6:19 Failed to recognize predicate '<EOF>'. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement +FAILED: ParseException line 5:0 missing EOF at 'union' near 'key' http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/unionSortBy.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/unionSortBy.q.out b/ql/src/test/results/clientnegative/unionSortBy.q.out index 4705f3b..7f01413 100644 --- a/ql/src/test/results/clientnegative/unionSortBy.q.out +++ b/ql/src/test/results/clientnegative/unionSortBy.q.out @@ -1 +1 @@ -FAILED: ParseException line 6:19 Failed to recognize predicate '<EOF>'. Failed rule: 'orderByClause clusterByClause distributeByClause sortByClause limitClause can only be applied to the whole union.' in statement +FAILED: ParseException line 5:0 missing EOF at 'union' near 'key' http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientnegative/windowing_after_orderby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/windowing_after_orderby.q.out b/ql/src/test/results/clientnegative/windowing_after_orderby.q.out new file mode 100644 index 0000000..03325bc --- /dev/null +++ b/ql/src/test/results/clientnegative/windowing_after_orderby.q.out @@ -0,0 +1,9 @@ +PREHOOK: query: create table empsalary (depname varchar(10), salary int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@empsalary +POSTHOOK: query: create table empsalary (depname varchar(10), salary int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@empsalary +FAILED: ParseException line 6:2 missing EOF at 'WINDOW' near 's' http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out b/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out index 9e0613f..e5df0ee 100644 --- a/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out +++ b/ql/src/test/results/clientpositive/cbo_rp_lineage2.q.out @@ -552,7 +552,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Input: default@src1 PREHOOK: Output: default@dest_l1 -{"version":"1.0","engine":"mr","database":"default","hash":"60b589744e2527dd235a6c8168d6a653","queryText":"INSERT OVERWRITE TABLE dest_l1\nSELECT j.*\nFROM (SELECT t1.key, p1.value\n FROM src1 t1\n LEFT OUTER JOIN src p1\n ON (t1.key = p1.key)\n UNION ALL\n SELECT t2.key, p2.value\n FROM src1 t2\n LEFT OUTER JOIN src p2\n ON (t2.key = p2.key)) j","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(key)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"value","edgeType":"PROJECTION"},{"sources":[4,2],"targets":[0,1],"expression":"(null-subquery1:j-subquery1:p1.key = null-subquery1:j-subquery1:t1.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(null-subquery2:j-subquery2:p2.key = null-subquery2:j-subquery2:t2.key)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.value"},{"id": 2,"vertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src.key"}]} +{"version":"1.0","engine":"mr","database":"default","hash":"60b589744e2527dd235a6c8168d6a653","queryText":"INSERT OVERWRITE TABLE dest_l1\nSELECT j.*\nFROM (SELECT t1.key, p1.value\n FROM src1 t1\n LEFT OUTER JOIN src p1\n ON (t1.key = p1.key)\n UNION ALL\n SELECT t2.key, p2.value\n FROM src1 t2\n LEFT OUTER JOIN src p2\n ON (t2.key = p2.key)) j","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(key)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"value","edgeType":"PROJECTION"},{"sources":[4,2],"targets":[0,1],"expression":"(j-subquery1:_u1-subquery1:p1.key = j-subquery1:_u1-subquery1:t1.key)","edgeType":"PREDICATE"},{"sources":[4,2],"targets":[0,1],"expression":"(j-subquery2:_u1-subquery2:p2.key = j-subquery2:_u1-subquery2:t2.key)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.key"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.value"},{"id":2,"v ertexType":"COLUMN","vertexId":"default.src1.key"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src.value"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src.key"}]} PREHOOK: query: drop table if exists emp PREHOOK: type: DROPTABLE PREHOOK: query: drop table if exists dept http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientpositive/complex_alias.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/complex_alias.q.out b/ql/src/test/results/clientpositive/complex_alias.q.out index 4cb6c83..361bc8d 100644 --- a/ql/src/test/results/clientpositive/complex_alias.q.out +++ b/ql/src/test/results/clientpositive/complex_alias.q.out @@ -17,7 +17,7 @@ POSTHOOK: Output: default@agg1 POSTHOOK: Lineage: agg1.col0 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: agg1.col1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: agg1.col2 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] -Warning: Shuffle Join JOIN[19][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT single_use_subq11.a1 AS a1, single_use_subq11.a2 AS a2 @@ -218,7 +218,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[19][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[20][tables = [single_use_subq12, single_use_subq11]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: SELECT single_use_subq11.a1 AS a1, single_use_subq11.a2 AS a2 FROM (SELECT Sum(agg1.col2) AS a1 http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientpositive/constant_prop_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/constant_prop_1.q.out b/ql/src/test/results/clientpositive/constant_prop_1.q.out index 8695e1d..aaa1dac 100644 --- a/ql/src/test/results/clientpositive/constant_prop_1.q.out +++ b/ql/src/test/results/clientpositive/constant_prop_1.q.out @@ -65,7 +65,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[13][tables = [sub, b]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [sub, b]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select a, key, value from ( @@ -102,10 +102,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 TableScan alias: src @@ -118,10 +118,10 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 Reduce Operator Tree: Limit @@ -241,7 +241,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Shuffle Join JOIN[13][tables = [sub, b]] in Stage 'Stage-2:MAPRED' is a cross product +Warning: Shuffle Join JOIN[14][tables = [sub, b]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select a, key, value from ( http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientpositive/input_part7.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/input_part7.q.out b/ql/src/test/results/clientpositive/input_part7.q.out index 459e384..3543d0e 100644 --- a/ql/src/test/results/clientpositive/input_part7.q.out +++ b/ql/src/test/results/clientpositive/input_part7.q.out @@ -168,8 +168,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [null-subquery1:a-subquery1:x, null-subquery2:a-subquery2:y] - /srcpart/ds=2008-04-08/hr=12 [null-subquery1:a-subquery1:x, null-subquery2:a-subquery2:y] + /srcpart/ds=2008-04-08/hr=11 [a-subquery1:_u1-subquery1:x, a-subquery2:_u1-subquery2:y] + /srcpart/ds=2008-04-08/hr=12 [a-subquery1:_u1-subquery1:x, a-subquery2:_u1-subquery2:y] Needs Tagging: false Reduce Operator Tree: Select Operator http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientpositive/llap/explainuser_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 840b757..931f1a2 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -2966,11 +2966,15 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Map 11 @@ -2985,11 +2989,15 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Map 12 @@ -3004,11 +3012,15 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Map 13 @@ -3055,44 +3067,48 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col6 - input vertices: - 0 Map 20 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1525 Data size: 16127 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col6 + input vertices: + 0 Map 20 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.a + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.b + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.c Execution mode: llap LLAP IO: no inputs Map 17 @@ -3107,44 +3123,48 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col6 - input vertices: - 0 Map 20 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1525 Data size: 16127 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col6 + input vertices: + 0 Map 20 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.a + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.b + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.c Execution mode: llap LLAP IO: no inputs Map 18 @@ -3159,44 +3179,48 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col6 - input vertices: - 0 Map 20 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1525 Data size: 16127 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col6 + input vertices: + 0 Map 20 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.a + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.b + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.c Execution mode: llap LLAP IO: no inputs Map 19 @@ -3211,44 +3235,48 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col1 (type: string) - 1 _col1 (type: string) - outputColumnNames: _col0, _col6 - input vertices: - 0 Map 20 - Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1525 Data size: 16127 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col1 (type: string) + 1 _col1 (type: string) + outputColumnNames: _col0, _col6 + input vertices: + 0 Map 20 Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.a - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.b - File Output Operator - compressed: false - Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.c + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1677 Data size: 17739 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.a + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.b + File Output Operator + compressed: false + Statistics: Num rows: 3409 Data size: 36062 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.c Execution mode: llap LLAP IO: no inputs Map 20 @@ -3323,11 +3351,15 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 525 Data size: 5503 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Map 6 @@ -3384,11 +3416,15 @@ STAGE PLANS: expressions: value (type: string) outputColumnNames: _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + Select Operator + expressions: _col1 (type: string) + outputColumnNames: _col1 Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1025 Data size: 10815 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs Reducer 10 @@ -4075,15 +4111,15 @@ Stage-4 Dependency Collection{} Stage-2 Reducer 4 llap - File Output Operator [FS_18] + File Output Operator [FS_20] table:{"name:":"default.dest1"} - Group By Operator [GBY_16] (rows=1 width=96) + Group By Operator [GBY_18] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_15] + Reduce Output Operator [RS_17] PartitionCols:_col0 - Group By Operator [GBY_14] (rows=1 width=280) + Group By Operator [GBY_16] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) Select Operator [SEL_9] (rows=501 width=272) Output:["_col0","_col1"] @@ -4091,29 +4127,29 @@ Stage-4 Output:["_col0","_col1"] TableScan [TS_6] (rows=500 width=10) Output:["key","value"] - Reduce Output Operator [RS_21] + Reduce Output Operator [RS_23] PartitionCols:_col0, _col1 - Group By Operator [GBY_20] (rows=1 width=464) + Group By Operator [GBY_22] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) Please refer to the previous Select Operator [SEL_9] <-Map 7 [CONTAINS] llap - Reduce Output Operator [RS_15] + Reduce Output Operator [RS_17] PartitionCols:_col0 - Group By Operator [GBY_14] (rows=1 width=280) + Group By Operator [GBY_16] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) - Select Operator [SEL_11] (rows=500 width=10) + Select Operator [SEL_12] (rows=500 width=10) Output:["_col0","_col1"] - TableScan [TS_10] (rows=500 width=10) + TableScan [TS_11] (rows=500 width=10) Output:["key","value"] - Reduce Output Operator [RS_21] + Reduce Output Operator [RS_23] PartitionCols:_col0, _col1 - Group By Operator [GBY_20] (rows=1 width=464) + Group By Operator [GBY_22] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Please refer to the previous Select Operator [SEL_11] + Please refer to the previous Select Operator [SEL_12] <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_15] + Reduce Output Operator [RS_17] PartitionCols:_col0 - Group By Operator [GBY_14] (rows=1 width=280) + Group By Operator [GBY_16] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) Select Operator [SEL_9] (rows=501 width=272) Output:["_col0","_col1"] @@ -4128,15 +4164,15 @@ Stage-4 Select Operator [SEL_1] (rows=500 width=10) TableScan [TS_0] (rows=500 width=10) default@src,s1,Tbl:COMPLETE,Col:COMPLETE - Reduce Output Operator [RS_21] + Reduce Output Operator [RS_23] PartitionCols:_col0, _col1 - Group By Operator [GBY_20] (rows=1 width=464) + Group By Operator [GBY_22] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) Please refer to the previous Select Operator [SEL_9] Reducer 5 llap - File Output Operator [FS_24] + File Output Operator [FS_26] table:{"name:":"default.dest2"} - Group By Operator [GBY_22] (rows=1 width=280) + Group By Operator [GBY_24] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 @@ -4183,15 +4219,15 @@ Stage-4 Dependency Collection{} Stage-2 Reducer 4 llap - File Output Operator [FS_14] + File Output Operator [FS_15] table:{"name:":"default.dest1"} - Group By Operator [GBY_12] (rows=1 width=96) + Group By Operator [GBY_13] (rows=1 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] <-Map 6 [CONTAINS] llap - Reduce Output Operator [RS_11] + Reduce Output Operator [RS_12] PartitionCols:_col0 - Group By Operator [GBY_10] (rows=1 width=280) + Group By Operator [GBY_11] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) Select Operator [SEL_9] (rows=501 width=11) Output:["_col0","_col1"] @@ -4199,17 +4235,15 @@ Stage-4 Output:["_col0","_col1"] TableScan [TS_6] (rows=500 width=10) Output:["key","value"] - Reduce Output Operator [RS_17] + Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=1 width=464) + Group By Operator [GBY_17] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Select Operator [SEL_15] (rows=501 width=11) - Output:["_col0","_col1"] - Please refer to the previous Select Operator [SEL_7] + Please refer to the previous Select Operator [SEL_9] <-Reducer 2 [CONTAINS] llap - Reduce Output Operator [RS_11] + Reduce Output Operator [RS_12] PartitionCols:_col0 - Group By Operator [GBY_10] (rows=1 width=280) + Group By Operator [GBY_11] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, substr(_col1, 5) Select Operator [SEL_9] (rows=501 width=11) Output:["_col0","_col1"] @@ -4224,17 +4258,15 @@ Stage-4 Select Operator [SEL_1] (rows=500 width=10) TableScan [TS_0] (rows=500 width=10) default@src,s1,Tbl:COMPLETE,Col:COMPLETE - Reduce Output Operator [RS_17] + Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 - Group By Operator [GBY_16] (rows=1 width=464) + Group By Operator [GBY_17] (rows=1 width=464) Output:["_col0","_col1","_col2","_col3"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1, substr(_col1, 5) - Select Operator [SEL_15] (rows=501 width=11) - Output:["_col0","_col1"] - Please refer to the previous Select Operator [SEL_5] + Please refer to the previous Select Operator [SEL_9] Reducer 5 llap - File Output Operator [FS_20] + File Output Operator [FS_21] table:{"name:":"default.dest2"} - Group By Operator [GBY_18] (rows=1 width=280) + Group By Operator [GBY_19] (rows=1 width=280) Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 http://git-wip-us.apache.org/repos/asf/hive/blob/2435e702/ql/src/test/results/clientpositive/optimize_nullscan.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/optimize_nullscan.q.out index 5f99e2a..1775a3f 100644 --- a/ql/src/test/results/clientpositive/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/optimize_nullscan.q.out @@ -1551,7 +1551,7 @@ STAGE PLANS: tag: 1 auto parallelism: false Path -> Alias: - nullscan://null/default.src/part_ [null-subquery2:a-subquery2:src] + nullscan://null/default.src/part_ [a-subquery2:_u1-subquery2:src] Path -> Partition: nullscan://null/default.src/part_ Partition @@ -1597,7 +1597,7 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - nullscan://null/default.src/part_ [null-subquery2:a-subquery2:src] + nullscan://null/default.src/part_ [a-subquery2:_u1-subquery2:src] Needs Tagging: true Reduce Operator Tree: Join Operator @@ -1693,7 +1693,7 @@ STAGE PLANS: MultiFileSpray: false Path -> Alias: #### A masked pattern was here #### - nullscan://null/default.src/part_ [null-subquery1:a-subquery1:src] + nullscan://null/default.src/part_ [a-subquery1:_u1-subquery1:src] Path -> Partition: #### A masked pattern was here #### Partition @@ -1760,7 +1760,7 @@ STAGE PLANS: name: default.src Truncated Path -> Alias: #### A masked pattern was here #### - nullscan://null/default.src/part_ [null-subquery1:a-subquery1:src] + nullscan://null/default.src/part_ [a-subquery1:_u1-subquery1:src] Stage: Stage-0 Fetch Operator
