HIVE-11815 : Correct the column/table names in subquery expression when creating a view (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8da2ed30 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8da2ed30 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8da2ed30 Branch: refs/heads/beeline-cli Commit: 8da2ed304891dc8483fe3d78eda4c9f70c54ae18 Parents: a12e5f5 Author: Pengcheng Xiong <[email protected]> Authored: Thu Sep 17 13:20:00 2015 -0700 Committer: Pengcheng Xiong <[email protected]> Committed: Thu Sep 17 13:20:00 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/parse/QBSubQuery.java | 7 -- .../hadoop/hive/ql/parse/SubQueryUtils.java | 11 -- .../queries/clientpositive/subquery_views.q | 22 +++- .../subquery_exists_implicit_gby.q.out | 8 +- .../subquery_nested_subquery.q.out | 4 +- .../subquery_notexists_implicit_gby.q.out | 8 +- .../subquery_windowing_corr.q.out | 7 +- .../results/clientpositive/subquery_views.q.out | 116 +++++++++++++++++++ 8 files changed, 141 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java index 92cbabc..f95ee8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java @@ -401,7 +401,6 @@ public class QBSubQuery implements ISubQueryJoinInfo { CNT_ALIAS, subQryCorrExprs, sqRR); - SubQueryUtils.setOriginDeep(ast, QBSubQuery.this.originalSQASTOrigin); return ast; } @@ -416,7 +415,6 @@ public class QBSubQuery implements ISubQueryJoinInfo { public ASTNode getJoinConditionAST() { ASTNode ast = SubQueryUtils.buildNotInNullJoinCond(getAlias(), CNT_ALIAS); - SubQueryUtils.setOriginDeep(ast, QBSubQuery.this.originalSQASTOrigin); return ast; } @@ -576,8 +574,6 @@ public class QBSubQuery implements ISubQueryJoinInfo { rewrite(outerQueryRR, forHavingClause, outerQueryAlias, insertClause, selectClause); - SubQueryUtils.setOriginDeep(subQueryAST, originalSQASTOrigin); - /* * Restriction.13.m :: In the case of an implied Group By on a * correlated SubQuery, the SubQuery always returns 1 row. @@ -696,8 +692,6 @@ public class QBSubQuery implements ISubQueryJoinInfo { } } - SubQueryUtils.setOriginDeep(joinConditionAST, originalSQASTOrigin); - SubQueryUtils.setOriginDeep(postJoinConditionAST, originalSQASTOrigin); } ASTNode updateOuterQueryFilter(ASTNode outerQryFilter) { @@ -711,7 +705,6 @@ public class QBSubQuery implements ISubQueryJoinInfo { return postJoinConditionAST; } ASTNode node = SubQueryUtils.andAST(outerQryFilter, postJoinConditionAST); - node.setOrigin(originalSQASTOrigin); return node; } http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java index 87a7ced..362a285 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SubQueryUtils.java @@ -467,17 +467,6 @@ public class SubQueryUtils { return check; } - static void setOriginDeep(ASTNode node, ASTNodeOrigin origin) { - if ( node == null ) { - return; - } - node.setOrigin(origin); - int childCnt = node.getChildCount(); - for(int i=0; i<childCnt; i++) { - setOriginDeep((ASTNode)node.getChild(i), origin); - } - } - /* * Set of functions to create the Null Check Query for Not-In SubQuery predicates. * For a SubQuery predicate like: http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/test/queries/clientpositive/subquery_views.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/subquery_views.q b/ql/src/test/queries/clientpositive/subquery_views.q index f15d41b..e646310 100644 --- a/ql/src/test/queries/clientpositive/subquery_views.q +++ b/ql/src/test/queries/clientpositive/subquery_views.q @@ -10,6 +10,8 @@ where exists where b.value = a.value and a.key = b.key and a.value > 'val_9') ; +describe extended cv1; + select * from cv1 where cv1.key in (select key from cv1 c where c.key > '95'); ; @@ -26,6 +28,8 @@ where b.key not in ) ; +describe extended cv2; + explain select * from cv2 where cv2.key in (select key from cv2 c where c.key < '11'); @@ -44,10 +48,26 @@ group by key, value having count(*) in (select count(*) from src s1 where s1.key > '9' group by s1.key ) ; +describe extended cv3; + select * from cv3; -- join of subquery views select * from cv3 -where cv3.key in (select key from cv1); \ No newline at end of file +where cv3.key in (select key from cv1); + +drop table tc; + +create table tc (`@d` int); + +insert overwrite table tc select 1 from src limit 1; + +drop view tcv; + +create view tcv as select * from tc b where exists (select a.`@d` from tc a where b.`@d`=a.`@d`); + +describe extended tcv; + +select * from tcv; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/test/results/clientnegative/subquery_exists_implicit_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/subquery_exists_implicit_gby.q.out b/ql/src/test/results/clientnegative/subquery_exists_implicit_gby.q.out index 4830c00..f7251e3 100644 --- a/ql/src/test/results/clientnegative/subquery_exists_implicit_gby.q.out +++ b/ql/src/test/results/clientnegative/subquery_exists_implicit_gby.q.out @@ -1,7 +1 @@ -FAILED: SemanticException Line 7:7 Invalid SubQuery expression 'key' in definition of SubQuery sq_1 [ -exists - (select count(*) - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_9' - ) -] used as sq_1 at Line 5:6: An Exists predicate on SubQuery with implicit Aggregation(no Group By clause) cannot be rewritten. (predicate will always return true). +FAILED: SemanticException [Error 10250]: Line 7:7 Invalid SubQuery expression 'key': An Exists predicate on SubQuery with implicit Aggregation(no Group By clause) cannot be rewritten. (predicate will always return true). http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/test/results/clientnegative/subquery_nested_subquery.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/subquery_nested_subquery.q.out b/ql/src/test/results/clientnegative/subquery_nested_subquery.q.out index ae3bc8f..140b093 100644 --- a/ql/src/test/results/clientnegative/subquery_nested_subquery.q.out +++ b/ql/src/test/results/clientnegative/subquery_nested_subquery.q.out @@ -1,3 +1 @@ -FAILED: SemanticException Line 3:53 Unsupported SubQuery Expression 'p_name' in definition of SubQuery sq_1 [ -x.p_name in (select y.p_name from part y where exists (select z.p_name from part z where y.p_name = z.p_name)) -] used as sq_1 at Line 3:15: Nested SubQuery expressions are not supported. +FAILED: SemanticException [Error 10249]: Line 3:53 Unsupported SubQuery Expression 'p_name': Nested SubQuery expressions are not supported. http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/test/results/clientnegative/subquery_notexists_implicit_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/subquery_notexists_implicit_gby.q.out b/ql/src/test/results/clientnegative/subquery_notexists_implicit_gby.q.out index 74422af..6d9fa0a 100644 --- a/ql/src/test/results/clientnegative/subquery_notexists_implicit_gby.q.out +++ b/ql/src/test/results/clientnegative/subquery_notexists_implicit_gby.q.out @@ -1,7 +1 @@ -FAILED: SemanticException Line 7:7 Invalid SubQuery expression 'key' in definition of SubQuery sq_1 [ -exists - (select sum(1) - from src a - where b.value = a.value and a.key = b.key and a.value > 'val_9' - ) -] used as sq_1 at Line 5:10: A Not Exists predicate on SubQuery with implicit Aggregation(no Group By clause) cannot be rewritten. (predicate will always return false). +FAILED: SemanticException [Error 10250]: Line 7:7 Invalid SubQuery expression 'key': A Not Exists predicate on SubQuery with implicit Aggregation(no Group By clause) cannot be rewritten. (predicate will always return false). http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/test/results/clientnegative/subquery_windowing_corr.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/subquery_windowing_corr.q.out b/ql/src/test/results/clientnegative/subquery_windowing_corr.q.out index 647a535..dcd3026 100644 --- a/ql/src/test/results/clientnegative/subquery_windowing_corr.q.out +++ b/ql/src/test/results/clientnegative/subquery_windowing_corr.q.out @@ -1,6 +1 @@ -FAILED: SemanticException Line 6:8 Unsupported SubQuery Expression '1' in definition of SubQuery sq_1 [ -a.p_size in - (select first_value(p_size) over(partition by p_mfgr order by p_size) - from part b - where a.p_brand = b.p_brand) -] used as sq_1 at Line 4:15: Correlated Sub Queries cannot contain Windowing clauses. +FAILED: SemanticException [Error 10249]: Line 6:8 Unsupported SubQuery Expression '1': Correlated Sub Queries cannot contain Windowing clauses. http://git-wip-us.apache.org/repos/asf/hive/blob/8da2ed30/ql/src/test/results/clientpositive/subquery_views.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/subquery_views.q.out b/ql/src/test/results/clientpositive/subquery_views.q.out index cfa7339..470fa83 100644 --- a/ql/src/test/results/clientpositive/subquery_views.q.out +++ b/ql/src/test/results/clientpositive/subquery_views.q.out @@ -26,6 +26,26 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@cv1 +PREHOOK: query: describe extended cv1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cv1 +POSTHOOK: query: describe extended cv1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cv1 +key string +value string + +#### A masked pattern was here #### +from src b +where exists + (select a.key + from src a + where b.value = a.value and a.key = b.key and a.value > 'val_9'), viewExpandedText:select `b`.`key`, `b`.`value` +from `default`.`src` `b` +where exists + (select `a`.`key` + from `default`.`src` `a` + where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_9'), tableType:VIRTUAL_VIEW) PREHOOK: query: select * from cv1 where cv1.key in (select key from cv1 c where c.key > '95') PREHOOK: type: QUERY @@ -69,6 +89,28 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@cv2 +PREHOOK: query: describe extended cv2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cv2 +POSTHOOK: query: describe extended cv2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cv2 +key string +value string + +#### A masked pattern was here #### +from src b +where b.key not in + (select a.key + from src a + where b.value = a.value and a.key = b.key and a.value > 'val_11' + ), viewExpandedText:select `b`.`key`, `b`.`value` +from `default`.`src` `b` +where `b`.`key` not in + (select `a`.`key` + from `default`.`src` `a` + where `b`.`value` = `a`.`value` and `a`.`key` = `b`.`key` and `a`.`value` > 'val_11' + ), tableType:VIRTUAL_VIEW) Warning: Shuffle Join JOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product Warning: Shuffle Join JOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Stage-6:MAPRED' is a cross product PREHOOK: query: explain @@ -425,6 +467,25 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@cv3 +PREHOOK: query: describe extended cv3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cv3 +POSTHOOK: query: describe extended cv3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cv3 +key string +value string +_c2 bigint + +#### A masked pattern was here #### +from src b +where b.key in (select key from src where src.key > '8') +group by key, value +having count(*) in (select count(*) from src s1 where s1.key > '9' group by s1.key ), viewExpandedText:select `b`.`key`, `b`.`value`, count(*) +from `default`.`src` `b` +where `b`.`key` in (select `src`.`key` from `default`.`src` where `src`.`key` > '8') +group by `b`.`key`, `b`.`value` +having count(*) in (select count(*) from `default`.`src` `s1` where `s1`.`key` > '9' group by `s1`.`key` ), tableType:VIRTUAL_VIEW) PREHOOK: query: select * from cv3 PREHOOK: type: QUERY PREHOOK: Input: default@cv3 @@ -473,3 +534,58 @@ POSTHOOK: Input: default@src 96 val_96 1 97 val_97 2 98 val_98 2 +PREHOOK: query: drop table tc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tc +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tc (`@d` int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tc +POSTHOOK: query: create table tc (`@d` int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tc +PREHOOK: query: insert overwrite table tc select 1 from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tc +POSTHOOK: query: insert overwrite table tc select 1 from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tc +POSTHOOK: Lineage: tc.@d SIMPLE [] +PREHOOK: query: drop view tcv +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view tcv +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view tcv as select * from tc b where exists (select a.`@d` from tc a where b.`@d`=a.`@d`) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@tc +PREHOOK: Output: database:default +PREHOOK: Output: default@tcv +POSTHOOK: query: create view tcv as select * from tc b where exists (select a.`@d` from tc a where b.`@d`=a.`@d`) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@tc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tcv +PREHOOK: query: describe extended tcv +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tcv +POSTHOOK: query: describe extended tcv +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tcv +@d int + +#### A masked pattern was here #### +PREHOOK: query: select * from tcv +PREHOOK: type: QUERY +PREHOOK: Input: default@tc +PREHOOK: Input: default@tcv +#### A masked pattern was here #### +POSTHOOK: query: select * from tcv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tc +POSTHOOK: Input: default@tcv +#### A masked pattern was here #### +1
