IMPALA-4142: qgen: Hive does not support CTEs inside sub-query blocks Changes:
Hive does not support WITH clauses inside sub-query blocks, the DefaultProfile actually already has an option to disable this called "use_nested_with" and the HiveProfile returns False for this method. However, there is a bug in the usage of "use_nested_with"; it is not checked every time a sub-query is created. Specifically, it is not checked when an in-line view is created, or when the query within a WITH clause is created (so it's possible to create WITH ... AS (WITH ...)). This patch fixes both of these issues since Hive throws a SQL compile exception for either case. Testing: * Added new unit tests * All unit tests pass * Tested locally against Hive * Tested against Impala via Leopard * Tested against Impala via the discrepancy checker Change-Id: Ie585bbbd88a32f502457a4ca9f2f54c7e1ade354 Reviewed-on: http://gerrit.cloudera.org:8080/4534 Reviewed-by: Taras Bobrovytsky <[email protected]> Tested-by: Taras Bobrovytsky <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b5b848f3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b5b848f3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b5b848f3 Branch: refs/heads/master Commit: b5b848f373f90fef674e666c29ca54c6ccb54858 Parents: 0780d2c Author: Sahil Takiar <[email protected]> Authored: Thu Sep 15 15:22:25 2016 -0700 Committer: Taras Bobrovytsky <[email protected]> Committed: Tue Sep 27 22:47:21 2016 +0000 ---------------------------------------------------------------------- tests/comparison/query_generator.py | 6 +- tests/comparison/tests/test_use_nested_with.py | 68 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b5b848f3/tests/comparison/query_generator.py ---------------------------------------------------------------------- diff --git a/tests/comparison/query_generator.py b/tests/comparison/query_generator.py index 4685934..a38aea4 100644 --- a/tests/comparison/query_generator.py +++ b/tests/comparison/query_generator.py @@ -249,7 +249,8 @@ class QueryGenerator(object): with_clause_inline_views = TableExprList() for with_clause_inline_view_idx \ in xrange(self.profile.get_with_clause_table_ref_count()): - query = self.create_query(table_exprs) + query = self.create_query(table_exprs, + allow_with_clause=self.profile.use_nested_with()) with_clause_alias_count = getattr(self.root_query, 'with_clause_alias_count', 0) + 1 self.root_query.with_clause_alias_count = with_clause_alias_count with_clause_inline_view = \ @@ -1116,7 +1117,8 @@ class QueryGenerator(object): def _create_inline_view(self, table_exprs, required_type=None): return InlineView(self.create_query( - table_exprs, required_select_item_type=required_type)) + table_exprs, required_select_item_type=required_type, + allow_with_clause=self.profile.use_nested_with())) def _create_join_clause(self, from_clause, table_exprs): join_type = self.profile.choose_join_type(JoinClause.JOINS_TYPES) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b5b848f3/tests/comparison/tests/test_use_nested_with.py ---------------------------------------------------------------------- diff --git a/tests/comparison/tests/test_use_nested_with.py b/tests/comparison/tests/test_use_nested_with.py new file mode 100644 index 0000000..418fdd7 --- /dev/null +++ b/tests/comparison/tests/test_use_nested_with.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from tests.comparison.common import TableExprList, Column, Table +from tests.comparison.db_types import Int +from tests.comparison.query_generator import QueryGenerator +from tests.comparison.query_profile import DefaultProfile + + +def test_use_nested_width_subquery(): + """ + Tests that setting DefaultProfile.use_nested_with to False works properly. Setting this + method to return False should prevent a WITH clause from being used inside a sub-query. + """ + + class MockQueryProfile(DefaultProfile): + """ + A mock QueryProfile that sets use_nested_with to False and forces the + QueryGenerator to created nested queries. + """ + + def __init__(self): + super(MockQueryProfile, self).__init__() + + # Force the QueryGenerator to create nested queries + self._bounds['MAX_NESTED_QUERY_COUNT'] = (4, 4) + + # Force the QueryGenerator to use WITH clauses whenever possible + self._probabilities['OPTIONAL_QUERY_CLAUSES']['WITH'] = 1 + + # Force the QueryGenerator to create inline views whenever possible + self._probabilities['MISC']['INLINE_VIEW'] = 1 + + def use_nested_with(self): + return False + + mock_query_gen = QueryGenerator(MockQueryProfile()) + + # Create two tables + table_expr_list = TableExprList() + + right_table = Table("right_table") + right_table.add_col(Column("right_table", "right_col", Int)) + table_expr_list.append(right_table) + + left_table = Table("left_table") + left_table.add_col(Column("left_table", "left_col", Int)) + table_expr_list.append(left_table) + + # Check that each nested_query doesn't have a with clause + for nested_query in mock_query_gen.create_query(table_expr_list).nested_queries: + assert nested_query.with_clause is None \ No newline at end of file
