This is an automated email from the ASF dual-hosted git repository. krisztiankasa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new ac40207 HIVE-25590: Able to create views referencing temporary tables and materialized views (Krisztian Kasa, reviewed by Zoltan Haindrich, Alessandro Solimando) ac40207 is described below commit ac4020789149115c71bcee9b85577fff7dd01f46 Author: Krisztian Kasa <kasakri...@gmail.com> AuthorDate: Fri Oct 8 18:18:34 2021 +0200 HIVE-25590: Able to create views referencing temporary tables and materialized views (Krisztian Kasa, reviewed by Zoltan Haindrich, Alessandro Solimando) --- .../view/create/AbstractCreateViewAnalyzer.java | 21 +++++++------------ .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 24 ++++++++-------------- .../queries/clientnegative/create_view_failure11.q | 5 +++++ .../queries/clientnegative/create_view_failure12.q | 7 +++++++ .../materialized_view_create_failure.q | 12 +++++++++++ .../clientnegative/create_view_failure10.q.out | 2 +- ...failure10.q.out => create_view_failure11.q.out} | 2 +- ...failure10.q.out => create_view_failure12.q.out} | 10 ++++++++- .../materialized_view_create_failure.q.out | 17 +++++++++++++++ ...aterialized_view_no_transactional_rewrite.q.out | 2 +- 10 files changed, 68 insertions(+), 34 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java index 1f9f103..bd0f6bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/AbstractCreateViewAnalyzer.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.ddl.view.create; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -42,7 +41,7 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; /** * Abstract ancestor of analyzers that can create a view. */ -abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer { +public abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer { AbstractCreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -85,26 +84,20 @@ abstract class AbstractCreateViewAnalyzer extends BaseSemanticAnalyzer { } } - protected void validateTablesUsed(SemanticAnalyzer analyzer) throws SemanticException { + public static void validateTablesUsed(SemanticAnalyzer analyzer) throws SemanticException { // Do not allow view to be defined on temp table or other materialized view - Set<String> tableAliases = analyzer.getQB().getTabAliases(); - for (String alias : tableAliases) { - if (SemanticAnalyzer.DUMMY_TABLE.equals(alias)) { + for (TableScanOperator ts : analyzer.getTopOps().values()) { + Table table = ts.getConf().getTableMetadata(); + if (SemanticAnalyzer.DUMMY_TABLE.equals(table.getTableName())) { continue; } - Table table = null; - try { - table = analyzer.getTableObjectByName(analyzer.getQB().getTabNameForAlias(alias)); - } catch (HiveException ex) { - throw new SemanticException(ex); - } if (table.isTemporary()) { - throw new SemanticException("View definition references temporary table " + alias); + throw new SemanticException("View definition references temporary table " + table.getCompleteName()); } if (table.isMaterializedView()) { - throw new SemanticException("View definition references materialized view " + alias); + throw new SemanticException("View definition references materialized view " + table.getCompleteName()); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b210b94..b2dc3c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -23,12 +23,12 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.DYNAMICPARTITIONCONV import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS; +import static org.apache.hadoop.hive.ql.ddl.view.create.AbstractCreateViewAnalyzer.validateTablesUsed; import static org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.NON_FK_FILTERED; import java.io.FileNotFoundException; import java.io.IOException; import java.security.AccessControlException; -import java.time.ZoneId; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; @@ -13943,25 +13943,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { throws SemanticException { try { // Do not allow view to be defined on temp table or other materialized view - Set<String> tableAliases = qb.getTabAliases(); - for (String alias : tableAliases) { - try { - if (DUMMY_TABLE.equals(alias)) { + validateTablesUsed(this); + if (createVwDesc.isRewriteEnabled()) { + for (TableScanOperator ts : topOps.values()) { + Table table = ts.getConf().getTableMetadata(); + if (SemanticAnalyzer.DUMMY_TABLE.equals(table.getTableName())) { continue; } - Table table = getTableObjectByName(qb.getTabNameForAlias(alias)); - if (table.isTemporary()) { - throw new SemanticException("View definition references temporary table " + alias); - } - if (table.isMaterializedView()) { - throw new SemanticException("View definition references materialized view " + alias); - } - if (createVwDesc.isRewriteEnabled() && !AcidUtils.isTransactionalTable(table)) { + if (!AcidUtils.isTransactionalTable(table)) { throw new SemanticException("Automatic rewriting for materialized view cannot " - + "be enabled if the materialized view uses non-transactional tables"); + + "be enabled if the materialized view uses non-transactional tables"); } - } catch (HiveException ex) { - throw new SemanticException(ex); } } diff --git a/ql/src/test/queries/clientnegative/create_view_failure11.q b/ql/src/test/queries/clientnegative/create_view_failure11.q new file mode 100644 index 0000000..c164e3f --- /dev/null +++ b/ql/src/test/queries/clientnegative/create_view_failure11.q @@ -0,0 +1,5 @@ +-- CREATE VIEW should fail if it references a temp table in a subquery +create temporary table tmp1 (c1 string, c2 string); + +create view tmp1_view as +select subq.c1 from (select c1, c2 from tmp1) subq; diff --git a/ql/src/test/queries/clientnegative/create_view_failure12.q b/ql/src/test/queries/clientnegative/create_view_failure12.q new file mode 100644 index 0000000..32339d7 --- /dev/null +++ b/ql/src/test/queries/clientnegative/create_view_failure12.q @@ -0,0 +1,7 @@ +-- CREATE VIEW should fail if it joins temp tables +create temporary table tmp1 (c1 string, c2 string); +create temporary table tmp2 (c1 string, c2 string); + +create view tmp1_view as +select tmp1.c1 from tmp1 +join tmp2 on tmp1.c1 = tmp2.c1; diff --git a/ql/src/test/queries/clientnegative/materialized_view_create_failure.q b/ql/src/test/queries/clientnegative/materialized_view_create_failure.q new file mode 100644 index 0000000..c0eeea9 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_create_failure.q @@ -0,0 +1,12 @@ +-- CREATE MATERIALIZED VIEW should fail if it references a temp table +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +create temporary table t1 (a int) stored as orc tblproperties ('transactional'='true'); +create table t2 (a int) stored as orc tblproperties ('transactional'='true'); + +create materialized view mv1 as +select t2.a from t1 +join t2 on (t1.a = t2.a); + +drop materialized view mv1; diff --git a/ql/src/test/results/clientnegative/create_view_failure10.q.out b/ql/src/test/results/clientnegative/create_view_failure10.q.out index 5aedfd3..7ba0844 100644 --- a/ql/src/test/results/clientnegative/create_view_failure10.q.out +++ b/ql/src/test/results/clientnegative/create_view_failure10.q.out @@ -6,4 +6,4 @@ POSTHOOK: query: create temporary table tmp1 (c1 string, c2 string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmp1 -FAILED: SemanticException View definition references temporary table tmp1 +FAILED: SemanticException View definition references temporary table default@tmp1 diff --git a/ql/src/test/results/clientnegative/create_view_failure10.q.out b/ql/src/test/results/clientnegative/create_view_failure11.q.out similarity index 95% copy from ql/src/test/results/clientnegative/create_view_failure10.q.out copy to ql/src/test/results/clientnegative/create_view_failure11.q.out index 5aedfd3..7ba0844 100644 --- a/ql/src/test/results/clientnegative/create_view_failure10.q.out +++ b/ql/src/test/results/clientnegative/create_view_failure11.q.out @@ -6,4 +6,4 @@ POSTHOOK: query: create temporary table tmp1 (c1 string, c2 string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmp1 -FAILED: SemanticException View definition references temporary table tmp1 +FAILED: SemanticException View definition references temporary table default@tmp1 diff --git a/ql/src/test/results/clientnegative/create_view_failure10.q.out b/ql/src/test/results/clientnegative/create_view_failure12.q.out similarity index 53% copy from ql/src/test/results/clientnegative/create_view_failure10.q.out copy to ql/src/test/results/clientnegative/create_view_failure12.q.out index 5aedfd3..651f794 100644 --- a/ql/src/test/results/clientnegative/create_view_failure10.q.out +++ b/ql/src/test/results/clientnegative/create_view_failure12.q.out @@ -6,4 +6,12 @@ POSTHOOK: query: create temporary table tmp1 (c1 string, c2 string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@tmp1 -FAILED: SemanticException View definition references temporary table tmp1 +PREHOOK: query: create temporary table tmp2 (c1 string, c2 string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmp2 +POSTHOOK: query: create temporary table tmp2 (c1 string, c2 string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmp2 +FAILED: SemanticException View definition references temporary table default@tmp1 diff --git a/ql/src/test/results/clientnegative/materialized_view_create_failure.q.out b/ql/src/test/results/clientnegative/materialized_view_create_failure.q.out new file mode 100644 index 0000000..e420202 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_create_failure.q.out @@ -0,0 +1,17 @@ +PREHOOK: query: create temporary table t1 (a int) stored as orc tblproperties ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create temporary table t1 (a int) stored as orc tblproperties ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t2 (a int) stored as orc tblproperties ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: create table t2 (a int) stored as orc tblproperties ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +FAILED: SemanticException View definition references temporary table default@t1 diff --git a/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out b/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out index abb3d96..6384c96 100644 --- a/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out +++ b/ql/src/test/results/clientnegative/materialized_view_no_transactional_rewrite.q.out @@ -17,4 +17,4 @@ POSTHOOK: Output: default@cmv_basetable POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] -FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Automatic rewriting for materialized view cannot be enabled if the materialized view uses non-transactional tables +FAILED: SemanticException Automatic rewriting for materialized view cannot be enabled if the materialized view uses non-transactional tables