HIVE-14249: Add simple materialized views with manual rebuilds (1) (Alan Gates, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/438109cb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/438109cb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/438109cb Branch: refs/heads/hive-14535 Commit: 438109cb90f126e6043533bba060f166bf55c954 Parents: 7f8263e Author: Alan Gates <ga...@apache.org> Authored: Wed Jul 20 12:37:31 2016 +0100 Committer: Jesus Camacho Rodriguez <jcama...@apache.org> Committed: Thu Sep 15 15:14:44 2016 +0100 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 13 +- .../apache/hive/jdbc/HiveDatabaseMetaData.java | 2 + .../apache/hadoop/hive/metastore/TableType.java | 2 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 + .../apache/hadoop/hive/ql/QueryProperties.java | 15 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 75 +++-- .../apache/hadoop/hive/ql/metadata/Table.java | 9 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 12 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 2 +- .../apache/hadoop/hive/ql/parse/EximUtil.java | 2 +- .../apache/hadoop/hive/ql/parse/HiveParser.g | 27 ++ .../hive/ql/parse/LoadSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/ParseContext.java | 27 +- .../org/apache/hadoop/hive/ql/parse/QB.java | 16 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 203 ++++++++----- .../hadoop/hive/ql/parse/StorageFormat.java | 17 +- .../hadoop/hive/ql/parse/TaskCompiler.java | 105 ++++--- .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 7 + .../hadoop/hive/ql/plan/CreateViewDesc.java | 54 ++++ .../hadoop/hive/ql/plan/HiveOperation.java | 2 + .../hadoop/hive/ql/plan/LoadFileDesc.java | 10 +- .../apache/hadoop/hive/ql/plan/PlanUtils.java | 41 +++ .../authorization/plugin/HiveOperationType.java | 1 + .../plugin/sqlstd/Operation2Privilege.java | 4 + ...ialized_view_authorization_create_no_grant.q | 15 + ...d_view_authorization_create_no_select_perm.q | 14 + ...materialized_view_authorization_drop_other.q | 14 + ...rialized_view_authorization_no_select_perm.q | 14 + ...alized_view_authorization_rebuild_no_grant.q | 20 ++ ...erialized_view_authorization_rebuild_other.q | 14 + .../clientnegative/materialized_view_delete.q | 10 + .../clientnegative/materialized_view_insert.q | 6 + .../clientnegative/materialized_view_load.q | 7 + .../materialized_view_replace_with_view.q | 8 + .../clientnegative/materialized_view_update.q | 10 + .../test/queries/clientnegative/view_delete.q | 10 + .../test/queries/clientnegative/view_update.q | 10 + .../clientpositive/authorization_view_sqlstd.q | 86 ------ .../materialized_view_authorization_sqlstd.q | 58 ++++ .../clientpositive/materialized_view_create.q | 31 ++ .../clientpositive/materialized_view_drop.q | 7 + .../clientpositive/materialized_view_rebuild.q | 13 + .../clientpositive/view_authorization_sqlstd.q | 89 ++++++ ...zed_view_authorization_create_no_grant.q.out | 26 ++ ...ew_authorization_create_no_select_perm.q.out | 20 ++ ...rialized_view_authorization_drop_other.q.out | 30 ++ ...ized_view_authorization_no_select_perm.q.out | 30 ++ ...ed_view_authorization_rebuild_no_grant.q.out | 42 +++ ...lized_view_authorization_rebuild_other.q.out | 30 ++ .../materialized_view_delete.q.out | 19 ++ .../materialized_view_insert.q.out | 19 ++ .../clientnegative/materialized_view_load.q.out | 19 ++ .../materialized_view_replace_with_view.q.out | 30 ++ .../materialized_view_update.q.out | 19 ++ .../results/clientnegative/view_delete.q.out | 19 ++ .../results/clientnegative/view_update.q.out | 19 ++ .../authorization_view_sqlstd.q.out | 269 ----------------- ...materialized_view_authorization_sqlstd.q.out | 195 ++++++++++++ .../materialized_view_create.q.out | 173 +++++++++++ .../clientpositive/materialized_view_drop.q.out | 39 +++ .../materialized_view_rebuild.q.out | 67 +++++ .../view_authorization_sqlstd.q.out | 295 +++++++++++++++++++ .../cli/operation/ClassicTableTypeMapping.java | 5 + 63 files changed, 1931 insertions(+), 521 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d6944ee..d68c4fb 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1104,10 +1104,15 @@ public class HiveConf extends Configuration { HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile", new StringSet("TextFile", "SequenceFile", "RCfile", "ORC"), "Default file format for CREATE TABLE statement. Users can explicitly override it by CREATE TABLE ... STORED AS [FORMAT]"), HIVEDEFAULTMANAGEDFILEFORMAT("hive.default.fileformat.managed", "none", - new StringSet("none", "TextFile", "SequenceFile", "RCfile", "ORC"), - "Default file format for CREATE TABLE statement applied to managed tables only. External tables will be \n" + - "created with format specified by hive.default.fileformat. Leaving this null will result in using hive.default.fileformat \n" + - "for all tables."), + new StringSet("none", "TextFile", "SequenceFile", "RCfile", "ORC"), + "Default file format for CREATE TABLE statement applied to managed tables only. External tables will be \n" + + "created with format specified by hive.default.fileformat. Leaving this null will result in using hive.default.fileformat \n" + + "for all tables."), + HIVEMATERIALIZEDVIEWFILEFORMAT("hive.materializedview.fileformat", "ORC", + new StringSet("none", "TextFile", "SequenceFile", "RCfile", "ORC"), + "File format for CREATE MATERIALIZED VIEW statement."), + HIVEMATERIALIZEDVIEWSERDE("hive.materializedview.serde", + "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "Serde used for materialized views"), HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "SequenceFile", new StringSet("TextFile", "SequenceFile", "RCfile", "Llap"), "Default file format for storing result of the query."), HIVECHECKFILEFORMAT("hive.fileformat.check", true, "Whether to check file format or not when loading data files"), http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java b/jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java index 9d73470..fa984f4 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveDatabaseMetaData.java @@ -728,6 +728,8 @@ public class HiveDatabaseMetaData implements DatabaseMetaData { return "VIEW"; } else if (hivetabletype.equals(TableType.EXTERNAL_TABLE.toString())) { return "EXTERNAL TABLE"; + } else if (hivetabletype.equals(TableType.MATERIALIZED_VIEW.toString())) { + return "MATERIALIZED VIEW"; } else { return hivetabletype; } http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java b/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java index 56eeaa0..e9e16d7 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java @@ -22,5 +22,5 @@ package org.apache.hadoop.hive.metastore; * Typesafe enum for types of tables described by the metastore. */ public enum TableType { - MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, INDEX_TABLE + MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, INDEX_TABLE, MATERIALIZED_VIEW } http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 001f852..af1583f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -450,6 +450,9 @@ public enum ErrorMsg { ACID_NOT_ENOUGH_HISTORY(10327, "Not enough history available for ({0},{1}). " + "Oldest available base: {2}", true), INVALID_COLUMN_NAME(10328, "Invalid column name"), + REPLACE_VIEW_WITH_MATERIALIZED(10400, "Attempt to replace view {0} with materialized view", true), + REPLACE_MATERIALIZED_WITH_VIEW(10401, "Attempt to replace materialized view {0} with view", true), + UPDATE_DELETE_VIEW(10402, "You cannot update or delete records in a view"), //========================== 20000 range starts here ========================// SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."), SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. " http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index 3bc9432..650792b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -65,6 +65,8 @@ public class QueryProperties { private boolean multiDestQuery; private boolean filterWithSubQuery; + // True if this statement creates or replaces a materialized view + private boolean isMaterializedView; public boolean isQuery() { return query; @@ -260,6 +262,19 @@ public class QueryProperties { return this.filterWithSubQuery; } + /** + * True indicates this statement create or replaces a materialized view, not that it is a query + * against a materialized view. + * @return + */ + public boolean isMaterializedView() { + return isMaterializedView; + } + + public void setMaterializedView(boolean isMaterializedView) { + this.isMaterializedView = isMaterializedView; + } + public void clear() { query = false; analyzeCommand = false; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 569c19e..68d5fde 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4041,7 +4041,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable { // Get the existing table Table oldtbl = db.getTable(crtTbl.getLikeTableName()); Table tbl; - if (oldtbl.getTableType() == TableType.VIRTUAL_VIEW) { + if (oldtbl.getTableType() == TableType.VIRTUAL_VIEW || + oldtbl.getTableType() == TableType.MATERIALIZED_VIEW) { String targetTableName = crtTbl.getTableName(); tbl=db.newTable(targetTableName); @@ -4187,39 +4188,50 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int createView(Hive db, CreateViewDesc crtView) throws HiveException { Table oldview = db.getTable(crtView.getViewName(), false); if (crtView.getOrReplace() && oldview != null) { - // replace existing view - // remove the existing partition columns from the field schema - oldview.setViewOriginalText(crtView.getViewOriginalText()); - oldview.setViewExpandedText(crtView.getViewExpandedText()); - oldview.setFields(crtView.getSchema()); - if (crtView.getComment() != null) { - oldview.setProperty("comment", crtView.getComment()); - } - if (crtView.getTblProps() != null) { - oldview.getTTable().getParameters().putAll(crtView.getTblProps()); - } - oldview.setPartCols(crtView.getPartCols()); - if (crtView.getInputFormat() != null) { - oldview.setInputFormatClass(crtView.getInputFormat()); - } - if (crtView.getOutputFormat() != null) { - oldview.setOutputFormatClass(crtView.getOutputFormat()); - } - oldview.checkValidity(null); - try { - db.alterTable(crtView.getViewName(), oldview, null); - } catch (InvalidOperationException e) { - throw new HiveException(e); + if (!crtView.isMaterialized()) { + // replace existing view + // remove the existing partition columns from the field schema + oldview.setViewOriginalText(crtView.getViewOriginalText()); + oldview.setViewExpandedText(crtView.getViewExpandedText()); + oldview.setFields(crtView.getSchema()); + if (crtView.getComment() != null) { + oldview.setProperty("comment", crtView.getComment()); + } + if (crtView.getTblProps() != null) { + oldview.getTTable().getParameters().putAll(crtView.getTblProps()); + } + oldview.setPartCols(crtView.getPartCols()); + if (crtView.getInputFormat() != null) { + oldview.setInputFormatClass(crtView.getInputFormat()); + } + if (crtView.getOutputFormat() != null) { + oldview.setOutputFormatClass(crtView.getOutputFormat()); + } + oldview.checkValidity(null); + try { + db.alterTable(crtView.getViewName(), oldview, null); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } + work.getOutputs().add(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); + } else { + // This is a replace, so we need an exclusive lock + work.getOutputs().add(new WriteEntity(oldview, WriteEntity.WriteType.DDL_EXCLUSIVE)); } - work.getOutputs().add(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); } else { // create new view Table tbl = db.newTable(crtView.getViewName()); - tbl.setTableType(TableType.VIRTUAL_VIEW); + if (crtView.isMaterialized()) { + tbl.setTableType(TableType.MATERIALIZED_VIEW); + } else { + tbl.setTableType(TableType.VIRTUAL_VIEW); + } tbl.setSerializationLib(null); tbl.clearSerDeInfo(); tbl.setViewOriginalText(crtView.getViewOriginalText()); - tbl.setViewExpandedText(crtView.getViewExpandedText()); + if (!crtView.isMaterialized()) { + tbl.setViewExpandedText(crtView.getViewExpandedText()); + } tbl.setFields(crtView.getSchema()); if (crtView.getComment() != null) { tbl.setProperty("comment", crtView.getComment()); @@ -4239,6 +4251,15 @@ public class DDLTask extends Task<DDLWork> implements Serializable { tbl.setOutputFormatClass(crtView.getOutputFormat()); } + if (crtView.isMaterialized()) { + // Short circuit the checks that the input format is valid, this is configured for all + // materialized views and doesn't change so we don't need to check it constantly. + tbl.getSd().setInputFormat(crtView.getInputFormat()); + tbl.getSd().setOutputFormat(crtView.getOutputFormat()); + tbl.getSd().setSerdeInfo(new SerDeInfo(crtView.getSerde(), crtView.getSerde(), + Collections.<String, String>emptyMap())); + } + db.createTable(tbl, crtView.getIfNotExists()); work.getOutputs().add(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); } http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index e0d35d3..ea90889 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -209,8 +209,11 @@ public class Table implements Serializable { } if (isView()) { + assert (getViewOriginalText() != null); + assert (getViewExpandedText() != null); + } else if (isMaterializedView()) { assert(getViewOriginalText() != null); - assert(getViewExpandedText() != null); + assert(getViewExpandedText() == null); } else { assert(getViewOriginalText() == null); assert(getViewExpandedText() == null); @@ -824,6 +827,10 @@ public class Table implements Serializable { return TableType.VIRTUAL_VIEW.equals(getTableType()); } + public boolean isMaterializedView() { + return TableType.MATERIALIZED_VIEW.equals(getTableType()); + } + /** * @return whether this table is actually an index table */ http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index e0e9b12..dd86abd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1000,7 +1000,9 @@ public abstract class BaseSemanticAnalyzer { assert (ast.getToken().getType() == HiveParser.TOK_TAB || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION || ast.getToken().getType() == HiveParser.TOK_TABTYPE - || ast.getToken().getType() == HiveParser.TOK_CREATETABLE); + || ast.getToken().getType() == HiveParser.TOK_CREATETABLE + || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW + || ast.getToken().getType() == HiveParser.TOK_REBUILD_MATERIALIZED_VIEW); int childIndex = 0; numDynParts = 0; @@ -1012,7 +1014,9 @@ public abstract class BaseSemanticAnalyzer { tableName = conf.getVar(HiveConf.ConfVars.HIVETESTMODEPREFIX) + tableName; } - if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { + if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE && + ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW && + ast.getToken().getType() != HiveParser.TOK_REBUILD_MATERIALIZED_VIEW) { tableHandle = db.getTable(tableName); } } catch (InvalidTableException ite) { @@ -1024,7 +1028,9 @@ public abstract class BaseSemanticAnalyzer { } // get partition metadata if partition specified - if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE) { + if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE && + ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW && + ast.getToken().getType() != HiveParser.TOK_REBUILD_MATERIALIZED_VIEW) { childIndex = 1; ASTNode partspec = (ASTNode) ast.getChild(1); partitions = new ArrayList<Partition>(); http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 9329e00..988c58e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1160,7 +1160,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { } } - storageFormat.fillDefaultStorageFormat(false); + storageFormat.fillDefaultStorageFormat(false, false); if (indexTableName == null) { indexTableName = MetaStoreUtils.getIndexTableName(qTabName[0], qTabName[1], indexName); indexTableName = qTabName[0] + "." + indexTableName; // on same database with base table http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index a3fcaa0..167f7a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -127,7 +127,7 @@ public class EximUtil { } static void validateTable(org.apache.hadoop.hive.ql.metadata.Table table) throws SemanticException { - if (table.isView()) { + if (table.isView() || table.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } if (table.isNonNative()) { http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index df596ff..92dfc90 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -250,6 +250,8 @@ TOK_ALTERVIEW_DROPPROPERTIES; TOK_ALTERVIEW_ADDPARTS; TOK_ALTERVIEW_DROPPARTS; TOK_ALTERVIEW_RENAME; +TOK_REBUILD_MATERIALIZED_VIEW; +TOK_CREATE_MATERIALIZED_VIEW; TOK_VIEWPARTCOLS; TOK_EXPLAIN; TOK_EXPLAIN_SQ_REWRITE; @@ -784,7 +786,9 @@ ddlStatement | showStatement | metastoreCheck | createViewStatement + | createMaterializedViewStatement | dropViewStatement + | dropMaterializedViewStatement | createFunctionStatement | createMacroStatement | createIndexStatement @@ -1030,6 +1034,7 @@ alterStatement @after { popMsg(state); } : KW_ALTER KW_TABLE tableName alterTableStatementSuffix -> ^(TOK_ALTERTABLE tableName alterTableStatementSuffix) | KW_ALTER KW_VIEW tableName KW_AS? alterViewStatementSuffix -> ^(TOK_ALTERVIEW tableName alterViewStatementSuffix) + | KW_ALTER KW_MATERIALIZED KW_VIEW tableName KW_REBUILD -> ^(TOK_REBUILD_MATERIALIZED_VIEW tableName) | KW_ALTER KW_INDEX alterIndexStatementSuffix -> alterIndexStatementSuffix | KW_ALTER (KW_DATABASE|KW_SCHEMA) alterDatabaseStatementSuffix -> alterDatabaseStatementSuffix ; @@ -1778,6 +1783,22 @@ createViewStatement ) ; +createMaterializedViewStatement +@init { + pushMsg("create materialized view statement", state); +} +@after { popMsg(state); } + : KW_CREATE KW_MATERIALIZED KW_VIEW (ifNotExists)? name=tableName + tableComment? tablePropertiesPrefixed? + KW_AS selectStatementWithCTE + -> ^(TOK_CREATE_MATERIALIZED_VIEW $name + ifNotExists? + tableComment? + tablePropertiesPrefixed? + selectStatementWithCTE + ) + ; + viewPartition @init { pushMsg("view partition specification", state); } @after { popMsg(state); } @@ -1791,6 +1812,12 @@ dropViewStatement : KW_DROP KW_VIEW ifExists? viewName -> ^(TOK_DROPVIEW viewName ifExists?) ; +dropMaterializedViewStatement +@init { pushMsg("drop materialized view statement", state); } +@after { popMsg(state); } + : KW_DROP KW_MATERIALIZED KW_VIEW ifExists? viewName -> ^(TOK_DROPTABLE viewName ifExists?) + ; + showFunctionIdentifier @init { pushMsg("identifier for show function statement", state); } @after { popMsg(state); } http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index a49b813..a7005f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -199,7 +199,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer { // initialize destination table/partition TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree); - if (ts.tableHandle.isView()) { + if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } if (ts.tableHandle.isNonNative()) { http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 4353d3a..35f34da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; @@ -116,6 +117,7 @@ public class ParseContext { private AnalyzeRewriteContext analyzeRewrite; private CreateTableDesc createTableDesc; + private CreateViewDesc createViewDesc; private boolean reduceSinkAddedBySortedDynPartition; private Map<SelectOperator, Table> viewProjectToViewSchema; @@ -129,20 +131,14 @@ public class ParseContext { /** * @param conf - * @param qb - * current QB - * @param ast - * current parse tree * @param opToPartPruner * map from table scan operator to partition pruner * @param opToPartList * @param topOps * list of operators for the top query - * @param opParseCtx - * operator parse context - contains a mapping from operator to - * operator parse state (row resolver etc.) * @param joinOps * context needed join processing (map join specifically) + * @param smbMapJoinOps * @param loadTableWork * list of destination tables being loaded * @param loadFileWork @@ -154,13 +150,19 @@ public class ParseContext { * @param destTableId * @param listMapJoinOpsNoReducer * list of map join operators with no reducer - * @param groupOpToInputTables * @param prunedPartitions * @param opToSamplePruner * operator to sample pruner map * @param globalLimitCtx * @param nameToSplitSample * @param rootTasks + * @param opToPartToSkewedPruner + * @param viewAliasToInput + * @param reduceSinkOperatorsAddedByEnforceBucketingSorting + * @param analyzeRewrite + * @param createTableDesc + * @param createViewDesc + * @param queryProperties */ public ParseContext( QueryState queryState, @@ -183,8 +185,8 @@ public class ParseContext { Map<String, ReadEntity> viewAliasToInput, List<ReduceSinkOperator> reduceSinkOperatorsAddedByEnforceBucketingSorting, AnalyzeRewriteContext analyzeRewrite, CreateTableDesc createTableDesc, - QueryProperties queryProperties, Map<SelectOperator, Table> viewProjectToTableSchema, - Set<FileSinkDesc> acidFileSinks) { + CreateViewDesc createViewDesc, QueryProperties queryProperties, + Map<SelectOperator, Table> viewProjectToTableSchema, Set<FileSinkDesc> acidFileSinks) { this.queryState = queryState; this.conf = queryState.getConf(); this.opToPartPruner = opToPartPruner; @@ -213,6 +215,7 @@ public class ParseContext { reduceSinkOperatorsAddedByEnforceBucketingSorting; this.analyzeRewrite = analyzeRewrite; this.createTableDesc = createTableDesc; + this.createViewDesc = createViewDesc; this.queryProperties = queryProperties; this.viewProjectToViewSchema = viewProjectToTableSchema; this.needViewColumnAuthorization = viewProjectToTableSchema != null @@ -582,6 +585,10 @@ public class ParseContext { this.createTableDesc = createTableDesc; } + public CreateViewDesc getCreateViewDesc() { + return createViewDesc; + } + public void setReduceSinkAddedBySortedDynPartition( final boolean reduceSinkAddedBySortedDynPartition) { this.reduceSinkAddedBySortedDynPartition = reduceSinkAddedBySortedDynPartition; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java index 7e732f3..59d537f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; /** * Implementation of the query block. @@ -63,6 +64,9 @@ public class QB { private boolean insideView; private Set<String> aliasInsideView; + // If this is a materialized view, this stores the view descriptor + private CreateViewDesc viewDesc; + // used by PTFs /* * This map maintains the PTFInvocationSpec for each PTF chain invocation in this QB. @@ -404,6 +408,18 @@ public class QB { return havingClauseSubQueryPredicate; } + public CreateViewDesc getViewDesc() { + return viewDesc; + } + + public void setViewDesc(CreateViewDesc viewDesc) { + this.viewDesc = viewDesc; + } + + public boolean isMaterializedView() { + return viewDesc != null && viewDesc.isMaterialized(); + } + void addEncryptedTargetTablePath(Path p) { if(encryptedTargetTablePaths == null) { encryptedTargetTablePaths = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b071ddf..ab2966b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -469,7 +469,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { listMapJoinOpsNoReducer, prunedPartitions, tabNameToTabObject, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, - analyzeRewrite, tableDesc, queryProperties, viewProjectToTableSchema, acidFileSinks); + analyzeRewrite, tableDesc, createVwDesc, queryProperties, viewProjectToTableSchema, acidFileSinks); } public CompilationOpContext getOpContext() { @@ -2060,7 +2060,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { switch (ast.getToken().getType()) { case HiveParser.TOK_TAB: { TableSpec ts = new TableSpec(db, conf, ast); - if (ts.tableHandle.isView()) { + if (ts.tableHandle.isView() || ts.tableHandle.isMaterializedView()) { throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg()); } @@ -2098,7 +2098,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { if ((!qb.getParseInfo().getIsSubQ()) && (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) { - if (qb.isCTAS()) { + if (qb.isCTAS() || qb.isMaterializedView()) { qb.setIsQuery(false); ctx.setResDir(null); ctx.setResFile(null); @@ -4403,12 +4403,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { /** * Returns the GenericUDAFInfo struct for the aggregation. * - * @param aggName - * The name of the UDAF. + * @param evaluator + * @param emode * @param aggParameters * The exprNodeDesc of the original parameters - * @param aggTree - * The ASTNode node of the UDAF in the query. * @return GenericUDAFInfo * @throws SemanticException * when the UDAF is not found or has problems. @@ -4714,6 +4712,9 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * Generate the GroupByOperator for the Query Block (parseInfo.getXXX(dest)). * The new GroupByOperator will be a child of the reduceSinkOperatorInfo. * + * @param parseInfo + * @param dest + * @param reduceSinkOperatorInfo * @param mode * The mode of the aggregation (MERGEPARTIAL, PARTIAL2) * @param genericUDAFEvaluators @@ -4723,7 +4724,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * list of grouping sets * @param groupingSetsPresent * whether grouping sets are present in this query - * @param groupingSetsConsumedCurrentMR + * @param groupingSetsNeedAdditionalMRJob * whether grouping sets are consumed by this group by * @return the new GroupByOperator */ @@ -6796,10 +6797,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // table command rather than taking the default value List<FieldSchema> field_schemas = null; CreateTableDesc tblDesc = qb.getTableDesc(); + CreateViewDesc viewDesc = qb.getViewDesc(); if (tblDesc != null) { field_schemas = new ArrayList<FieldSchema>(); destTableIsTemporary = tblDesc.isTemporary(); destTableIsMaterialization = tblDesc.isMaterialization(); + } else if (viewDesc != null) { + field_schemas = new ArrayList<FieldSchema>(); + destTableIsTemporary = false; } boolean first = true; @@ -6858,6 +6863,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // update the create table descriptor with the resulting schema. if (tblDesc != null) { tblDesc.setCols(new ArrayList<FieldSchema>(field_schemas)); + } else if (viewDesc != null) { + viewDesc.setSchema(new ArrayList<FieldSchema>(field_schemas)); } boolean isDestTempFile = true; @@ -6869,11 +6876,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE); - loadFileWork.add(new LoadFileDesc(tblDesc, queryTmpdir, dest_path, isDfsDir, cols, + loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc, queryTmpdir, dest_path, isDfsDir, cols, colTypes)); if (tblDesc == null) { - if (qb.getIsQuery()) { + if (viewDesc != null) { + table_desc = PlanUtils.getTableDesc(viewDesc, cols, colTypes); + } else if (qb.getIsQuery()) { String fileFormat; if (SessionState.get().getIsUsingThriftJDBCBinarySerDe()) { fileFormat = "SequenceFile"; @@ -10434,7 +10443,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * * @param source * @param dest - * @param outputColNames + * @param colExprMap + * @param outputInternalColNames * - a list to which the new internal column names will be added, in * the same order as in the dest row resolver */ @@ -10687,10 +10697,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } // 3. analyze create view command - if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW - || (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { + if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW || + ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW || + ast.getToken().getType() == HiveParser.TOK_REBUILD_MATERIALIZED_VIEW || + (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && + ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { child = analyzeCreateView(ast, qb); - queryState.setCommandType(HiveOperation.CREATEVIEW); if (child == null) { return false; } @@ -10835,7 +10847,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { listMapJoinOpsNoReducer, prunedPartitions, tabNameToTabObject, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, - analyzeRewrite, tableDesc, queryProperties, viewProjectToTableSchema, acidFileSinks); + analyzeRewrite, tableDesc, createVwDesc, queryProperties, viewProjectToTableSchema, acidFileSinks); // 5. Take care of view creation if (createVwDesc != null) { @@ -10846,32 +10858,34 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // validate the create view statement at this point, the createVwDesc gets // all the information for semanticcheck - validateCreateView(createVwDesc); + validateCreateView(); - // Since we're only creating a view (not executing it), we don't need to - // optimize or translate the plan (and in fact, those procedures can - // interfere with the view creation). So skip the rest of this method. - ctx.setResDir(null); - ctx.setResFile(null); + if (!createVwDesc.isMaterialized()) { + // Since we're only creating a view (not executing it), we don't need to + // optimize or translate the plan (and in fact, those procedures can + // interfere with the view creation). So skip the rest of this method. + ctx.setResDir(null); + ctx.setResFile(null); - try { - PlanUtils.addInputsForView(pCtx); - } catch (HiveException e) { - throw new SemanticException(e); - } + try { + PlanUtils.addInputsForView(pCtx); + } catch (HiveException e) { + throw new SemanticException(e); + } - // Generate lineage info for create view statements - // if LineageLogger hook is configured. - if (HiveConf.getVar(conf, HiveConf.ConfVars.POSTEXECHOOKS).contains( - "org.apache.hadoop.hive.ql.hooks.LineageLogger")) { - ArrayList<Transform> transformations = new ArrayList<Transform>(); - transformations.add(new HiveOpConverterPostProc()); - transformations.add(new Generator()); - for (Transform t : transformations) { - pCtx = t.transform(pCtx); + // Generate lineage info for create view statements + // if LineageLogger hook is configured. + if (HiveConf.getVar(conf, HiveConf.ConfVars.POSTEXECHOOKS).contains( + "org.apache.hadoop.hive.ql.hooks.LineageLogger")) { + ArrayList<Transform> transformations = new ArrayList<Transform>(); + transformations.add(new HiveOpConverterPostProc()); + transformations.add(new Generator()); + for (Transform t : transformations) { + pCtx = t.transform(pCtx); + } } + return; } - return; } // 6. Generate table access stats if required @@ -11002,6 +11016,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { private void saveViewDefinition() throws SemanticException { + if (createVwDesc.isMaterialized() && createVwDesc.getOrReplace()) { + // This is a rebuild, there's nothing to do here. + return; + } + // Make a copy of the statement's result schema, since we may // modify it below as part of imposing view column names. List<FieldSchema> derivedSchema = @@ -11112,7 +11131,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } createVwDesc.setSchema(derivedSchema); - createVwDesc.setViewExpandedText(expandedText); + if (!createVwDesc.isMaterialized()) { + // materialized views don't store the expanded text as they won't be rewritten at query time. + createVwDesc.setViewExpandedText(expandedText); + } } static List<FieldSchema> convertRowSchemaToViewSchema(RowResolver rr) throws SemanticException { @@ -11496,8 +11518,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { RowFormatParams rowFormatParams = new RowFormatParams(); StorageFormat storageFormat = new StorageFormat(conf); - LOG.info("Creating table " + dbDotTab + " position=" - + ast.getCharPositionInLine()); + LOG.info("Creating table " + dbDotTab + " position=" + ast.getCharPositionInLine()); int numCh = ast.getChildCount(); /* @@ -11539,6 +11560,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { command_type = CTLT; } break; + case HiveParser.TOK_QUERY: // CTAS if (command_type == CTLT) { throw new SemanticException(ErrorMsg.CTAS_CTLT_COEXISTENCE.getMsg()); @@ -11629,7 +11651,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException("Unrecognized command."); } - storageFormat.fillDefaultStorageFormat(isExt); + storageFormat.fillDefaultStorageFormat(isExt, false); if ((command_type == CTAS) && (storageFormat.getStorageHandler() != null)) { throw new SemanticException(ErrorMsg.CREATE_NON_NATIVE_AS.getMsg()); @@ -11782,6 +11804,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { qb.setTableDesc(tableDesc); return selectStmt; + default: throw new SemanticException("Unrecognized command."); } @@ -11797,8 +11820,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { outputs.add(new WriteEntity(t, WriteEntity.WriteType.DDL_NO_LOCK)); } - private ASTNode analyzeCreateView(ASTNode ast, QB qb) - throws SemanticException { + private ASTNode analyzeCreateView(ASTNode ast, QB qb) throws SemanticException { String[] qualTabName = getQualifiedTableName((ASTNode) ast.getChild(0)); String dbDotTable = getDotName(qualTabName); List<FieldSchema> cols = null; @@ -11809,6 +11831,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { ASTNode selectStmt = null; Map<String, String> tblProps = null; List<String> partColNames = null; + boolean isRebuild = ast.getToken().getType() == HiveParser.TOK_REBUILD_MATERIALIZED_VIEW; + boolean isMaterialized = ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW + || isRebuild; + StorageFormat storageFormat = storageFormat = new StorageFormat(conf); + storageFormat.fillDefaultStorageFormat(false, isMaterialized); LOG.info("Creating view " + dbDotTable + " position=" + ast.getCharPositionInLine()); @@ -11852,18 +11879,40 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { orReplace = true; } - StorageFormat defaultFmt = new StorageFormat(conf); - defaultFmt.fillDefaultStorageFormat(false); - createVwDesc = new CreateViewDesc( - dbDotTable, cols, comment, defaultFmt.getInputFormat(), - defaultFmt.getOutputFormat(), tblProps, partColNames, - ifNotExists, orReplace, isAlterViewAs); - unparseTranslator.enable(); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - createVwDesc), conf)); - addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW); + if (isMaterialized) { + createVwDesc = new CreateViewDesc(dbDotTable, cols, comment, tblProps, partColNames, + ifNotExists, orReplace || isRebuild, isAlterViewAs, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), storageFormat.getSerde()); + addDbAndTabToOutputs(qualTabName, TableType.MATERIALIZED_VIEW); + queryState.setCommandType(HiveOperation.CREATE_MATERIALIZED_VIEW); + qb.setViewDesc(createVwDesc); + } else { + createVwDesc = new CreateViewDesc( + dbDotTable, cols, comment, tblProps, partColNames, + ifNotExists, orReplace, isAlterViewAs, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), storageFormat.getSerde()); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), + createVwDesc), conf)); + + addDbAndTabToOutputs(qualTabName, TableType.VIRTUAL_VIEW); + queryState.setCommandType(HiveOperation.CREATEVIEW); + } + + if (isRebuild) { + // We need to go lookup the table and get the select statement and then parse it. + try { + Table tab = db.getTable(qualTabName[0], qualTabName[1]); + String viewText = tab.getViewOriginalText(); + ParseDriver pd = new ParseDriver(); + ASTNode tree = pd.parse(viewText, ctx, false); + selectStmt = ParseUtils.findRootNonNullToken(tree); + } catch (Exception e) { + throw new SemanticException(e); + } + } + return selectStmt; } @@ -11874,7 +11923,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // validate the create view statement // the statement could be CREATE VIEW, REPLACE VIEW, or ALTER VIEW AS SELECT // check semantic conditions - private void validateCreateView(CreateViewDesc createVwDesc) + private void validateCreateView() throws SemanticException { try { Table oldView = getTable(createVwDesc.getViewName(), false); @@ -11903,8 +11952,21 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { //replace view if (createVwDesc.getOrReplace() && oldView != null) { + // Don't allow swapping between virtual and materialized view in replace + if (oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && createVwDesc.isMaterialized()) { + throw new SemanticException(ErrorMsg.REPLACE_VIEW_WITH_MATERIALIZED, + oldView.getTableName()); + } + + if (oldView.getTableType().equals(TableType.MATERIALIZED_VIEW) && + !createVwDesc.isMaterialized()) { + throw new SemanticException(ErrorMsg.REPLACE_MATERIALIZED_WITH_VIEW, + oldView.getTableName()); + } + // Existing table is not a view - if (!oldView.getTableType().equals(TableType.VIRTUAL_VIEW)) { + if (!oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && + !oldView.getTableType().equals(TableType.MATERIALIZED_VIEW)) { String tableNotViewErrorMsg = "The following is an existing table, not a view: " + createVwDesc.getViewName(); @@ -11912,26 +11974,28 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { ErrorMsg.EXISTING_TABLE_IS_NOT_VIEW.getMsg(tableNotViewErrorMsg)); } - // if old view has partitions, it could not be replaced - String partitionViewErrorMsg = - "The following view has partition, it could not be replaced: " + - createVwDesc.getViewName(); - try { - if ((createVwDesc.getPartCols() == null || - createVwDesc.getPartCols().isEmpty() || - !createVwDesc.getPartCols().equals(oldView.getPartCols())) && - !oldView.getPartCols().isEmpty() && - !db.getPartitions(oldView).isEmpty()) { + if (!createVwDesc.isMaterialized()) { + // if old view has partitions, it could not be replaced + String partitionViewErrorMsg = + "The following view has partition, it could not be replaced: " + + createVwDesc.getViewName(); + try { + if ((createVwDesc.getPartCols() == null || + createVwDesc.getPartCols().isEmpty() || + !createVwDesc.getPartCols().equals(oldView.getPartCols())) && + !oldView.getPartCols().isEmpty() && + !db.getPartitions(oldView).isEmpty()) { + throw new SemanticException( + ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); + } + } catch (HiveException e) { throw new SemanticException( - ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); + ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); } - } catch (HiveException e) { - throw new SemanticException( - ErrorMsg.REPLACE_VIEW_WITH_PARTITION.getMsg(partitionViewErrorMsg)); } } } catch (HiveException e) { - throw new SemanticException(e.getMessage()); + throw new SemanticException(e.getMessage(), e); } } @@ -13040,6 +13104,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() && !qb.getParseInfo().getDestToOrderBy().isEmpty()); queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit()); + queryProperties.setMaterializedView(qb.getViewDesc() != null); } } private void warn(String msg) { http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 48aca4d..d3b955c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -104,13 +104,22 @@ public class StorageFormat { } } - protected void fillDefaultStorageFormat(boolean isExternal) throws SemanticException { + protected void fillDefaultStorageFormat(boolean isExternal, boolean isMaterializedView) + throws SemanticException { if ((inputFormat == null) && (storageHandler == null)) { - String defaultFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT); - String defaultManagedFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTMANAGEDFILEFORMAT); + String defaultFormat; + String defaultManagedFormat; + if (isMaterializedView) { + defaultFormat = defaultManagedFormat = + HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMATERIALIZEDVIEWFILEFORMAT); + serde = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMATERIALIZEDVIEWSERDE); + } else { + defaultFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT); + defaultManagedFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEDEFAULTMANAGEDFILEFORMAT); + } if (!isExternal && !"none".equals(defaultManagedFormat)) { - defaultFormat = defaultManagedFormat; + defaultFormat = defaultManagedFormat; } if (StringUtils.isBlank(defaultFormat)) { http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index fb2b992..f781390 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContex import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; +import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.LoadFileDesc; @@ -76,8 +77,6 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.thrift.ThriftFormatter; import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe; -import akka.util.Collections; - import com.google.common.collect.Interner; import com.google.common.collect.Interners; @@ -226,18 +225,25 @@ public abstract class TaskCompiler { boolean oneLoadFile = true; for (LoadFileDesc lfd : loadFileWork) { - if (pCtx.getQueryProperties().isCTAS()) { + if (pCtx.getQueryProperties().isCTAS() || pCtx.getQueryProperties().isMaterializedView()) { assert (oneLoadFile); // should not have more than 1 load file for // CTAS // make the movetask's destination directory the table's destination. Path location; - String loc = pCtx.getCreateTable().getLocation(); + String loc = pCtx.getQueryProperties().isCTAS() ? + pCtx.getCreateTable().getLocation() : null; if (loc == null) { // get the table's default location + // Always use default location for materialized view Path targetPath; try { - String[] names = Utilities.getDbTableName( - pCtx.getCreateTable().getTableName()); + String protoName = null; + if (pCtx.getQueryProperties().isCTAS()) { + protoName = pCtx.getCreateTable().getTableName(); + } else if (pCtx.getQueryProperties().isMaterializedView()) { + protoName = pCtx.getCreateViewDesc().getViewName(); + } + String[] names = Utilities.getDbTableName(protoName); if (!db.databaseExists(names[0])) { throw new SemanticException("ERROR: The database " + names[0] + " does not exist."); @@ -314,41 +320,15 @@ public abstract class TaskCompiler { crtTblDesc.validate(conf); - // clear the mapredWork output file from outputs for CTAS - // DDLWork at the tail of the chain will have the output - Iterator<WriteEntity> outIter = outputs.iterator(); - while (outIter.hasNext()) { - switch (outIter.next().getType()) { - case DFS_DIR: - case LOCAL_DIR: - outIter.remove(); - break; - default: - break; - } - } Task<? extends Serializable> crtTblTask = TaskFactory.get(new DDLWork( inputs, outputs, crtTblDesc), conf); - - // find all leaf tasks and make the DDLTask as a dependent task of all of - // them - HashSet<Task<? extends Serializable>> leaves = new LinkedHashSet<Task<? extends Serializable>>(); - getLeafTasks(rootTasks, leaves); - assert (leaves.size() > 0); - for (Task<? extends Serializable> task : leaves) { - if (task instanceof StatsTask) { - // StatsTask require table to already exist - for (Task<? extends Serializable> parentOfStatsTask : task.getParentTasks()) { - parentOfStatsTask.addDependentTask(crtTblTask); - } - for (Task<? extends Serializable> parentOfCrtTblTask : crtTblTask.getParentTasks()) { - parentOfCrtTblTask.removeDependentTask(task); - } - crtTblTask.addDependentTask(task); - } else { - task.addDependentTask(crtTblTask); - } - } + patchUpAfterCTASorMaterializedView(rootTasks, outputs, crtTblTask); + } else if (pCtx.getQueryProperties().isMaterializedView()) { + // generate a DDL task and make it a dependent task of the leaf + CreateViewDesc viewDesc = pCtx.getCreateViewDesc(); + Task<? extends Serializable> crtViewTask = TaskFactory.get(new DDLWork( + inputs, outputs, viewDesc), conf); + patchUpAfterCTASorMaterializedView(rootTasks, outputs, crtViewTask); } if (globalLimitCtx.isEnable() && pCtx.getFetchTask() != null) { @@ -376,6 +356,44 @@ public abstract class TaskCompiler { } } + private void patchUpAfterCTASorMaterializedView(final List<Task<? extends Serializable>> rootTasks, + final HashSet<WriteEntity> outputs, + Task<? extends Serializable> createTask) { + // clear the mapredWork output file from outputs for CTAS + // DDLWork at the tail of the chain will have the output + Iterator<WriteEntity> outIter = outputs.iterator(); + while (outIter.hasNext()) { + switch (outIter.next().getType()) { + case DFS_DIR: + case LOCAL_DIR: + outIter.remove(); + break; + default: + break; + } + } + + // find all leaf tasks and make the DDLTask as a dependent task of all of + // them + HashSet<Task<? extends Serializable>> leaves = + new LinkedHashSet<Task<? extends Serializable>>(); + getLeafTasks(rootTasks, leaves); + assert (leaves.size() > 0); + for (Task<? extends Serializable> task : leaves) { + if (task instanceof StatsTask) { + // StatsTask require table to already exist + for (Task<? extends Serializable> parentOfStatsTask : task.getParentTasks()) { + parentOfStatsTask.addDependentTask(createTask); + } + for (Task<? extends Serializable> parentOfCrtTblTask : createTask.getParentTasks()) { + parentOfCrtTblTask.removeDependentTask(task); + } + createTask.addDependentTask(task); + } else { + task.addDependentTask(createTask); + } + } + } /** * A helper function to generate a column stats task on top of map-red task. The column stats @@ -385,7 +403,11 @@ public abstract class TaskCompiler { * This method generates a plan with a column stats task on top of map-red task and sets up the * appropriate metadata to be used during execution. * - * @param qb + * @param analyzeRewrite + * @param loadTableWork + * @param loadFileWork + * @param rootTasks + * @param outerQueryLimit */ @SuppressWarnings("unchecked") protected void genColumnStatsTask(AnalyzeRewriteContext analyzeRewrite, @@ -494,7 +516,8 @@ public abstract class TaskCompiler { pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks, pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(), pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(), - pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getQueryProperties(), pCtx.getViewProjectToTableSchema(), + pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getCreateViewDesc(), + pCtx.getQueryProperties(), pCtx.getViewProjectToTableSchema(), pCtx.getAcidSinks()); clone.setFetchTask(pCtx.getFetchTask()); clone.setLineageInfo(pCtx.getLineageInfo()); http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java index 34d83ef..5b874e4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -141,6 +142,12 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer { throw new SemanticException(e.getMessage(), e); } + if (mTable.getTableType() == TableType.VIRTUAL_VIEW || + mTable.getTableType() == TableType.MATERIALIZED_VIEW) { + LOG.error("Table " + getDotName(tableName) + " is a view or materialized view"); + throw new SemanticException(ErrorMsg.UPDATE_DELETE_VIEW.getMsg()); + } + List<FieldSchema> partCols = mTable.getPartCols(); List<String> bucketingCols = mTable.getBucketCols(); http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java index 81c4f77..3ddbb1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java @@ -48,6 +48,8 @@ public class CreateViewDesc extends DDLDesc implements Serializable { private boolean ifNotExists; private boolean orReplace; private boolean isAlterViewAs; + private boolean isMaterialized; + private String serde; // only used for materialized views /** * For serialization only. @@ -55,6 +57,17 @@ public class CreateViewDesc extends DDLDesc implements Serializable { public CreateViewDesc() { } + /** + * Used to create a virtual view descriptor. + * @param viewName + * @param schema + * @param comment + * @param tblProps + * @param partColNames + * @param ifNotExists + * @param orReplace + * @param isAlterViewAs + */ public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, String inputFormat, String outputFormat, Map<String, String> tblProps, @@ -70,6 +83,39 @@ public class CreateViewDesc extends DDLDesc implements Serializable { this.ifNotExists = ifNotExists; this.orReplace = orReplace; this.isAlterViewAs = isAlterViewAs; + this.isMaterialized = false; + } + + /** + * Used to create a materialized view descriptor + * @param viewName + * @param schema + * @param comment + * @param tblProps + * @param partColNames + * @param ifNotExists + * @param orReplace + * @param isAlterViewAs + * @param inputFormat + * @param outputFormat + * @param serde + */ + public CreateViewDesc(String viewName, List<FieldSchema> schema, String comment, + Map<String, String> tblProps, List<String> partColNames, + boolean ifNotExists, boolean orReplace, boolean isAlterViewAs, + String inputFormat, String outputFormat, String serde) { + this.viewName = viewName; + this.schema = schema; + this.tblProps = tblProps; + this.partColNames = partColNames; + this.comment = comment; + this.ifNotExists = ifNotExists; + this.orReplace = orReplace; + this.isAlterViewAs = isAlterViewAs; + this.isMaterialized = true; + this.inputFormat = inputFormat; + this.outputFormat = outputFormat; + this.serde = serde; } @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) @@ -194,4 +240,12 @@ public class CreateViewDesc extends DDLDesc implements Serializable { this.outputFormat = outputFormat; } + public boolean isMaterialized() { + return isMaterialized; + } + + public String getSerde() { + return serde; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java index d2746ae..d5b1c9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java @@ -78,6 +78,8 @@ public enum HiveOperation { CREATEMACRO("CREATEMACRO", null, null), DROPMACRO("DROPMACRO", null, null), CREATEVIEW("CREATEVIEW", new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.CREATE}), + CREATE_MATERIALIZED_VIEW("CREATEVIEW", new Privilege[]{Privilege.SELECT}, new + Privilege[]{Privilege.CREATE}), DROPVIEW("DROPVIEW", null, new Privilege[]{Privilege.DROP}), CREATEINDEX("CREATEINDEX", null, null), DROPINDEX("DROPINDEX", null, null), http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java index df153a2..9a868a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java @@ -39,15 +39,19 @@ public class LoadFileDesc extends LoadDesc implements Serializable { public LoadFileDesc() { } - public LoadFileDesc(final CreateTableDesc createTableDesc, final Path sourcePath, - final Path targetDir, - final boolean isDfsDir, final String columns, final String columnTypes) { + public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc, + final Path sourcePath, final Path targetDir, final boolean isDfsDir, + final String columns, final String columnTypes) { this(sourcePath, targetDir, isDfsDir, columns, columnTypes); if (createTableDesc != null && createTableDesc.getDatabaseName() != null && createTableDesc.getTableName() != null) { destinationCreateTable = (createTableDesc.getTableName().contains(".") ? "" : createTableDesc .getDatabaseName() + ".") + createTableDesc.getTableName(); + } else if (createViewDesc != null) { + // The work is already done in analyzeCreateView to assure that the view name is fully + // qualified. + destinationCreateTable = createViewDesc.getViewName(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 5dc3aa6..b15ad34 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -380,6 +380,47 @@ public final class PlanUtils { return ret; } + /** + * Generate a table descriptor from a createViewDesc. + */ + public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, String colTypes) { + TableDesc ret; + + try { + Class serdeClass = JavaUtils.loadClass(crtViewDesc.getSerde()); + ret = getTableDesc(serdeClass, new String(LazySerDeParameters.DefaultSeparators), cols, + colTypes, false, false); + + // set other table properties + /* + TODO - I don't think I need any of this + Properties properties = ret.getProperties(); + + if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) { + properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, + crtTblDesc.getTableName()); + } + + if (crtTblDesc.getTblProps() != null) { + properties.putAll(crtTblDesc.getTblProps()); + } + */ + + // replace the default input & output file format with those found in + // crtTblDesc + Class<? extends InputFormat> inClass = + (Class<? extends InputFormat>)JavaUtils.loadClass(crtViewDesc.getInputFormat()); + Class<? extends HiveOutputFormat> outClass = + (Class<? extends HiveOutputFormat>)JavaUtils.loadClass(crtViewDesc.getOutputFormat()); + + ret.setInputFileFormatClass(inClass); + ret.setOutputFileFormatClass(outClass); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to find class in getTableDesc: " + e.getMessage(), e); + } + return ret; + } + /** * Generate the table descriptor of MetadataTypedColumnsetSerDe with the * separatorCode. MetaDataTypedColumnsetSerDe is used because LazySimpleSerDe http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index c507f67..884b129 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -82,6 +82,7 @@ public enum HiveOperationType { CREATEMACRO, DROPMACRO, CREATEVIEW, + CREATE_MATERIALIZED_VIEW, DROPVIEW, CREATEINDEX, DROPINDEX, http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java index bbe28ab..ddf1e66 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java @@ -332,6 +332,10 @@ public class Operation2Privilege { new PrivRequirement(SEL_GRANT_AR, IOType.INPUT), new PrivRequirement(OWNER_PRIV_AR, HivePrivilegeObjectType.DATABASE))); + op2Priv.put(HiveOperationType.CREATE_MATERIALIZED_VIEW, PrivRequirement.newPrivRequirementList( + new PrivRequirement(SEL_GRANT_AR, IOType.INPUT), + new PrivRequirement(OWNER_PRIV_AR, HivePrivilegeObjectType.DATABASE))); + op2Priv.put(HiveOperationType.SHOWFUNCTIONS, PrivRequirement.newIOPrivRequirement (null, null)); op2Priv.put(HiveOperationType.SHOWINDEXES, PrivRequirement.newIOPrivRequirement http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_grant.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_grant.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_grant.q new file mode 100644 index 0000000..079baff --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_grant.q @@ -0,0 +1,15 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvcng_gtable (a int, b varchar(256), c decimal(10,2)); + +insert into amvcng_gtable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +grant select on table amvcng_gtable to user user2; + +set user.name=user2; + +create materialized view amvcng_gmat_view as select a, c from amvcng_gtable; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_select_perm.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_select_perm.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_select_perm.q new file mode 100644 index 0000000..4de525c --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_create_no_select_perm.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvnsp_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + + +set user.name=user2; + +create materialized view amvnsp_mat_view as select a, c from amvnsp_table; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_authorization_drop_other.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_drop_other.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_drop_other.q new file mode 100644 index 0000000..c9a5930 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_drop_other.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvdo_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvdo_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvdo_mat_view as select a, c from amvdo_table; + +set user.name=user2; +drop materialized view amvdo_mat_view; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_authorization_no_select_perm.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_no_select_perm.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_no_select_perm.q new file mode 100644 index 0000000..8428152 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_no_select_perm.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvnsp_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvnsp_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvnsp_mat_view as select a, c from amvnsp_table; + +set user.name=user2; +select * from amvnsp_mat_view; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q new file mode 100644 index 0000000..a2e7d38 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q @@ -0,0 +1,20 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvrng_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +grant select on table amvrng_table to user user2 with grant option; + +set user.name=user2; +create materialized view amvrng_mat_view as select a, c from amvrng_table; + +set user.name=user1; +revoke grant option for select on table amvrng_table from user user2; + +set user.name=user2; +alter materialized view amvrng_mat_view rebuild; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q new file mode 100644 index 0000000..7c2d145 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvro_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvro_mat_view as select a, c from amvro_table; + +set user.name=user2; +alter materialized view amvro_mat_view rebuild; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_delete.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_delete.q b/ql/src/test/queries/clientnegative/materialized_view_delete.q new file mode 100644 index 0000000..f557df9 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_delete.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table dmv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view dmv_mat_view as select a, b, c from dmv_basetable; + +delete from dmv_mat_view where b = 'fred'; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_insert.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_insert.q b/ql/src/test/queries/clientnegative/materialized_view_insert.q new file mode 100644 index 0000000..2daae52 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_insert.q @@ -0,0 +1,6 @@ +create table imv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view imv_mat_view as select a, b, c from imv_basetable; + +insert into imv_mat_view values (1, 'fred', 3.14); http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_load.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_load.q b/ql/src/test/queries/clientnegative/materialized_view_load.q new file mode 100644 index 0000000..a8d0dfd --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_load.q @@ -0,0 +1,7 @@ +create table lmv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view lmv_mat_view as select a, b, c from lmv_basetable; + +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE lmv_mat_view; + http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_replace_with_view.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_replace_with_view.q b/ql/src/test/queries/clientnegative/materialized_view_replace_with_view.q new file mode 100644 index 0000000..7c972f5 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_replace_with_view.q @@ -0,0 +1,8 @@ +create table rmvwv_basetable (a int, b varchar(256), c decimal(10,2)); + +insert into rmvwv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view rmvwv_mat_view as select a, b, c from rmvwv_basetable; + +create or replace view rmvwv_mat_view as select a, c from rmvwv_basetable; + http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/materialized_view_update.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_update.q b/ql/src/test/queries/clientnegative/materialized_view_update.q new file mode 100644 index 0000000..8245ef0 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_update.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table umv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view umv_mat_view as select a, b, c from umv_basetable; + +update umv_mat_view set b = 'joe' where b = 'fred'; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/view_delete.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/view_delete.q b/ql/src/test/queries/clientnegative/view_delete.q new file mode 100644 index 0000000..c9dc985 --- /dev/null +++ b/ql/src/test/queries/clientnegative/view_delete.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table dv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create view dv_view as select a, b, c from dv_basetable; + +delete from dv_view where b = 'fred'; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientnegative/view_update.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/view_update.q b/ql/src/test/queries/clientnegative/view_update.q new file mode 100644 index 0000000..3a54ccb --- /dev/null +++ b/ql/src/test/queries/clientnegative/view_update.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table uv_basetable (a int, b varchar(256), c decimal(10,2)); + + +create view uv_view as select a, b, c from uv_basetable; + +update uv_view set b = 'joe' where b = 'fred'; http://git-wip-us.apache.org/repos/asf/hive/blob/438109cb/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q b/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q deleted file mode 100644 index 14044bf..0000000 --- a/ql/src/test/queries/clientpositive/authorization_view_sqlstd.q +++ /dev/null @@ -1,86 +0,0 @@ -set hive.test.authz.sstd.hs2.mode=true; -set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; -set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; -set hive.security.authorization.enabled=true; -set user.name=user1; - --- Test view authorization , and 'show grant' variants - -create table t1(i int, j int, k int); -show grant user user1 on table t1; - --- protecting certain columns -create view vt1 as select i,k from t1; - --- protecting certain rows -create view vt2 as select * from t1 where i > 1; - -show grant user user1 on all; - ---view grant to user --- try with and without table keyword - -grant select on vt1 to user user2; -grant insert on table vt1 to user user3; - -set user.name=user2; -show grant user user2 on table vt1; -set user.name=user3; -show grant user user3 on table vt1; - - -set user.name=user2; - -explain authorization select * from vt1; -select * from vt1; - --- verify input objects required does not include table --- even if view is within a sub query -select * from (select * from vt1) a; - -select * from vt1 union all select * from vt1; - -set user.name=user1; - -grant all on table vt2 to user user2; - -set user.name=user2; -show grant user user2 on table vt2; -show grant user user2 on all; -set user.name=user1; - -revoke all on vt2 from user user2; - -set user.name=user2; -show grant user user2 on table vt2; - - -set user.name=hive_admin_user; -set role admin; -show grant on table vt2; - -set user.name=user1; -revoke select on table vt1 from user user2; - -set user.name=user2; -show grant user user2 on table vt1; -show grant user user2 on all; - -set user.name=user3; --- grant privileges on roles for view, after next statement -show grant user user3 on table vt1; - -set user.name=hive_admin_user; -show current roles; -set role ADMIN; -create role role_v; -grant role_v to user user4 ; -show role grant user user4; -show roles; - -grant all on table vt2 to role role_v; -show grant role role_v on table vt2; - -revoke delete on table vt2 from role role_v; -show grant role role_v on table vt2; -show grant on table vt2;