Repository: hive Updated Branches: refs/heads/master a3ed7d6fb -> 3104d4756
HIVE-19694 : Create Materialized View statement should check for MV name conflicts before running MV's SQL statement. (Miklos Gergely via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3104d475 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3104d475 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3104d475 Branch: refs/heads/master Commit: 3104d4756a6ff4c7853e4a17bbef13317770efd2 Parents: a3ed7d6 Author: Miklos Gergely <[email protected]> Authored: Tue Jul 31 13:24:35 2018 -0700 Committer: Ashutosh Chauhan <[email protected]> Committed: Tue Jul 31 13:24:35 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/CalcitePlanner.java | 4 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 113 ++++--------------- .../materialized_view_name_collusion.q | 10 ++ .../materialized_view_name_collusion.q.out | 19 ++++ 4 files changed, 50 insertions(+), 96 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index a70aea0..61396e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1008,7 +1008,6 @@ public class CalcitePlanner extends SemanticAnalyzer { table.setMaterializedTable(true); LOG.info(cteName + " will be materialized into " + location); - cte.table = table; cte.source = analyzer; ctx.addMaterializedTable(cteName, table); @@ -1567,8 +1566,7 @@ public class CalcitePlanner extends SemanticAnalyzer { } ASTNode selExprList = qb.getParseInfo().getSelForClause(dest); - RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, inputRR, qb, - selExprList); + RowResolver out_rwsch = handleInsertStatementSpec(colList, dest, inputRR, qb, selExprList); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b28cf98..474c793 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -169,7 +169,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverterPos import org.apache.hadoop.hive.ql.optimizer.lineage.Generator; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType; -import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec; @@ -271,7 +270,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; @@ -280,7 +278,6 @@ import org.apache.hadoop.security.UserGroupInformation; import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.math.IntMath; import com.google.common.math.LongMath; @@ -386,9 +383,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { CreateTableDesc tableDesc; - /** Not thread-safe. */ - final ASTSearcher astSearcher = new ASTSearcher(); - protected AnalyzeRewriteContext analyzeRewrite; private WriteEntity acidAnalyzeTable; @@ -1156,15 +1150,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { return this.nameToSplitSample; } - /** - * Convert a string to Text format and write its bytes in the same way TextOutputFormat would do. - * This is needed to properly encode non-ascii characters. - */ - private static void writeAsText(String text, FSDataOutputStream out) throws IOException { - Text to = new Text(text); - out.write(to.getBytes(), 0, to.getLength()); - } - private void assertCombineInputFormat(Tree numerator, String message) throws SemanticException { String inputFormat = conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez") ? HiveConf.getVar(conf, HiveConf.ConfVars.HIVETEZINPUTFORMAT): @@ -1323,7 +1308,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<CTEClause> parents = new ArrayList<CTEClause>(); // materialized - Table table; SemanticAnalyzer source; List<Task<?>> getTasks() { @@ -1432,7 +1416,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { table.setMaterializedTable(true); LOG.info(cteName + " will be materialized into " + location); - cte.table = table; cte.source = analyzer; ctx.addMaterializedTable(cteName, table); @@ -1551,15 +1534,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { return alias; } - private String extractLateralViewAlias(ASTNode lateralView) { - // Lateral view AST has the following shape: - // ^(TOK_LATERAL_VIEW - // ^(TOK_SELECT ^(TOK_SELEXPR ^(TOK_FUNCTION Identifier params) identifier* tableAlias))) - ASTNode selExpr = (ASTNode) lateralView.getChild(0).getChild(0); - ASTNode astTableAlias = (ASTNode) Iterables.getLast(selExpr.getChildren()); - return astTableAlias.getChild(0).getText(); - } - /** * Phase 1: (including, but not limited to): * @@ -1928,7 +1902,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { /** * This is phase1 of supporting specifying schema in insert statement * insert into foo(z,y) select a,b from bar; - * @see #handleInsertStatementSpec(java.util.List, String, RowResolver, RowResolver, QB, ASTNode) + * @see #handleInsertStatementSpec(java.util.List, String, RowResolver, QB, ASTNode) * @throws SemanticException */ private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp, Phase1Ctx ctx_1) throws SemanticException { @@ -4615,7 +4589,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } selectStar = selectStar && exprList.getChildCount() == posn + 1; - out_rwsch = handleInsertStatementSpec(col_list, dest, out_rwsch, inputRR, qb, selExprList); + out_rwsch = handleInsertStatementSpec(col_list, dest, out_rwsch, qb, selExprList); ArrayList<String> columnNames = new ArrayList<String>(); Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>(); @@ -4667,7 +4641,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } - boolean defaultConstraintsFetch = true; for (int i = 0; i < targetTableColNames.size(); i++) { String f = targetTableColNames.get(i); if(targetCol2Projection.containsKey(f)) { @@ -4726,7 +4699,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * @throws SemanticException */ public RowResolver handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest, - RowResolver outputRR, RowResolver inputRR, QB qb, + RowResolver outputRR, QB qb, ASTNode selExprList) throws SemanticException { //(z,x) List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query @@ -4816,17 +4789,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } /** - * Convert exprNodeDesc array to Typeinfo array. - */ - static ArrayList<TypeInfo> getTypeInfo(ArrayList<ExprNodeDesc> exprs) { - ArrayList<TypeInfo> result = new ArrayList<TypeInfo>(); - for (ExprNodeDesc expr : exprs) { - result.add(expr.getTypeInfo()); - } - return result; - } - - /** * Convert exprNodeDesc array to ObjectInspector array. */ static ArrayList<ObjectInspector> getWritableObjectInspector(ArrayList<ExprNodeDesc> exprs) { @@ -4838,18 +4800,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } /** - * Convert exprNodeDesc array to Typeinfo array. - */ - static ObjectInspector[] getStandardObjectInspector(ArrayList<TypeInfo> exprs) { - ObjectInspector[] result = new ObjectInspector[exprs.size()]; - for (int i = 0; i < exprs.size(); i++) { - result[i] = TypeInfoUtils - .getStandardWritableObjectInspectorFromTypeInfo(exprs.get(i)); - } - return result; - } - - /** * Returns the GenericUDAFEvaluator for the aggregation. This is called once * for each GroupBy aggregation. */ @@ -5594,7 +5544,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<String> outputKeyColumnNames = new ArrayList<String>(); List<String> outputValueColumnNames = new ArrayList<String>(); - ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, dest, + ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, reduceSinkInputRowResolver, reduceSinkOutputRowResolver, outputKeyColumnNames, colExprMap); @@ -5662,7 +5612,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { return rsOp; } - private ArrayList<ExprNodeDesc> getReduceKeysForReduceSink(List<ASTNode> grpByExprs, String dest, + private ArrayList<ExprNodeDesc> getReduceKeysForReduceSink(List<ASTNode> grpByExprs, RowResolver reduceSinkInputRowResolver, RowResolver reduceSinkOutputRowResolver, List<String> outputKeyColumnNames, Map<String, ExprNodeDesc> colExprMap) throws SemanticException { @@ -5800,7 +5750,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<String> outputValueColumnNames = new ArrayList<String>(); List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest); - ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, dest, + ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, reduceSinkInputRowResolver, reduceSinkOutputRowResolver, outputKeyColumnNames, colExprMap); @@ -6062,7 +6012,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI; containsDistinctAggr = containsDistinctAggr || isDistinct; - boolean isStar = value.getType() == HiveParser.TOK_FUNCTIONSTAR; Mode amode = groupByDescModeToUDAFMode(mode, isDistinct); GenericUDAFEvaluator genericUDAFEvaluator = genericUDAFEvaluators .get(entry.getKey()); @@ -6268,15 +6217,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { return curr; } - static ArrayList<GenericUDAFEvaluator> getUDAFEvaluators( - ArrayList<AggregationDesc> aggs) { - ArrayList<GenericUDAFEvaluator> result = new ArrayList<GenericUDAFEvaluator>(); - for (int i = 0; i < aggs.size(); i++) { - result.add(aggs.get(i).getGenericUDAFEvaluator()); - } - return result; - } - /** * Generate a Group-By plan using a 2 map-reduce jobs (5 operators will be * inserted): @@ -11825,19 +11765,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { this.qb = qb; } - boolean analyzeCreateTable(ASTNode child) throws SemanticException { - if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) { - // if it is not CTAS, we don't need to go further and just return - if ((child = analyzeCreateTable(ast, qb, null)) == null) { - return true; - } - } else { - queryState.setCommandType(HiveOperation.QUERY); - } - - return false; - } - @Override @SuppressWarnings("nls") public void analyzeInternal(ASTNode ast) throws SemanticException { @@ -11857,13 +11784,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * Planner specific stuff goes in here. */ static class PlannerContext { - protected ASTNode child; - protected Phase1Ctx ctx_1; - - void setParseTreeAttr(ASTNode child, Phase1Ctx ctx_1) { - this.child = child; - this.ctx_1 = ctx_1; - } void setCTASToken(ASTNode child) { } @@ -12201,8 +12121,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { getMetaData(qb, createVwDesc == null); LOG.info("Completed getting MetaData in Semantic Analysis"); - plannerCtx.setParseTreeAttr(child, ctx_1); - return true; } @@ -13593,10 +13511,23 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { storageFormat.fillDefaultStorageFormat(false, isMaterialized); - if (ifNotExists && orReplace){ + if (ifNotExists && orReplace) { throw new SemanticException("Can't combine IF NOT EXISTS and OR REPLACE."); } + if (isMaterialized && !ifNotExists) { + // Verify that the table does not already exist + // dumpTable is only used to check the conflict for non-temporary tables + try { + Table dumpTable = db.newTable(dbDotTable); + if (null != db.getTable(dumpTable.getDbName(), dumpTable.getTableName(), false) && !ctx.isExplainSkipExecution()) { + throw new SemanticException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(dbDotTable)); + } + } catch (HiveException e) { + throw new SemanticException(e); + } + } + if (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY) { isAlterViewAs = true; @@ -13629,10 +13560,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { return selectStmt; } - CreateViewDesc getCreateViewDesc() { - return this.createVwDesc; - } - // validate the (materialized) view statement // check semantic conditions private void validateCreateView() http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q b/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q new file mode 100644 index 0000000..567ac2a --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_name_collusion.q @@ -0,0 +1,10 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; + +create table mvnc_basetable (a int, b varchar(256), c decimal(10,2)); + + +create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable; + +create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable; http://git-wip-us.apache.org/repos/asf/hive/blob/3104d475/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out b/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out new file mode 100644 index 0000000..9a2cb3e --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_name_collusion.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create table mvnc_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mvnc_basetable +POSTHOOK: query: create table mvnc_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mvnc_basetable +PREHOOK: query: create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@mvnc_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@mvnc_mat_view +POSTHOOK: query: create materialized view mvnc_mat_view disable rewrite as select a, b, c from mvnc_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@mvnc_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mvnc_mat_view +FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Table already exists: default.mvnc_mat_view
