HIVE-12367 : Lock/unlock database should add current database to inputs and outputs of authz hook (Dapeng Sun via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bfec2e9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bfec2e9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bfec2e9 Branch: refs/heads/llap Commit: 6bfec2e97c4e434646aa9aeffd98c9939313fa6e Parents: a71edcf Author: Dapeng Sun <s...@apache.org> Authored: Tue Dec 15 19:39:00 2015 -0800 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Thu Mar 24 19:22:28 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/ql/Driver.java | 20 ++++++++++++++++++++ .../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +++++++++++++-- .../clientnegative/dbtxnmgr_nodblock.q.out | 2 ++ .../clientnegative/dbtxnmgr_nodbunlock.q.out | 2 ++ .../lockneg_query_tbl_in_locked_db.q.out | 6 ++++++ .../lockneg_try_db_lock_conflict.q.out | 6 ++++++ .../lockneg_try_drop_locked_db.q.out | 4 ++++ .../lockneg_try_lock_db_in_use.q.out | 6 ++++++ 8 files changed, 59 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index d7e4ac7..7276e31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1416,6 +1416,10 @@ public class Driver implements CommandProcessor { if (!checkConcurrency()) { return false; } + // Lock operations themselves don't require the lock. + if (isExplicitLockOperation()){ + return false; + } if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_LOCK_MAPRED_ONLY)) { return true; } @@ -1438,6 +1442,22 @@ public class Driver implements CommandProcessor { return false; } + private boolean isExplicitLockOperation() { + HiveOperation currentOpt = plan.getOperation(); + if (currentOpt != null) { + switch (currentOpt) { + case LOCKDB: + case UNLOCKDB: + case LOCKTABLE: + case UNLOCKTABLE: + return true; + default: + return false; + } + } + return false; + } + private CommandProcessorResponse createProcessorResponse(int ret) { queryDisplay.setErrorMessage(errorMessage); return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError); http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 0c087ed..fe9b8cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -2444,8 +2444,12 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { String dbName = unescapeIdentifier(ast.getChild(0).getText()); String mode = unescapeIdentifier(ast.getChild(1).getText().toUpperCase()); - //inputs.add(new ReadEntity(dbName)); - //outputs.add(new WriteEntity(dbName)); + inputs.add(new ReadEntity(getDatabase(dbName))); + // Lock database operation is to acquire the lock explicitly, the operation + // itself doesn't need to be locked. Set the WriteEntity as WriteType: + // DDL_NO_LOCK here, otherwise it will conflict with Hive's transaction. + outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK)); + LockDatabaseDesc lockDatabaseDesc = new LockDatabaseDesc(dbName, mode, HiveConf.getVar(conf, ConfVars.HIVEQUERYID)); lockDatabaseDesc.setQueryStr(ctx.getCmd()); @@ -2457,6 +2461,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { private void analyzeUnlockDatabase(ASTNode ast) throws SemanticException { String dbName = unescapeIdentifier(ast.getChild(0).getText()); + inputs.add(new ReadEntity(getDatabase(dbName))); + // Unlock database operation is to release the lock explicitly, the + // operation itself don't need to be locked. Set the WriteEntity as + // WriteType: DDL_NO_LOCK here, otherwise it will conflict with + // Hive's transaction. + outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK)); + UnlockDatabaseDesc unlockDatabaseDesc = new UnlockDatabaseDesc(dbName); DDLWork work = new DDLWork(getInputs(), getOutputs(), unlockDatabaseDesc); rootTasks.add(TaskFactory.get(work, conf)); http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out index b8bbe93..e783251 100644 --- a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out +++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out @@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:drop_nodblock PREHOOK: query: lock database drop_nodblock shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:drop_nodblock +PREHOOK: Output: database:drop_nodblock FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out index ea7a3eb..d7a39f0 100644 --- a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out +++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out @@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:drop_nodbunlock PREHOOK: query: unlock database drop_nodbunlock PREHOOK: type: UNLOCKDATABASE +PREHOOK: Input: database:drop_nodbunlock +PREHOOK: Output: database:drop_nodbunlock FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out index 134a57b..2c46159 100644 --- a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out +++ b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out @@ -34,8 +34,12 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: lock database lockneg1 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg1 +PREHOOK: Output: database:lockneg1 POSTHOOK: query: lock database lockneg1 shared POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg1 +POSTHOOK: Output: database:lockneg1 PREHOOK: query: show locks database lockneg1 PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks database lockneg1 @@ -53,4 +57,6 @@ POSTHOOK: Input: lockneg1@tstsrcpart@ds=2008-04-08/hr=11 500 PREHOOK: query: unlock database lockneg1 PREHOOK: type: UNLOCKDATABASE +PREHOOK: Input: database:lockneg1 +PREHOOK: Output: database:lockneg1 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Database lockneg1 is not locked http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out index 0a5d98a..fe12d83 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out @@ -6,9 +6,15 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:lockneg4 PREHOOK: query: lock database lockneg4 exclusive PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg4 +PREHOOK: Output: database:lockneg4 POSTHOOK: query: lock database lockneg4 exclusive POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg4 +POSTHOOK: Output: database:lockneg4 PREHOOK: query: lock database lockneg4 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg4 +PREHOOK: Output: database:lockneg4 Unable to acquire EXPLICIT, SHARED lock lockneg4 after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out index 5ac4e3c..e66965e 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out @@ -6,8 +6,12 @@ POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:lockneg9 PREHOOK: query: lock database lockneg9 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg9 +PREHOOK: Output: database:lockneg9 POSTHOOK: query: lock database lockneg9 shared POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg9 +POSTHOOK: Output: database:lockneg9 PREHOOK: query: show locks database lockneg9 PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks database lockneg9 http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out index 5486151..e5c8f3e 100644 --- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out +++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out @@ -34,13 +34,19 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: lock database lockneg2 shared PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg2 +PREHOOK: Output: database:lockneg2 POSTHOOK: query: lock database lockneg2 shared POSTHOOK: type: LOCKDATABASE +POSTHOOK: Input: database:lockneg2 +POSTHOOK: Output: database:lockneg2 PREHOOK: query: show locks PREHOOK: type: SHOWLOCKS POSTHOOK: query: show locks POSTHOOK: type: SHOWLOCKS PREHOOK: query: lock database lockneg2 exclusive PREHOOK: type: LOCKDATABASE +PREHOOK: Input: database:lockneg2 +PREHOOK: Output: database:lockneg2 Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts. FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask