HIVE-19859: Inspect lock components for DBHiveLock while verifying whether transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/59f8aae2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/59f8aae2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/59f8aae2 Branch: refs/heads/branch-3 Commit: 59f8aae2c5ce26d14874b98d3135a81640085435 Parents: 1cd543f Author: Jesus Camacho Rodriguez <[email protected]> Authored: Tue Jun 12 09:23:28 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Tue Jun 12 16:10:12 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/ql/Driver.java | 31 +++++++++++++++----- .../hadoop/hive/ql/lockmgr/DbLockManager.java | 18 ++++++++++-- .../apache/hadoop/hive/ql/lockmgr/HiveLock.java | 26 ++++++++++++++++ 3 files changed, 66 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 8b5262a..1f6bb6a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -64,6 +64,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.LockComponent; +import org.apache.hadoop.hive.metastore.api.LockType; import org.apache.hadoop.hive.metastore.api.Schema; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; @@ -806,13 +808,27 @@ public class Driver implements IDriver { } Set<String> nonSharedLocks = new HashSet<>(); for (HiveLock lock : ctx.getHiveLocks()) { - if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE || - lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) { - if (lock.getHiveLockObject().getPaths().length == 2) { - // Pos 0 of lock paths array contains dbname, pos 1 contains tblname - nonSharedLocks.add( - Warehouse.getQualifiedName( - lock.getHiveLockObject().getPaths()[0], lock.getHiveLockObject().getPaths()[1])); + if (lock.mayContainComponents()) { + // The lock may have multiple components, e.g., DbHiveLock, hence we need + // to check for each of them + for (LockComponent lckCmp : lock.getHiveLockComponents()) { + if (lckCmp.getType() == LockType.EXCLUSIVE || + lckCmp.getType() == LockType.SHARED_WRITE) { + nonSharedLocks.add( + Warehouse.getQualifiedName( + lckCmp.getDbname(), lckCmp.getTablename())); + } + } + } else { + // The lock has a single components, e.g., SimpleHiveLock or ZooKeeperHiveLock + if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE || + lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) { + if (lock.getHiveLockObject().getPaths().length == 2) { + // Pos 0 of lock paths array contains dbname, pos 1 contains tblname + nonSharedLocks.add( + Warehouse.getQualifiedName( + lock.getHiveLockObject().getPaths()[0], lock.getHiveLockObject().getPaths()[1])); + } } } } @@ -1937,6 +1953,7 @@ public class Driver implements IDriver { try { if (!isValidTxnListState()) { + LOG.info("Compiling after acquiring locks"); // Snapshot was outdated when locks were acquired, hence regenerate context, // txn list and retry // TODO: Lock acquisition should be moved before analyze, this is a bit hackish. http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java index ab94475..1a04278 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.lockmgr; +import com.google.common.collect.ImmutableList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.DDLTask; import org.slf4j.Logger; @@ -114,7 +115,7 @@ public final class DbLockManager implements HiveLockManager{ res = txnManager.getMS().checkLock(res.getLockid()); } long retryDuration = System.currentTimeMillis() - startRetry; - DbHiveLock hl = new DbHiveLock(res.getLockid(), queryId, lock.getTxnid()); + DbHiveLock hl = new DbHiveLock(res.getLockid(), queryId, lock.getTxnid(), lock.getComponent()); if(locks.size() > 0) { boolean logMsg = false; for(DbHiveLock l : locks) { @@ -310,14 +311,17 @@ public final class DbLockManager implements HiveLockManager{ long lockId; String queryId; long txnId; + List<LockComponent> components; DbHiveLock(long id) { lockId = id; } - DbHiveLock(long id, String queryId, long txnId) { + + DbHiveLock(long id, String queryId, long txnId, List<LockComponent> components) { lockId = id; this.queryId = queryId; this.txnId = txnId; + this.components = ImmutableList.copyOf(components); } @Override @@ -331,6 +335,16 @@ public final class DbLockManager implements HiveLockManager{ } @Override + public boolean mayContainComponents() { + return true; + } + + @Override + public List<LockComponent> getHiveLockComponents() { + return components; + } + + @Override public boolean equals(Object other) { if (other instanceof DbHiveLock) { return lockId == ((DbHiveLock)other).lockId; http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java index 5373137..8239245 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java @@ -18,7 +18,33 @@ package org.apache.hadoop.hive.ql.lockmgr; +import com.google.common.collect.ImmutableList; +import org.apache.hadoop.hive.metastore.api.LockComponent; + +import java.util.List; + public abstract class HiveLock { + public abstract HiveLockObject getHiveLockObject(); + public abstract HiveLockMode getHiveLockMode(); + + /** + * Returns true if for this lock implementation, a single lock can in turn + * lock multiple objects, e.g., multi-statement transaction. + */ + public boolean mayContainComponents() { + return false; + } + + /** + * Returns the lock components if a single lock can in turn + * lock multiple objects, e.g., multi-statement transaction. + * + * Returns an empty list if the lock does not have multiple + * components. + */ + public List<LockComponent> getHiveLockComponents() { + return ImmutableList.of(); + } }
