Repository: hive
Updated Branches:
  refs/heads/master e7cee3001 -> 230ae3629


HIVE-19859: Inspect lock components for DBHiveLock while verifying whether 
transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh 
Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/230ae362
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/230ae362
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/230ae362

Branch: refs/heads/master
Commit: 230ae362943585aa77f469b7563b41c6a2987997
Parents: e7cee30
Author: Jesus Camacho Rodriguez <jcama...@apache.org>
Authored: Tue Jun 12 09:23:28 2018 -0700
Committer: Jesus Camacho Rodriguez <jcama...@apache.org>
Committed: Tue Jun 12 12:55:46 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/ql/Driver.java  | 31 +++++++++++++++-----
 .../hadoop/hive/ql/lockmgr/DbLockManager.java   | 18 ++++++++++--
 .../apache/hadoop/hive/ql/lockmgr/HiveLock.java | 26 ++++++++++++++++
 3 files changed, 66 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/230ae362/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index c9d18a6..abeb7fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockType;
 import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
@@ -809,13 +811,27 @@ public class Driver implements IDriver {
     }
     Set<String> nonSharedLocks = new HashSet<>();
     for (HiveLock lock : ctx.getHiveLocks()) {
-      if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
-          lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
-        if (lock.getHiveLockObject().getPaths().length == 2) {
-          // Pos 0 of lock paths array contains dbname, pos 1 contains tblname
-          nonSharedLocks.add(
-              Warehouse.getQualifiedName(
-                  lock.getHiveLockObject().getPaths()[0], 
lock.getHiveLockObject().getPaths()[1]));
+      if (lock.mayContainComponents()) {
+        // The lock may have multiple components, e.g., DbHiveLock, hence we 
need
+        // to check for each of them
+        for (LockComponent lckCmp : lock.getHiveLockComponents()) {
+          if (lckCmp.getType() == LockType.EXCLUSIVE ||
+              lckCmp.getType() == LockType.SHARED_WRITE) {
+            nonSharedLocks.add(
+                Warehouse.getQualifiedName(
+                    lckCmp.getDbname(), lckCmp.getTablename()));
+          }
+        }
+      } else {
+        // The lock has a single components, e.g., SimpleHiveLock or 
ZooKeeperHiveLock
+        if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
+            lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
+          if (lock.getHiveLockObject().getPaths().length == 2) {
+            // Pos 0 of lock paths array contains dbname, pos 1 contains 
tblname
+            nonSharedLocks.add(
+                Warehouse.getQualifiedName(
+                    lock.getHiveLockObject().getPaths()[0], 
lock.getHiveLockObject().getPaths()[1]));
+          }
         }
       }
     }
@@ -1938,6 +1954,7 @@ public class Driver implements IDriver {
 
       try {
         if (!isValidTxnListState()) {
+          LOG.info("Compiling after acquiring locks");
           // Snapshot was outdated when locks were acquired, hence regenerate 
context,
           // txn list and retry
           // TODO: Lock acquisition should be moved before analyze, this is a 
bit hackish.

http://git-wip-us.apache.org/repos/asf/hive/blob/230ae362/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
index ab94475..1a04278 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hive.ql.lockmgr;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.DDLTask;
 import org.slf4j.Logger;
@@ -114,7 +115,7 @@ public final class DbLockManager implements HiveLockManager{
         res = txnManager.getMS().checkLock(res.getLockid());
       }
       long retryDuration = System.currentTimeMillis() - startRetry;
-      DbHiveLock hl = new DbHiveLock(res.getLockid(), queryId, 
lock.getTxnid());
+      DbHiveLock hl = new DbHiveLock(res.getLockid(), queryId, 
lock.getTxnid(), lock.getComponent());
       if(locks.size() > 0) {
         boolean logMsg = false;
         for(DbHiveLock l : locks) {
@@ -310,14 +311,17 @@ public final class DbLockManager implements 
HiveLockManager{
     long lockId;
     String queryId;
     long txnId;
+    List<LockComponent> components;
 
     DbHiveLock(long id) {
       lockId = id;
     }
-    DbHiveLock(long id, String queryId, long txnId) {
+
+    DbHiveLock(long id, String queryId, long txnId, List<LockComponent> 
components) {
       lockId = id;
       this.queryId = queryId;
       this.txnId = txnId;
+      this.components = ImmutableList.copyOf(components);
     }
 
     @Override
@@ -331,6 +335,16 @@ public final class DbLockManager implements 
HiveLockManager{
     }
 
     @Override
+    public boolean mayContainComponents() {
+      return true;
+    }
+
+    @Override
+    public List<LockComponent> getHiveLockComponents() {
+      return components;
+    }
+
+    @Override
     public boolean equals(Object other) {
       if (other instanceof DbHiveLock) {
         return lockId == ((DbHiveLock)other).lockId;

http://git-wip-us.apache.org/repos/asf/hive/blob/230ae362/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
index 5373137..8239245 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
@@ -18,7 +18,33 @@
 
 package org.apache.hadoop.hive.ql.lockmgr;
 
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+
+import java.util.List;
+
 public abstract class HiveLock {
+
   public abstract HiveLockObject getHiveLockObject();
+
   public abstract HiveLockMode   getHiveLockMode();
+
+  /**
+   * Returns true if for this lock implementation, a single lock can in turn
+   * lock multiple objects, e.g., multi-statement transaction.
+   */
+  public boolean mayContainComponents() {
+    return false;
+  }
+
+  /**
+   * Returns the lock components if a single lock can in turn
+   * lock multiple objects, e.g., multi-statement transaction.
+   *
+   * Returns an empty list if the lock does not have multiple
+   * components.
+   */
+  public List<LockComponent> getHiveLockComponents() {
+    return ImmutableList.of();
+  }
 }

Reply via email to