[2/2] hive git commit: HIVE-19859: Inspect lock components for DBHiveLock while verifying whether transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-19859: Inspect lock components for DBHiveLock while verifying whether
transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh
Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/59f8aae2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/59f8aae2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/59f8aae2
Branch: refs/heads/branch-3
Commit: 59f8aae2c5ce26d14874b98d3135a81640085435
Parents: 1cd543f
Author: Jesus Camacho Rodriguez
Authored: Tue Jun 12 09:23:28 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Tue Jun 12 16:10:12 2018 -0700
--
.../java/org/apache/hadoop/hive/ql/Driver.java | 31 +++-
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 18 ++--
.../apache/hadoop/hive/ql/lockmgr/HiveLock.java | 26
3 files changed, 66 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 8b5262a..1f6bb6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
@@ -806,13 +808,27 @@ public class Driver implements IDriver {
}
Set nonSharedLocks = new HashSet<>();
for (HiveLock lock : ctx.getHiveLocks()) {
- if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
- lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
-if (lock.getHiveLockObject().getPaths().length == 2) {
- // Pos 0 of lock paths array contains dbname, pos 1 contains tblname
- nonSharedLocks.add(
- Warehouse.getQualifiedName(
- lock.getHiveLockObject().getPaths()[0],
lock.getHiveLockObject().getPaths()[1]));
+ if (lock.mayContainComponents()) {
+// The lock may have multiple components, e.g., DbHiveLock, hence we
need
+// to check for each of them
+for (LockComponent lckCmp : lock.getHiveLockComponents()) {
+ if (lckCmp.getType() == LockType.EXCLUSIVE ||
+ lckCmp.getType() == LockType.SHARED_WRITE) {
+nonSharedLocks.add(
+Warehouse.getQualifiedName(
+lckCmp.getDbname(), lckCmp.getTablename()));
+ }
+}
+ } else {
+// The lock has a single components, e.g., SimpleHiveLock or
ZooKeeperHiveLock
+if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
+lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
+ if (lock.getHiveLockObject().getPaths().length == 2) {
+// Pos 0 of lock paths array contains dbname, pos 1 contains
tblname
+nonSharedLocks.add(
+Warehouse.getQualifiedName(
+lock.getHiveLockObject().getPaths()[0],
lock.getHiveLockObject().getPaths()[1]));
+ }
}
}
}
@@ -1937,6 +1953,7 @@ public class Driver implements IDriver {
try {
if (!isValidTxnListState()) {
+ LOG.info("Compiling after acquiring locks");
// Snapshot was outdated when locks were acquired, hence regenerate
context,
// txn list and retry
// TODO: Lock acquisition should be moved before analyze, this is a
bit hackish.
http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
index ab94475..1a04278 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.lockmgr;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.slf4j.Logger;
@@ -114,7 +115,7 @@ public final class DbLockManager implements HiveLockManager{
hive git commit: HIVE-19859: Inspect lock components for DBHiveLock while verifying whether transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master e7cee3001 -> 230ae3629
HIVE-19859: Inspect lock components for DBHiveLock while verifying whether
transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh
Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/230ae362
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/230ae362
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/230ae362
Branch: refs/heads/master
Commit: 230ae362943585aa77f469b7563b41c6a2987997
Parents: e7cee30
Author: Jesus Camacho Rodriguez
Authored: Tue Jun 12 09:23:28 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Tue Jun 12 12:55:46 2018 -0700
--
.../java/org/apache/hadoop/hive/ql/Driver.java | 31 +++-
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 18 ++--
.../apache/hadoop/hive/ql/lockmgr/HiveLock.java | 26
3 files changed, 66 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/230ae362/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index c9d18a6..abeb7fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
@@ -809,13 +811,27 @@ public class Driver implements IDriver {
}
Set nonSharedLocks = new HashSet<>();
for (HiveLock lock : ctx.getHiveLocks()) {
- if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
- lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
-if (lock.getHiveLockObject().getPaths().length == 2) {
- // Pos 0 of lock paths array contains dbname, pos 1 contains tblname
- nonSharedLocks.add(
- Warehouse.getQualifiedName(
- lock.getHiveLockObject().getPaths()[0],
lock.getHiveLockObject().getPaths()[1]));
+ if (lock.mayContainComponents()) {
+// The lock may have multiple components, e.g., DbHiveLock, hence we
need
+// to check for each of them
+for (LockComponent lckCmp : lock.getHiveLockComponents()) {
+ if (lckCmp.getType() == LockType.EXCLUSIVE ||
+ lckCmp.getType() == LockType.SHARED_WRITE) {
+nonSharedLocks.add(
+Warehouse.getQualifiedName(
+lckCmp.getDbname(), lckCmp.getTablename()));
+ }
+}
+ } else {
+// The lock has a single components, e.g., SimpleHiveLock or
ZooKeeperHiveLock
+if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
+lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
+ if (lock.getHiveLockObject().getPaths().length == 2) {
+// Pos 0 of lock paths array contains dbname, pos 1 contains
tblname
+nonSharedLocks.add(
+Warehouse.getQualifiedName(
+lock.getHiveLockObject().getPaths()[0],
lock.getHiveLockObject().getPaths()[1]));
+ }
}
}
}
@@ -1938,6 +1954,7 @@ public class Driver implements IDriver {
try {
if (!isValidTxnListState()) {
+ LOG.info("Compiling after acquiring locks");
// Snapshot was outdated when locks were acquired, hence regenerate
context,
// txn list and retry
// TODO: Lock acquisition should be moved before analyze, this is a
bit hackish.
http://git-wip-us.apache.org/repos/asf/hive/blob/230ae362/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
index ab94475..1a04278 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.lockmgr;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.slf4j.Logger;
@@ -1
