[gobblin] branch master updated: [GOBBLIn-1595]Fix the dead lock during hive registration (#3450)

[zihanli58]

This is an automated email from the ASF dual-hosted git repository.

zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 26f6930  [GOBBLIn-1595]Fix the dead lock during hive registration 
(#3450)
26f6930 is described below

commit 26f69301b0ad160ca869140c31a4ccc400949225
Author: Zihan Li <zi...@linkedin.com>
AuthorDate: Thu Jan 13 11:42:02 2022 -0800

    [GOBBLIn-1595]Fix the dead lock during hive registration (#3450)
---
 gobblin-aws/build.gradle                           |  4 ++-
 .../hive/metastore/HiveMetaStoreBasedRegister.java | 35 ++++++++++------------
 2 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/gobblin-aws/build.gradle b/gobblin-aws/build.gradle
index 0d9717e..1c0fd5f 100644
--- a/gobblin-aws/build.gradle
+++ b/gobblin-aws/build.gradle
@@ -58,7 +58,9 @@ dependencies {
   compile externalDependency.hadoopYarnClient
   compile externalDependency.avroMapredH2
   compile externalDependency.findBugsAnnotations
-  compile externalDependency.helix
+  compile (externalDependency.helix) {
+    exclude group: 'io.dropwizard.metrics', module: 'metrics-core'
+  }
 
   testCompile project(path: ':gobblin-cluster', configuration: 'tests')
   testCompile project(":gobblin-example")
diff --git 
a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
 
b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
index 56e8276..3e21e44 100644
--- 
a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
+++ 
b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
@@ -276,7 +276,7 @@ public class HiveMetaStoreBasedRegister extends 
HiveRegister {
       Table table) throws TException, IOException{
     try (AutoCloseableHiveLock lock = this.locks.getTableLock(dbName, 
tableName)) {
       try {
-        if(!existsTable(dbName, tableName)) {
+        if(!existsTable(dbName, tableName, client)) {
           try (Timer.Context context = 
this.metricContext.timer(CREATE_HIVE_TABLE).time()) {
             client.createTable(getTableWithCreateTimeNow(table));
             log.info(String.format("Created Hive table %s in db %s", 
tableName, dbName));
@@ -472,39 +472,34 @@ public class HiveMetaStoreBasedRegister extends 
HiveRegister {
   private void createOrAlterTable(IMetaStoreClient client, Table table, 
HiveSpec spec) throws TException, IOException {
     String dbName = table.getDbName();
     String tableName = table.getTableName();
-    boolean tableExistenceInCache;
-    if (this.optimizedChecks) {
-      try {
-        this.tableAndDbExistenceCache.get(dbName + ":" + tableName, new 
Callable<Boolean>() {
-          @Override
-          public Boolean call() throws Exception {
-            return ensureHiveTableExistenceBeforeAlternation(tableName, 
dbName, client, table);
-          }
-        });
-      } catch (ExecutionException ee) {
-        throw new IOException("Table existence checking throwing execution 
exception.", ee);
-      }
-    } else {
-      this.ensureHiveTableExistenceBeforeAlternation(tableName, dbName, 
client, table);
-    }
+    this.ensureHiveTableExistenceBeforeAlternation(tableName, dbName, client, 
table);
     alterTableIfNeeded(tableName, dbName, client, table, spec);
   }
 
-  @Override
-  public boolean existsTable(String dbName, String tableName) throws 
IOException {
+  public boolean existsTable(String dbName, String tableName, IMetaStoreClient 
client) throws IOException {
     if (this.optimizedChecks && 
this.tableAndDbExistenceCache.getIfPresent(dbName + ":" + tableName ) != null ) 
{
       return true;
     }
-    try (AutoReturnableObject<IMetaStoreClient> client = 
this.clientPool.getClient()) {
+    try {
+      boolean exists;
       try (Timer.Context context = 
this.metricContext.timer(TABLE_EXISTS).time()) {
-        return client.get().tableExists(dbName, tableName);
+        exists =  client.tableExists(dbName, tableName);
       }
+      this.tableAndDbExistenceCache.put(dbName + ":" + tableName, exists);
+      return exists;
     } catch (TException e) {
       throw new IOException(String.format("Unable to check existence of table 
%s in db %s", tableName, dbName), e);
     }
   }
 
   @Override
+  public boolean existsTable(String dbName, String tableName) throws 
IOException {
+    try (AutoReturnableObject<IMetaStoreClient> client = 
this.clientPool.getClient()) {
+      return existsTable(dbName, tableName, client.get());
+    }
+  }
+
+  @Override
   public boolean existsPartition(String dbName, String tableName, List<Column> 
partitionKeys,
       List<String> partitionValues) throws IOException {
     try (AutoReturnableObject<IMetaStoreClient> client = 
this.clientPool.getClient()) {