Repository: tajo Updated Branches: refs/heads/TAJO-1027 [created] a51940a2a
TAJO-995: HiveMetaStoreClient wrapper should retry the connection Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/7c98709f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/7c98709f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/7c98709f Branch: refs/heads/TAJO-1027 Commit: 7c98709f0fcb06dfb675acae3d6489a6126f55b5 Parents: 242d6ad Author: jinossy <[email protected]> Authored: Wed Aug 6 17:43:35 2014 +0900 Committer: jinossy <[email protected]> Committed: Wed Aug 6 17:43:35 2014 +0900 ---------------------------------------------------------------------- .../tajo/catalog/store/HCatalogStore.java | 11 ++++++----- .../catalog/store/HCatalogStoreClientPool.java | 20 ++++++++++++++++---- .../apache/tajo/catalog/store/HCatalogUtil.java | 6 ++++++ 3 files changed, 28 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/7c98709f/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 6f48348..d27f19c 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -48,6 +48,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.InternalException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; +import org.apache.thrift.TException; import java.io.IOException; import java.util.*; @@ -77,13 +78,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { @Override public boolean existTable(final String databaseName, final String tableName) throws CatalogException { boolean exist = false; - org.apache.hadoop.hive.ql.metadata.Table table = null; + org.apache.hadoop.hive.ql.metadata.Table table; HCatalogStoreClientPool.HCatalogStoreClient client = null; // get table try { client = clientPool.getClient(); - table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName); + table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); if (table != null) { exist = true; } @@ -118,7 +119,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { // get hive table schema try { client = clientPool.getClient(); - table = HCatUtil.getTable(client.getHiveClient(), databaseName, tableName); + table = HCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); path = table.getPath(); } catch (NoSuchObjectException nsoe) { throw new CatalogException("Table not found. - tableName:" + tableName, nsoe); @@ -291,7 +292,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { try { client = clientPool.getClient(); return client.getHiveClient().getAllTables(databaseName); - } catch (MetaException e) { + } catch (TException e) { throw new CatalogException(e); } finally { if(client != null) client.release(); @@ -401,7 +402,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { try { client = clientPool.getClient(); return client.getHiveClient().getAllDatabases(); - } catch (MetaException e) { + } catch (TException e) { throw new CatalogException(e); } finally { if (client != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/7c98709f/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java index 2fff67c..8ccb100 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStoreClientPool.java @@ -16,7 +16,9 @@ package org.apache.tajo.catalog.store; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.*; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.log4j.Logger; import java.util.Iterator; @@ -41,12 +43,22 @@ public class HCatalogStoreClientPool { * connection pool. */ public class HCatalogStoreClient { - private final HiveMetaStoreClient hiveClient; + private final IMetaStoreClient hiveClient; public AtomicBoolean isInUse = new AtomicBoolean(false); private HCatalogStoreClient(HiveConf hiveConf) { try { - this.hiveClient = new HiveMetaStoreClient(hiveConf); + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook(Table table) throws MetaException { + /* metadata hook implementation, or null if this + * storage handler does not need any metadata notifications + */ + return null; + } + }; + + this.hiveClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName()); clientPool.add(this); LOG.info("MetaStoreClient created (size = " + clientPool.size() + ")"); } catch (Exception e) { @@ -58,7 +70,7 @@ public class HCatalogStoreClientPool { /** * Returns the internal HiveMetaStoreClient object. */ - public HiveMetaStoreClient getHiveClient() { + public IMetaStoreClient getHiveClient() { return hiveClient; } http://git-wip-us.apache.org/repos/asf/tajo/blob/7c98709f/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java index 9e60768..54fdb9d 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogUtil.java @@ -20,9 +20,11 @@ package org.apache.tajo.catalog.store; import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hcatalog.common.HCatException; import org.apache.hcatalog.data.schema.HCatFieldSchema; @@ -30,6 +32,7 @@ import org.apache.hcatalog.data.schema.HCatSchema; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; +import org.apache.thrift.TException; import parquet.hadoop.mapred.DeprecatedParquetOutputFormat; public class HCatalogUtil { @@ -137,4 +140,7 @@ public class HCatalogUtil { } } + public static Table getTable(IMetaStoreClient client, String dbName, String tableName) throws TException { + return new Table(client.getTable(dbName, tableName)); + } }
