Repository: hive Updated Branches: refs/heads/master bfa69e107 -> 0772f186f
HIVE-20194 : HiveMetastoreClient should use reflection to instantiate embedded HMS instance (Alexander Kolbasov reviewed by Alan Gates and Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0772f186 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0772f186 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0772f186 Branch: refs/heads/master Commit: 0772f186f90db378dd6eacd5116038b2ee355cd4 Parents: bfa69e1 Author: Alexander Kolbasov <[email protected]> Authored: Tue Aug 14 10:40:05 2018 -0700 Committer: Vihang Karajgaonkar <[email protected]> Committed: Tue Aug 14 10:40:05 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/HiveMetaStore.java | 18 +++++-- .../hive/metastore/HiveMetaStoreClient.java | 51 +++++++++++++++++++- .../HiveMetaStoreClientPreCatalog.java | 8 +-- 3 files changed, 69 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0772f186/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index a53d4be..d34e944 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8979,10 +8979,22 @@ public class HiveMetaStore extends ThriftHiveMetastore { return RetryingHMSHandler.getProxy(conf, baseHandler, local); } - static Iface newRetryingHMSHandler(String name, Configuration conf, boolean local) + /** + * Create retrying HMS handler for embedded metastore. + * + * <h1>IMPORTANT</h1> + * + * This method is called indirectly by HiveMetastoreClient and HiveMetaStoreClientPreCatalog + * using reflection. It can not be removed and its arguments can't be changed without matching + * change in HiveMetastoreClient and HiveMetaStoreClientPreCatalog. + * + * @param conf configuration to use + * @throws MetaException + */ + static Iface newRetryingHMSHandler(Configuration conf) throws MetaException { - HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false); - return RetryingHMSHandler.getProxy(conf, baseHandler, local); + HMSHandler baseHandler = new HiveMetaStore.HMSHandler("hive client", conf, false); + return RetryingHMSHandler.getProxy(conf, baseHandler, true); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/0772f186/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 91c86a7..5ae00af 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -107,6 +107,14 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { public final static ClientCapabilities TEST_VERSION = new ClientCapabilities( Lists.newArrayList(ClientCapability.INSERT_ONLY_TABLES, ClientCapability.TEST_CAPABILITY)); + // Name of the HiveMetaStore class. It is used to initialize embedded metastore + private static final String HIVE_METASTORE_CLASS = + "org.apache.hadoop.hive.metastore.HiveMetaStore"; + + // Method used to create Hive Metastore client. It is called as + // HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + private static final String HIVE_METASTORE_CREATE_HANDLER_METHOD = "newRetryingHMSHandler"; + ThriftHiveMetastore.Iface client = null; private TTransport transport = null; private boolean isConnected = false; @@ -164,9 +172,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } + + client = callEmbeddedMetastore(this.conf); + // instantiate the metastore server handler directly instead of connecting // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); isConnected = true; snapshotActiveConf(); return; @@ -224,6 +234,45 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { open(); } + /** + * Instantiate the metastore server handler directly instead of connecting + * through the network + * + * @param conf Configuration object passed to embedded metastore + * @return embedded client instance + * @throws MetaException + */ + static ThriftHiveMetastore.Iface callEmbeddedMetastore(Configuration conf) throws MetaException { + // Instantiate the metastore server handler directly instead of connecting + // through the network + // + // The code below simulates the following code + // + // client = HiveMetaStore.newRetryingHMSHandler(this.conf); + // + // using reflection API. This is done to avoid dependency of MetastoreClient on Hive Metastore. + // Note that newRetryingHMSHandler is static method, so we pass null as the object reference. + // + try { + Class<?> clazz = Class.forName(HIVE_METASTORE_CLASS); + //noinspection JavaReflectionMemberAccess + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, + Configuration.class); + method.setAccessible(true); + return (ThriftHiveMetastore.Iface) method.invoke(null, conf); + } catch (InvocationTargetException e) { + if (e.getCause() != null) { + MetaStoreUtils.logAndThrowMetaException((Exception)e.getCause()); + } + MetaStoreUtils.logAndThrowMetaException(e); + } catch (ClassNotFoundException + | NoSuchMethodException + | IllegalAccessException e) { + MetaStoreUtils.logAndThrowMetaException(e); + } + return null; + } + private void resolveUris() throws MetaException { String metastoreUrisString[] = MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS).split(","); http://git-wip-us.apache.org/repos/asf/hive/blob/0772f186/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 34055d2..35abd00 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.metastore; +import static org.apache.hadoop.hive.metastore.HiveMetaStoreClient.callEmbeddedMetastore; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; import java.io.IOException; import java.lang.reflect.Constructor; @@ -162,9 +162,9 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos throw new MetaException("Embedded metastore is not allowed here. Please configure " + ConfVars.THRIFT_URIS.toString() + "; it is currently set to [" + msUri + "]"); } - // instantiate the metastore server handler directly instead of connecting - // through the network - client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true); + + client = callEmbeddedMetastore(this.conf); + isConnected = true; snapshotActiveConf(); return;
