HIVE-9508: MetaStore client socket connection should have a lifetime (Thiruvel Thirumoolan reviewed by Vaibhav Gumashta)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2a12c9a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2a12c9a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2a12c9a Branch: refs/heads/llap Commit: e2a12c9a630d037f3e3aaf42acc873eac86bc9f3 Parents: 652febc Author: Vaibhav Gumashta <[email protected]> Authored: Tue May 5 10:44:16 2015 -0700 Committer: Vaibhav Gumashta <[email protected]> Committed: Tue May 5 10:44:16 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 6 ++++ .../hive/metastore/TestHiveMetaStore.java | 30 ++++++++++++++++++++ .../hive/metastore/RetryingMetaStoreClient.java | 27 +++++++++++++++--- 3 files changed, 59 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 5d4dbea..69fda45 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -119,6 +119,7 @@ public class HiveConf extends Configuration { HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, + HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, HiveConf.ConfVars.METASTOREPWD, HiveConf.ConfVars.METASTORECONNECTURLHOOK, HiveConf.ConfVars.METASTORECONNECTURLKEY, @@ -398,6 +399,11 @@ public class HiveConf extends Configuration { METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", "600s", new TimeValidator(TimeUnit.SECONDS), "MetaStore Client socket timeout in seconds"), + METASTORE_CLIENT_SOCKET_LIFETIME("hive.metastore.client.socket.lifetime", "0s", + new TimeValidator(TimeUnit.SECONDS), + "MetaStore Client socket lifetime in seconds. After this time is exceeded, client\n" + + "reconnects on the next MetaStore operation. A value of 0s means the connection\n" + + "has an infinite lifetime."), METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine", "password to use against metastore database"), METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "", http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 130fd67..dffeb34 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -2867,4 +2867,34 @@ public abstract class TestHiveMetaStore extends TestCase { ownerName, ownerType, createTime, functionType, resources); client.createFunction(func); } + + public void testRetriableClientWithConnLifetime() throws Exception { + + HiveConf conf = new HiveConf(hiveConf); + conf.setLong(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME.name(), 60); + long timeout = 65 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect + + // Test a normal retriable client + IMetaStoreClient client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName()); + client.getAllDatabases(); + client.close(); + + // Connect after the lifetime, there should not be any failures + client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName()); + Thread.sleep(timeout); + client.getAllDatabases(); + client.close(); + } + + private HiveMetaHookLoader getHookLoader() { + HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() { + @Override + public HiveMetaHook getHook( + org.apache.hadoop.hive.metastore.api.Table tbl) + throws MetaException { + return null; + } + }; + return hookLoader; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java index 77da6f7..1b6487a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java @@ -54,9 +54,9 @@ public class RetryingMetaStoreClient implements InvocationHandler { private final int retryLimit; private final long retryDelaySeconds; private final Map<String, Long> metaCallTimeMap; - - - + private final long connectionLifeTimeInMillis; + private long lastConnectionTime; + private boolean localMetaStore; protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader, Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { @@ -64,6 +64,11 @@ public class RetryingMetaStoreClient implements InvocationHandler { this.retryDelaySeconds = hiveConf.getTimeVar( HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS); this.metaCallTimeMap = metaCallTimeMap; + this.connectionLifeTimeInMillis = + hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000; + this.lastConnectionTime = System.currentTimeMillis(); + String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS); + localMetaStore = (msUri == null) || msUri.trim().isEmpty(); reloginExpiringKeytabUser(); this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] { @@ -104,8 +109,9 @@ public class RetryingMetaStoreClient implements InvocationHandler { while (true) { try { reloginExpiringKeytabUser(); - if(retriesMade > 0){ + if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) { base.reconnect(); + lastConnectionTime = System.currentTimeMillis(); } if (metaCallTimeMap == null) { ret = method.invoke(base, args); @@ -171,6 +177,19 @@ public class RetryingMetaStoreClient implements InvocationHandler { return methodSb.toString(); } + private boolean hasConnectionLifeTimeReached(Method method) { + if (connectionLifeTimeInMillis <= 0 || localMetaStore || + method.getName().equalsIgnoreCase("close")) { + return false; + } + boolean shouldReconnect = + (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis; + if (LOG.isDebugEnabled()) { + LOG.debug("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect); + } + return shouldReconnect; + } + /** * Relogin if login user is logged in using keytab * Relogin is actually done by ugi code only if sufficient time has passed
