This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 338d6d5 [GOBBLIN-912] Enable TTL caching on Hive Metastore client
connection
338d6d5 is described below
commit 338d6d52c1336fceec5f9a41a7dfd58ab8cbd155
Author: Zihan Li <[email protected]>
AuthorDate: Wed Oct 16 21:57:35 2019 -0700
[GOBBLIN-912] Enable TTL caching on Hive Metastore client connection
Closes #2766 from ZihanLi58/ttlHMS
---
.../gobblin/hive/HiveMetastoreClientPool.java | 26 ++++++++++++++++++++++
.../hive/metastore/HiveMetaStoreBasedRegister.java | 2 +-
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git
a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveMetastoreClientPool.java
b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveMetastoreClientPool.java
index 6721399..49915ed 100644
---
a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveMetastoreClientPool.java
+++
b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveMetastoreClientPool.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.apache.gobblin.util.PropertiesUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -59,6 +60,19 @@ public class HiveMetastoreClientPool {
public static final String POOL_CACHE_TTL_MINUTES_KEY =
"hive.metaStorePoolCache.ttl";
+ public static final String POOL_EVICTION_POLICY_CLASS_NAME =
"pool.eviction.policy.class.name";
+
+ public static final String DEFAULT_POOL_EVICTION_POLICY_CLASS_NAME =
"org.apache.commons.pool2.impl.DefaultEvictionPolicy";
+
+ public static final String POOL_MIN_EVICTABLE_IDLE_TIME_MILLIS =
"pool.min.evictable.idle.time.millis";
+
+ public static final long DEFAULT_POOL_MIN_EVICTABLE_IDLE_TIME_MILLIS =
600000L;
+
+ public static final String POOL_TIME_BETWEEN_EVICTION_MILLIS =
"pool.time.between eviction.millis";
+
+ public static final long DEFAULT_POOL_TIME_BETWEEN_EVICTION_MILLIS = 60000L;
+
+
private static Cache<Optional<String>, HiveMetastoreClientPool> poolCache =
null;
private static final Cache<Optional<String>, HiveMetastoreClientPool>
createPoolCache(final Properties properties) {
@@ -107,6 +121,14 @@ public class HiveMetastoreClientPool {
/**
* Constructor for {@link HiveMetastoreClientPool}.
+ * By default we will using the default eviction strategy for the client
pool. Client will be evicted if the following conditions are met:
+ * * <ul>
+ * * <li>the object has been idle longer than
+ * * {@link GenericObjectPool#getMinEvictableIdleTimeMillis()}</li>
+ * * <li>there are more than {@link GenericObjectPool#getMinIdle()} idle
objects in
+ * * the pool and the object has been idle for longer than
+ * * {@link GenericObjectPool#getSoftMinEvictableIdleTimeMillis()} </li>
+ * * </ul>
* @deprecated It is recommended to use the static {@link #get} method
instead. Use this constructor only if you
* different pool configurations are required.
*/
@@ -119,6 +141,10 @@ public class HiveMetastoreClientPool {
this.factory = new HiveMetaStoreClientFactory(metastoreURI);
this.pool = new GenericObjectPool<>(this.factory, config);
+ //Set the eviction policy for the client pool
+
this.pool.setEvictionPolicyClassName(properties.getProperty(POOL_EVICTION_POLICY_CLASS_NAME,
DEFAULT_POOL_EVICTION_POLICY_CLASS_NAME));
+
this.pool.setMinEvictableIdleTimeMillis(PropertiesUtils.getPropAsLong(properties,
POOL_MIN_EVICTABLE_IDLE_TIME_MILLIS,
DEFAULT_POOL_MIN_EVICTABLE_IDLE_TIME_MILLIS));
+
this.pool.setTimeBetweenEvictionRunsMillis(PropertiesUtils.getPropAsLong(properties,
POOL_TIME_BETWEEN_EVICTION_MILLIS, DEFAULT_POOL_TIME_BETWEEN_EVICTION_MILLIS));
this.hiveConf = this.factory.getHiveConf();
}
diff --git
a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
index 41ceaf3..5e8f999 100644
---
a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
+++
b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreBasedRegister.java
@@ -230,7 +230,7 @@ public class HiveMetaStoreBasedRegister extends
HiveRegister {
if(shouldUpdateLatestSchema) {
updateSchema(spec, table);
}
- if (needToUpdateTable(existingTable, spec.getTable())) {
+ if (needToUpdateTable(existingTable,
HiveMetaStoreUtils.getHiveTable(table))) {
try (Timer.Context context =
this.metricContext.timer(ALTER_TABLE).time()) {
client.alter_table(dbName, tableName,
getNewTblByMergingExistingTblProps(table, existingTable));
}
