hive git commit: HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai)

2018-04-11 Thread vgumashta 
Repository: hive
Updated Branches:
  refs/heads/branch-3 f57c33607 -> a3d9c46df


HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during 
prewarm (Vaibhav Gumashta reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3d9c46d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3d9c46d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3d9c46d

Branch: refs/heads/branch-3
Commit: a3d9c46df453f787c3eaeeb2c33373997d17dec4
Parents: f57c336
Author: Vaibhav Gumashta 
Authored: Wed Apr 11 15:39:30 2018 -0700
Committer: Vaibhav Gumashta 
Committed: Wed Apr 11 15:46:26 2018 -0700

--
 .../hive/metastore/cache/CachedStore.java   | 198 +++
 1 file changed, 114 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a3d9c46d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index c47856d..1ce86bb 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -18,23 +18,21 @@
 package org.apache.hadoop.hive.metastore.cache;
 
 
-import java.io.Closeable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.EmptyStackException;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Stack;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -100,7 +98,6 @@ import 
org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
 import org.apache.hadoop.hive.metastore.api.SchemaVersion;
 import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.TableMeta;
 import org.apache.hadoop.hive.metastore.api.Type;
@@ -146,6 +143,7 @@ public class CachedStore implements RawStore, Configurable {
   // Time after which metastore cache is updated from metastore DB by the 
background update thread
   private static long cacheRefreshPeriodMS = DEFAULT_CACHE_REFRESH_PERIOD;
   private static AtomicBoolean isCachePrewarmed = new AtomicBoolean(false);
+  private static TablesPendingPrewarm tblsPendingPrewarm = new 
TablesPendingPrewarm();
   private RawStore rawStore = null;
   private Configuration conf;
   private PartitionExpressionProxy expressionProxy = null;
@@ -153,10 +151,6 @@ public class CachedStore implements RawStore, Configurable 
{
 
   static final private Logger LOG = 
LoggerFactory.getLogger(CachedStore.class.getName());
 
-  public CachedStore() {
-
-  }
-
   @Override
   public void setConf(Configuration conf) {
 setConfInternal(conf);
@@ -211,12 +205,13 @@ public class CachedStore implements RawStore, 
Configurable {
   Collection catalogsToCache;
   try {
 catalogsToCache = catalogsToCache(rawStore);
-LOG.info("Going to cache catalogs: " +
-org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
+LOG.info("Going to cache catalogs: "
++ org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
 List catalogs = new ArrayList<>(catalogsToCache.size());
-for (String catName : catalogsToCache) 
catalogs.add(rawStore.getCatalog(catName));
+for (String catName : catalogsToCache)
+  catalogs.add(rawStore.getCatalog(catName));
 sharedCache.populateCatalogsInCache(catalogs);
-  } catch (MetaException|NoSuchObjectException e) {
+  } catch (MetaException | NoSuchObjectException e) {
 LOG.warn("Failed to populate catalogs in cache, going to try again", 
e);
 // try again
 continue;
@@ -232,8 +227,8 @@ public class CachedStore implements RawStore, Configurable {
   databases.add(rawStore.getDatabase(catName, dbName));
 } catch

hive git commit: HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai)

2018-04-11 Thread vgumashta 
Repository: hive
Updated Branches:
  refs/heads/master 42187fdbc -> b3fe6522e


HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during 
prewarm (Vaibhav Gumashta reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b3fe6522
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b3fe6522
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b3fe6522

Branch: refs/heads/master
Commit: b3fe6522e651fa4f00f1a1a75e6f12c132eacf21
Parents: 42187fd
Author: Vaibhav Gumashta 
Authored: Wed Apr 11 15:39:30 2018 -0700
Committer: Vaibhav Gumashta 
Committed: Wed Apr 11 15:39:30 2018 -0700

--
 .../hive/metastore/cache/CachedStore.java   | 198 +++
 1 file changed, 114 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b3fe6522/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index c47856d..1ce86bb 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -18,23 +18,21 @@
 package org.apache.hadoop.hive.metastore.cache;
 
 
-import java.io.Closeable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.EmptyStackException;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Stack;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -100,7 +98,6 @@ import 
org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
 import org.apache.hadoop.hive.metastore.api.SchemaVersion;
 import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.TableMeta;
 import org.apache.hadoop.hive.metastore.api.Type;
@@ -146,6 +143,7 @@ public class CachedStore implements RawStore, Configurable {
   // Time after which metastore cache is updated from metastore DB by the 
background update thread
   private static long cacheRefreshPeriodMS = DEFAULT_CACHE_REFRESH_PERIOD;
   private static AtomicBoolean isCachePrewarmed = new AtomicBoolean(false);
+  private static TablesPendingPrewarm tblsPendingPrewarm = new 
TablesPendingPrewarm();
   private RawStore rawStore = null;
   private Configuration conf;
   private PartitionExpressionProxy expressionProxy = null;
@@ -153,10 +151,6 @@ public class CachedStore implements RawStore, Configurable 
{
 
   static final private Logger LOG = 
LoggerFactory.getLogger(CachedStore.class.getName());
 
-  public CachedStore() {
-
-  }
-
   @Override
   public void setConf(Configuration conf) {
 setConfInternal(conf);
@@ -211,12 +205,13 @@ public class CachedStore implements RawStore, 
Configurable {
   Collection catalogsToCache;
   try {
 catalogsToCache = catalogsToCache(rawStore);
-LOG.info("Going to cache catalogs: " +
-org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
+LOG.info("Going to cache catalogs: "
++ org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
 List catalogs = new ArrayList<>(catalogsToCache.size());
-for (String catName : catalogsToCache) 
catalogs.add(rawStore.getCatalog(catName));
+for (String catName : catalogsToCache)
+  catalogs.add(rawStore.getCatalog(catName));
 sharedCache.populateCatalogsInCache(catalogs);
-  } catch (MetaException|NoSuchObjectException e) {
+  } catch (MetaException | NoSuchObjectException e) {
 LOG.warn("Failed to populate catalogs in cache, going to try again", 
e);
 // try again
 continue;
@@ -232,8 +227,8 @@ public class CachedStore implements RawStore, Configurable {
   databases.add(rawStore.getDatabase(catName, dbName));
 } catch