sam-an-cloudera commented on a change in pull request #662: Hive-21787:
Metastore table cache enhancement
URL: https://github.com/apache/hive/pull/662#discussion_r293636400
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
##########
@@ -113,16 +134,116 @@ public int getPosition() {
}
}
+ static class TableWrapperSizeUpdater implements Runnable {
+ private Set<String> setToUpdate;
+ private ReentrantReadWriteLock lock;
+ private Cache<String, TableWrapper> cache;
+
+ TableWrapperSizeUpdater(Set<String> set, ReentrantReadWriteLock lock1,
Cache<String, TableWrapper> cache1) {
+ setToUpdate = set;
+ lock = lock1;
+ cache = cache1;
+ }
+
+ @Override public void run() {
+ for (String s : setToUpdate) {
+ refreshTableWrapperInCache(s);
+ }
+ setToUpdate.clear();
+ }
+
+ void refreshTableWrapperInCache(String tblKey) {
+ try {
+ lock.writeLock().lock();
+ TableWrapper tw = cache.getIfPresent(tblKey);
+ if (tw != null) {
+ //cache will re-weigh the TableWrapper and record new weight.
+ cache.put(tblKey, tw);
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+ }
+
+ /**
+ * Builder class for changing parameter of shared cache
+ */
+ public static class Builder {
+ private Map<String, Integer> tableSizeMap = null;
+ private int concurrencyLevel = -1;
+ private int refreshInterval = 10000;
+ private Configuration conf;
+
+ Builder tableSizeMap(Map<String, Integer> mp) {
+ this.tableSizeMap = mp;
+ return this;
+ }
+
+ Builder configuration(Configuration c) {
+ this.conf = c;
+ return this;
+ }
+
+ Builder concurrencyLevel(int cl) {
+ this.concurrencyLevel = cl;
+ return this;
+ }
+
+ Builder refreshInterval(int numMillis) {
+ this.refreshInterval = numMillis;
+ return this;
+ }
+
+ public SharedCache build(SharedCache sc) {
+ sc.tableSizeMap = this.tableSizeMap;
+ sc.initialize(conf, refreshInterval, concurrencyLevel);
+ return sc;
+ }
+ }
+
+ public void initialize(Configuration conf, int refreshInterval, int
concurrencyLevel) {
+ maxCacheSizeInBytes = MetastoreConf.getSizeVar(conf,
MetastoreConf.ConfVars.CACHED_RAW_STORE_MAX_CACHE_MEMORY);
- public void initialize(long maxSharedCacheSizeInBytes) {
- maxCacheSizeInBytes = maxSharedCacheSizeInBytes;
// Create estimators
if ((maxCacheSizeInBytes > 0) && (sizeEstimators == null)) {
sizeEstimators =
IncrementalObjectSizeEstimator.createEstimators(SharedCache.class);
}
+
+ if (tableCache == null) {
+ CacheBuilder<String, TableWrapper> b = CacheBuilder.newBuilder()
+ .maximumWeight(maxCacheSizeInBytes > 0 ? maxCacheSizeInBytes : 1024
* 1024)
+ .weigher(new Weigher<String, TableWrapper>() {
+ @Override public int weigh(String key, TableWrapper value) {
+ return value.getSize();
+ }
+ }).removalListener(new RemovalListener<String, TableWrapper>() {
+ @Override public void onRemoval(RemovalNotification<String,
TableWrapper> notification) {
Review comment:
I tried add it, but then I found this will cause regression in some existing
test methods because of the following. When we alter table, the update call
will first decrement the SD reference count. So if in the removal listener, it
gets decremented again. so for the same SD it gets decremented twice. Some test
method in TestCachedStoreUpdateUsingEvents.java relies on the fact the SD is
decremented only once. I need to think of a better way to control this. Right
now I will revert the file and not decrement in removal listener.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
