daijyc commented on a change in pull request #662: Hive-21787: Metastore table
cache enhancement
URL: https://github.com/apache/hive/pull/662#discussion_r291863243
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
##########
@@ -113,16 +134,116 @@ public int getPosition() {
}
}
+ static class TableWrapperSizeUpdater implements Runnable {
+ private Set<String> setToUpdate;
+ private ReentrantReadWriteLock lock;
+ private Cache<String, TableWrapper> cache;
+
+ TableWrapperSizeUpdater(Set<String> set, ReentrantReadWriteLock lock1,
Cache<String, TableWrapper> cache1) {
+ setToUpdate = set;
+ lock = lock1;
+ cache = cache1;
+ }
+
+ @Override public void run() {
+ for (String s : setToUpdate) {
+ refreshTableWrapperInCache(s);
+ }
+ setToUpdate.clear();
+ }
+
+ void refreshTableWrapperInCache(String tblKey) {
+ try {
+ lock.writeLock().lock();
+ TableWrapper tw = cache.getIfPresent(tblKey);
+ if (tw != null) {
+ //cache will re-weigh the TableWrapper and record new weight.
+ cache.put(tblKey, tw);
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+ }
+
+ /**
+ * Builder class for changing parameter of shared cache
+ */
+ public static class Builder {
+ private Map<String, Integer> tableSizeMap = null;
+ private int concurrencyLevel = -1;
+ private int refreshInterval = 10000;
+ private Configuration conf;
+
+ Builder tableSizeMap(Map<String, Integer> mp) {
+ this.tableSizeMap = mp;
+ return this;
+ }
+
+ Builder configuration(Configuration c) {
+ this.conf = c;
+ return this;
+ }
+
+ Builder concurrencyLevel(int cl) {
+ this.concurrencyLevel = cl;
+ return this;
+ }
+
+ Builder refreshInterval(int numMillis) {
+ this.refreshInterval = numMillis;
+ return this;
+ }
+
+ public SharedCache build(SharedCache sc) {
+ sc.tableSizeMap = this.tableSizeMap;
+ sc.initialize(conf, refreshInterval, concurrencyLevel);
+ return sc;
+ }
+ }
+
+ public void initialize(Configuration conf, int refreshInterval, int
concurrencyLevel) {
+ maxCacheSizeInBytes = MetastoreConf.getSizeVar(conf,
MetastoreConf.ConfVars.CACHED_RAW_STORE_MAX_CACHE_MEMORY);
- public void initialize(long maxSharedCacheSizeInBytes) {
- maxCacheSizeInBytes = maxSharedCacheSizeInBytes;
// Create estimators
if ((maxCacheSizeInBytes > 0) && (sizeEstimators == null)) {
sizeEstimators =
IncrementalObjectSizeEstimator.createEstimators(SharedCache.class);
}
+
+ if (tableCache == null) {
+ CacheBuilder<String, TableWrapper> b = CacheBuilder.newBuilder()
+ .maximumWeight(maxCacheSizeInBytes > 0 ? maxCacheSizeInBytes : 1024
* 1024)
+ .weigher(new Weigher<String, TableWrapper>() {
+ @Override public int weigh(String key, TableWrapper value) {
+ return value.getSize();
+ }
+ }).removalListener(new RemovalListener<String, TableWrapper>() {
+ @Override public void onRemoval(RemovalNotification<String,
TableWrapper> notification) {
Review comment:
When a table evict, we also need to decrease sd reference count associate
with it or its partitions.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
