Repository: hbase Updated Branches: refs/heads/branch-1.2 dfc9616ae -> 3bce24dee
HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3bce24de Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3bce24de Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3bce24de Branch: refs/heads/branch-1.2 Commit: 3bce24dee6edc3af9d3190a506e9d6df4c115108 Parents: dfc9616 Author: chenheng <chenh...@apache.org> Authored: Thu Aug 4 21:13:42 2016 +0800 Committer: chenheng <chenh...@apache.org> Committed: Thu Aug 4 21:37:48 2016 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/LruBlockCache.java | 36 ++++++++++++++++++-- .../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 2 ++ .../hbase/io/hfile/TestLruBlockCache.java | 5 +++ 3 files changed, 40 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 2781833..52ecf00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -112,6 +112,10 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { */ static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor"; + /** + * Hard capacity limit of cache, will reject any put if size > this * acceptable + */ + static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.hard.capacity.limit.factor"; static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.single.percentage"; static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.multi.percentage"; static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.memory.percentage"; @@ -138,6 +142,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { static final float DEFAULT_MULTI_FACTOR = 0.50f; static final float DEFAULT_MEMORY_FACTOR = 0.25f; + /** default hard capacity limit */ + static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f; + static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false; /** Statistics thread */ @@ -171,6 +178,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Cache access count (sequential ID) */ private final AtomicLong count; + /** hard capacity limit */ + private float hardCapacityLimitFactor; + /** Cache statistics */ private final CacheStats stats; @@ -228,6 +238,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR, DEFAULT_MEMORY_FACTOR, + DEFAULT_HARD_CAPACITY_LIMIT_FACTOR, false, DEFAULT_MAX_BLOCK_SIZE ); @@ -243,6 +254,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR), conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR), conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR), + conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR), conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE), conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE) ); @@ -269,7 +281,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel, float minFactor, float acceptableFactor, float singleFactor, - float multiFactor, float memoryFactor, boolean forceInMemory, long maxBlockSize) { + float multiFactor, float memoryFactor, float hardLimitFactor, + boolean forceInMemory, long maxBlockSize) { this.maxBlockSize = maxBlockSize; if(singleFactor + multiFactor + memoryFactor != 1 || singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) { @@ -297,6 +310,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { this.elements = new AtomicLong(0); this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel); this.size = new AtomicLong(this.overhead); + this.hardCapacityLimitFactor = hardLimitFactor; if(evictionThread) { this.evictionThread = new EvictionThread(this); this.evictionThread.start(); // FindBugs SC_START_IN_CTOR @@ -359,6 +373,22 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { LOG.warn(msg); return; } + long currentSize = size.get(); + long currentAcceptableSize = acceptableSize(); + long hardLimitSize = (long) (hardCapacityLimitFactor * currentAcceptableSize); + if (currentSize >= hardLimitSize) { + stats.failInsert(); + if (LOG.isTraceEnabled()) { + LOG.trace("LruBlockCache current size " + StringUtils.byteDesc(currentSize) + + " has exceeded acceptable size " + StringUtils.byteDesc(currentAcceptableSize) + " too many." + + " the hard limit size is " + StringUtils.byteDesc(hardLimitSize) + ", failed to put cacheKey:" + + cacheKey + " into LruBlockCache."); + } + if (!evictionInProgress) { + runEviction(); + } + return; + } cb = new LruCachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory); long newSize = updateSizeMetrics(cb, false); map.put(cacheKey, cb); @@ -367,7 +397,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { long size = map.size(); assertCounterSanity(size, val); } - if (newSize > acceptableSize() && !evictionInProgress) { + if (newSize > currentAcceptableSize && !evictionInProgress) { runEviction(); } } @@ -904,7 +934,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public final static long CACHE_FIXED_OVERHEAD = ClassSize.align( (4 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) + - (5 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN) + (6 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN) + ClassSize.OBJECT); @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index c89ef92..f6cae6d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -152,6 +152,8 @@ public class TestCacheOnWrite { // default blockcaches.add(new CacheConfig(conf).getBlockCache()); + //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287 + TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f); // memory BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration()); blockcaches.add(lru); http://git-wip-us.apache.org/repos/asf/hbase/blob/3bce24de/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index fd239ab..00e0e20 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -267,6 +267,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 16 * 1024 * 1024); @@ -388,6 +389,7 @@ public class TestLruBlockCache { 0.2f, // single 0.3f, // multi 0.5f, // memory + 1.2f, // limit true, 16 * 1024 * 1024); @@ -494,6 +496,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 16 * 1024 * 1024); @@ -557,6 +560,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 1024); CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long"); @@ -596,6 +600,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false, 16 * 1024 * 1024);
