Repository: hbase Updated Branches: refs/heads/master ef712df94 -> 4483b3cbe
HBASE-15338: Add a option to disable the data block cache for testing the performance of underlying file system Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4483b3cb Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4483b3cb Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4483b3cb Branch: refs/heads/master Commit: 4483b3cbed0c9cc17116af2407d946e605d6d627 Parents: ef712df Author: Liu Shaohui <[email protected]> Authored: Mon Mar 7 16:37:07 2016 +0800 Committer: Liu Shaohui <[email protected]> Committed: Mon Mar 7 16:37:07 2016 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/CacheConfig.java | 24 ++++--- .../hadoop/hbase/io/hfile/HFileReaderImpl.java | 2 +- .../hadoop/hbase/io/hfile/TestCacheConfig.java | 71 +++++++++++++++++++- 3 files changed, 87 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/4483b3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 6fe3927..2680c3d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -45,6 +45,12 @@ public class CacheConfig { private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName()); /** + * Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be + * cached if the block cache is enabled. + */ + public static final String CACHE_DATA_ON_READ_KEY = "hbase.block.data.cacheonread"; + + /** * Configuration key to cache data blocks on write. There are separate * switches for bloom blocks and non-root index blocks. */ @@ -136,8 +142,9 @@ public class CacheConfig { private static final String EXTERNAL_BLOCKCACHE_KEY = "hbase.blockcache.use.external"; private static final boolean EXTERNAL_BLOCKCACHE_DEFAULT = false; - private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY="hbase.blockcache.external.class"; - private static final String DROP_BEHIND_CACHE_COMPACTION_KEY="hbase.hfile.drop.behind.compaction"; + private static final String EXTERNAL_BLOCKCACHE_CLASS_KEY = "hbase.blockcache.external.class"; + private static final String DROP_BEHIND_CACHE_COMPACTION_KEY = + "hbase.hfile.drop.behind.compaction"; private static final boolean DROP_BEHIND_CACHE_COMPACTION_DEFAULT = true; /** @@ -219,7 +226,8 @@ public class CacheConfig { */ public CacheConfig(Configuration conf, HColumnDescriptor family) { this(CacheConfig.instantiateBlockCache(conf), - family.isBlockCacheEnabled(), + conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ) + && family.isBlockCacheEnabled(), family.isInMemory(), // For the following flags we enable them regardless of per-schema settings // if they are enabled in the global configuration. @@ -236,7 +244,7 @@ public class CacheConfig { DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), - conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) + conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); } @@ -247,7 +255,7 @@ public class CacheConfig { */ public CacheConfig(Configuration conf) { this(CacheConfig.instantiateBlockCache(conf), - DEFAULT_CACHE_DATA_ON_READ, + conf.getBoolean(CACHE_DATA_ON_READ_KEY, DEFAULT_CACHE_DATA_ON_READ), DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set // strictly from conf conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), @@ -258,7 +266,7 @@ public class CacheConfig { conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), - conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) + conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); } @@ -432,7 +440,7 @@ public class CacheConfig { * @return true if data blocks should be compressed in the cache, false if not */ public boolean shouldCacheDataCompressed() { - return isBlockCacheEnabled() && this.cacheDataCompressed; + return isBlockCacheEnabled() && this.cacheDataOnRead && this.cacheDataCompressed; } /** @@ -442,7 +450,7 @@ public class CacheConfig { if (!isBlockCacheEnabled()) return false; switch (category) { case DATA: - return this.cacheDataCompressed; + return this.cacheDataOnRead && this.cacheDataCompressed; default: return false; } http://git-wip-us.apache.org/repos/asf/hbase/blob/4483b3cb/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 331b8ba..8f5040e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -1426,7 +1426,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, this.isPrimaryReplicaReader()); - cacheBlock &= cacheConf.shouldCacheDataOnRead(); + cacheBlock &= cacheConf.shouldCacheBlockOnRead(BlockType.META.getCategory()); if (cacheConf.isBlockCacheEnabled()) { HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, false, true, true, BlockType.META, null); http://git-wip-us.apache.org/repos/asf/hbase/blob/4483b3cb/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index ad08794..2d06e293 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.io.hfile; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -34,9 +35,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.nio.ByteBuff; @@ -211,6 +214,72 @@ public class TestCacheConfig { } @Test + public void testDisableCacheDataBlock() throws IOException { + Configuration conf = HBaseConfiguration.create(); + CacheConfig cacheConfig = new CacheConfig(conf); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); + assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); + assertFalse(cacheConfig.shouldCacheDataCompressed()); + assertFalse(cacheConfig.shouldCacheDataOnWrite()); + assertTrue(cacheConfig.shouldCacheDataOnRead()); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); + assertFalse(cacheConfig.shouldCacheBloomsOnWrite()); + assertFalse(cacheConfig.shouldCacheIndexesOnWrite()); + + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_DATA_BLOCKS_COMPRESSED_KEY, true); + conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true); + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true); + + cacheConfig = new CacheConfig(conf); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); + assertTrue(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); + assertTrue(cacheConfig.shouldCacheDataCompressed()); + assertTrue(cacheConfig.shouldCacheDataOnWrite()); + assertTrue(cacheConfig.shouldCacheDataOnRead()); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); + assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); + assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); + + conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, false); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + + cacheConfig = new CacheConfig(conf); + assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); + assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); + assertFalse(cacheConfig.shouldCacheDataCompressed()); + assertFalse(cacheConfig.shouldCacheDataOnWrite()); + assertFalse(cacheConfig.shouldCacheDataOnRead()); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); + assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); + assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); + + conf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY, true); + conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); + + HColumnDescriptor family = new HColumnDescriptor("testDisableCacheDataBlock"); + family.setBlockCacheEnabled(false); + + cacheConfig = new CacheConfig(conf, family); + assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA)); + assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA)); + assertFalse(cacheConfig.shouldCacheDataCompressed()); + assertFalse(cacheConfig.shouldCacheDataOnWrite()); + assertFalse(cacheConfig.shouldCacheDataOnRead()); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.INDEX)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.META)); + assertTrue(cacheConfig.shouldCacheBlockOnRead(BlockCategory.BLOOM)); + assertTrue(cacheConfig.shouldCacheBloomsOnWrite()); + assertTrue(cacheConfig.shouldCacheIndexesOnWrite()); + } + + @Test public void testCacheConfigDefaultLRUBlockCache() { CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.isBlockCacheEnabled()); @@ -351,4 +420,4 @@ public class TestCacheConfig { blocks.get(BlockType.DATA) == null? 0: blocks.get(BlockType.DATA).intValue()); } -} \ No newline at end of file +}
