HBASE-15385 PREFETCH_BLOCKS_ON_OPEN in HColumnDescriptor is ignored
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa215a67 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa215a67 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa215a67 Branch: refs/heads/HBASE-14850 Commit: fa215a67e20da8c1a450b16db27c73ee3f9d02c0 Parents: 3e1bdcc Author: stack <[email protected]> Authored: Wed Apr 20 09:38:30 2016 -0700 Committer: stack <[email protected]> Committed: Wed Apr 20 09:38:30 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/io/hfile/CacheConfig.java | 7 +++++-- .../apache/hadoop/hbase/io/hfile/HFileReaderImpl.java | 6 +++--- .../org/apache/hadoop/hbase/io/hfile/TestPrefetch.java | 13 +++++++++++++ src/main/asciidoc/_chapters/performance.adoc | 5 +++++ 4 files changed, 26 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 2680c3d..ffb9424 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -246,11 +246,14 @@ public class CacheConfig { HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); + LOG.info("Created cacheConfig for " + family.getNameAsString() + ": " + this); } /** * Create a cache configuration using the specified configuration object and - * defaults for family level settings. + * defaults for family level settings. Only use if no column family context. Prefer + * {@link CacheConfig#CacheConfig(Configuration, HColumnDescriptor)} + * @see #CacheConfig(Configuration, HColumnDescriptor) * @param conf hbase configuration */ public CacheConfig(Configuration conf) { @@ -268,6 +271,7 @@ public class CacheConfig { HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY, DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); + LOG.info("Created cacheConfig: " + this); } /** @@ -303,7 +307,6 @@ public class CacheConfig { this.prefetchOnOpen = prefetchOnOpen; this.cacheDataInL1 = cacheDataInL1; this.dropBehindCompaction = dropBehindCompaction; - LOG.info(this); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 0b1ac83..9a63d2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -253,7 +253,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { try { end = getTrailer().getLoadOnOpenDataOffset(); if (LOG.isTraceEnabled()) { - LOG.trace("File=" + path.toString() + ", offset=" + offset + ", end=" + end); + LOG.trace("Prefetch=" + path.toString() + ", offset=" + offset + ", end=" + end); } // TODO: Could we use block iterator in here? Would that get stuff into the cache? HFileBlock prevBlock = null; @@ -279,11 +279,11 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } catch (IOException e) { // IOExceptions are probably due to region closes (relocation, etc.) if (LOG.isTraceEnabled()) { - LOG.trace("File=" + path.toString() + ", offset=" + offset + ", end=" + end, e); + LOG.trace("Prefetch=" + path.toString() + ", offset=" + offset + ", end=" + end, e); } } catch (Exception e) { // Other exceptions are interesting - LOG.warn("File=" + path.toString() + ", offset=" + offset + ", end=" + end, e); + LOG.warn("Prefetch=" + path.toString() + ", offset=" + offset + ", end=" + end, e); } finally { PrefetchExecutor.complete(path); } http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index e31ac52..4c3db03 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -27,6 +27,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.fs.HFileSystem; @@ -60,6 +63,16 @@ public class TestPrefetch { cacheConf = new CacheConfig(conf); } + @Test + public void testPrefetchSetInHCDWorks() { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("f")); + hcd.setPrefetchBlocksOnOpen(true); + Configuration c = HBaseConfiguration.create(); + assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false)); + CacheConfig cc = new CacheConfig(c, hcd); + assertTrue(cc.shouldPrefetchOnOpen()); + } + @Test(timeout=60000) public void testPrefetch() throws Exception { Path storeFile = writeStoreFile(); http://git-wip-us.apache.org/repos/asf/hbase/blob/fa215a67/src/main/asciidoc/_chapters/performance.adoc ---------------------------------------------------------------------- diff --git a/src/main/asciidoc/_chapters/performance.adoc b/src/main/asciidoc/_chapters/performance.adoc index 01956d5..efb6ace 100644 --- a/src/main/asciidoc/_chapters/performance.adoc +++ b/src/main/asciidoc/_chapters/performance.adoc @@ -207,6 +207,11 @@ tableDesc.addFamily(cfDesc); See the API documentation for link:https://hbase.apache.org/devapidocs/org/apache/hadoop/hbase/io/hfile/CacheConfig.html[CacheConfig]. +To see prefetch in operation, enable TRACE level logging on +`org.apache.hadoop.hbase.io.hfile.HFileReaderImpl` in hbase-2.0+ +or on `org.apache.hadoop.hbase.io.hfile.HFileReaderV2` in earlier versions, hbase-1.x, of HBase. + + [[perf.rs.memstore.size]] === `hbase.regionserver.global.memstore.size`
