Repository: hbase Updated Branches: refs/heads/branch-1.4 3f831ebdc -> e177f71fd
HBASE-18641 Include block content verification logic used in lruCache in bucketCache - revert due to test failure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e177f71f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e177f71f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e177f71f Branch: refs/heads/branch-1.4 Commit: e177f71fd13f00faa104172a0a42bb6a7f5f9040 Parents: 3f831eb Author: tedyu <yuzhih...@gmail.com> Authored: Fri Sep 8 12:45:24 2017 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Fri Sep 8 12:45:24 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/BlockCacheUtil.java | 11 ----------- .../hadoop/hbase/io/hfile/LruBlockCache.java | 13 +++++++++++-- .../hadoop/hbase/io/hfile/bucket/BucketCache.java | 17 ++--------------- 3 files changed, 13 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e177f71f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index a3d46ed..e5bb83b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.io.hfile; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.NavigableMap; import java.util.NavigableSet; import java.util.concurrent.ConcurrentSkipListMap; @@ -27,7 +26,6 @@ import java.util.concurrent.ConcurrentSkipListSet; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; -import org.apache.hadoop.hbase.util.Bytes; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.JsonMappingException; @@ -173,15 +171,6 @@ public class BlockCacheUtil { return cbsbf; } - public static int compareCacheBlock(Cacheable left, Cacheable right) { - ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength()); - left.serialize(l); - ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength()); - right.serialize(r); - return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(), - r.array(), r.arrayOffset(), r.limit()); - } - /** * Use one of these to keep a running account of cached blocks by file. Throw it away when done. * This is different than metrics in that it is stats on current state of a cache. http://git-wip-us.apache.org/repos/asf/hbase/blob/e177f71f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 69fea76..3fbe44b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -373,7 +373,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { LruCachedBlock cb = map.get(cacheKey); if (cb != null) { // compare the contents, if they are not equal, we are in big trouble - if (BlockCacheUtil.compareCacheBlock(buf, cb.getBuffer()) != 0) { + if (compare(buf, cb.getBuffer()) != 0) { throw new RuntimeException("Cached block contents differ, which should not have happened." + "cacheKey:" + cacheKey); } @@ -433,6 +433,15 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { } } + private int compare(Cacheable left, Cacheable right) { + ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength()); + left.serialize(l); + ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength()); + right.serialize(r); + return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(), + r.array(), r.arrayOffset(), r.limit()); + } + /** * Cache the block with the specified name and buffer. * <p> @@ -568,7 +577,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { boolean wait = getCurrentSize() < acceptableSize(); boolean inMemory = block.getPriority() == BlockPriority.MEMORY; ((BucketCache)victimHandler).cacheBlockWithWait(block.getCacheKey(), block.getBuffer(), - inMemory, true, wait); + inMemory, wait); } else { victimHandler.cacheBlock(block.getCacheKey(), block.getBuffer()); } http://git-wip-us.apache.org/repos/asf/hbase/blob/e177f71f/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index b8d7453..6d50bef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -402,7 +402,7 @@ public class BucketCache implements BlockCache, HeapSize { @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory, final boolean cacheDataInL1) { - cacheBlockWithWait(cacheKey, cachedItem, inMemory, cacheDataInL1, wait_when_cache); + cacheBlockWithWait(cacheKey, cachedItem, inMemory, wait_when_cache); } /** @@ -413,26 +413,13 @@ public class BucketCache implements BlockCache, HeapSize { * @param wait if true, blocking wait when queue is full */ public void cacheBlockWithWait(BlockCacheKey cacheKey, Cacheable cachedItem, boolean inMemory, - boolean cacheDataInL1, boolean wait) { + boolean wait) { if (LOG.isTraceEnabled()) LOG.trace("Caching key=" + cacheKey + ", item=" + cachedItem); if (!cacheEnabled) { return; } if (backingMap.containsKey(cacheKey)) { - /* - * Compare already cached block only if lruBlockCache is not used to cache data blocks - */ - if (!cacheDataInL1) { - Cacheable existingBlock = getBlock(cacheKey, false, false, false); - if (BlockCacheUtil.compareCacheBlock(cachedItem, existingBlock) != 0) { - throw new RuntimeException("Cached block contents differ, which should not have happened." - + "cacheKey:" + cacheKey); - } - } - String msg = "Caching an already cached block: " + cacheKey; - msg += ". This is harmless and can happen in rare cases (see HBASE-8547)"; - LOG.warn(msg); return; }