This is an automated email from the ASF dual-hosted git repository.
taklwu pushed a commit to branch HBASE-30018
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-30018 by this push:
new 64a4f639c32 HBASE-30196 Add diagnostic cached-block iteration
capability for pluggable cache engines (#8384)
64a4f639c32 is described below
commit 64a4f639c3259680b3b8604e8a0ac67f4f487e01
Author: Vladimir Rodionov <[email protected]>
AuthorDate: Wed Jun 24 14:43:40 2026 -0700
HBASE-30196 Add diagnostic cached-block iteration capability for pluggable
cache engines (#8384)
Signed-off-by: Andor Molnár <[email protected]>
Signed-off-by: Tak Lon (Stephen) Wu <[email protected]>
---
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 44 ++++++++++++--
.../cache/BlockCacheBackedCacheAccessService.java | 15 ++++-
.../hbase/io/hfile/cache/CacheAccessService.java | 10 +++-
.../hbase/io/hfile/cache/CacheAccessServices.java | 24 ++++++++
.../io/hfile/cache/NoOpCacheAccessService.java | 5 ++
.../cache/TopologyBackedCacheAccessService.java | 13 ++++-
.../TestAvoidCellReferencesIntoShippedBlocks.java | 27 ++++++---
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 67 ++++++++++-----------
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 68 +++++++++++++---------
.../io/hfile/TestForceCacheImportantBlocks.java | 10 +++-
.../apache/hadoop/hbase/io/hfile/TestHFile.java | 34 +++++++----
.../hadoop/hbase/io/hfile/TestHFileReaderImpl.java | 3 +-
.../apache/hadoop/hbase/io/hfile/TestPrefetch.java | 32 +++++-----
.../hbase/io/hfile/TestRowIndexV1RoundTrip.java | 3 +-
.../hfile/cache/CacheAccessServiceTestFactory.java | 29 +++++++++
.../regionserver/TestSecureBulkLoadManager.java | 4 +-
16 files changed, 274 insertions(+), 114 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 840ec45fbab..a90bc700b5d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.conf.ConfigurationManager;
import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver;
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import
org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService;
import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices;
import org.apache.yetus.audience.InterfaceAudience;
@@ -190,13 +191,40 @@ public class CacheConfig implements
PropagatingConfigurationObserver {
* @param conf hbase configuration
*/
public CacheConfig(Configuration conf) {
- this(conf, null);
+ this(conf, (CacheAccessService) null);
}
+ /**
+ * Create a cache configuration using the specified configuration object and
block cache. Only use
+ * in tests
+ * @param conf hbase configuration
+ * @param blockCache block cache to use for this configuration
+ */
public CacheConfig(Configuration conf, BlockCache blockCache) {
this(conf, null, blockCache, ByteBuffAllocator.HEAP);
}
+ /**
+ * Create a cache configuration using the specified configuration object and
cache access service.
+ * Only use in tests
+ * @param conf hbase configuration
+ * @param service cache access service to use for this configuration
+ */
+ public CacheConfig(Configuration conf, CacheAccessService service) {
+ this(conf, null, service, ByteBuffAllocator.HEAP);
+ }
+
+ public CacheConfig(Configuration conf, ColumnFamilyDescriptor family,
CacheAccessService service,
+ ByteBuffAllocator byteBuffAllocator) {
+ initFromConf(conf, family);
+ this.byteBuffAllocator = byteBuffAllocator;
+ this.cacheAccessService = service != null ? service :
CacheAccessServices.disabled();
+ this.blockCache = service instanceof BlockCacheBackedCacheAccessService
+ ? ((BlockCacheBackedCacheAccessService) service).getBlockCache()
+ : null;
+
+ }
+
/**
* Create a cache configuration using the specified configuration object and
family descriptor.
* @param conf hbase configuration
@@ -204,6 +232,15 @@ public class CacheConfig implements
PropagatingConfigurationObserver {
*/
public CacheConfig(Configuration conf, ColumnFamilyDescriptor family,
BlockCache blockCache,
ByteBuffAllocator byteBuffAllocator) {
+ initFromConf(conf, family);
+ this.blockCache = blockCache;
+ this.byteBuffAllocator = byteBuffAllocator;
+ this.cacheAccessService = blockCache != null
+ ? CacheAccessServices.fromBlockCache(blockCache)
+ : CacheAccessServices.disabled();
+ }
+
+ private void initFromConf(Configuration conf, ColumnFamilyDescriptor family)
{
if (family == null || family.isBlockCacheEnabled()) {
this.cacheDataOnRead = conf.getBoolean(CACHE_DATA_ON_READ_KEY,
DEFAULT_CACHE_DATA_ON_READ);
this.inMemory = family == null ? DEFAULT_IN_MEMORY : family.isInMemory();
@@ -233,11 +270,6 @@ public class CacheConfig implements
PropagatingConfigurationObserver {
this.heapUsageThreshold =
conf.getDouble(PREFETCH_HEAP_USAGE_THRESHOLD,
DEFAULT_PREFETCH_HEAP_USAGE_THRESHOLD);
}
- this.blockCache = blockCache;
- this.byteBuffAllocator = byteBuffAllocator;
- this.cacheAccessService = blockCache != null
- ? CacheAccessServices.fromBlockCache(blockCache)
- : CacheAccessServices.disabled();
}
/**
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheAccessService.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheAccessService.java
index 91441f5deb0..e55b03f6fd8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheAccessService.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/BlockCacheBackedCacheAccessService.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.io.hfile.cache;
+import java.util.Iterator;
import java.util.Objects;
import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
@@ -26,6 +27,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.CacheStats;
import org.apache.hadoop.hbase.io.hfile.Cacheable;
+import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.io.hfile.HFileInfo;
import org.apache.yetus.audience.InterfaceAudience;
@@ -51,7 +53,8 @@ import org.apache.yetus.audience.InterfaceAudience;
* </p>
*/
@InterfaceAudience.Private
-public class BlockCacheBackedCacheAccessService implements CacheAccessService {
+public class BlockCacheBackedCacheAccessService
+ implements CacheAccessService, Iterable<CachedBlock> {
private final BlockCache blockCache;
@@ -321,4 +324,14 @@ public class BlockCacheBackedCacheAccessService implements
CacheAccessService {
Objects.requireNonNull(conf, "conf must not be null");
return blockCache.shouldCacheBlock(key, maxTimestamp, conf);
}
+
+ @Override
+ public Iterator<CachedBlock> iterator() {
+ return blockCache.iterator();
+ }
+
+ @Override
+ public long getCurrentSize() {
+ return blockCache.getCurrentSize();
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessService.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessService.java
index ce565518b98..d5dfa54ea27 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessService.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessService.java
@@ -22,6 +22,7 @@ import java.util.Optional;
import java.util.function.Consumer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.CacheStats;
@@ -73,7 +74,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* </p>
*/
@InterfaceAudience.Private
-public interface CacheAccessService {
+public interface CacheAccessService extends ConfigurationObserver {
/**
* Returns a human-readable name for this cache access service instance.
@@ -325,6 +326,12 @@ public interface CacheAccessService {
*/
long size();
+ /**
+ * Returns the occupied size of the block cache, in bytes.
+ * @return occupied space in cache, in bytes
+ */
+ long getCurrentSize();
+
/**
* Returns the currently occupied size of cached data blocks, in bytes.
* <p>
@@ -422,6 +429,7 @@ public interface CacheAccessService {
* </p>
* @param config new configuration
*/
+ @Override
default void onConfigurationChange(Configuration config) {
// noop
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java
index a4de6035728..d4e739fcdcc 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServices.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.hbase.io.hfile.cache;
import java.util.Objects;
+import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory;
+import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -117,4 +119,26 @@ public final class CacheAccessServices {
public static CacheAccessService disabled() {
return new NoOpCacheAccessService();
}
+
+ /**
+ * Returns an iterable cached-block view for the supplied cache access
service when available.
+ * <p>
+ * Cached-block iteration is intended for tests, diagnostics, and admin
views only. It must not be
+ * used by normal read/write paths. Not every {@link CacheAccessService}
implementation is
+ * required to expose cached-block iteration.
+ * </p>
+ * @param cacheAccessService cache access service
+ * @return optional iterable cached-block view
+ * @throws NullPointerException if {@code cacheAccessService} is {@code null}
+ */
+ @SuppressWarnings("unchecked")
+ public static Optional<Iterable<CachedBlock>>
+ asCachedBlockIterable(CacheAccessService cacheAccessService) {
+ Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be
null");
+ if (cacheAccessService instanceof Iterable) {
+ return Optional.of((Iterable<CachedBlock>) cacheAccessService);
+ }
+ return Optional.empty();
+ }
+
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/NoOpCacheAccessService.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/NoOpCacheAccessService.java
index 938541e93ee..e956c466e34 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/NoOpCacheAccessService.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/NoOpCacheAccessService.java
@@ -306,4 +306,9 @@ public final class NoOpCacheAccessService implements
CacheAccessService {
Objects.requireNonNull(conf, "conf must not be null");
return Optional.empty();
}
+
+ @Override
+ public long getCurrentSize() {
+ return 0;
+ }
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java
index 70f575c1136..b7c278d3ead 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/cache/TopologyBackedCacheAccessService.java
@@ -287,8 +287,8 @@ public class TopologyBackedCacheAccessService implements
CacheAccessService {
}
/**
- * Returns aggregate occupied cache size across participating engines.
- * @return aggregate occupied cache size
+ * Returns aggregate total size of the block cache, in bytes.
+ * @return size of cache, in bytes
*/
@Override
public long size() {
@@ -299,6 +299,15 @@ public class TopologyBackedCacheAccessService implements
CacheAccessService {
return size;
}
+ /**
+ * Returns aggregate occupied size of the block cache, in bytes.
+ * @return occupied space in cache, in bytes
+ */
+ @Override
+ public long getCurrentSize() {
+ return getCurrentDataSize();
+ }
+
/**
* Returns aggregate occupied data-block size across participating engines.
* @return aggregate occupied data-block size
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
index 31ce704bf1b..51599c7f0f7 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAvoidCellReferencesIntoShippedBlocks.java
@@ -46,6 +46,9 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.CachedBlock;
import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import
org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices;
import org.apache.hadoop.hbase.regionserver.DelegatingInternalScanner;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStore;
@@ -139,7 +142,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
- final BlockCache cache = cacheConf.getBlockCache().get();
+ final CacheAccessService cache = cacheConf.getCacheAccessService();
// insert data. 5 Rows are added
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, data);
@@ -211,9 +214,9 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
private static class ScannerThread extends Thread {
private final Table table;
- private final BlockCache cache;
+ private final CacheAccessService cache;
- public ScannerThread(Table table, BlockCache cache) {
+ public ScannerThread(Table table, CacheAccessService cache) {
this.table = table;
this.cache = cache;
}
@@ -230,7 +233,8 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
}
}
List<BlockCacheKey> cacheList = new ArrayList<>();
- Iterator<CachedBlock> iterator = cache.iterator();
+ Iterator<CachedBlock> iterator =
+
CacheAccessServices.asCachedBlockIterable(cache).orElseThrow().iterator();
// evict all the blocks
while (iterator.hasNext()) {
CachedBlock next = iterator.next();
@@ -304,7 +308,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
CacheConfig cacheConf = store.getCacheConfig();
cacheConf.setCacheDataOnWrite(true);
cacheConf.setEvictOnClose(true);
- final BlockCache cache = cacheConf.getBlockCache().get();
+ final CacheAccessService cache = cacheConf.getCacheAccessService();
// insert data. 5 Rows are added
Put put = new Put(ROW);
put.addColumn(FAMILY, QUALIFIER, data);
@@ -364,10 +368,12 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
try (ScanPerNextResultScanner scanner =
new
ScanPerNextResultScanner(TEST_UTIL.getAsyncConnection().getTable(tableName),
s)) {
Thread evictorThread = new Thread() {
+ @SuppressWarnings("unchecked")
@Override
public void run() {
List<BlockCacheKey> cacheList = new ArrayList<>();
- Iterator<CachedBlock> iterator = cache.iterator();
+ Iterator<CachedBlock> iterator =
+
CacheAccessServices.asCachedBlockIterable(cache).orElseThrow().iterator();
// evict all the blocks
while (iterator.hasNext()) {
CachedBlock next = iterator.next();
@@ -383,7 +389,7 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
Thread.sleep(1);
} catch (InterruptedException e1) {
}
- iterator = cache.iterator();
+ iterator =
CacheAccessServices.asCachedBlockIterable(cache).orElseThrow().iterator();
int refBlockCount = 0;
while (iterator.hasNext()) {
iterator.next();
@@ -407,7 +413,8 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
while (true) {
newBlockRefCount = 0;
newCacheList.clear();
- iterator = cache.iterator();
+ iterator =
+
CacheAccessServices.asCachedBlockIterable(cache).orElseThrow().iterator();
while (iterator.hasNext()) {
CachedBlock next = iterator.next();
BlockCacheKey cacheKey = new
BlockCacheKey(next.getFilename(), next.getOffset());
@@ -443,7 +450,9 @@ public class TestAvoidCellReferencesIntoShippedBlocks {
/**
* For {@link BucketCache},we only evict Block if there is no rpc referenced.
*/
- private void evictBlock(BlockCache blockCache, BlockCacheKey blockCacheKey) {
+ private void evictBlock(CacheAccessService cache, BlockCacheKey
blockCacheKey) {
+ // TODO: will be refactored later once we get CacheEngine refactoring done
+ BlockCache blockCache = ((BlockCacheBackedCacheAccessService)
cache).getBlockCache();
assertTrue(blockCache instanceof CombinedBlockCache);
BlockCache[] blockCaches = blockCache.getBlockCaches();
for (BlockCache currentBlockCache : blockCaches) {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index eea755218b6..03c9f52d37a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -41,6 +41,7 @@ import
org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import
org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService;
import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory;
import org.apache.hadoop.hbase.io.hfile.cache.NoOpCacheAccessService;
import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
import org.apache.hadoop.hbase.nio.ByteBuff;
@@ -159,31 +160,31 @@ public class TestCacheConfig {
}
/**
- * @param bc The block cache instance.
+ * @param service The cache access service instance.
* @param cc Cache config.
* @param doubling If true, addition of element ups counter by 2, not 1,
because element added to
* onheap and offheap caches.
* @param sizing True if we should run sizing test (doesn't always apply).
*/
- void basicBlockCacheOps(final BlockCache bc, final CacheConfig cc, final
boolean doubling,
- final boolean sizing) {
+ void basicBlockCacheOps(final CacheAccessService service, final CacheConfig
cc,
+ final boolean doubling, final boolean sizing) {
assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory());
BlockCacheKey bck = new BlockCacheKey("f", 0);
Cacheable c = new DataCacheEntry();
// Do asserts on block counting.
- long initialBlockCount = bc.getBlockCount();
- bc.cacheBlock(bck, c, cc.isInMemory());
- assertEquals(doubling ? 2 : 1, bc.getBlockCount() - initialBlockCount);
- bc.evictBlock(bck);
- assertEquals(initialBlockCount, bc.getBlockCount());
+ long initialBlockCount = service.getBlockCount();
+ service.cacheBlock(bck, c, cc.isInMemory());
+ assertEquals(doubling ? 2 : 1, service.getBlockCount() -
initialBlockCount);
+ service.evictBlock(bck);
+ assertEquals(initialBlockCount, service.getBlockCount());
// Do size accounting. Do it after the above 'warm-up' because it looks
like some
// buffers do lazy allocation so sizes are off on first go around.
if (sizing) {
- long originalSize = bc.getCurrentSize();
- bc.cacheBlock(bck, c, cc.isInMemory());
- assertTrue(bc.getCurrentSize() > originalSize);
- bc.evictBlock(bck);
- long size = bc.getCurrentSize();
+ long originalSize = service.getCurrentDataSize();
+ service.cacheBlock(bck, c, cc.isInMemory());
+ assertTrue(service.getCurrentDataSize() > originalSize);
+ service.evictBlock(bck);
+ long size = service.getCurrentDataSize();
assertEquals(originalSize, size);
}
}
@@ -254,7 +255,8 @@ public class TestCacheConfig {
ColumnFamilyDescriptor columnFamilyDescriptor =
ColumnFamilyDescriptorBuilder
.newBuilder(Bytes.toBytes("testDisableCacheDataBlock")).setBlockCacheEnabled(false).build();
- cacheConfig = new CacheConfig(conf, columnFamilyDescriptor, null,
ByteBuffAllocator.HEAP);
+ cacheConfig = new CacheConfig(conf, columnFamilyDescriptor,
(CacheAccessService) null,
+ ByteBuffAllocator.HEAP);
assertFalse(cacheConfig.shouldCacheBlockOnRead(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheCompressed(BlockCategory.DATA));
assertFalse(cacheConfig.shouldCacheDataCompressed());
@@ -271,9 +273,9 @@ public class TestCacheConfig {
public void testCacheConfigDefaultLRUBlockCache() {
CacheConfig cc = new CacheConfig(this.conf);
assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory());
- BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf);
- basicBlockCacheOps(blockCache, cc, false, true);
- assertTrue(blockCache instanceof LruBlockCache);
+ CacheAccessService service =
CacheAccessServiceTestFactory.fromConfiguration(this.conf);
+ basicBlockCacheOps(service, cc, false, true);
+ assertTrue(CacheAccessServiceTestFactory.blockCache(service) instanceof
LruBlockCache);
}
/**
@@ -303,11 +305,11 @@ public class TestCacheConfig {
final int bcSize = 100;
this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
CacheConfig cc = new CacheConfig(this.conf);
- BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf);
- basicBlockCacheOps(blockCache, cc, false, false);
- assertTrue(blockCache instanceof CombinedBlockCache);
+ CacheAccessService service =
CacheAccessServiceTestFactory.fromConfiguration(this.conf);
+ basicBlockCacheOps(service, cc, false, false);
+ assertTrue(CacheAccessServiceTestFactory.blockCache(service) instanceof
CombinedBlockCache);
// TODO: Assert sizes allocated are right and proportions.
- CombinedBlockCache cbc = (CombinedBlockCache) blockCache;
+ CombinedBlockCache cbc = (CombinedBlockCache)
CacheAccessServiceTestFactory.blockCache(service);
BlockCache[] bcs = cbc.getBlockCaches();
assertTrue(bcs[0] instanceof LruBlockCache);
LruBlockCache lbc = (LruBlockCache) bcs[0];
@@ -335,11 +337,11 @@ public class TestCacheConfig {
assertTrue(lruExpectedSize < bcExpectedSize);
this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
CacheConfig cc = new CacheConfig(this.conf);
- BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf);
- basicBlockCacheOps(blockCache, cc, false, false);
- assertTrue(blockCache instanceof CombinedBlockCache);
+ CacheAccessService service =
CacheAccessServiceTestFactory.fromConfiguration(this.conf);
+ basicBlockCacheOps(service, cc, false, false);
+ assertTrue(CacheAccessServiceTestFactory.blockCache(service) instanceof
CombinedBlockCache);
// TODO: Assert sizes allocated are right and proportions.
- CombinedBlockCache cbc = (CombinedBlockCache) blockCache;
+ CombinedBlockCache cbc = (CombinedBlockCache)
CacheAccessServiceTestFactory.blockCache(service);
FirstLevelBlockCache lbc = cbc.l1Cache;
assertEquals(lruExpectedSize, lbc.getMaxSize());
BlockCache bc = cbc.l2Cache;
@@ -388,19 +390,18 @@ public class TestCacheConfig {
@Test
public void testIndexOnlyLruBlockCache() {
- CacheConfig cc = new CacheConfig(this.conf);
conf.set(BlockCacheFactory.BLOCKCACHE_POLICY_KEY, "IndexOnlyLRU");
- BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf);
- assertTrue(blockCache instanceof IndexOnlyLruBlockCache);
+ CacheAccessService cache =
CacheAccessServiceTestFactory.fromConfiguration(this.conf);
+ assertTrue(CacheAccessServiceTestFactory.blockCache(cache) instanceof
IndexOnlyLruBlockCache);
// reject data block
- long initialBlockCount = blockCache.getBlockCount();
+ long initialBlockCount = cache.getBlockCount();
BlockCacheKey bck = new BlockCacheKey("bck", 0);
Cacheable c = new DataCacheEntry();
- blockCache.cacheBlock(bck, c, true);
+ cache.cacheBlock(bck, c, true);
// accept index block
Cacheable indexCacheEntry = new IndexCacheEntry();
- blockCache.cacheBlock(bck, indexCacheEntry, true);
- assertEquals(initialBlockCount + 1, blockCache.getBlockCount());
+ cache.cacheBlock(bck, indexCacheEntry, true);
+ assertEquals(initialBlockCount + 1, cache.getBlockCount());
}
@Test
@@ -440,7 +441,7 @@ public class TestCacheConfig {
void testCacheAccessServiceIsNoOpWhenBlockCacheIsNull() {
Configuration conf = this.conf;
- CacheConfig cacheConfig = new CacheConfig(conf, null);
+ CacheConfig cacheConfig = new CacheConfig(conf);
CacheAccessService service = cacheConfig.getCacheAccessService();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index 1fc359e6319..a016cbef03a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -50,6 +50,10 @@ import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import
org.apache.hadoop.hbase.io.hfile.cache.BlockCacheBackedCacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServices;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
@@ -88,7 +92,7 @@ public class TestCacheOnWrite {
private FileSystem fs;
private Random rand = new Random(12983177L);
private Path storeFilePath;
- private BlockCache blockCache;
+ private CacheAccessService cache;
private String testDescription;
private final CacheOnWriteType cowType;
@@ -148,46 +152,47 @@ public class TestCacheOnWrite {
}
public TestCacheOnWrite(CacheOnWriteType cowType, Compression.Algorithm
compress,
- boolean cacheCompressedData, BlockCache blockCache) {
+ boolean cacheCompressedData, CacheAccessService blockCache) {
this.cowType = cowType;
this.compress = compress;
this.cacheCompressedData = cacheCompressedData;
- this.blockCache = blockCache;
+ this.cache = blockCache;
testDescription = "[cacheOnWrite=" + cowType + ", compress=" + compress
+ ", cacheCompressedData=" + cacheCompressedData + "]";
LOG.info(testDescription);
}
- private static List<BlockCache> getBlockCaches() throws IOException {
+ private static List<CacheAccessService> getCacheServices() throws
IOException {
Configuration conf = TEST_UTIL.getConfiguration();
- List<BlockCache> blockcaches = new ArrayList<>();
+ List<CacheAccessService> caches = new ArrayList<>();
// default
- blockcaches.add(BlockCacheFactory.createBlockCache(conf));
+ caches.add(CacheAccessServiceTestFactory.fromConfiguration(conf));
// set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f
due to HBASE-16287
TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME,
2.0f);
// memory
- BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024,
TEST_UTIL.getConfiguration());
- blockcaches.add(lru);
+ CacheAccessService lru =
+ CacheAccessServiceTestFactory.lru(128 * 1024 * 1024, 64 * 1024,
TEST_UTIL.getConfiguration());
+ caches.add(lru);
// bucket cache
FileSystem.get(conf).mkdirs(TEST_UTIL.getDataTestDir());
int[] bucketSizes =
{ INDEX_BLOCK_SIZE, DATA_BLOCK_SIZE, BLOOM_BLOCK_SIZE, 64 * 1024, 128 *
1024 };
- BlockCache bucketcache =
- new BucketCache("offheap", 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5,
64 * 100, null);
- blockcaches.add(bucketcache);
- return blockcaches;
+ CacheAccessService bucketcache =
CacheAccessServiceTestFactory.bucket("offheap",
+ 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5, 64 * 100, null);
+ caches.add(bucketcache);
+ return caches;
}
public static Stream<Arguments> parameters() throws IOException {
List<Arguments> params = new ArrayList<>();
- for (BlockCache blockCache : getBlockCaches()) {
+ for (CacheAccessService cache : getCacheServices()) {
for (CacheOnWriteType cowType : CacheOnWriteType.values()) {
for (Compression.Algorithm compress :
HBaseCommonTestingUtil.COMPRESSION_ALGORITHMS) {
for (boolean cacheCompressedData : new boolean[] { false, true }) {
- params.add(Arguments.of(cowType, compress, cacheCompressedData,
blockCache));
+ params.add(Arguments.of(cowType, compress, cacheCompressedData,
cache));
}
}
}
@@ -195,23 +200,25 @@ public class TestCacheOnWrite {
return params.stream();
}
- private void clearBlockCache(BlockCache blockCache) throws
InterruptedException {
+ private void clearBlockCache(CacheAccessService cache) throws
InterruptedException {
+ // TODO: HBASE-30018 refactor later
+ BlockCache blockCache = ((BlockCacheBackedCacheAccessService)
cache).getBlockCache();
if (blockCache instanceof LruBlockCache) {
((LruBlockCache) blockCache).clearCache();
} else {
// BucketCache may not return all cached blocks(blocks in write queue),
so check it here.
- for (int clearCount = 0; blockCache.getBlockCount() > 0; clearCount++) {
+ for (int clearCount = 0; cache.getBlockCount() > 0; clearCount++) {
if (clearCount > 0) {
- LOG.warn("clear block cache " + blockCache + " " + clearCount + "
times, "
- + blockCache.getBlockCount() + " blocks remaining");
+ LOG.warn("clear block cache " + cache + " " + clearCount + " times, "
+ + cache.getBlockCount() + " blocks remaining");
Thread.sleep(10);
}
for (CachedBlock block : Lists.newArrayList(blockCache)) {
BlockCacheKey key = new BlockCacheKey(block.getFilename(),
block.getOffset());
// CombinedBucketCache may need evict two times.
- for (int evictCount = 0; blockCache.evictBlock(key); evictCount++) {
+ for (int evictCount = 0; cache.evictBlock(key); evictCount++) {
if (evictCount > 1) {
- LOG.warn("evict block " + block + " in " + blockCache + " " +
evictCount
+ LOG.warn("evict block " + block + " in " + cache + " " +
evictCount
+ " times, maybe a bug here");
}
}
@@ -233,13 +240,13 @@ public class TestCacheOnWrite {
cowType.shouldBeCached(BlockType.LEAF_INDEX));
conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
cowType.shouldBeCached(BlockType.BLOOM_CHUNK));
- cacheConf = new CacheConfig(conf, blockCache);
+ cacheConf = new CacheConfig(conf, ((BlockCacheBackedCacheAccessService)
cache).getBlockCache());
fs = HFileSystem.get(conf);
}
@AfterEach
public void tearDown() throws IOException, InterruptedException {
- clearBlockCache(blockCache);
+ clearBlockCache(cache);
}
@AfterAll
@@ -277,7 +284,7 @@ public class TestCacheOnWrite {
HFileBlock block =
reader.readBlock(offset, -1, false, true, false, true, null,
encodingInCache);
BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
offset);
- HFileBlock fromCache = (HFileBlock) blockCache.getBlock(blockCacheKey,
true, false, true);
+ HFileBlock fromCache = (HFileBlock) cache.getBlock(blockCacheKey, true,
false, true);
boolean isCached = fromCache != null;
cachedBlocksOffset.add(offset);
cachedBlocks.put(offset, fromCache == null ? null : Pair.newPair(block,
fromCache));
@@ -434,7 +441,8 @@ public class TestCacheOnWrite {
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(cfBytes)
.setCompressionType(compress).setBloomFilterType(BLOOM_TYPE).setMaxVersions(maxVersions)
.setDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()).build();
- HRegion region = TEST_UTIL.createTestRegion(table, cfd, blockCache);
+ HRegion region = TEST_UTIL.createTestRegion(table, cfd,
+ ((BlockCacheBackedCacheAccessService) cache).getBlockCache());
int rowIdx = 0;
long ts = EnvironmentEdgeManager.currentTime();
for (int iFile = 0; iFile < 5; ++iFile) {
@@ -466,8 +474,8 @@ public class TestCacheOnWrite {
region.flush(true);
}
- clearBlockCache(blockCache);
- assertEquals(0, blockCache.getBlockCount());
+ clearBlockCache(cache);
+ assertEquals(0, cache.getBlockCount());
region.compact(false);
LOG.debug("compactStores() returned");
@@ -475,8 +483,10 @@ public class TestCacheOnWrite {
boolean dataBlockCached = false;
boolean bloomBlockCached = false;
boolean indexBlockCached = false;
+ Iterable<CachedBlock> cachedBlocks =
+ CacheAccessServices.asCachedBlockIterable(cache).orElseThrow();
- for (CachedBlock block : blockCache) {
+ for (CachedBlock block : cachedBlocks) {
if (DATA_BLOCK_TYPES.contains(block.getBlockType())) {
dataBlockCached = true;
} else if (BLOOM_BLOCK_TYPES.contains(block.getBlockType())) {
@@ -489,8 +499,8 @@ public class TestCacheOnWrite {
// Data blocks should be cached in instances where we are caching blocks
on write. In the case
// of testing
// BucketCache, we cannot verify block type as it is not stored in the
cache.
- boolean cacheOnCompactAndNonBucketCache =
- cacheBlocksOnCompaction && !(blockCache instanceof BucketCache);
+ boolean cacheOnCompactAndNonBucketCache = cacheBlocksOnCompaction
+ && !(((BlockCacheBackedCacheAccessService) cache).getBlockCache()
instanceof BucketCache);
String assertErrorMessage = "\nTest description: " + testDescription
+ "\ncacheBlocksOnCompaction: " + cacheBlocksOnCompaction + "\n";
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
index 7fe499b11fd..c7a0dd2e489 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.IOTests;
@@ -96,13 +98,15 @@ public class TestForceCacheImportantBlocks {
public void testCacheBlocks() throws IOException {
// Set index block size to be the same as normal block size.
TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY,
BLOCK_SIZE);
- BlockCache blockCache =
BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration());
+ CacheAccessService cache =
+
CacheAccessServiceTestFactory.fromConfiguration(TEST_UTIL.getConfiguration());
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS)
.setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE)
.setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build();
- HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache);
- CacheStats stats = blockCache.getStats();
+ HRegion region =
+ TEST_UTIL.createTestRegion(TABLE, cfd,
CacheAccessServiceTestFactory.blockCache(cache));
+ CacheStats stats = cache.getStats();
writeTestData(region);
assertEquals(0, stats.getHitCount());
assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum());
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
index 5d470a60a3d..0168fa7cc69 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
@@ -43,6 +43,7 @@ import java.util.Optional;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -76,6 +77,8 @@ import org.apache.hadoop.hbase.io.encoding.IndexBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory;
import org.apache.hadoop.hbase.monitoring.ThreadLocalServerSideScanMetrics;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.RefCnt;
@@ -175,7 +178,8 @@ public class TestHFile {
fillByteBuffAllocator(alloc, bufCount);
Path storeFilePath = writeStoreFile();
// Open the file reader with LRUBlockCache
- BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true,
conf);
+ CacheAccessService lru =
+ CacheAccessServiceTestFactory.lru(1024 * 1024 * 32, blockSize, true,
conf);
CacheConfig cacheConfig = new CacheConfig(conf, null, lru, alloc);
HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig,
true, conf);
long offset = 0;
@@ -216,10 +220,15 @@ public class TestHFile {
counter.incrementAndGet();
}
});
- BlockCache cache = Mockito.mock(BlockCache.class);
+ CacheAccessService cache = Mockito.mock(CacheAccessService.class);
Mockito.when(cache.shouldCacheBlock(Mockito.any(), Mockito.anyLong(),
Mockito.any()))
.thenReturn(Optional.of(false));
Mockito.when(cache.isCacheEnabled()).thenReturn(true);
+ Mockito.doAnswer(invocation -> {
+ Consumer<CacheAccessService> action = invocation.getArgument(0);
+ action.accept(cache);
+ return null;
+ }).when(cache).ifEnabled(Mockito.<Consumer<CacheAccessService>> any());
Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
"testWriterCacheOnWriteSkipDoesNotLeak");
HFileContext context = new
HFileContextBuilder().withBlockSize(blockSize).build();
@@ -263,8 +272,8 @@ public class TestHFile {
Path storeFilePath = writeStoreFile();
// Initialize the block cache and HFile reader
- BlockCache lru = BlockCacheFactory.createBlockCache(conf);
- assertTrue(lru instanceof LruBlockCache);
+ CacheAccessService lru =
CacheAccessServiceTestFactory.fromConfiguration(conf);
+ assertTrue(CacheAccessServiceTestFactory.blockCache(lru) instanceof
LruBlockCache);
CacheConfig cacheConfig = new CacheConfig(conf, null, lru,
ByteBuffAllocator.HEAP);
HFileReaderImpl reader =
(HFileReaderImpl) HFile.createReader(fs, storeFilePath, cacheConfig,
true, conf);
@@ -337,14 +346,14 @@ public class TestHFile {
assertBytesReadFromCache(true, DataBlockEncoding.FAST_DIFF);
}
- private BlockCache initCombinedBlockCache(final String l1CachePolicy) {
+ private CacheAccessService initCombinedBlockCacheBackedService(final String
l1CachePolicy) {
Configuration that = HBaseConfiguration.create(conf);
that.setFloat(BUCKET_CACHE_SIZE_KEY, 32); // 32MB for bucket cache.
that.set(BUCKET_CACHE_IOENGINE_KEY, "offheap");
that.set(BLOCKCACHE_POLICY_KEY, l1CachePolicy);
- BlockCache bc = BlockCacheFactory.createBlockCache(that);
+ CacheAccessService bc =
CacheAccessServiceTestFactory.fromConfiguration(that);
assertNotNull(bc);
- assertTrue(bc instanceof CombinedBlockCache);
+ assertTrue(CacheAccessServiceTestFactory.blockCache(bc) instanceof
CombinedBlockCache);
return bc;
}
@@ -358,7 +367,7 @@ public class TestHFile {
fillByteBuffAllocator(alloc, bufCount);
Path storeFilePath = writeStoreFile();
// Open the file reader with CombinedBlockCache
- BlockCache combined = initCombinedBlockCache("LRU");
+ CacheAccessService combined = initCombinedBlockCacheBackedService("LRU");
conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc);
HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig,
true, conf);
@@ -428,7 +437,8 @@ public class TestHFile {
myConf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY,
cacheConfigCacheBlockOnRead);
// Open the file reader with LRUBlockCache
- BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true,
myConf);
+ CacheAccessService lru =
+ CacheAccessServiceTestFactory.lru(1024 * 1024 * 32, blockSize, true,
myConf);
CacheConfig cacheConfig = new CacheConfig(myConf, null, lru, alloc);
HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig,
true, myConf);
long offset = 0;
@@ -482,7 +492,7 @@ public class TestHFile {
fillByteBuffAllocator(alloc, bufCount);
Path storeFilePath = writeStoreFile();
// Open the file reader with CombinedBlockCache
- BlockCache combined = initCombinedBlockCache("LRU");
+ CacheAccessService combined = initCombinedBlockCacheBackedService("LRU");
Configuration myConf = new Configuration(conf);
myConf.setBoolean(CacheConfig.CACHE_DATA_ON_READ_KEY,
cacheConfigCacheBlockOnRead);
@@ -539,7 +549,7 @@ public class TestHFile {
private void readStoreFile(Path storeFilePath, Configuration conf,
ByteBuffAllocator alloc)
throws Exception {
// Open the file reader with block cache disabled.
- CacheConfig cache = new CacheConfig(conf, null, null, alloc);
+ CacheConfig cache = new CacheConfig(conf, null, (CacheAccessService) null,
alloc);
HFile.Reader reader = HFile.createReader(fs, storeFilePath, cache, true,
conf);
long offset = 0;
while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
@@ -1158,7 +1168,7 @@ public class TestHFile {
fillByteBuffAllocator(alloc, bufCount);
Path storeFilePath = writeStoreFile();
// Open the file reader with CombinedBlockCache
- BlockCache combined = initCombinedBlockCache(l1CachePolicy);
+ CacheAccessService combined =
initCombinedBlockCacheBackedService(l1CachePolicy);
conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc);
HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig,
true, conf);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
index a6e3e998b1f..e7c1ef61453 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -118,7 +119,7 @@ public class TestHFileReaderImpl {
@Test
public void testReadWorksWhenCacheCorrupt() throws Exception {
- BlockCache mockedCache = mock(BlockCache.class);
+ CacheAccessService mockedCache = mock(CacheAccessService.class);
when(mockedCache.getBlock(any(), anyBoolean(), anyBoolean(), anyBoolean(),
any()))
.thenThrow(new RuntimeException("Injected error"));
Path p = makeNewFile();
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
index 598c4f2fe86..4cb393da39c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessServiceTestFactory;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
@@ -102,7 +104,7 @@ public class TestPrefetch {
private Configuration conf;
private CacheConfig cacheConf;
private FileSystem fs;
- private BlockCache blockCache;
+ private CacheAccessService cache;
@RegisterExtension
private static OpenTelemetryExtension OTEL_EXT =
OpenTelemetryExtension.create();
@@ -112,8 +114,8 @@ public class TestPrefetch {
conf = TEST_UTIL.getConfiguration();
conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true);
fs = HFileSystem.get(conf);
- blockCache = BlockCacheFactory.createBlockCache(conf);
- cacheConf = new CacheConfig(conf, blockCache);
+ cache = CacheAccessServiceTestFactory.fromConfiguration(conf);
+ cacheConf = new CacheConfig(conf, cache);
}
@Test
@@ -122,7 +124,7 @@ public class TestPrefetch {
.newBuilder(Bytes.toBytes("f")).setPrefetchBlocksOnOpen(true).build();
Configuration c = HBaseConfiguration.create();
assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false));
- CacheConfig cc = new CacheConfig(c, columnFamilyDescriptor, blockCache,
ByteBuffAllocator.HEAP);
+ CacheConfig cc = new CacheConfig(c, columnFamilyDescriptor, cache,
ByteBuffAllocator.HEAP);
assertTrue(cc.shouldPrefetchOnOpen());
}
@@ -137,7 +139,7 @@ public class TestPrefetch {
.setBlockCacheEnabled(false).build();
HFileContext meta = new
HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build();
CacheConfig cacheConfig =
- new CacheConfig(conf, columnFamilyDescriptor, blockCache,
ByteBuffAllocator.HEAP);
+ new CacheConfig(conf, columnFamilyDescriptor, cache,
ByteBuffAllocator.HEAP);
Path storeFile = writeStoreFile("testPrefetchBlockCacheDisabled", meta,
cacheConfig);
readStoreFile(storeFile, (r, o) -> {
HFileBlock block = null;
@@ -148,7 +150,7 @@ public class TestPrefetch {
}
return block;
}, (key, block) -> {
- boolean isCached = blockCache.getBlock(key, true, false, true) != null;
+ boolean isCached = cache.getBlock(key, true, false, true) != null;
if (
block.getBlockType() == BlockType.DATA || block.getBlockType() ==
BlockType.ROOT_INDEX
|| block.getBlockType() == BlockType.INTERMEDIATE_INDEX
@@ -169,7 +171,7 @@ public class TestPrefetch {
Configuration newConf = new Configuration(conf);
newConf.setDouble(CacheConfig.PREFETCH_HEAP_USAGE_THRESHOLD, 0.1);
CacheConfig cacheConfig =
- new CacheConfig(newConf, columnFamilyDescriptor, blockCache,
ByteBuffAllocator.HEAP);
+ new CacheConfig(newConf, columnFamilyDescriptor, cache,
ByteBuffAllocator.HEAP);
Path storeFile = writeStoreFile("testPrefetchHeapUsageAboveThreshold",
meta, cacheConfig);
MutableInt cachedCount = new MutableInt(0);
MutableInt unCachedCount = new MutableInt(0);
@@ -182,7 +184,7 @@ public class TestPrefetch {
}
return block;
}, (key, block) -> {
- boolean isCached = blockCache.getBlock(key, true, false, true) != null;
+ boolean isCached = cache.getBlock(key, true, false, true) != null;
if (
block.getBlockType() == BlockType.DATA || block.getBlockType() ==
BlockType.ROOT_INDEX
|| block.getBlockType() == BlockType.INTERMEDIATE_INDEX
@@ -253,7 +255,7 @@ public class TestPrefetch {
}
return block;
}, (key, block) -> {
- boolean isCached = blockCache.getBlock(key, true, false, true) != null;
+ boolean isCached = cache.getBlock(key, true, false, true) != null;
if (
block.getBlockType() == BlockType.DATA || block.getBlockType() ==
BlockType.ROOT_INDEX
|| block.getBlockType() == BlockType.INTERMEDIATE_INDEX
@@ -273,7 +275,7 @@ public class TestPrefetch {
}
return block;
}, (key, block) -> {
- boolean isCached = blockCache.getBlock(key, true, false, true) != null;
+ boolean isCached = cache.getBlock(key, true, false, true) != null;
if (block.getBlockType() == BlockType.DATA) {
assertFalse(block.isUnpacked());
} else if (
@@ -315,7 +317,7 @@ public class TestPrefetch {
@Test
public void testPrefetchCompressed() throws Exception {
conf.setBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, true);
- cacheConf = new CacheConfig(conf, blockCache);
+ cacheConf = new CacheConfig(conf, cache);
HFileContext context = new
HFileContextBuilder().withCompression(Compression.Algorithm.GZ)
.withBlockSize(DATA_BLOCK_SIZE).build();
Path storeFile = writeStoreFile("TestPrefetchCompressed", context);
@@ -408,7 +410,7 @@ public class TestPrefetch {
private void testPrefetchWhenRefs(boolean compactionEnabled,
Consumer<Cacheable> test)
throws Exception {
- cacheConf = new CacheConfig(conf, blockCache);
+ cacheConf = new CacheConfig(conf, cache);
HFileContext context = new
HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build();
Path tableDir = new Path(TEST_UTIL.getDataTestDir(),
"testPrefetchSkipRefs");
RegionInfo region =
@@ -438,14 +440,14 @@ public class TestPrefetch {
HFileBlock block = reader.readBlock(offset, -1, false, true, false,
true, null, null, true);
BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
offset);
if (block.getBlockType() == BlockType.DATA) {
- test.accept(blockCache.getBlock(blockCacheKey, true, false, true));
+ test.accept(cache.getBlock(blockCacheKey, true, false, true));
}
offset += block.getOnDiskSizeWithHeader();
}
}
private void testPrefetchWhenHFileLink(Consumer<Cacheable> test) throws
Exception {
- cacheConf = new CacheConfig(conf, blockCache);
+ cacheConf = new CacheConfig(conf, cache);
HFileContext context = new
HFileContextBuilder().withBlockSize(DATA_BLOCK_SIZE).build();
Path testDir = TEST_UTIL.getDataTestDir("testPrefetchWhenHFileLink");
final RegionInfo hri =
@@ -493,7 +495,7 @@ public class TestPrefetch {
HFileBlock block = reader.readBlock(offset, -1, false, true, false,
true, null, null, true);
BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(),
offset);
if (block.getBlockType() == BlockType.DATA) {
- test.accept(blockCache.getBlock(blockCacheKey, true, false, true));
+ test.accept(cache.getBlock(blockCacheKey, true, false, true));
}
offset += block.getOnDiskSizeWithHeader();
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestRowIndexV1RoundTrip.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestRowIndexV1RoundTrip.java
index 8431a18bf65..83e63f0dd07 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestRowIndexV1RoundTrip.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestRowIndexV1RoundTrip.java
@@ -37,6 +37,7 @@ import
org.apache.hadoop.hbase.SizeCachedNoTagsByteBufferKeyValue;
import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue;
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -109,7 +110,7 @@ public class TestRowIndexV1RoundTrip {
cacheConfig = new CacheConfig(conf);
} else {
ByteBuffAllocator allocator = ByteBuffAllocator.create(conf, true);
- cacheConfig = new CacheConfig(conf, null, null, allocator);
+ cacheConfig = new CacheConfig(conf, null, (CacheAccessService) null,
allocator);
}
HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConfig,
false, conf);
HFileScanner scanner = reader.getScanner(conf, false, false);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
index 069fc6fb6bc..7aec8d9d195 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/cache/CacheAccessServiceTestFactory.java
@@ -640,4 +640,33 @@ public final class CacheAccessServiceTestFactory {
new BucketCache(ioEngineName, capacity, blockSize, bucketSizes,
writerThreadNum, writerQLen,
persistencePath, ioErrorsTolerationDuration, conf, onlineRegions));
}
+
+ /**
+ * Returns the legacy {@link BlockCache} backing the supplied {@link
CacheAccessService}.
+ * <p>
+ * This helper is intended for tests only. It supports transitional test
migration where
+ * production code is exercised through {@link CacheAccessService}, but the
test still needs
+ * direct access to the underlying legacy {@link BlockCache} for
implementation-specific
+ * assertions, cached-block iteration, metrics inspection, or other
diagnostic checks.
+ * </p>
+ * <p>
+ * Only {@link BlockCacheBackedCacheAccessService} is supported. Services
backed by future
+ * topology/cache-engine implementations are not required to expose a legacy
{@link BlockCache}.
+ * Tests that use this method should therefore be treated as compatibility
tests, not as tests of
+ * the final pluggable-cache architecture.
+ * </p>
+ * @param cacheAccessService cache access service
+ * @return backing legacy block cache
+ * @throws NullPointerException if {@code cacheAccessService} is {@code
null}
+ * @throws IllegalArgumentException if {@code cacheAccessService} is not
backed by a legacy
+ * {@link BlockCache}
+ */
+ public static BlockCache blockCache(CacheAccessService cacheAccessService) {
+ Objects.requireNonNull(cacheAccessService, "cacheAccessService must not be
null");
+ if (cacheAccessService instanceof BlockCacheBackedCacheAccessService) {
+ return ((BlockCacheBackedCacheAccessService)
cacheAccessService).getBlockCache();
+ }
+ throw new IllegalArgumentException("CacheAccessService is not backed by a
legacy BlockCache: "
+ + cacheAccessService.getClass().getName());
+ }
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
index 0cf6b758f4d..b770f4676d0 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSecureBulkLoadManager.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.io.hfile.cache.CacheAccessService;
import
org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
@@ -228,7 +229,8 @@ public class TestSecureBulkLoadManager {
ColumnFamilyDescriptor family = desc.getColumnFamily(FAMILY);
Compression.Algorithm compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
- CacheConfig writerCacheConf = new CacheConfig(conf, family, null,
ByteBuffAllocator.HEAP);
+ CacheConfig writerCacheConf =
+ new CacheConfig(conf, family, (CacheAccessService) null,
ByteBuffAllocator.HEAP);
writerCacheConf.setCacheDataOnWrite(false);
HFileContext hFileContext = new
HFileContextBuilder().withIncludesMvcc(false)
.withIncludesTags(true).withCompression(compression).withCompressTags(family.isCompressTags())
