wchevreuil commented on code in PR #5829:
URL: https://github.com/apache/hbase/pull/5829#discussion_r1577636485
##########
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java:
##########
@@ -245,6 +248,184 @@ public void testColdDataFiles() {
}
}
+ @Test
+ public void testPickColdDataFiles() {
+ Map<String, String> coldDataFiles = dataTieringManager.getColdFilesList();
+ assertEquals(1, coldDataFiles.size());
+ // hStoreFiles[3] is the cold file.
+ assert
(coldDataFiles.containsKey(hStoreFiles.get(3).getFileInfo().getActiveFileName()));
+ }
+
+ /*
+ * Verify that two cold blocks(both) are evicted when bucket reaches its
capacity. The hot file
+ * remains in the cache.
+ */
+ @Test
+ public void testBlockEvictions() throws Exception {
+ long capacitySize = 64 * 1024;
+ int writeThreads = 3;
+ int writerQLen = 64;
+ int[] bucketSizes = new int[] { 8 * 1024 + 1024 };
+
+ // Setup: Create a bucket cache with lower capacity
+ BucketCache bucketCache = new BucketCache("file:" + testDir +
"/bucket.cache", capacitySize,
+ 8192, bucketSizes, writeThreads, writerQLen, testDir +
"/bucket.persistence",
+ DEFAULT_ERROR_TOLERATION_DURATION, defaultConf);
+
+ // Create three Cache keys with cold data files and a block with hot data.
+ // hStoreFiles.get(3) is a cold data file, while hStoreFiles.get(0) is a
hot file.
+ Set<BlockCacheKey> cacheKeys = new HashSet<>();
+ cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 0, true,
BlockType.DATA));
+ cacheKeys.add(new BlockCacheKey(hStoreFiles.get(3).getPath(), 8192, true,
BlockType.DATA));
+ cacheKeys.add(new BlockCacheKey(hStoreFiles.get(0).getPath(), 0, true,
BlockType.DATA));
+
+ // Create dummy data to be cached and fill the cache completely.
+ CacheTestUtils.HFileBlockPair[] blocks =
CacheTestUtils.generateHFileBlocks(8192, 3);
+
+ int blocksIter = 0;
+ for (BlockCacheKey key : cacheKeys) {
+ bucketCache.cacheBlock(key, blocks[blocksIter++].getBlock());
+ // Ensure that the block is persisted to the file.
+ Waiter.waitFor(defaultConf, 1000000, 100,
+ () -> (bucketCache.getBackingMap().containsKey(key)));
+ }
+
+ // Verify that the bucket cache contains 3 blocks.
+ assertEquals(3, bucketCache.getBackingMap().keySet().size());
+
+ // Add an additional block into cache with hot data which should trigger
the eviction
+ BlockCacheKey newKey = new BlockCacheKey(hStoreFiles.get(2).getPath(), 0,
true, BlockType.DATA);
+ CacheTestUtils.HFileBlockPair[] newBlock =
CacheTestUtils.generateHFileBlocks(8192, 1);
+
+ bucketCache.cacheBlock(newKey, newBlock[0].getBlock());
+ Waiter.waitFor(defaultConf, 1000000, 100,
+ () -> (bucketCache.getBackingMap().containsKey(newKey)));
+
+ // Verify that the bucket cache now contains 2 hot blocks blocks only.
+ // Both cold blocks of 8KB will be evicted to make room for 1 block of 8KB
+ an additional
+ // space.
+ validateBlocks(bucketCache.getBackingMap().keySet(), 2, 2, 0);
Review Comment:
Yeah, we had reviewed this internally, not at all an issue. It's due to the
internals of BucketAllocator when calculating buckets limits.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
