This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new e1d5ef02163 HBASE-29215 View block cache as JSON failing for
BucketCache implementation (#6851)
e1d5ef02163 is described below
commit e1d5ef02163ef2e698b7478616be338f12ec15c1
Author: Chandra Sekhar K <[email protected]>
AuthorDate: Wed Mar 26 17:00:47 2025 +0530
HBASE-29215 View block cache as JSON failing for BucketCache implementation
(#6851)
Signed-off-by: Wellington Chevreuil <[email protected]>
(cherry picked from f4bcb76a76c05733b2e0b6448b26412ca1e374a4)
Change-Id: Id24e36dd8ad3b03131e35bd3b8d7b39df4d8fe06
---
.../apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java | 4 ++--
.../hadoop/hbase/io/hfile/TestMemcachedBlockCache.java | 2 ++
.../org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java | 10 ++++++----
.../org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java | 12 ++++++++++++
.../hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java | 2 ++
.../org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java | 2 ++
.../apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java | 2 ++
7 files changed, 28 insertions(+), 6 deletions(-)
diff --git
a/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
b/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
index efb8b97bccb..7e323ae669d 100644
---
a/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
+++
b/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
@@ -82,8 +82,8 @@ public class MemcachedBlockCache implements BlockCache {
public static final boolean MEMCACHED_OPTIMIZE_DEFAULT = false;
public static final int STAT_THREAD_PERIOD = 60 * 5;
- private final MemcachedClient client;
- private final HFileBlockTranscoder tc = new HFileBlockTranscoder();
+ private transient final MemcachedClient client;
+ private transient final HFileBlockTranscoder tc = new HFileBlockTranscoder();
private final CacheStats cacheStats = new CacheStats("MemcachedBlockCache");
private final AtomicLong cachedCount = new AtomicLong();
private final AtomicLong notCachedCount = new AtomicLong();
diff --git
a/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java
b/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java
index 96ff85e8414..60370f5208b 100644
---
a/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java
+++
b/hbase-external-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestMemcachedBlockCache.java
@@ -135,6 +135,8 @@ public class TestMemcachedBlockCache {
assertEquals(expected.getBlockType(), actual.getBlockType());
assertEquals(expected.getSerializedLength(),
actual.getSerializedLength());
}
+
+ CacheTestUtils.testConvertToJSON(cache);
}
@Test
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 2a5124cd3c8..d651f055bb7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -167,14 +167,14 @@ public class BucketCache implements BlockCache, HeapSize {
* together with the region those belong to and the total cached size for the
* region.TestBlockEvictionOnRegionMovement
*/
- final Map<String, Pair<String, Long>> fullyCachedFiles = new
ConcurrentHashMap<>();
+ transient final Map<String, Pair<String, Long>> fullyCachedFiles = new
ConcurrentHashMap<>();
/**
* Map of region -> total size of the region prefetched on this region
server. This is the total
* size of hFiles for this region prefetched on this region server
*/
final Map<String, Long> regionCachedSize = new ConcurrentHashMap<>();
- private BucketCachePersister cachePersister;
+ private transient BucketCachePersister cachePersister;
/**
* Flag if the cache is enabled or not... We shut it off if there are IO
errors for some time, so
@@ -222,6 +222,8 @@ public class BucketCache implements BlockCache, HeapSize {
// reset after a successful read/write.
private volatile long ioErrorStartTime = -1;
+ private transient Configuration conf;
+
/**
* A ReentrantReadWriteLock to lock on a particular block identified by
offset. The purpose of
* this is to avoid freeing the block which is being read.
@@ -230,7 +232,7 @@ public class BucketCache implements BlockCache, HeapSize {
*/
transient final IdReadWriteLock<Long> offsetLock = new
IdReadWriteLock<>(ReferenceType.SOFT);
- NavigableSet<BlockCacheKey> blocksByHFile = new ConcurrentSkipListSet<>(
+ transient NavigableSet<BlockCacheKey> blocksByHFile = new
ConcurrentSkipListSet<>(
Comparator.comparing(BlockCacheKey::getHfileName).thenComparingLong(BlockCacheKey::getOffset));
/** Statistics thread schedule pool (for heavy debugging, could remove) */
@@ -283,7 +285,7 @@ public class BucketCache implements BlockCache, HeapSize {
private long allocFailLogPrevTs; // time of previous log event for
allocation failure.
private static final int ALLOCATION_FAIL_LOG_TIME_PERIOD = 60000; // Default
1 minute.
- private Map<String, HRegion> onlineRegions;
+ private transient Map<String, HRegion> onlineRegions;
private long orphanBlockGracePeriod = 0;
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
index 728969c1275..b8c9794665f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.io.hfile;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -161,6 +162,8 @@ public class CacheTestUtils {
}
}
+ testConvertToJSON(toBeTested);
+
}
public static void hammerSingleKey(final BlockCache toBeTested, int
numThreads, int numQueries)
@@ -351,4 +354,13 @@ public class CacheTestUtils {
}
}
}
+
+ public static void testConvertToJSON(BlockCache toBeTested) {
+ try {
+ String json = BlockCacheUtil.toJSON(toBeTested);
+ assertNotNull(json);
+ } catch (Exception e) {
+ fail("conversion to JSON should not fail, exception is:" + e);
+ }
+ }
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java
index b065c7c6374..00f43fcced9 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruAdaptiveBlockCache.java
@@ -220,6 +220,8 @@ public class TestLruAdaptiveBlockCache {
assertEquals(buf.heapSize(), block.heapSize());
}
+ CacheTestUtils.testConvertToJSON(cache);
+
// Expect no evictions
assertEquals(0, cache.getStats().getEvictionCount());
Thread t = new LruAdaptiveBlockCache.StatisticsThread(cache);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index ecd5fa2dd9a..59cacc154a5 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -223,6 +223,8 @@ public class TestLruBlockCache {
assertEquals(buf.heapSize(), block.heapSize());
}
+ CacheTestUtils.testConvertToJSON(cache);
+
// Expect no evictions
assertEquals(0, cache.getStats().getEvictionCount());
Thread t = new LruBlockCache.StatisticsThread(cache);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
index 31166bd5fa4..3c6b4647ef5 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
@@ -97,6 +97,8 @@ public class TestTinyLfuBlockCache {
// Expect no evictions
assertEquals(0, cache.getStats().getEvictionCount());
+
+ CacheTestUtils.testConvertToJSON(cache);
}
@Test
