This is an automated email from the ASF dual-hosted git repository.
liuxiaocs7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new b8be9177d07 HBASE-30204 Split TestClearRegionBlockCache by cache type
to avoid pe… (#8310)
b8be9177d07 is described below
commit b8be9177d07ac9c726a3bea8f6ecaa5530d298d1
Author: Xiao Liu <[email protected]>
AuthorDate: Fri Jun 5 20:03:58 2026 +0800
HBASE-30204 Split TestClearRegionBlockCache by cache type to avoid pe…
(#8310)
Signed-off-by: Duo Zhang <[email protected]>
---
...che.java => ClearRegionBlockCacheTestBase.java} | 135 ++++++++---------
.../regionserver/TestClearRegionBlockCache.java | 168 ++-------------------
.../TestClearRegionBlockCacheWithBucketCache.java | 41 +++++
3 files changed, 116 insertions(+), 228 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/ClearRegionBlockCacheTestBase.java
similarity index 56%
copy from
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
copy to
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/ClearRegionBlockCacheTestBase.java
index fade7bdbaba..c9815707208 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/ClearRegionBlockCacheTestBase.java
@@ -21,14 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.stream.Stream;
-import org.apache.hadoop.conf.Configuration;
+import java.util.List;
import org.apache.hadoop.hbase.CacheEvictionStats;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.AsyncAdmin;
@@ -37,68 +34,60 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.params.provider.Arguments;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@Tag(LargeTests.TAG)
-@HBaseParameterizedTestTemplate(name = "{index}: {0}")
-public class TestClearRegionBlockCache {
-
- private static final Logger LOG =
LoggerFactory.getLogger(TestClearRegionBlockCache.class);
- private static final TableName TABLE_NAME =
TableName.valueOf("testClearRegionBlockCache");
+import org.junit.jupiter.api.Test;
+
+public abstract class ClearRegionBlockCacheTestBase {
+
private static final byte[] FAMILY = Bytes.toBytes("family");
private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("5")
};
private static final int NUM_RS = 2;
- private final HBaseTestingUtil HTU = new HBaseTestingUtil();
-
- private Configuration CONF = HTU.getConfiguration();
- private Table table;
- private HRegionServer rs1, rs2;
- private SingleProcessHBaseCluster cluster;
+ private static TableName tableName;
+ private static HBaseTestingUtil htu;
+ private static HRegionServer rs1;
+ private static HRegionServer rs2;
- private final String cacheType;
-
- public TestClearRegionBlockCache(String cacheType) {
- this.cacheType = cacheType;
+ protected static void setUpCluster(TableName testTableName) throws Exception
{
+ setUpCluster(testTableName, false);
}
- public static Stream<Arguments> parameters() {
- return Stream.of(Arguments.of("lru"), Arguments.of("bucket"));
+ protected static void setUpBucketCacheCluster(TableName testTableName)
throws Exception {
+ setUpCluster(testTableName, true);
}
- @BeforeEach
- public void setup() throws Exception {
- if (cacheType.equals("bucket")) {
- CONF.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
- CONF.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 30);
+ private static void setUpCluster(TableName testTableName, boolean
bucketCache) throws Exception {
+ tableName = testTableName;
+ htu = new HBaseTestingUtil();
+ if (bucketCache) {
+ htu.getConfiguration().set(HConstants.BUCKET_CACHE_IOENGINE_KEY,
"offheap");
+ htu.getConfiguration().setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 30);
}
+ htu.startMiniCluster(NUM_RS);
+ rs1 = htu.getMiniHBaseCluster().getRegionServer(0);
+ rs2 = htu.getMiniHBaseCluster().getRegionServer(1);
- cluster = HTU.startMiniCluster(NUM_RS);
- rs1 = cluster.getRegionServer(0);
- rs2 = cluster.getRegionServer(1);
-
- // Create table
- table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);
+ try (Table table = htu.createTable(testTableName, FAMILY, SPLIT_KEY)) {
+ htu.loadNumericRows(table, FAMILY, 1, 10);
+ htu.flush(testTableName);
+ }
+ }
- HTU.loadNumericRows(table, FAMILY, 1, 10);
- HTU.flush(TABLE_NAME);
+ protected static void tearDownCluster() throws Exception {
+ if (htu != null) {
+ htu.shutdownMiniCluster();
+ }
}
- @AfterEach
- public void teardown() throws Exception {
- HTU.shutdownMiniCluster();
+ @BeforeEach
+ void clearBlockCacheBeforeTest() {
+ clearRegionBlockCache(rs1);
+ clearRegionBlockCache(rs2);
}
- @TestTemplate
- public void testClearBlockCache() throws Exception {
+ @Test
+ void testClearBlockCache() throws Exception {
BlockCache blockCache1 = rs1.getBlockCache().get();
BlockCache blockCache2 = rs2.getBlockCache().get();
@@ -108,21 +97,21 @@ public class TestClearRegionBlockCache {
// scan will cause blocks to be added in BlockCache
scanAllRegionsForRS(rs1);
assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
- HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
+ htu.getNumHFilesForRS(rs1, tableName, FAMILY));
clearRegionBlockCache(rs1);
scanAllRegionsForRS(rs2);
assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
- HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
+ htu.getNumHFilesForRS(rs2, tableName, FAMILY));
clearRegionBlockCache(rs2);
assertEquals(initialBlockCount1, blockCache1.getBlockCount(), "" +
blockCache1.getBlockCount());
assertEquals(initialBlockCount2, blockCache2.getBlockCount(), "" +
blockCache2.getBlockCount());
}
- @TestTemplate
- public void testClearBlockCacheFromAdmin() throws Exception {
- Admin admin = HTU.getAdmin();
+ @Test
+ void testClearBlockCacheFromAdmin() throws Exception {
+ Admin admin = htu.getAdmin();
BlockCache blockCache1 = rs1.getBlockCache().get();
BlockCache blockCache2 = rs2.getBlockCache().get();
@@ -132,22 +121,23 @@ public class TestClearRegionBlockCache {
// scan will cause blocks to be added in BlockCache
scanAllRegionsForRS(rs1);
assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
- HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
+ htu.getNumHFilesForRS(rs1, tableName, FAMILY));
scanAllRegionsForRS(rs2);
assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
- HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
+ htu.getNumHFilesForRS(rs2, tableName, FAMILY));
- CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME);
- assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1,
TABLE_NAME, FAMILY)
- + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
+ CacheEvictionStats stats = admin.clearBlockCache(tableName);
+ assertEquals(
+ htu.getNumHFilesForRS(rs1, tableName, FAMILY) +
htu.getNumHFilesForRS(rs2, tableName, FAMILY),
+ stats.getEvictedBlocks());
assertEquals(initialBlockCount1, blockCache1.getBlockCount());
assertEquals(initialBlockCount2, blockCache2.getBlockCount());
}
- @TestTemplate
- public void testClearBlockCacheFromAsyncAdmin() throws Exception {
+ @Test
+ void testClearBlockCacheFromAsyncAdmin() throws Exception {
try (AsyncConnection conn =
- ConnectionFactory.createAsyncConnection(HTU.getConfiguration()).get()) {
+ ConnectionFactory.createAsyncConnection(htu.getConfiguration()).get()) {
AsyncAdmin admin = conn.getAdmin();
BlockCache blockCache1 = rs1.getBlockCache().get();
@@ -158,29 +148,32 @@ public class TestClearRegionBlockCache {
// scan will cause blocks to be added in BlockCache
scanAllRegionsForRS(rs1);
assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
- HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
+ htu.getNumHFilesForRS(rs1, tableName, FAMILY));
scanAllRegionsForRS(rs2);
assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
- HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
+ htu.getNumHFilesForRS(rs2, tableName, FAMILY));
- CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME).get();
- assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1,
TABLE_NAME, FAMILY)
- + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
+ CacheEvictionStats stats = admin.clearBlockCache(tableName).get();
+ assertEquals(htu.getNumHFilesForRS(rs1, tableName, FAMILY)
+ + htu.getNumHFilesForRS(rs2, tableName, FAMILY),
stats.getEvictedBlocks());
assertEquals(initialBlockCount1, blockCache1.getBlockCount());
assertEquals(initialBlockCount2, blockCache2.getBlockCount());
}
}
private void scanAllRegionsForRS(HRegionServer rs) throws IOException {
- for (Region region : rs.getRegions(TABLE_NAME)) {
- RegionScanner scanner = region.getScanner(new Scan());
- while (scanner.next(new ArrayList<Cell>()))
- ;
+ for (Region region : rs.getRegions(tableName)) {
+ try (RegionScanner scanner = region.getScanner(new Scan())) {
+ List<Cell> cells = new ArrayList<>();
+ while (scanner.next(cells)) {
+ cells.clear();
+ }
+ }
}
}
private void clearRegionBlockCache(HRegionServer rs) {
- for (Region region : rs.getRegions(TABLE_NAME)) {
+ for (Region region : rs.getRegions(tableName)) {
rs.clearRegionBlockCache(region);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
index fade7bdbaba..791b4bddf41 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCache.java
@@ -17,171 +17,25 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.stream.Stream;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CacheEvictionStats;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
-import org.apache.hadoop.hbase.HBaseTestingUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.AsyncAdmin;
-import org.apache.hadoop.hbase.client.AsyncConnection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.TestTemplate;
-import org.junit.jupiter.params.provider.Arguments;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
@Tag(LargeTests.TAG)
-@HBaseParameterizedTestTemplate(name = "{index}: {0}")
-public class TestClearRegionBlockCache {
-
- private static final Logger LOG =
LoggerFactory.getLogger(TestClearRegionBlockCache.class);
- private static final TableName TABLE_NAME =
TableName.valueOf("testClearRegionBlockCache");
- private static final byte[] FAMILY = Bytes.toBytes("family");
- private static final byte[][] SPLIT_KEY = new byte[][] { Bytes.toBytes("5")
};
- private static final int NUM_RS = 2;
-
- private final HBaseTestingUtil HTU = new HBaseTestingUtil();
-
- private Configuration CONF = HTU.getConfiguration();
- private Table table;
- private HRegionServer rs1, rs2;
- private SingleProcessHBaseCluster cluster;
-
- private final String cacheType;
-
- public TestClearRegionBlockCache(String cacheType) {
- this.cacheType = cacheType;
- }
-
- public static Stream<Arguments> parameters() {
- return Stream.of(Arguments.of("lru"), Arguments.of("bucket"));
- }
-
- @BeforeEach
- public void setup() throws Exception {
- if (cacheType.equals("bucket")) {
- CONF.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
- CONF.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 30);
- }
-
- cluster = HTU.startMiniCluster(NUM_RS);
- rs1 = cluster.getRegionServer(0);
- rs2 = cluster.getRegionServer(1);
-
- // Create table
- table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);
-
- HTU.loadNumericRows(table, FAMILY, 1, 10);
- HTU.flush(TABLE_NAME);
- }
-
- @AfterEach
- public void teardown() throws Exception {
- HTU.shutdownMiniCluster();
- }
+public class TestClearRegionBlockCache extends ClearRegionBlockCacheTestBase {
- @TestTemplate
- public void testClearBlockCache() throws Exception {
- BlockCache blockCache1 = rs1.getBlockCache().get();
- BlockCache blockCache2 = rs2.getBlockCache().get();
-
- long initialBlockCount1 = blockCache1.getBlockCount();
- long initialBlockCount2 = blockCache2.getBlockCount();
-
- // scan will cause blocks to be added in BlockCache
- scanAllRegionsForRS(rs1);
- assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
- HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
- clearRegionBlockCache(rs1);
-
- scanAllRegionsForRS(rs2);
- assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
- HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
- clearRegionBlockCache(rs2);
-
- assertEquals(initialBlockCount1, blockCache1.getBlockCount(), "" +
blockCache1.getBlockCount());
- assertEquals(initialBlockCount2, blockCache2.getBlockCount(), "" +
blockCache2.getBlockCount());
- }
-
- @TestTemplate
- public void testClearBlockCacheFromAdmin() throws Exception {
- Admin admin = HTU.getAdmin();
-
- BlockCache blockCache1 = rs1.getBlockCache().get();
- BlockCache blockCache2 = rs2.getBlockCache().get();
- long initialBlockCount1 = blockCache1.getBlockCount();
- long initialBlockCount2 = blockCache2.getBlockCount();
-
- // scan will cause blocks to be added in BlockCache
- scanAllRegionsForRS(rs1);
- assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
- HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
- scanAllRegionsForRS(rs2);
- assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
- HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
-
- CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME);
- assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1,
TABLE_NAME, FAMILY)
- + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
- assertEquals(initialBlockCount1, blockCache1.getBlockCount());
- assertEquals(initialBlockCount2, blockCache2.getBlockCount());
- }
-
- @TestTemplate
- public void testClearBlockCacheFromAsyncAdmin() throws Exception {
- try (AsyncConnection conn =
- ConnectionFactory.createAsyncConnection(HTU.getConfiguration()).get()) {
- AsyncAdmin admin = conn.getAdmin();
-
- BlockCache blockCache1 = rs1.getBlockCache().get();
- BlockCache blockCache2 = rs2.getBlockCache().get();
- long initialBlockCount1 = blockCache1.getBlockCount();
- long initialBlockCount2 = blockCache2.getBlockCount();
-
- // scan will cause blocks to be added in BlockCache
- scanAllRegionsForRS(rs1);
- assertEquals(blockCache1.getBlockCount() - initialBlockCount1,
- HTU.getNumHFilesForRS(rs1, TABLE_NAME, FAMILY));
- scanAllRegionsForRS(rs2);
- assertEquals(blockCache2.getBlockCount() - initialBlockCount2,
- HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
-
- CacheEvictionStats stats = admin.clearBlockCache(TABLE_NAME).get();
- assertEquals(stats.getEvictedBlocks(), HTU.getNumHFilesForRS(rs1,
TABLE_NAME, FAMILY)
- + HTU.getNumHFilesForRS(rs2, TABLE_NAME, FAMILY));
- assertEquals(initialBlockCount1, blockCache1.getBlockCount());
- assertEquals(initialBlockCount2, blockCache2.getBlockCount());
- }
- }
+ private static final TableName TABLE_NAME =
+ TableName.valueOf("testClearRegionBlockCacheWithLruBlockCache");
- private void scanAllRegionsForRS(HRegionServer rs) throws IOException {
- for (Region region : rs.getRegions(TABLE_NAME)) {
- RegionScanner scanner = region.getScanner(new Scan());
- while (scanner.next(new ArrayList<Cell>()))
- ;
- }
+ @BeforeAll
+ public static void setUp() throws Exception {
+ setUpCluster(TABLE_NAME);
}
- private void clearRegionBlockCache(HRegionServer rs) {
- for (Region region : rs.getRegions(TABLE_NAME)) {
- rs.clearRegionBlockCache(region);
- }
+ @AfterAll
+ public static void tearDown() throws Exception {
+ tearDownCluster();
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCacheWithBucketCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCacheWithBucketCache.java
new file mode 100644
index 00000000000..45537b994f2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClearRegionBlockCacheWithBucketCache.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+
+@Tag(LargeTests.TAG)
+public class TestClearRegionBlockCacheWithBucketCache extends
ClearRegionBlockCacheTestBase {
+
+ private static final TableName TABLE_NAME =
+ TableName.valueOf("testClearRegionBlockCacheWithBucketCache");
+
+ @BeforeAll
+ public static void setUp() throws Exception {
+ setUpBucketCacheCluster(TABLE_NAME);
+ }
+
+ @AfterAll
+ public static void tearDown() throws Exception {
+ tearDownCluster();
+ }
+}
