This is an automated email from the ASF dual-hosted git repository.
Apache9 pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 2748f782d9d HBASE-30258 Run getAllRegionLocations more times in
TestMetaRegionReplicaReplication to make it more stable (#8416)
2748f782d9d is described below
commit 2748f782d9d7e3c33d47032bc5ea8c67ad1b66a2
Author: Duo Zhang <[email protected]>
AuthorDate: Fri Jun 26 22:22:43 2026 +0800
HBASE-30258 Run getAllRegionLocations more times in
TestMetaRegionReplicaReplication to make it more stable (#8416)
Signed-off-by: Xiao Liu <[email protected]>
(cherry picked from commit f49bda0570e35a336727641932a65dd41bd63c70)
---
.../hadoop/hbase/regionserver/HRegionServer.java | 4 +-
.../TestMetaRegionReplicaReplication.java | 84 ++++++++++++++--------
2 files changed, 56 insertions(+), 32 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 60cc81b6b72..7b6f192314c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3335,7 +3335,9 @@ public class HRegionServer extends
HBaseServerBase<RSRpcServices>
movedRegionInfoCache.put(encodedName, new MovedRegionInfo(destination,
closeSeqNum));
}
- void removeFromMovedRegions(String encodedName) {
+ // public for being called in tests
+ @InterfaceAudience.Private
+ public void removeFromMovedRegions(String encodedName) {
movedRegionInfoCache.invalidate(encodedName);
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java
index d9a1f9fc186..b4a0b4e76f9 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java
@@ -18,15 +18,17 @@
package org.apache.hadoop.hbase.replication.regionserver;
import static
org.apache.hadoop.hbase.client.RegionLocator.LOCATOR_META_REPLICAS_MODE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
@@ -374,7 +376,8 @@ public class TestMetaRegionReplicaReplication {
private void primaryIncreaseReplicaNoChange(final long[] before, final
long[] after) {
// There are read requests increase for primary meta replica.
- assertTrue(after[RegionInfo.DEFAULT_REPLICA_ID] >
before[RegionInfo.DEFAULT_REPLICA_ID]);
+ assertThat(after[RegionInfo.DEFAULT_REPLICA_ID],
+ greaterThan(before[RegionInfo.DEFAULT_REPLICA_ID]));
// No change for replica regions
for (int i = 1; i < after.length; i++) {
@@ -385,7 +388,7 @@ public class TestMetaRegionReplicaReplication {
private void primaryIncreaseReplicaIncrease(final long[] before, final
long[] after) {
// There are read requests increase for all meta replica regions,
for (int i = 0; i < after.length; i++) {
- assertTrue(after[i] > before[i]);
+ assertThat(after[i], greaterThan(before[i]));
}
}
@@ -398,6 +401,22 @@ public class TestMetaRegionReplicaReplication {
}
}
+ private void runAtMostNTimes(int times, Callable<?> action, Runnable
assertion) throws Exception {
+ for (int i = 0; i < times - 1; i++) {
+ action.call();
+ try {
+ assertion.run();
+ // return if the assertion passes, otherwise try again
+ return;
+ } catch (AssertionError e) {
+ LOG.warn("Assertion failed, times = {}, try again", i, e);
+ }
+ }
+ // try last time
+ action.call();
+ assertion.run();
+ }
+
@Test
public void testHBaseMetaReplicaGets() throws Exception {
TableName tn = TableName.valueOf(testName);
@@ -430,38 +449,43 @@ public class TestMetaRegionReplicaReplication {
}
}
}
+ }
- getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicas);
+ getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicas);
- Configuration c = new Configuration(HTU.getConfiguration());
- c.setBoolean(HConstants.USE_META_REPLICAS, true);
- c.set(LOCATOR_META_REPLICAS_MODE, "LoadBalance");
- Connection connection = ConnectionFactory.createConnection(c);
+ Configuration c = new Configuration(HTU.getConfiguration());
+ c.setBoolean(HConstants.USE_META_REPLICAS, true);
+ c.set(LOCATOR_META_REPLICAS_MODE, "LoadBalance");
+ try (Connection connection = ConnectionFactory.createConnection(c);
Table tableForGet = connection.getTable(tn);
+ RegionLocator locator = tableForGet.getRegionLocator()) {
byte[][] getRows = new byte[HBaseTestingUtil.KEYS.length][];
-
- int i = 0;
- for (byte[] key : HBaseTestingUtil.KEYS) {
- getRows[i] = key;
- i++;
+ for (int i = 1; i < HBaseTestingUtil.KEYS.length; i++) {
+ getRows[i] = HBaseTestingUtil.KEYS[i];
}
getRows[0] = Bytes.toBytes("aaa");
- doNGets(tableForGet, getRows);
-
- getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicasAfterGet);
// There are more reads against all meta replica regions, including the
primary region.
- primaryIncreaseReplicaIncrease(readReqsForMetaReplicas,
readReqsForMetaReplicasAfterGet);
-
- RegionLocator locator = tableForGet.getRegionLocator();
-
- for (int j = 0; j < numOfMetaReplica * 3; j++) {
- locator.getAllRegionLocations();
- }
-
- getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterGetAllLocations);
- primaryIncreaseReplicaIncrease(readReqsForMetaReplicasAfterGet,
- readReqsForMetaReplicasAfterGetAllLocations);
+ // Since in load balance mode, we will randomly select region replicas,
it is possible that we
+ // missed read some replicas, so here we run it at most 3 times to make
it more stable
+ runAtMostNTimes(3, () -> {
+ locator.clearRegionLocationCache();
+ doNGets(tableForGet, getRows);
+ getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterGet);
+ return null;
+ }, () -> primaryIncreaseReplicaIncrease(readReqsForMetaReplicas,
+ readReqsForMetaReplicasAfterGet));
+
+ // Same as above, we need to run it multiple times to make it stable.
And numOfMetaReplicas is
+ // much less than HBaseTestingUtil.KEYS.length, so here we need to run
more times.
+ runAtMostNTimes(30, () -> {
+ for (int i = 0; i < numOfMetaReplica; i++) {
+ locator.getAllRegionLocations();
+ }
+ getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterGetAllLocations);
+ return null;
+ }, () -> primaryIncreaseReplicaIncrease(readReqsForMetaReplicasAfterGet,
+ readReqsForMetaReplicasAfterGetAllLocations));
// move one of regions so it meta cache may be invalid.
HTU.moveRegionAndWait(userRegion.getRegionInfo(),
destRs.getServerName());
@@ -477,10 +501,8 @@ public class TestMetaRegionReplicaReplication {
// Move region again.
HTU.moveRegionAndWait(userRegion.getRegionInfo(), srcRs.getServerName());
- // Wait until moveRegion cache timeout.
- while
(destRs.getMovedRegion(userRegion.getRegionInfo().getEncodedName()) != null) {
- Thread.sleep(1000);
- }
+ // Remove it from the move region cache
+
destRs.removeFromMovedRegions(userRegion.getRegionInfo().getEncodedName());
getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterSecondMove);
