This is an automated email from the ASF dual-hosted git repository.
Apache9 pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new e26f9e77442 HBASE-30258 Run getAllRegionLocations more times in
TestMetaRegionReplicaReplication to make it more stable (#8416)
e26f9e77442 is described below
commit e26f9e77442279975c945468ce0b4abaf5b98351
Author: Duo Zhang <[email protected]>
AuthorDate: Fri Jun 26 22:22:43 2026 +0800
HBASE-30258 Run getAllRegionLocations more times in
TestMetaRegionReplicaReplication to make it more stable (#8416)
Signed-off-by: Xiao Liu <[email protected]>
(cherry picked from commit f49bda0570e35a336727641932a65dd41bd63c70)
---
.../hadoop/hbase/regionserver/HRegionServer.java | 4 +-
.../TestMetaRegionReplicaReplicationEndpoint.java | 81 ++++++++++++++--------
2 files changed, 55 insertions(+), 30 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 65ac9e333d4..c2491cd02e4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -3827,7 +3827,9 @@ public class HRegionServer extends Thread
movedRegionInfoCache.put(encodedName, new MovedRegionInfo(destination,
closeSeqNum));
}
- void removeFromMovedRegions(String encodedName) {
+ // public for being called in tests
+ @InterfaceAudience.Private
+ public void removeFromMovedRegions(String encodedName) {
movedRegionInfoCache.invalidate(encodedName);
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplicationEndpoint.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplicationEndpoint.java
index ca3103b7c11..a0238f6f4c4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplicationEndpoint.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplicationEndpoint.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hbase.replication.regionserver;
import static
org.apache.hadoop.hbase.client.RegionLocator.LOCATOR_META_REPLICAS_MODE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -29,6 +31,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
+import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hadoop.conf.Configuration;
@@ -538,7 +541,8 @@ public class TestMetaRegionReplicaReplicationEndpoint {
private void primaryIncreaseReplicaNoChange(final long[] before, final
long[] after) {
// There are read requests increase for primary meta replica.
- assertTrue(after[RegionInfo.DEFAULT_REPLICA_ID] >
before[RegionInfo.DEFAULT_REPLICA_ID]);
+ assertThat(after[RegionInfo.DEFAULT_REPLICA_ID],
+ greaterThan(before[RegionInfo.DEFAULT_REPLICA_ID]));
// No change for replica regions
for (int i = 1; i < after.length; i++) {
@@ -559,7 +563,7 @@ public class TestMetaRegionReplicaReplicationEndpoint {
private void primaryIncreaseReplicaIncrease(final long[] before, final
long[] after) {
// There are read requests increase for all meta replica regions,
for (int i = 0; i < after.length; i++) {
- assertTrue(after[i] > before[i]);
+ assertThat(after[i], greaterThan(before[i]));
}
}
@@ -572,6 +576,22 @@ public class TestMetaRegionReplicaReplicationEndpoint {
}
}
+ private void runAtMostNTimes(int times, Callable<?> action, Runnable
assertion) throws Exception {
+ for (int i = 0; i < times - 1; i++) {
+ action.call();
+ try {
+ assertion.run();
+ // return if the assertion passes, otherwise try again
+ return;
+ } catch (AssertionError e) {
+ LOG.warn("Assertion failed, times = {}, try again", i, e);
+ }
+ }
+ // try last time
+ action.call();
+ assertion.run();
+ }
+
@Test
public void testHBaseMetaReplicaGets() throws Exception {
TableName tn = TableName.valueOf(methodName);
@@ -604,37 +624,42 @@ public class TestMetaRegionReplicaReplicationEndpoint {
}
}
}
+ }
- getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicas);
+ getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicas);
- Configuration c = new Configuration(HTU.getConfiguration());
- c.set(LOCATOR_META_REPLICAS_MODE, "LoadBalance");
- Connection connection = ConnectionFactory.createConnection(c);
+ Configuration c = new Configuration(HTU.getConfiguration());
+ c.set(LOCATOR_META_REPLICAS_MODE, "LoadBalance");
+ try (Connection connection = ConnectionFactory.createConnection(c);
Table tableForGet = connection.getTable(tn);
+ RegionLocator locator = tableForGet.getRegionLocator()) {
byte[][] getRows = new byte[HBaseTestingUtility.KEYS.length][];
-
- int i = 0;
- for (byte[] key : HBaseTestingUtility.KEYS) {
- getRows[i] = key;
- i++;
+ for (int i = 1; i < HBaseTestingUtility.KEYS.length; i++) {
+ getRows[i] = HBaseTestingUtility.KEYS[i];
}
getRows[0] = Bytes.toBytes("aaa");
- doNGets(tableForGet, getRows);
-
- getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicasAfterGet);
// There are more reads against all meta replica regions, including the
primary region.
- primaryIncreaseReplicaIncrease(readReqsForMetaReplicas,
readReqsForMetaReplicasAfterGet);
-
- RegionLocator locator = tableForGet.getRegionLocator();
-
- for (int j = 0; j < numOfMetaReplica * 3; j++) {
- locator.getAllRegionLocations();
- }
-
- getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterGetAllLocations);
- primaryIncreaseReplicaIncrease(readReqsForMetaReplicasAfterGet,
- readReqsForMetaReplicasAfterGetAllLocations);
+ // Since in load balance mode, we will randomly select region replicas,
it is possible that we
+ // missed read some replicas, so here we run it at most 3 times to make
it more stable
+ runAtMostNTimes(3, () -> {
+ locator.clearRegionLocationCache();
+ doNGets(tableForGet, getRows);
+ getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterGet);
+ return null;
+ }, () -> primaryIncreaseReplicaIncrease(readReqsForMetaReplicas,
+ readReqsForMetaReplicasAfterGet));
+
+ // Same as above, we need to run it multiple times to make it stable.
And numOfMetaReplicas is
+ // much less than HBaseTestingUtil.KEYS.length, so here we need to run
more times.
+ runAtMostNTimes(30, () -> {
+ for (int i = 0; i < numOfMetaReplica; i++) {
+ locator.getAllRegionLocations();
+ }
+ getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterGetAllLocations);
+ return null;
+ }, () -> primaryIncreaseReplicaIncrease(readReqsForMetaReplicasAfterGet,
+ readReqsForMetaReplicasAfterGetAllLocations));
// move one of regions so it meta cache may be invalid.
HTU.moveRegionAndWait(userRegion.getRegionInfo(),
destRs.getServerName());
@@ -650,10 +675,8 @@ public class TestMetaRegionReplicaReplicationEndpoint {
// Move region again.
HTU.moveRegionAndWait(userRegion.getRegionInfo(), srcRs.getServerName());
- // Wait until moveRegion cache timeout.
- while
(destRs.getMovedRegion(userRegion.getRegionInfo().getEncodedName()) != null) {
- Thread.sleep(1000);
- }
+ // Remove it from the move region cache
+
destRs.removeFromMovedRegions(userRegion.getRegionInfo().getEncodedName());
getMetaReplicaReadRequests(metaRegions,
readReqsForMetaReplicasAfterSecondMove);
