HBASE-19553, Old replica regions should be cleared from AM memory after primary 
region split or merge

Signed-off-by: Ted Yu <te...@apache.org>
Signed-off-by: Huaxiang Sun <huaxiang...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/29b03a50
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/29b03a50
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/29b03a50

Branch: refs/heads/branch-1.4
Commit: 29b03a5023de55d96768f53f471e09aa97598750
Parents: 569f16e
Author: Pankaj Kumar <pankaj...@huawei.com>
Authored: Thu Jan 25 13:25:22 2018 +0800
Committer: Andrew Purtell <apurt...@apache.org>
Committed: Fri Feb 23 17:03:03 2018 -0800

----------------------------------------------------------------------
 .../hadoop/hbase/master/AssignmentManager.java  |  17 ++++
 .../apache/hadoop/hbase/client/TestAdmin1.java  | 102 +++++++++++++++++--
 2 files changed, 111 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/29b03a50/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
index b8c088a..c7339ef 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
@@ -4394,6 +4394,8 @@ public class AssignmentManager extends ZooKeeperListener {
       LOG.warn("Couldn't assign all replica(s) of region " + mergedHri+ " 
because of " +
                 ie.getMessage());
     }
+    // Remove merged region's replica from AM's memory
+    clearReplicaRegions(c);
   }
 
   private void doSplittingOfReplicas(final HRegionInfo parentHri, final 
HRegionInfo hri_a,
@@ -4437,6 +4439,21 @@ public class AssignmentManager extends ZooKeeperListener 
{
     } catch (InterruptedException e) {
       LOG.warn("Caught exception " + e + " while trying to assign replica(s) 
of daughter(s)");
     }
+    // Remove parent region's replica from AM's memory
+    clearReplicaRegions(c);
+  }
+
+  /*
+   * Clear the replica regions after region split or merge.
+   */
+  private void clearReplicaRegions(Collection<List<HRegionInfo>> regionInfos) {
+    for (List<HRegionInfo> regionInfoList : regionInfos) {
+      for (HRegionInfo regionInfo : regionInfoList) {
+        if (!RegionReplicaUtil.isDefaultReplica(regionInfo)) {
+          regionStates.deleteRegion(regionInfo);
+        }
+      }
+    }
   }
 
   private void prepareDaughterReplicaForAssignment(HRegionInfo daughterHri, 
HRegionInfo parentHri,

http://git-wip-us.apache.org/repos/asf/hbase/blob/29b03a50/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
----------------------------------------------------------------------
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index 279b304..5885024 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotEnabledException;
 import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.executor.EventHandler;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -57,7 +58,9 @@ import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.zookeeper.ZKTableStateClientSideReader;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
+import org.apache.hadoop.hbase.master.AssignmentManager;
 import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.RequestConverter;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
@@ -1146,15 +1149,8 @@ public class TestAdmin1 {
     // regions.
     // Set up a table with 3 regions and replication set to 3
     TableName tableName = 
TableName.valueOf("testSplitAndMergeWithReplicaTable");
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    desc.setRegionReplication(3);
     byte[] cf = "f".getBytes();
-    HColumnDescriptor hcd = new HColumnDescriptor(cf);
-    desc.addFamily(hcd);
-    byte[][] splitRows = new byte[2][];
-    splitRows[0] = new byte[]{(byte)'4'};
-    splitRows[1] = new byte[]{(byte)'7'};
-    TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
+    createReplicaTable(tableName, cf);
     List<HRegion> oldRegions;
     do {
       oldRegions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
@@ -1243,6 +1239,96 @@ public class TestAdmin1 {
   }
 
   /**
+   * Test case to validate whether parent's replica regions are cleared from 
AM's memory after
+   * SPLIT/MERGE.
+   */
+  @Test
+  public void testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge() 
throws Exception {
+    final TableName tableName =
+        
TableName.valueOf("testRegionStateCleanupFromAMMemoryAfterRegionSplitAndMerge");
+    createReplicaTable(tableName, "f".getBytes());
+    final int regionReplication = 
admin.getTableDescriptor(tableName).getRegionReplication();
+
+    List<Pair<HRegionInfo, ServerName>> regions = 
MetaTableAccessor.getTableRegionsAndLocations(
+      TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConnection(), tableName);
+    assertEquals(9, regions.size());
+    final int primaryRegionCount = regions.size() / regionReplication;
+
+    final AssignmentManager am = 
TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
+    List<HRegionInfo> splitRegions =
+        
am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
+    assertEquals(0, splitRegions.size());
+
+    // Validate region split
+    byte[] regionName = regions.get(0).getFirst().getRegionName();
+    try {
+      TEST_UTIL.getHBaseAdmin().split(regionName, Bytes.toBytes('2'));
+    } catch (IllegalArgumentException ex) {
+      fail("Exception occured during region split" + ex);
+    }
+
+    // Wait for replica region to become online
+    TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
+      @Override
+      public boolean evaluate() throws IOException {
+        return 
am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
+            .size() == (primaryRegionCount + 1) * regionReplication;
+      }
+    });
+
+    regions = 
MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
+      TEST_UTIL.getConnection(), tableName);
+    assertEquals(12, regions.size());
+    final int primaryRegionCountAfterSplit = regions.size() / 
regionReplication;
+
+    // Split region after region split
+    splitRegions =
+        
am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.SPLIT);
+    // Parent's replica region should be removed from AM's memory.
+    assertEquals(1, splitRegions.size());
+
+    // Validate region merge
+    HRegionInfo regionA = regions.get(3).getFirst();
+    HRegionInfo regionB = regions.get(6).getFirst();
+    try {
+      TEST_UTIL.getHBaseAdmin().mergeRegions(regionA.getRegionName(), 
regionB.getRegionName(),
+        true);
+    } catch (IllegalArgumentException ex) {
+      fail("Exception occured during region merge" + ex);
+    }
+
+    // Wait for replica regions to become online
+    TEST_UTIL.waitFor(60000, 500, new Waiter.Predicate<IOException>() {
+      @Override
+      public boolean evaluate() throws IOException {
+        return 
am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OPEN)
+            .size() == (primaryRegionCountAfterSplit - 1) * regionReplication;
+      }
+    });
+
+    regions = 
MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getZooKeeperWatcher(),
+      TEST_UTIL.getConnection(), tableName);
+    assertEquals(9, regions.size());
+    // Offline region after region merge
+    List<HRegionInfo> offlineRegions =
+        
am.getRegionStates().getRegionByStateOfTable(tableName).get(RegionState.State.OFFLINE);
+    // Parent's replica region should be removed from AM's memory.
+    assertEquals(0, offlineRegions.size());
+  }
+
+  private byte[] createReplicaTable(TableName tableName, byte[] cf) throws 
IOException {
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.setRegionReplication(3);
+    HColumnDescriptor hcd = new HColumnDescriptor(cf);
+    desc.addFamily(hcd);
+    byte[][] splitRows = new byte[2][];
+    splitRows[0] = new byte[] { (byte) '4' };
+    splitRows[1] = new byte[] { (byte) '7' };
+    TEST_UTIL.getHBaseAdmin().createTable(desc, splitRows);
+    return cf;
+  }
+
+  /**
    * HADOOP-2156
    * @throws IOException
    */

Reply via email to