This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-24950 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a1cdb720097fe66f87211400d938f85ffed55771 Author: Duo Zhang <[email protected]> AuthorDate: Wed Sep 9 12:38:32 2020 +0800 HBASE-24606 Implement meta merge (#2311) Signed-off-by: Guanghao Zhang <[email protected]> --- .../org/apache/hadoop/hbase/MetaTableAccessor.java | 1 - .../hbase/master/assignment/AssignmentManager.java | 2 +- .../hbase/master/assignment/RegionStateStore.java | 11 ++-- ...etaSplit.java => TestSimpleMetaSplitMerge.java} | 73 +++++++++++++++++----- 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 46602d8..d2fd5e8 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -994,7 +994,6 @@ public final class MetaTableAccessor { LOG.debug("Overwritten regions: {} ", regionInfos); } - public static Put addRegionInfo(final Put p, final RegionInfo hri) throws IOException { p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(p.getRow()) .setFamily(HConstants.CATALOG_FAMILY).setQualifier(HConstants.REGIONINFO_QUALIFIER) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 64bcafd..fc621ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1857,7 +1857,7 @@ public class AssignmentManager { * References removed). */ public void markRegionAsMerged(final RegionInfo child, final ServerName serverName, - RegionInfo [] mergeParents) + RegionInfo[] mergeParents) throws IOException { final RegionStateNode node = regionStates.getOrCreateRegionStateNode(child); node.setState(State.MERGED); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index b2aa339..65cc23d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -313,12 +313,15 @@ public class RegionStateStore { private Result getRegionCatalogResult(RegionInfo region) throws IOException { Get get = new Get(CatalogFamilyFormat.getMetaKeyForRegion(region)).addFamily(HConstants.CATALOG_FAMILY); - try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) { - return table.get(get); + if (region.isMetaRegion()) { + return masterRegion.get(get); + } else { + try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) { + return table.get(get); + } } } - private static Put addSequenceNum(Put p, long openSeqNum, int replicaId) throws IOException { return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(p.getRow()) .setFamily(HConstants.CATALOG_FAMILY) @@ -501,7 +504,7 @@ public class RegionStateStore { } // ============================================================================================ - // Delete Region State helpers + // Delete Region State helpers // ============================================================================================ public void deleteRegion(final RegionInfo regionInfo) throws IOException { deleteRegions(Collections.singletonList(regionInfo)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java similarity index 59% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java index 110309c..7cbd245 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java @@ -20,17 +20,23 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.List; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -38,11 +44,11 @@ import org.junit.Test; import org.junit.experimental.categories.Category; @Category({ MiscTests.class, MediumTests.class }) -public class TestSimpleMetaSplit { +public class TestSimpleMetaSplitMerge { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSimpleMetaSplit.class); + HBaseClassTestRule.forClass(TestSimpleMetaSplitMerge.class); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); @@ -70,29 +76,66 @@ public class TestSimpleMetaSplit { UTIL.shutdownMiniCluster(); } + private void assertMetaRegionCount(int count) { + // do not count it from client as it will reset the location cache for meta table + assertEquals(count, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(TableName.META_TABLE_NAME).size()); + } + + private void clearCache(TableName tableName) throws IOException { + try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) { + locator.clearRegionLocationCache(); + } + } + + private void assertValue(TableName tableName, String row) throws IOException { + try (Table table = UTIL.getConnection().getTable(tableName)) { + Result result = table.get(new Get(Bytes.toBytes(row))); + assertEquals(row, Bytes.toString(result.getValue(CF, CQ))); + } + } + @Test - public void test() throws IOException { + public void test() throws Exception { try (Table table = UTIL.getConnection().getTable(TD1.getTableName())) { table.put(new Put(Bytes.toBytes("row1")).addColumn(CF, CQ, Bytes.toBytes("row1"))); } try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) { table.put(new Put(Bytes.toBytes("row2")).addColumn(CF, CQ, Bytes.toBytes("row2"))); } + Admin admin = UTIL.getAdmin(); // split meta - UTIL.getAdmin().split(TableName.META_TABLE_NAME, Bytes.toBytes("b")); - // do not count it from client as it will reset the location cache for meta table - assertEquals(2, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() - .getRegionsOfTable(TableName.META_TABLE_NAME).size()); + admin.split(TableName.META_TABLE_NAME, Bytes.toBytes("b")); + assertMetaRegionCount(2); // clear the cache for table 'b' - try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TD2.getTableName())) { - locator.clearRegionLocationCache(); - } + clearCache(TD2.getTableName()); // make sure that we could get the location of the TD2 from the second meta region - try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) { - Result result = table.get(new Get(Bytes.toBytes("row2"))); - assertEquals("row2", Bytes.toString(result.getValue(CF, CQ))); - } + assertValue(TD2.getTableName(), "row2"); // assert from client side - assertEquals(2, UTIL.getAdmin().getRegions(TableName.META_TABLE_NAME).size()); + List<RegionInfo> regions = admin.getRegions(TableName.META_TABLE_NAME); + assertEquals(2, regions.size()); + // compact to make sure we can merge + for (JVMClusterUtil.RegionServerThread t : UTIL.getMiniHBaseCluster() + .getRegionServerThreads()) { + for (HRegion r : t.getRegionServer().getOnlineRegionsLocalContext()) { + if (TableName.isMetaTableName(r.getRegionInfo().getTable())) { + r.compact(true); + for (HStore store : r.getStores()) { + store.closeAndArchiveCompactedFiles(); + } + } + } + } + // merge the 2 regions back to 1 + admin.mergeRegionsAsync(regions.stream().map(RegionInfo::getRegionName).toArray(byte[][]::new), + false).get(); + assertMetaRegionCount(1); + // clear the cache for table 'a' and 'b' + clearCache(TD1.getTableName()); + clearCache(TD2.getTableName()); + + // make sure that we could still get the locations from the new meta region + assertValue(TD2.getTableName(), "row2"); + assertValue(TD1.getTableName(), "row1"); } }
