Repository: hbase Updated Branches: refs/heads/master dc1065a85 -> a3c5a7448
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 2630ce0..81846df 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -39,7 +39,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; @@ -66,27 +66,23 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.exceptions.UnexpectedStateException; +import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionState.State; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.RegionStates; -import org.apache.hadoop.hbase.master.NoSuchProcedureException; -import org.apache.hadoop.hbase.procedure2.Procedure; -import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -102,11 +98,11 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.junit.rules.TestRule; /** * The below tests are testing split region against a running cluster @@ -114,9 +110,8 @@ import org.junit.rules.TestRule; @Category({RegionServerTests.class, LargeTests.class}) @SuppressWarnings("deprecation") public class TestSplitTransactionOnCluster { - private static final Log LOG = LogFactory.getLog(TestSplitTransactionOnCluster.class); - @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). - withLookingForStuckThread(true).build(); + private static final Log LOG = + LogFactory.getLog(TestSplitTransactionOnCluster.class); private Admin admin = null; private MiniHBaseCluster cluster = null; private static final int NB_SERVERS = 3; @@ -155,11 +150,8 @@ public class TestSplitTransactionOnCluster { throws IOException, InterruptedException { assertEquals(1, regions.size()); HRegionInfo hri = regions.get(0).getRegionInfo(); - try { - cluster.getMaster().getAssignmentManager().waitForAssignment(hri, 600000); - } catch (NoSuchProcedureException e) { - LOG.info("Presume the procedure has been cleaned up so just proceed: " + e.toString()); - } + cluster.getMaster().getAssignmentManager() + .waitOnRegionToClearRegionsInTransition(hri, 600000); return hri; } @@ -168,12 +160,24 @@ public class TestSplitTransactionOnCluster { final Region region, final byte[] midKey) throws IOException { long procId = cluster.getMaster().splitRegion(region.getRegionInfo(), midKey, 0, 0); - // wait for the split to complete or get interrupted. If the split completes successfully, - // the procedure will return true; if the split fails, the procedure would throw exception. - ProcedureTestingUtility.waitProcedure(cluster.getMaster().getMasterProcedureExecutor(), procId); + // wait + if (procId != -1) { + // wait for the split to complete or get interrupted. If the split completes successfully, + // the procedure will return true; if the split fails, the procedure would throw exception. + // + while (!rsServer.isProcedureFinished(procId)) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new IOException("Split region interrupted."); + } + } + } else { + throw new IOException ("Request split region failed."); + } } - @Test + @Test(timeout = 60000) public void testRITStateForRollback() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final HMaster master = cluster.getMaster(); @@ -211,10 +215,7 @@ public class TestSplitTransactionOnCluster { observer.latch.await(); LOG.info("Waiting for region to come out of RIT"); - while (!cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)) { - Threads.sleep(100); - } - assertTrue(cluster.getMaster().getAssignmentManager().getRegionStates().isRegionOnline(hri)); + cluster.getMaster().getAssignmentManager().waitOnRegionToClearRegionsInTransition(hri, 60000); } finally { admin.setBalancerRunning(true, false); master.setCatalogJanitorEnabled(true); @@ -223,7 +224,7 @@ public class TestSplitTransactionOnCluster { } } - @Test + @Test(timeout = 60000) public void testSplitFailedCompactionAndSplit() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); // Create table then get the single region for our new table. @@ -283,8 +284,8 @@ public class TestSplitTransactionOnCluster { } } - @Test - public void testSplitRollbackOnRegionClosing() throws IOException, InterruptedException { + @Test (timeout = 300000) + public void testExistingZnodeBlocksSplitAndWeRollback() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); // Create table then get the single region for our new table. @@ -320,7 +321,8 @@ public class TestSplitTransactionOnCluster { assertEquals(regionCount, ProtobufUtil.getOnlineRegions( server.getRSRpcServices()).size()); } - regionStates.updateRegionState(hri, State.OPEN); + regionStates.regionOnline(hri, server.getServerName()); + // Now try splitting and it should work. split(hri, server, regionCount); // Get daughters @@ -339,7 +341,8 @@ public class TestSplitTransactionOnCluster { * @throws IOException * @throws InterruptedException */ - @Test + @Ignore // TODO: revisit this test when the new AM and SSH is implement + @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit()throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); @@ -416,7 +419,7 @@ public class TestSplitTransactionOnCluster { } } - @Test + @Test(timeout = 180000) public void testSplitShouldNotThrowNPEEvenARegionHasEmptySplitFiles() throws Exception { TableName userTableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(userTableName); @@ -441,7 +444,6 @@ public class TestSplitTransactionOnCluster { List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager().getRegionStates() .getRegionsOfTable(userTableName); - assertEquals(1, regionsOfTable.size()); HRegionInfo hRegionInfo = regionsOfTable.get(0); Put p = new Put("row6".getBytes()); p.addColumn("col".getBytes(), "ql".getBytes(), "val".getBytes()); @@ -459,13 +461,10 @@ public class TestSplitTransactionOnCluster { .getRegionsOfTable(userTableName); while (regionsOfTable.size() != 2) { - Thread.sleep(1000); + Thread.sleep(2000); regionsOfTable = cluster.getMaster() .getAssignmentManager().getRegionStates() .getRegionsOfTable(userTableName); - LOG.debug("waiting 2 regions to be available, got " + regionsOfTable.size() + - ": " + regionsOfTable); - } Assert.assertEquals(2, regionsOfTable.size()); @@ -489,7 +488,7 @@ public class TestSplitTransactionOnCluster { * @throws NodeExistsException * @throws KeeperException */ - @Test + @Test (timeout = 300000) public void testMasterRestartAtRegionSplitPendingCatalogJanitor() throws IOException, InterruptedException, NodeExistsException, KeeperException, ServiceException { @@ -512,35 +511,25 @@ public class TestSplitTransactionOnCluster { // Get region pre-split. HRegionServer server = cluster.getRegionServer(tableRegionIndex); printOutRegions(server, "Initial regions: "); - // Call split. + this.admin.splitRegion(hri.getRegionName()); - List<HRegion> daughters = checkAndGetDaughters(tableName); - // Before cleanup, get a new master. + checkAndGetDaughters(tableName); + HMaster master = abortAndWaitForMaster(); - // Now call compact on the daughters and clean up any references. - for (HRegion daughter: daughters) { - daughter.compact(true); - assertFalse(daughter.hasReferences()); - } - // BUT calling compact on the daughters is not enough. The CatalogJanitor looks - // in the filesystem, and the filesystem content is not same as what the Region - // is reading from. Compacted-away files are picked up later by the compacted - // file discharger process. It runs infrequently. Make it run so CatalogJanitor - // doens't find any references. - for (RegionServerThread rst: cluster.getRegionServerThreads()) { - boolean oldSetting = rst.getRegionServer().compactedFileDischarger.setUseExecutor(false); - rst.getRegionServer().compactedFileDischarger.run(); - rst.getRegionServer().compactedFileDischarger.setUseExecutor(oldSetting); - } - cluster.getMaster().setCatalogJanitorEnabled(true); - LOG.info("Starting run of CatalogJanitor"); - cluster.getMaster().getCatalogJanitor().run(); - ProcedureTestingUtility.waitAllProcedures(cluster.getMaster().getMasterProcedureExecutor()); + + this.admin = TESTING_UTIL.getAdmin(); + + // Update the region to be offline and split, so that HRegionInfo#equals + // returns true in checking rebuilt region states map. + hri.setOffline(true); + hri.setSplit(true); RegionStates regionStates = master.getAssignmentManager().getRegionStates(); + assertTrue("Split parent should be in SPLIT state", + regionStates.isRegionInState(hri, State.SPLIT)); ServerName regionServerOfRegion = regionStates.getRegionServerOfRegion(hri); - assertEquals(null, regionServerOfRegion); + assertTrue(regionServerOfRegion == null); } finally { - TESTING_UTIL.getAdmin().setBalancerRunning(true, false); + this.admin.setBalancerRunning(true, false); cluster.getMaster().setCatalogJanitorEnabled(true); t.close(); } @@ -640,7 +629,7 @@ public class TestSplitTransactionOnCluster { * If a table has regions that have no store files in a region, they should split successfully * into two regions with no store files. */ - @Test + @Test(timeout = 60000) public void testSplitRegionWithNoStoreFiles() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); @@ -679,10 +668,10 @@ public class TestSplitTransactionOnCluster { fail("Split execution should have succeeded with no exceptions thrown"); } - // Postcondition: split the table with no store files into two regions, but still have no + // Postcondition: split the table with no store files into two regions, but still have not // store files List<HRegion> daughters = cluster.getRegions(tableName); - assertEquals(2, daughters.size()); + assertTrue(daughters.size() == 2); // check dirs HBaseFsck.debugLsr(conf, new Path("/")); @@ -696,13 +685,10 @@ public class TestSplitTransactionOnCluster { RegionStates regionStates = am.getRegionStates(); long start = EnvironmentEdgeManager.currentTime(); while (!regionStates.isRegionInState(hri, State.SPLIT)) { - LOG.debug("Waiting for SPLIT state on: " + hri); assertFalse("Timed out in waiting split parent to be in state SPLIT", EnvironmentEdgeManager.currentTime() - start > 60000); Thread.sleep(500); } - assertTrue(regionStates.isRegionInState(daughters.get(0).getRegionInfo(), State.OPEN)); - assertTrue(regionStates.isRegionInState(daughters.get(1).getRegionInfo(), State.OPEN)); // We should not be able to assign it again am.assign(hri, true); @@ -711,12 +697,7 @@ public class TestSplitTransactionOnCluster { assertTrue(regionStates.isRegionInState(hri, State.SPLIT)); // We should not be able to unassign it either - try { - am.unassign(hri); - fail("Should have thrown exception"); - } catch (UnexpectedStateException e) { - // Expected - } + am.unassign(hri, null); assertFalse("Split region can't be unassigned", regionStates.isRegionInTransition(hri)); assertTrue(regionStates.isRegionInState(hri, State.SPLIT)); @@ -958,14 +939,11 @@ public class TestSplitTransactionOnCluster { if (enabled.get() && req.getTransition(0).getTransitionCode().equals( TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) { RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates(); - for (RegionStates.RegionStateNode regionState: - regionStates.getRegionsInTransition()) { - /* TODO!!!! + for (RegionState regionState: regionStates.getRegionsInTransition()) { // Find the merging_new region and remove it if (regionState.isSplittingNew()) { regionStates.deleteRegion(regionState.getRegion()); } - */ } } return resp; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java index 9ccfeef..5bc4c9b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java @@ -34,7 +34,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -59,17 +58,13 @@ import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; @Category({ VerySlowRegionServerTests.class, LargeTests.class }) public class TestLogRolling extends AbstractTestLogRolling { private static final Log LOG = LogFactory.getLog(TestLogRolling.class); - @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). - withLookingForStuckThread(true).build(); @BeforeClass public static void setUpBeforeClass() throws Exception { http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSecureAsyncWALReplay.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSecureAsyncWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSecureAsyncWALReplay.java index d31d8cb..5b8b404 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSecureAsyncWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSecureAsyncWALReplay.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.regionserver.wal; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -26,14 +25,10 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider.AsyncWriter; import org.apache.hadoop.hbase.wal.WAL.Reader; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; @Category({ RegionServerTests.class, MediumTests.class }) public class TestSecureAsyncWALReplay extends TestAsyncWALReplay { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). - withLookingForStuckThread(true).build(); @BeforeClass public static void setUpBeforeClass() throws Exception { http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java index 2758d4d..e2aa580 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java @@ -22,22 +22,16 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; @Category({ RegionServerTests.class, MediumTests.class }) public class TestWALReplay extends AbstractTestWALReplay { - @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). - withLookingForStuckThread(true).build(); @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -54,4 +48,4 @@ public class TestWALReplay extends AbstractTestWALReplay { HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1); return wal; } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController3.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController3.java index d8666b6..4bb97d3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController3.java @@ -57,11 +57,7 @@ import org.junit.experimental.categories.Category; import org.junit.rules.TestName; /** - * Performs checks for reference counting w.r.t. TableAuthManager which is used by - * AccessController. - * - * NOTE: Only one test in here. In AMv2, there is problem deleting because - * we are missing auth. For now disabled. See the cleanup method. + * Performs checks for reference counting w.r.t. TableAuthManager which is used by AccessController. */ @Category({SecurityTests.class, MediumTests.class}) public class TestAccessController3 extends SecureTestUtil { @@ -204,7 +200,7 @@ public class TestAccessController3 extends SecureTestUtil { TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { rs = thread.getRegionServer(); } - // cleanUp(); + cleanUp(); TEST_UTIL.shutdownMiniCluster(); assertTrue("region server should have aborted due to FaultyAccessController", rs.isAborted()); } @@ -266,16 +262,12 @@ public class TestAccessController3 extends SecureTestUtil { private static void cleanUp() throws Exception { // Clean the _acl_ table - // TODO: Skipping delete because of access issues w/ AMv2. - // AMv1 seems to crash servers on exit too for same lack of - // auth perms but it gets hung up. - /* try { deleteTable(TEST_UTIL, TEST_TABLE); } catch (TableNotFoundException ex) { // Test deleted the table, no problem LOG.info("Test deleted table " + TEST_TABLE); - }*/ + } // Verify all table/namespace permissions are erased assertEquals(0, AccessControlLists.getTablePermissions(conf, TEST_TABLE).size()); assertEquals( http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java index f6e328e..02bd49b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java @@ -61,9 +61,8 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.coprocessor.MasterObserver; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.mob.MobFileName; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; @@ -92,7 +91,6 @@ public class BaseTestHBaseFsck { protected final static String FAM_STR = "fam"; protected final static byte[] FAM = Bytes.toBytes(FAM_STR); protected final static int REGION_ONLINE_TIMEOUT = 800; - protected static AssignmentManager assignmentManager; protected static RegionStates regionStates; protected static ExecutorService tableExecutorService; protected static ScheduledThreadPoolExecutor hbfsckExecutorService; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java index ca8bc91..b6a185b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.io.hfile.TestHFile; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java index 4188146..74ef414 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java @@ -19,30 +19,6 @@ package org.apache.hadoop.hbase.util; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -69,8 +45,11 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.io.hfile.TestHFile; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.RegionStates; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.SplitTableRegionProcedure; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction; @@ -93,7 +72,26 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -@Ignore // Turning off because needs fsck. +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.*; +import static org.junit.Assert.*; + @Category({MiscTests.class, LargeTests.class}) public class TestHBaseFsckOneRS extends BaseTestHBaseFsck { @Rule @@ -1597,6 +1595,72 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck { } } + @Test (timeout=180000) + public void testCleanUpDaughtersNotInMetaAfterFailedSplit() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + try { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(Bytes.toBytes("f"))); + createTable(TEST_UTIL, desc, null); + + tbl = connection.getTable(desc.getTableName()); + for (int i = 0; i < 5; i++) { + Put p1 = new Put(("r" + i).getBytes()); + p1.addColumn(Bytes.toBytes("f"), "q1".getBytes(), "v".getBytes()); + tbl.put(p1); + } + admin.flush(desc.getTableName()); + List<HRegion> regions = cluster.getRegions(desc.getTableName()); + int serverWith = cluster.getServerWith(regions.get(0).getRegionInfo().getRegionName()); + HRegionServer regionServer = cluster.getRegionServer(serverWith); + byte[] parentRegionName = regions.get(0).getRegionInfo().getRegionName(); + cluster.getServerWith(parentRegionName); + // Create daughters without adding to META table + MasterProcedureEnv env = cluster.getMaster().getMasterProcedureExecutor().getEnvironment(); + SplitTableRegionProcedure splitR = new SplitTableRegionProcedure( + env, regions.get(0).getRegionInfo(), Bytes.toBytes("r3")); + splitR.prepareSplitRegion(env); + splitR.setRegionStateToSplitting(env); + splitR.closeParentRegionForSplit(env); + splitR.createDaughterRegions(env); + + AssignmentManager am = cluster.getMaster().getAssignmentManager(); + for (RegionState state : am.getRegionStates().getRegionsInTransition()) { + am.regionOffline(state.getRegion()); + } + + Map<HRegionInfo, ServerName> regionsMap = new HashMap<>(); + regionsMap.put(regions.get(0).getRegionInfo(), regionServer.getServerName()); + am.assign(regionsMap); + am.waitForAssignment(regions.get(0).getRegionInfo()); + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { + HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED }); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(tableName).size()); + + // fix hole + assertErrors( + doFsck(conf, false, true, false, false, false, false, false, false, false, false, false, + false, null), + new HBaseFsck.ErrorReporter.ERROR_CODE[] { + HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED, + HBaseFsck.ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED }); + + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(5, countRows()); + } finally { + if (tbl != null) { + tbl.close(); + tbl = null; + } + cleanupTable(tableName); + } + } + /** * This creates fixes a bad table with a hole in meta. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java index 3d0647e..403bf5e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java @@ -31,14 +31,13 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -59,7 +58,6 @@ import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.*; import static org.junit.Assert.*; -@Ignore @Category({MiscTests.class, LargeTests.class}) public class TestHBaseFsckReplicas extends BaseTestHBaseFsck { @Rule http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java index 7f891d8..6f90bb2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java @@ -19,13 +19,7 @@ package org.apache.hadoop.hbase.util; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors; -import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import com.google.common.collect.Multimap; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.SynchronousQueue; @@ -41,6 +35,8 @@ import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -50,15 +46,14 @@ import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import com.google.common.collect.Multimap; +import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.*; +import static org.junit.Assert.*; -@Ignore // Until after HBASE-14614 goes in. @Category({MiscTests.class, LargeTests.class}) public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck { @Rule @@ -83,7 +78,8 @@ public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck { hbfsckExecutorService = new ScheduledThreadPoolExecutor(POOL_SIZE); - assignmentManager = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); + AssignmentManager assignmentManager = + TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); regionStates = assignmentManager.getRegionStates(); connection = (ClusterConnection) TEST_UTIL.getConnection(); @@ -112,7 +108,7 @@ public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck { public void testFixAssignmentsWhenMETAinTransition() throws Exception { MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); admin.closeRegion(cluster.getServerHoldingMeta(), HRegionInfo.FIRST_META_REGIONINFO); - assignmentManager.offlineRegion(HRegionInfo.FIRST_META_REGIONINFO); + regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO); new MetaTableLocator().deleteMetaLocation(cluster.getMaster().getZooKeeper()); assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO)); HBaseFsck hbck = doFsck(conf, true); @@ -397,6 +393,7 @@ public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck { // Mess it up by creating an overlap MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster master = cluster.getMaster(); HRegionInfo hriOverlap1 = createRegion(tbl.getTableDescriptor(), Bytes.toBytes("A"), Bytes.toBytes("AB")); TEST_UTIL.assignRegion(hriOverlap1); @@ -442,7 +439,7 @@ public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck { try (Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService)) { Put put = new Put(regionName); put.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, - Bytes.toBytes(serverName.getAddress().toString())); + Bytes.toBytes(serverName.getHostAndPort())); meta.put(put); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java index 1560efe..3be7787 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HBaseFsck; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -53,7 +52,7 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore { private static final Log LOG = LogFactory.getLog(TestOfflineMetaRebuildBase.class); @SuppressWarnings("deprecation") - @Ignore @Test(timeout = 120000) // To fix post HBASE-14614 + @Test(timeout = 120000) public void testMetaRebuild() throws Exception { wipeOutMeta(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java index 60c4b25..b8565e3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildHole.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.HBaseFsck; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -39,7 +38,6 @@ import org.junit.experimental.categories.Category; * This builds a table, removes info from meta, and then fails when attempting * to rebuild meta. */ -@Ignore @Category({MiscTests.class, MediumTests.class}) public class TestOfflineMetaRebuildHole extends OfflineMetaRebuildTestCore { private final static Log LOG = LogFactory.getLog(TestOfflineMetaRebuildHole.class); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java index 85e0560..ae72935 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildOverlap.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.HBaseFsck; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -42,7 +41,6 @@ import com.google.common.collect.Multimap; * This builds a table, builds an overlap, and then fails when attempting to * rebuild meta. */ -@Ignore @Category({MiscTests.class, MediumTests.class}) public class TestOfflineMetaRebuildOverlap extends OfflineMetaRebuildTestCore { private final static Log LOG = LogFactory.getLog(TestOfflineMetaRebuildOverlap.class);
