http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index 59e8fb3..32bce26 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -27,11 +27,8 @@ import static org.mockito.Mockito.spy; import java.io.IOException; import java.util.Map; -import java.util.NavigableMap; import java.util.SortedMap; -import java.util.SortedSet; import java.util.TreeMap; -import java.util.concurrent.ConcurrentSkipListMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,141 +36,267 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaMockingUtil; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; +import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination; +import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator; -import org.apache.hadoop.hbase.master.assignment.MockMasterServices; -import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; -import org.apache.hadoop.hbase.regionserver.ChunkCreator; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MultiResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ResultOrException; import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.ChunkCreator; import org.apache.hadoop.hbase.regionserver.MemStoreLABImpl; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; -import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Triple; -import org.junit.After; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.junit.rules.TestRule; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; @Category({MasterTests.class, SmallTests.class}) public class TestCatalogJanitor { private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class); - @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). - withTimeout(this.getClass()).withLookingForStuckThread(true).build(); - @Rule public final TestName name = new TestName(); - private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); - private MockMasterServices masterServices; - private CatalogJanitor janitor; + + @Rule + public TestName name = new TestName(); @BeforeClass - public static void beforeClass() throws Exception { + public static void setup() throws Exception { ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); } + /** + * Mock MasterServices for tests below. + */ + class MockMasterServices extends MockNoopMasterServices { + private final ClusterConnection connection; + private final MasterFileSystem mfs; + private final AssignmentManager asm; + private final ServerManager sm; + + MockMasterServices(final HBaseTestingUtility htu) throws IOException { + super(htu.getConfiguration()); + + ClientProtos.ClientService.BlockingInterface ri = + Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); + MutateResponse.Builder builder = MutateResponse.newBuilder(); + builder.setProcessed(true); + try { + Mockito.when(ri.mutate( + (RpcController)Mockito.any(), (MutateRequest)Mockito.any())). + thenReturn(builder.build()); + } catch (ServiceException se) { + throw ProtobufUtil.handleRemoteException(se); + } + try { + Mockito.when(ri.multi( + (RpcController)Mockito.any(), (MultiRequest)Mockito.any())). + thenAnswer(new Answer<MultiResponse>() { + @Override + public MultiResponse answer(InvocationOnMock invocation) throws Throwable { + return buildMultiResponse( (MultiRequest)invocation.getArguments()[1]); + } + }); + } catch (ServiceException se) { + throw ProtobufUtil.getRemoteException(se); + } + // Mock an ClusterConnection and a AdminProtocol implementation. Have the + // ClusterConnection return the HRI. Have the HRI return a few mocked up responses + // to make our test work. + this.connection = + HConnectionTestingUtility.getMockedConnectionAndDecorate(getConfiguration(), + Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri, + ServerName.valueOf("example.org,12345,6789"), + HRegionInfo.FIRST_META_REGIONINFO); + // Set hbase.rootdir into test dir. + FileSystem.get(getConfiguration()); + Path rootdir = FSUtils.getRootDir(getConfiguration()); + FSUtils.setRootDir(getConfiguration(), rootdir); + Mockito.mock(AdminProtos.AdminService.BlockingInterface.class); + + this.mfs = new MasterFileSystem(this); + this.asm = Mockito.mock(AssignmentManager.class); + this.sm = Mockito.mock(ServerManager.class); + } - @Before - public void setup() throws IOException { - setRootDirAndCleanIt(HTU, this.name.getMethodName()); - NavigableMap<ServerName, SortedSet<byte []>> regionsToRegionServers = - new ConcurrentSkipListMap<ServerName, SortedSet<byte []>>(); - this.masterServices = - new MockMasterServices(HTU.getConfiguration(), regionsToRegionServers); - this.masterServices.start(10, null); - this.janitor = new CatalogJanitor(masterServices); - } + @Override + public AssignmentManager getAssignmentManager() { + return this.asm; + } + + @Override + public MasterFileSystem getMasterFileSystem() { + return this.mfs; + } + + @Override + public ClusterConnection getConnection() { + return this.connection; + } + + @Override + public ServerName getServerName() { + return ServerName.valueOf("mockserver.example.org", 1234, -1L); + } + + @Override + public ServerManager getServerManager() { + return this.sm; + } + + @Override + public CoordinatedStateManager getCoordinatedStateManager() { + BaseCoordinatedStateManager m = Mockito.mock(BaseCoordinatedStateManager.class); + SplitLogManagerCoordination c = Mockito.mock(SplitLogManagerCoordination.class); + Mockito.when(m.getSplitLogManagerCoordination()).thenReturn(c); + SplitLogManagerDetails d = Mockito.mock(SplitLogManagerDetails.class); + Mockito.when(c.getDetails()).thenReturn(d); + return m; + } - @After - public void teardown() { - this.janitor.cancel(true); - this.masterServices.stop("DONE"); + @Override + public TableDescriptors getTableDescriptors() { + return new TableDescriptors() { + @Override + public HTableDescriptor remove(TableName tablename) throws IOException { + // noop + return null; + } + + @Override + public Map<String, HTableDescriptor> getAll() throws IOException { + // noop + return null; + } + + @Override public Map<String, HTableDescriptor> getAllDescriptors() throws IOException { + // noop + return null; + } + + @Override + public HTableDescriptor get(TableName tablename) + throws IOException { + return createHTableDescriptor(); + } + + @Override + public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException { + return null; + } + + @Override + public void add(HTableDescriptor htd) throws IOException { + // noop + } + + @Override + public void setCacheOn() throws IOException { + } + + @Override + public void setCacheOff() throws IOException { + } + }; + } } - /** - * Test clearing a split parent. - */ @Test public void testCleanParent() throws IOException, InterruptedException { - TableDescriptor td = createTableDescriptorForCurrentMethod(); - // Create regions. - HRegionInfo parent = - new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); - HRegionInfo splita = - new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); - HRegionInfo splitb = - new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); - // Test that when both daughter regions are in place, that we do not remove the parent. - Result r = createResult(parent, splita, splitb); - // Add a reference under splitA directory so we don't clear out the parent. - Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); - Path tabledir = FSUtils.getTableDir(rootdir, td.getTableName()); - Path parentdir = new Path(tabledir, parent.getEncodedName()); - Path storedir = HStore.getStoreHomedir(tabledir, splita, td.getColumnFamilies()[0].getName()); - Reference ref = Reference.createTopReference(Bytes.toBytes("ccc")); - long now = System.currentTimeMillis(); - // Reference name has this format: StoreFile#REF_NAME_PARSER - Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); - FileSystem fs = this.masterServices.getMasterFileSystem().getFileSystem(); - Path path = ref.write(fs, p); - assertTrue(fs.exists(path)); - LOG.info("Created reference " + path); - // Add a parentdir for kicks so can check it gets removed by the catalogjanitor. - fs.mkdirs(parentdir); - assertFalse(this.janitor.cleanParent(parent, r)); - ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); - assertTrue(fs.exists(parentdir)); - // Remove the reference file and try again. - assertTrue(fs.delete(p, true)); - assertTrue(this.janitor.cleanParent(parent, r)); - // Parent cleanup is run async as a procedure. Make sure parentdir is removed. - ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); - assertTrue(!fs.exists(parentdir)); + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testCleanParent"); + MasterServices services = new MockMasterServices(htu); + try { + CatalogJanitor janitor = new CatalogJanitor(services); + // Create regions. + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = + new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + Bytes.toBytes("eee")); + HRegionInfo splita = + new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc")); + HRegionInfo splitb = + new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), + Bytes.toBytes("eee")); + // Test that when both daughter regions are in place, that we do not + // remove the parent. + Result r = createResult(parent, splita, splitb); + // Add a reference under splitA directory so we don't clear out the parent. + Path rootdir = services.getMasterFileSystem().getRootDir(); + Path tabledir = + FSUtils.getTableDir(rootdir, htd.getTableName()); + Path storedir = HStore.getStoreHomedir(tabledir, splita, + htd.getColumnFamilies()[0].getName()); + Reference ref = Reference.createTopReference(Bytes.toBytes("ccc")); + long now = System.currentTimeMillis(); + // Reference name has this format: StoreFile#REF_NAME_PARSER + Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); + FileSystem fs = services.getMasterFileSystem().getFileSystem(); + Path path = ref.write(fs, p); + assertTrue(fs.exists(path)); + assertFalse(janitor.cleanParent(parent, r)); + // Remove the reference file and try again. + assertTrue(fs.delete(p, true)); + assertTrue(janitor.cleanParent(parent, r)); + } finally { + services.stop("shutdown"); + } } /** * Make sure parent gets cleaned up even if daughter is cleaned up before it. + * @throws IOException + * @throws InterruptedException */ @Test public void testParentCleanedEvenIfDaughterGoneFirst() throws IOException, InterruptedException { - parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), - Bytes.toBytes("eee")); + parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst( + "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee")); } /** * Make sure last parent with empty end key gets cleaned up even if daughter is cleaned up before it. + * @throws IOException + * @throws InterruptedException */ @Test public void testLastParentCleanedEvenIfDaughterGoneFirst() throws IOException, InterruptedException { - parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(this.name.getMethodName(), - new byte[0]); - } - - /** - * @return A TableDescriptor with a tableName of current method name and a column - * family that is MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME) - */ - private TableDescriptor createTableDescriptorForCurrentMethod() { - return TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName())). - addFamily(new HColumnDescriptor(MockMasterServices.DEFAULT_COLUMN_FAMILY_NAME)). - build(); + parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst( + "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]); } /** @@ -181,40 +304,50 @@ public class TestCatalogJanitor { * * @param rootDir the test case name, used as the HBase testing utility root * @param lastEndKey the end key of the split parent + * @throws IOException + * @throws InterruptedException */ private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst( final String rootDir, final byte[] lastEndKey) throws IOException, InterruptedException { - TableDescriptor td = createTableDescriptorForCurrentMethod(); + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, rootDir); + MasterServices services = new MockMasterServices(htu); + CatalogJanitor janitor = new CatalogJanitor(services); + final HTableDescriptor htd = createHTableDescriptor(); + // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. - HRegionInfo parent = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), lastEndKey); + + // Parent + HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + lastEndKey); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a - HRegionInfo splita = - new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); + HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc")); Thread.sleep(1001); // Make daughters of daughter a; splitaa and splitab. - HRegionInfo splitaa = - new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb")); - HRegionInfo splitab = - new HRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc")); + HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + Bytes.toBytes("bbb")); + HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"), + Bytes.toBytes("ccc")); // Daughter b - HRegionInfo splitb = - new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), lastEndKey); + HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), + lastEndKey); Thread.sleep(1001); // Make Daughters of daughterb; splitba and splitbb. - HRegionInfo splitba = - new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd")); - HRegionInfo splitbb = - new HRegionInfo(td.getTableName(), Bytes.toBytes("ddd"), lastEndKey); + HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), + Bytes.toBytes("ddd")); + HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"), + lastEndKey); // First test that our Comparator works right up in CatalogJanitor. - SortedMap<HRegionInfo, Result> regions = - new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator()); + // Just fo kicks. + SortedMap<HRegionInfo, Result> regions = new TreeMap<>(new CatalogJanitor.SplitParentFirstComparator()); // Now make sure that this regions map sorts as we expect it to. regions.put(parent, createResult(parent, splita, splitb)); regions.put(splitb, createResult(splitb, splitba, splitbb)); @@ -232,9 +365,10 @@ public class TestCatalogJanitor { index++; } - // Now play around with the cleanParent function. Create a ref from splita up to the parent. + // Now play around with the cleanParent function. Create a ref from splita + // up to the parent. Path splitaRef = - createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false); + createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); // Make sure actual super parent sticks around because splita has a ref. assertFalse(janitor.cleanParent(parent, regions.get(parent))); @@ -246,13 +380,13 @@ public class TestCatalogJanitor { // the daughter splita can be split (can't split if still references). // BUT make the timing such that the daughter gets cleaned up before we // can get a chance to let go of the parent. - FileSystem fs = FileSystem.get(HTU.getConfiguration()); + FileSystem fs = FileSystem.get(htu.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); // Create the refs from daughters of splita. Path splitaaRef = - createReferences(this.masterServices, td, splita, splitaa, Bytes.toBytes("bbb"), false); + createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false); Path splitabRef = - createReferences(this.masterServices, td, splita, splitab, Bytes.toBytes("bbb"), true); + createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true); // Test splita. It should stick around because references from splitab, etc. assertFalse(janitor.cleanParent(splita, regions.get(splita))); @@ -264,67 +398,74 @@ public class TestCatalogJanitor { // Super parent should get cleaned up now both splita and splitb are gone. assertTrue(janitor.cleanParent(parent, regions.get(parent))); + + services.stop("test finished"); + janitor.cancel(true); } /** * CatalogJanitor.scan() should not clean parent regions if their own - * parents are still referencing them. This ensures that grandparent regions + * parents are still referencing them. This ensures that grandfather regions * do not point to deleted parent regions. */ @Test public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { - TableDescriptor td = createTableDescriptorForCurrentMethod(); + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents"); + MasterServices services = new MockMasterServices(htu); + + final HTableDescriptor htd = createHTableDescriptor(); + // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. // Parent - HRegionInfo parent = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), - HConstants.EMPTY_BYTE_ARRAY, true); + HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + new byte[0], true); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a - HRegionInfo splita = - new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); + HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + Bytes.toBytes("ccc"), true); Thread.sleep(1001); - // Make daughters of daughter a; splitaa and splitab. - HRegionInfo splitaa = - new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); - HRegionInfo splitab = - new HRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); + HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), + Bytes.toBytes("bbb"), false); + HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), false); // Daughter b - HRegionInfo splitb = - new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY); + HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), + new byte[0]); Thread.sleep(1001); - // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab. final Map<HRegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator()); splitParents.put(parent, createResult(parent, splita, splitb)); splita.setOffline(true); //simulate that splita goes offline when it is split - splitParents.put(splita, createResult(splita, splitaa, splitab)); + splitParents.put(splita, createResult(splita, splitaa,splitab)); final Map<HRegionInfo, Result> mergedRegions = new TreeMap<>(); - CatalogJanitor spy = spy(this.janitor); - doReturn(new Triple<>(10, mergedRegions, splitParents)).when(spy). - getMergedRegionsAndSplitParents(); + CatalogJanitor janitor = spy(new CatalogJanitor(services)); + doReturn(new Triple<>(10, mergedRegions, splitParents)).when(janitor) + .getMergedRegionsAndSplitParents(); - // Create ref from splita to parent - LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog()); + //create ref from splita to parent Path splitaRef = - createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false); - LOG.info("Created reference " + splitaRef); + createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); - // Parent and splita should not be removed because a reference from splita to parent. - assertEquals(0, spy.scan()); + //parent and A should not be removed + assertEquals(0, janitor.scan()); - // Now delete the ref - FileSystem fs = FileSystem.get(HTU.getConfiguration()); + //now delete the ref + FileSystem fs = FileSystem.get(htu.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); //now, both parent, and splita can be deleted - assertEquals(2, spy.scan()); + assertEquals(2, janitor.scan()); + + services.stop("test finished"); + janitor.cancel(true); } /** @@ -334,7 +475,7 @@ public class TestCatalogJanitor { @Test public void testSplitParentFirstComparator() { SplitParentFirstComparator comp = new SplitParentFirstComparator(); - TableDescriptor td = createTableDescriptorForCurrentMethod(); + final HTableDescriptor htd = createHTableDescriptor(); /* Region splits: * @@ -355,12 +496,15 @@ public class TestCatalogJanitor { */ // root region - HRegionInfo rootRegion = new HRegionInfo(td.getTableName(), - HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, true); - HRegionInfo firstRegion = new HRegionInfo(td.getTableName(), - HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true); - HRegionInfo lastRegion = new HRegionInfo(td.getTableName(), - Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true); + HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(), + HConstants.EMPTY_START_ROW, + HConstants.EMPTY_END_ROW, true); + HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(), + HConstants.EMPTY_START_ROW, + Bytes.toBytes("bbb"), true); + HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("bbb"), + HConstants.EMPTY_END_ROW, true); assertTrue(comp.compare(rootRegion, rootRegion) == 0); assertTrue(comp.compare(firstRegion, firstRegion) == 0); @@ -370,15 +514,19 @@ public class TestCatalogJanitor { assertTrue(comp.compare(firstRegion, lastRegion) < 0); //first region split into a, b - HRegionInfo firstRegiona = new HRegionInfo(td.getTableName(), - HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true); - HRegionInfo firstRegionb = new HRegionInfo(td.getTableName(), - Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true); + HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(), + HConstants.EMPTY_START_ROW, + Bytes.toBytes("aaa"), true); + HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("aaa"), + Bytes.toBytes("bbb"), true); //last region split into a, b - HRegionInfo lastRegiona = new HRegionInfo(td.getTableName(), - Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true); - HRegionInfo lastRegionb = new HRegionInfo(td.getTableName(), - Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true); + HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("bbb"), + Bytes.toBytes("ddd"), true); + HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("ddd"), + HConstants.EMPTY_END_ROW, true); assertTrue(comp.compare(firstRegiona, firstRegiona) == 0); assertTrue(comp.compare(firstRegionb, firstRegionb) == 0); @@ -401,47 +549,58 @@ public class TestCatalogJanitor { assertTrue(comp.compare(firstRegionb, lastRegiona) < 0); assertTrue(comp.compare(firstRegionb, lastRegionb) < 0); - HRegionInfo lastRegionaa = new HRegionInfo(td.getTableName(), - Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); - HRegionInfo lastRegionab = new HRegionInfo(td.getTableName(), - Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false); + HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("bbb"), + Bytes.toBytes("ccc"), false); + HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("ccc"), + Bytes.toBytes("ddd"), false); assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0); assertTrue(comp.compare(lastRegiona, lastRegionab) < 0); assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0); + } @Test public void testArchiveOldRegion() throws Exception { + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testCleanParent"); + MasterServices services = new MockMasterServices(htu); + + // create the janitor + CatalogJanitor janitor = new CatalogJanitor(services); + // Create regions. - TableDescriptor td = createTableDescriptorForCurrentMethod(); - HRegionInfo parent = new HRegionInfo(td.getTableName(), + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); - HRegionInfo splita = new HRegionInfo(td.getTableName(), + HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); - HRegionInfo splitb = new HRegionInfo(td.getTableName(), - Bytes.toBytes("ccc"), Bytes.toBytes("eee")); + HRegionInfo splitb = new HRegionInfo(htd.getTableName(), + Bytes.toBytes("ccc"), + Bytes.toBytes("eee")); // Test that when both daughter regions are in place, that we do not // remove the parent. Result parentMetaRow = createResult(parent, splita, splitb); - FileSystem fs = FileSystem.get(HTU.getConfiguration()); - Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); + FileSystem fs = FileSystem.get(htu.getConfiguration()); + Path rootdir = services.getMasterFileSystem().getRootDir(); // have to set the root directory since we use it in HFileDisposer to figure out to get to the // archive directory. Otherwise, it just seems to pick the first root directory it can find (so // the single test passes, but when the full suite is run, things get borked). FSUtils.setRootDir(fs.getConf(), rootdir); - Path tabledir = FSUtils.getTableDir(rootdir, td.getTableName()); - Path storedir = HStore.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName()); - Path storeArchive = - HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(), parent, - tabledir, td.getColumnFamilies()[0].getName()); + Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName()); + Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName()); + Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent, + tabledir, htd.getColumnFamilies()[0].getName()); LOG.debug("Table dir:" + tabledir); LOG.debug("Store dir:" + storedir); LOG.debug("Store archive dir:" + storeArchive); // add a couple of store files that we can check for - FileStatus[] mockFiles = addMockStoreFiles(2, this.masterServices, storedir); + FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir); // get the current store files for comparison FileStatus[] storeFiles = fs.listStatus(storedir); int index = 0; @@ -454,10 +613,6 @@ public class TestCatalogJanitor { // do the cleaning of the parent assertTrue(janitor.cleanParent(parent, parentMetaRow)); - Path parentDir = new Path(tabledir, parent.getEncodedName()); - // Cleanup procedure runs async. Wait till it done. - ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); - assertTrue(!fs.exists(parentDir)); LOG.debug("Finished cleanup of parent region"); // and now check to make sure that the files have actually been archived @@ -469,6 +624,8 @@ public class TestCatalogJanitor { // cleanup FSUtils.delete(fs, rootdir, true); + services.stop("Test finished"); + janitor.cancel(true); } /** @@ -488,62 +645,69 @@ public class TestCatalogJanitor { */ @Test public void testDuplicateHFileResolution() throws Exception { - TableDescriptor td = createTableDescriptorForCurrentMethod(); + HBaseTestingUtility htu = new HBaseTestingUtility(); + setRootDirAndCleanIt(htu, "testCleanParent"); + MasterServices services = new MockMasterServices(htu); + + // create the janitor + + CatalogJanitor janitor = new CatalogJanitor(services); // Create regions. - HRegionInfo parent = new HRegionInfo(td.getTableName(), + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); + htd.addFamily(new HColumnDescriptor("f")); + HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("eee")); - HRegionInfo splita = new HRegionInfo(td.getTableName(), + HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc")); - HRegionInfo splitb = new HRegionInfo(td.getTableName(), + HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), Bytes.toBytes("eee")); // Test that when both daughter regions are in place, that we do not // remove the parent. Result r = createResult(parent, splita, splitb); - FileSystem fs = FileSystem.get(HTU.getConfiguration()); - Path rootdir = this.masterServices.getMasterFileSystem().getRootDir(); - // Have to set the root directory since we use it in HFileDisposer to figure out to get to the + + FileSystem fs = FileSystem.get(htu.getConfiguration()); + + Path rootdir = services.getMasterFileSystem().getRootDir(); + // have to set the root directory since we use it in HFileDisposer to figure out to get to the // archive directory. Otherwise, it just seems to pick the first root directory it can find (so // the single test passes, but when the full suite is run, things get borked). FSUtils.setRootDir(fs.getConf(), rootdir); Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable()); - Path storedir = HStore.getStoreHomedir(tabledir, parent, td.getColumnFamilies()[0].getName()); + Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName()); System.out.println("Old root:" + rootdir); System.out.println("Old table:" + tabledir); System.out.println("Old store:" + storedir); - Path storeArchive = - HFileArchiveUtil.getStoreArchivePath(this.masterServices.getConfiguration(), parent, - tabledir, td.getColumnFamilies()[0].getName()); + Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent, + tabledir, htd.getColumnFamilies()[0].getName()); System.out.println("Old archive:" + storeArchive); // enable archiving, make sure that files get archived - addMockStoreFiles(2, this.masterServices, storedir); + addMockStoreFiles(2, services, storedir); // get the current store files for comparison FileStatus[] storeFiles = fs.listStatus(storedir); - // Do the cleaning of the parent + // do the cleaning of the parent assertTrue(janitor.cleanParent(parent, r)); - Path parentDir = new Path(tabledir, parent.getEncodedName()); - ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); - assertTrue(!fs.exists(parentDir)); - // And now check to make sure that the files have actually been archived + // and now check to make sure that the files have actually been archived FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive); assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs); // now add store files with the same names as before to check backup // enable archiving, make sure that files get archived - addMockStoreFiles(2, this.masterServices, storedir); + addMockStoreFiles(2, services, storedir); - // Do the cleaning of the parent + // do the cleaning of the parent assertTrue(janitor.cleanParent(parent, r)); - // Cleanup procedure runs async. Wait till it done. - ProcedureTestingUtility.waitAllProcedures(masterServices.getMasterProcedureExecutor()); - assertTrue(!fs.exists(parentDir)); // and now check to make sure that the files have actually been archived archivedStoreFiles = fs.listStatus(storeArchive); assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true); + + // cleanup + services.stop("Test finished"); + janitor.cancel(true); } private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir) @@ -565,7 +729,8 @@ public class TestCatalogJanitor { return storeFiles; } - private String setRootDirAndCleanIt(final HBaseTestingUtility htu, final String subdir) + private String setRootDirAndCleanIt(final HBaseTestingUtility htu, + final String subdir) throws IOException { Path testdir = htu.getDataTestDir(subdir); FileSystem fs = FileSystem.get(htu.getConfiguration()); @@ -574,14 +739,24 @@ public class TestCatalogJanitor { return FSUtils.getRootDir(htu.getConfiguration()).toString(); } + /** + * @param services Master services instance. + * @param htd + * @param parent + * @param daughter + * @param midkey + * @param top True if we are to write a 'top' reference. + * @return Path to reference we created. + * @throws IOException + */ private Path createReferences(final MasterServices services, - final TableDescriptor td, final HRegionInfo parent, + final HTableDescriptor htd, final HRegionInfo parent, final HRegionInfo daughter, final byte [] midkey, final boolean top) throws IOException { Path rootdir = services.getMasterFileSystem().getRootDir(); Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable()); Path storedir = HStore.getStoreHomedir(tabledir, daughter, - td.getColumnFamilies()[0].getName()); + htd.getColumnFamilies()[0].getName()); Reference ref = top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey); long now = System.currentTimeMillis(); @@ -597,4 +772,30 @@ public class TestCatalogJanitor { throws IOException { return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b); } -} \ No newline at end of file + + private HTableDescriptor createHTableDescriptor() { + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t")); + htd.addFamily(new HColumnDescriptor("f")); + return htd; + } + + private MultiResponse buildMultiResponse(MultiRequest req) { + MultiResponse.Builder builder = MultiResponse.newBuilder(); + RegionActionResult.Builder regionActionResultBuilder = + RegionActionResult.newBuilder(); + ResultOrException.Builder roeBuilder = ResultOrException.newBuilder(); + for (RegionAction regionAction: req.getRegionActionList()) { + regionActionResultBuilder.clear(); + for (ClientProtos.Action action: regionAction.getActionList()) { + roeBuilder.clear(); + roeBuilder.setResult(ClientProtos.Result.getDefaultInstance()); + roeBuilder.setIndex(action.getIndex()); + regionActionResultBuilder.addResultOrException(roeBuilder.build()); + } + builder.addRegionActionResult(regionActionResultBuilder.build()); + } + return builder.build(); + } + +} +
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index fcd2cf9..b78bfd1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -84,7 +84,6 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination; import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java index d6210b9..eb4ce99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableState; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java index 0073cdf..74f2c91 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterBalanceThrottling.java @@ -34,11 +34,9 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; -@Ignore // SimpleLoadBalancer seems borked whether AMv2 or not. Disabling till gets attention. @Category({MasterTests.class, MediumTests.class}) public class TestMasterBalanceThrottling { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -122,9 +120,8 @@ public class TestMasterBalanceThrottling { @Override public void run() { while (!stop.get()) { - maxCount.set(Math.max(maxCount.get(), - master.getAssignmentManager().getRegionStates() - .getRegionsInTransition().size())); + maxCount.set(Math.max(maxCount.get(), master.getAssignmentManager().getRegionStates() + .getRegionsInTransitionCount())); try { Thread.sleep(10); } catch (InterruptedException e) { @@ -139,7 +136,7 @@ public class TestMasterBalanceThrottling { } private void unbalance(HMaster master, TableName tableName) throws Exception { - while (master.getAssignmentManager().getRegionStates().getRegionsInTransition().size() > 0) { + while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) { Thread.sleep(100); } HRegionServer biasedServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); @@ -147,7 +144,7 @@ public class TestMasterBalanceThrottling { master.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(biasedServer.getServerName().getServerName())); } - while (master.getAssignmentManager().getRegionStates().getRegionsInTransition().size() > 0) { + while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) { Thread.sleep(100); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 29c0576..f57d6b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -43,8 +43,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.assignment.RegionStates; -import org.apache.hadoop.hbase.master.assignment.RegionStateStore; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -57,12 +55,10 @@ import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @Category({FlakeyTests.class, LargeTests.class}) -@Ignore // Needs to be rewritten for AMv2. Uses tricks not ordained when up on AMv2. public class TestMasterFailover { private static final Log LOG = LogFactory.getLog(TestMasterFailover.class); @@ -256,22 +252,23 @@ public class TestMasterFailover { // Put the online region in pending_close. It is actually already opened. // This is to simulate that the region close RPC is not sent out before failover RegionState oldState = regionStates.getRegionState(hriOnline); - RegionState newState = new RegionState(hriOnline, State.CLOSING, oldState.getServerName()); - stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState); + RegionState newState = new RegionState( + hriOnline, State.PENDING_CLOSE, oldState.getServerName()); + stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState); // Put the offline region in pending_open. It is actually not opened yet. // This is to simulate that the region open RPC is not sent out before failover oldState = new RegionState(hriOffline, State.OFFLINE); - newState = new RegionState(hriOffline, State.OPENING, newState.getServerName()); - stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState); + newState = new RegionState(hriOffline, State.PENDING_OPEN, newState.getServerName()); + stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState); HRegionInfo failedClose = new HRegionInfo(offlineTable.getTableName(), null, null); createRegion(failedClose, rootdir, conf, offlineTable); MetaTableAccessor.addRegionToMeta(master.getConnection(), failedClose); - oldState = new RegionState(failedClose, State.CLOSING); + oldState = new RegionState(failedClose, State.PENDING_CLOSE); newState = new RegionState(failedClose, State.FAILED_CLOSE, newState.getServerName()); - stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState); + stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState); HRegionInfo failedOpen = new HRegionInfo(offlineTable.getTableName(), null, null); createRegion(failedOpen, rootdir, conf, offlineTable); @@ -279,9 +276,9 @@ public class TestMasterFailover { // Simulate a region transitioning to failed open when the region server reports the // transition as FAILED_OPEN - oldState = new RegionState(failedOpen, State.OPENING); + oldState = new RegionState(failedOpen, State.PENDING_OPEN); newState = new RegionState(failedOpen, State.FAILED_OPEN, newState.getServerName()); - stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState); + stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState); HRegionInfo failedOpenNullServer = new HRegionInfo(offlineTable.getTableName(), null, null); LOG.info("Failed open NUll server " + failedOpenNullServer.getEncodedName()); @@ -292,7 +289,7 @@ public class TestMasterFailover { // the region oldState = new RegionState(failedOpenNullServer, State.OFFLINE); newState = new RegionState(failedOpenNullServer, State.FAILED_OPEN, null); - stateStore.updateRegionState(HConstants.NO_SEQNUM, -1, newState, oldState); + stateStore.updateRegionState(HConstants.NO_SEQNUM, newState, oldState); // Stop the master log("Aborting master"); @@ -381,12 +378,12 @@ public class TestMasterFailover { assertEquals("hbase:meta should be onlined on RS", metaState.getState(), State.OPEN); - // Update meta state as OPENING, then kill master + // Update meta state as PENDING_OPEN, then kill master // that simulates, that RS successfully deployed, but // RPC was lost right before failure. // region server should expire (how it can be verified?) MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(), - rs.getServerName(), State.OPENING); + rs.getServerName(), State.PENDING_OPEN); Region meta = rs.getFromOnlineRegions(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); rs.removeFromOnlineRegions(meta, null); ((HRegion)meta).close(); @@ -413,12 +410,12 @@ public class TestMasterFailover { assertEquals("hbase:meta should be onlined on RS", metaState.getState(), State.OPEN); - // Update meta state as CLOSING, then kill master + // Update meta state as PENDING_CLOSE, then kill master // that simulates, that RS successfully deployed, but // RPC was lost right before failure. // region server should expire (how it can be verified?) MetaTableLocator.setMetaLocation(activeMaster.getZooKeeper(), - rs.getServerName(), State.CLOSING); + rs.getServerName(), State.PENDING_CLOSE); log("Aborting master"); activeMaster.abort("test-kill"); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index 8a216c5..0084d44 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -56,13 +55,12 @@ public class TestMasterMetrics { KeeperException, InterruptedException { super(conf, cp); } -/* + @Override protected void tryRegionServerReport( long reportStartTime, long reportEndTime) { // do nothing } -*/ } @BeforeClass @@ -83,7 +81,7 @@ public class TestMasterMetrics { } } - @Ignore @Test(timeout = 300000) + @Test(timeout = 300000) public void testClusterRequests() throws Exception { // sending fake request to master to see how metric value has changed @@ -116,7 +114,7 @@ public class TestMasterMetrics { master.stopMaster(); } - @Ignore @Test + @Test public void testDefaultMasterMetrics() throws Exception { MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource(); metricsHelper.assertGauge( "numRegionServers", 2, masterSource); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java index 8b0874a..6c737e9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.master; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Collection; @@ -173,7 +172,7 @@ public class TestMasterOperationsForRegionReplicas { } validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection()); - /* DISABLED!!!!! FOR NOW!!!! + // Now shut the whole cluster down, and verify the assignments are kept so that the // availability constraints are met. TEST_UTIL.getConfiguration().setBoolean("hbase.master.startup.retainassign", true); @@ -193,19 +192,17 @@ public class TestMasterOperationsForRegionReplicas { TEST_UTIL.getMiniHBaseCluster().startRegionServer(); } - // Check on alter table + //check on alter table ADMIN.disableTable(tableName); assert(ADMIN.isTableDisabled(tableName)); //increase the replica desc.setRegionReplication(numReplica + 1); ADMIN.modifyTable(tableName, desc); ADMIN.enableTable(tableName); - LOG.info(ADMIN.getTableDescriptor(tableName).toString()); assert(ADMIN.isTableEnabled(tableName)); List<HRegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster() .getAssignmentManager().getRegionStates().getRegionsOfTable(tableName); - assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica=" + numReplica, - regions.size() == numRegions * (numReplica + 1)); + assert(regions.size() == numRegions * (numReplica + 1)); //decrease the replica(earlier, table was modified to have a replica count of numReplica + 1) ADMIN.disableTable(tableName); @@ -232,7 +229,6 @@ public class TestMasterOperationsForRegionReplicas { assert(defaultReplicas.size() == numRegions); Collection<Integer> counts = new HashSet<>(defaultReplicas.values()); assert(counts.size() == 1 && counts.contains(new Integer(numReplica))); - */ } finally { ADMIN.disableTable(tableName); ADMIN.deleteTable(tableName); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java index 23efdb2..b59e6ff 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java @@ -18,12 +18,15 @@ */ package org.apache.hadoop.hbase.master; +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; + import java.io.IOException; import java.io.StringWriter; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -32,23 +35,23 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.assignment.AssignmentManager; -import org.apache.hadoop.hbase.master.assignment.RegionStates; -import org.apache.hadoop.hbase.regionserver.MetricsRegionServer; -import org.apache.hadoop.hbase.regionserver.MetricsRegionServerWrapperStub; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServer; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerWrapperStub; +import org.apache.hadoop.hbase.tmpl.master.AssignmentManagerStatusTmpl; +import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; /** * Tests for the master status page and its template. @@ -87,7 +90,7 @@ public class TestMasterStatusServlet { // Fake AssignmentManager and RIT AssignmentManager am = Mockito.mock(AssignmentManager.class); RegionStates rs = Mockito.mock(RegionStates.class); - List<RegionState> regionsInTransition = new ArrayList<>(); + Set<RegionState> regionsInTransition = new HashSet<>(); regionsInTransition.add(new RegionState(FAKE_HRI, RegionState.State.CLOSING, 12345L, FAKE_HOST)); Mockito.doReturn(rs).when(am).getRegionStates(); Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransition(); @@ -154,4 +157,45 @@ public class TestMasterStatusServlet { .setDeadServers(deadServers) .render(new StringWriter(), master); } + + @Test + public void testAssignmentManagerTruncatedList() throws IOException { + AssignmentManager am = Mockito.mock(AssignmentManager.class); + RegionStates rs = Mockito.mock(RegionStates.class); + + // Add 100 regions as in-transition + TreeSet<RegionState> regionsInTransition = new TreeSet<>(RegionStates.REGION_STATE_COMPARATOR); + for (byte i = 0; i < 100; i++) { + HRegionInfo hri = new HRegionInfo(FAKE_TABLE.getTableName(), + new byte[]{i}, new byte[]{(byte) (i+1)}); + regionsInTransition.add( + new RegionState(hri, RegionState.State.CLOSING, 12345L, FAKE_HOST)); + } + // Add hbase:meta in transition as well + regionsInTransition.add( + new RegionState(HRegionInfo.FIRST_META_REGIONINFO, + RegionState.State.CLOSING, 123L, FAKE_HOST)); + Mockito.doReturn(rs).when(am).getRegionStates(); + Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransition(); + Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransitionOrderedByTimestamp(); + + // Render to a string + StringWriter sw = new StringWriter(); + new AssignmentManagerStatusTmpl() + // NOT IMPLEMENTED!!!! .setLimit(50) + .render(sw, am); + String result = sw.toString(); + // Should always include META + assertTrue(result.contains(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())); + + /* BROKEN BY HBASE-13839 Fix AssgnmentManagerTmpl.jamon issues (coloring, content etc.) FIX!! + // Make sure we only see 50 of them + Matcher matcher = Pattern.compile("CLOSING").matcher(result); + int count = 0; + while (matcher.find()) { + count++; + } + assertEquals(50, count); + */ + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java index 8641b20..782c400 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java @@ -81,7 +81,7 @@ public class TestMasterWalManager { // Create a ZKW to use in the test ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, walPath), - new SplitLogTask.Owned(inRecoveryServerName).toByteArray(), + new SplitLogTask.Owned(inRecoveryServerName, mwm.getLogRecoveryMode()).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); String staleRegionPath = ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, staleRegion); ZKUtil.createWithParents(zkw, staleRegionPath); http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java index 68160df..a845a73 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMetaShutdownHandler.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZKUtil; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java index fe5883b..daf6d43 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionState.java @@ -35,19 +35,14 @@ public class TestRegionState { public TestName name = new TestName(); @Test - public void testSerializeDeserialize() { - final TableName tableName = TableName.valueOf("testtb"); - for (RegionState.State state: RegionState.State.values()) { - testSerializeDeserialize(tableName, state); - } - } - - private void testSerializeDeserialize(final TableName tableName, final RegionState.State state) { - RegionState state1 = new RegionState(new HRegionInfo(tableName), state); + public void test() { + RegionState state1 = new RegionState( + new HRegionInfo(TableName.valueOf(name.getMethodName())), RegionState.State.OPENING); ClusterStatusProtos.RegionState protobuf1 = state1.convert(); RegionState state2 = RegionState.convert(protobuf1); ClusterStatusProtos.RegionState protobuf2 = state1.convert(); - assertEquals("RegionState does not match " + state, state1, state2); - assertEquals("Protobuf does not match " + state, protobuf1, protobuf2); + + assertEquals(state1, state2); + assertEquals(protobuf1, protobuf2); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java new file mode 100644 index 0000000..17004ec --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionStates.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; + +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.IOException; +import static org.junit.Assert.assertTrue; +import static junit.framework.Assert.assertFalse; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category({MasterTests.class, SmallTests.class}) +public class TestRegionStates { + @Test (timeout=10000) + public void testCanMakeProgressThoughMetaIsDown() + throws IOException, InterruptedException, BrokenBarrierException { + MasterServices server = mock(MasterServices.class); + when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1")); + Connection connection = mock(ClusterConnection.class); + // Set up a table that gets 'stuck' when we try to fetch a row from the meta table. + // It is stuck on a CyclicBarrier latch. We use CyclicBarrier because it will tell us when + // thread is waiting on latch. + Table metaTable = Mockito.mock(Table.class); + final CyclicBarrier latch = new CyclicBarrier(2); + when(metaTable.get((Get)Mockito.any())).thenAnswer(new Answer<Result>() { + @Override + public Result answer(InvocationOnMock invocation) throws Throwable { + latch.await(); + throw new java.net.ConnectException("Connection refused"); + } + }); + when(connection.getTable(TableName.META_TABLE_NAME)).thenReturn(metaTable); + when(server.getConnection()).thenReturn((ClusterConnection)connection); + Configuration configuration = mock(Configuration.class); + when(server.getConfiguration()).thenReturn(configuration); + TableStateManager tsm = mock(TableStateManager.class); + ServerManager sm = mock(ServerManager.class); + when(sm.isServerOnline(isA(ServerName.class))).thenReturn(true); + + RegionStateStore rss = mock(RegionStateStore.class); + final RegionStates regionStates = new RegionStates(server, tsm, sm, rss); + final ServerName sn = mockServer("one", 1); + regionStates.updateRegionState(HRegionInfo.FIRST_META_REGIONINFO, State.SPLITTING_NEW, sn); + Thread backgroundThread = new Thread("Get stuck setting server offline") { + @Override + public void run() { + regionStates.serverOffline(sn); + } + }; + assertTrue(latch.getNumberWaiting() == 0); + backgroundThread.start(); + while (latch.getNumberWaiting() == 0); + // Verify I can do stuff with synchronized RegionStates methods, that I am not locked out. + // Below is a call that is synchronized. Can I do it and not block? + regionStates.getRegionServerOfRegion(HRegionInfo.FIRST_META_REGIONINFO); + // Done. Trip the barrier on the background thread. + latch.await(); + } + + @Test + public void testWeDontReturnDrainingServersForOurBalancePlans() throws Exception { + MasterServices server = mock(MasterServices.class); + when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1")); + Configuration configuration = mock(Configuration.class); + when(server.getConfiguration()).thenReturn(configuration); + TableStateManager tsm = mock(TableStateManager.class); + ServerManager sm = mock(ServerManager.class); + when(sm.isServerOnline(isA(ServerName.class))).thenReturn(true); + + RegionStateStore rss = mock(RegionStateStore.class); + RegionStates regionStates = new RegionStates(server, tsm, sm, rss); + + ServerName one = mockServer("one", 1); + ServerName two = mockServer("two", 1); + ServerName three = mockServer("three", 1); + + when(sm.getDrainingServersList()).thenReturn(Arrays.asList(three)); + + regionStates.regionOnline(createFakeRegion(), one); + regionStates.regionOnline(createFakeRegion(), two); + regionStates.regionOnline(createFakeRegion(), three); + + + Map<TableName, Map<ServerName, List<HRegionInfo>>> result = + regionStates.getAssignmentsByTable(); + for (Map<ServerName, List<HRegionInfo>> map : result.values()) { + assertFalse(map.keySet().contains(three)); + } + } + + private HRegionInfo createFakeRegion() { + HRegionInfo info = mock(HRegionInfo.class); + when(info.getEncodedName()).thenReturn(UUID.randomUUID().toString()); + return info; + } + + private ServerName mockServer(String fakeHost, int fakePort) { + ServerName serverName = mock(ServerName.class); + when(serverName.getHostname()).thenReturn(fakeHost); + when(serverName.getPort()).thenReturn(fakePort); + return serverName; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java index 351fca4..7c41c0f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java @@ -35,14 +35,12 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; import org.junit.After; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -109,7 +107,6 @@ public class TestRestartCluster { * This tests retaining assignments on a cluster restart */ @Test (timeout=300000) - @Ignore // Does not work in new AMv2 currently. public void testRetainAssignmentOnRestart() throws Exception { UTIL.startMiniCluster(2); while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { @@ -198,7 +195,7 @@ public class TestRestartCluster { Threads.sleep(100); } - snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getConnection()); + snapshot =new SnapshotOfRegionAssignmentFromMeta(master.getConnection()); snapshot.initialize(); Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap(); @@ -207,8 +204,7 @@ public class TestRestartCluster { if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue; ServerName oldServer = regionToRegionServerMap.get(entry.getKey()); ServerName currentServer = entry.getValue(); - LOG.info("Key=" + entry.getKey() + " oldServer=" + oldServer + ", currentServer=" + currentServer); - assertEquals(entry.getKey().toString(), oldServer.getAddress(), currentServer.getAddress()); + assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort()); assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode()); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java index 58be83b..ec7ffe6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestWarmupRegion.java @@ -19,10 +19,7 @@ package org.apache.hadoop.hbase.master; import static org.apache.hadoop.hbase.regionserver.HRegion.warmupHRegion; -import static org.junit.Assert.assertTrue; - import java.io.IOException; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -32,20 +29,21 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.CompactionState; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.CompactionState; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.After; +import org.junit.experimental.categories.Category; +import org.junit.BeforeClass; import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.After; import org.junit.Test; -import org.junit.experimental.categories.Category; /** * Run tests that use the HBase clients; {@link org.apache.hadoop.hbase.client.HTable}. @@ -160,8 +158,6 @@ public class TestWarmupRegion { for (int i = 0; i < 10; i++) { HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(serverid); byte [] destName = Bytes.toBytes(rs.getServerName().toString()); - assertTrue(destName != null); - LOG.info("i=" + i ); TEST_UTIL.getMiniHBaseCluster().getMaster().move(info.getEncodedNameAsBytes(), destName); serverid = (serverid + 1) % 2; } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c5a744/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java deleted file mode 100644 index 07b989b..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/AssignmentTestingUtil.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.assignment; - -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; -import org.apache.hadoop.hbase.util.Threads; - -import static org.junit.Assert.assertEquals; - [email protected] [email protected] -public abstract class AssignmentTestingUtil { - private static final Log LOG = LogFactory.getLog(AssignmentTestingUtil.class); - - private AssignmentTestingUtil() {} - - public static void waitForRegionToBeInTransition(final HBaseTestingUtility util, - final HRegionInfo hri) throws Exception { - while (!getMaster(util).getAssignmentManager().getRegionStates().isRegionInTransition(hri)) { - Threads.sleep(10); - } - } - - public static void waitForRsToBeDead(final HBaseTestingUtility util, - final ServerName serverName) throws Exception { - util.waitFor(60000, new ExplainingPredicate<Exception>() { - @Override - public boolean evaluate() { - return getMaster(util).getServerManager().isServerDead(serverName); - } - - @Override - public String explainFailure() { - return "Server " + serverName + " is not dead"; - } - }); - } - - public static void stopRs(final HBaseTestingUtility util, final ServerName serverName) - throws Exception { - LOG.info("STOP REGION SERVER " + serverName); - util.getMiniHBaseCluster().stopRegionServer(serverName); - waitForRsToBeDead(util, serverName); - } - - public static void killRs(final HBaseTestingUtility util, final ServerName serverName) - throws Exception { - LOG.info("KILL REGION SERVER " + serverName); - util.getMiniHBaseCluster().killRegionServer(serverName); - waitForRsToBeDead(util, serverName); - } - - public static void crashRs(final HBaseTestingUtility util, final ServerName serverName, - final boolean kill) throws Exception { - if (kill) { - killRs(util, serverName); - } else { - stopRs(util, serverName); - } - } - - public static ServerName crashRsWithRegion(final HBaseTestingUtility util, - final HRegionInfo hri, final boolean kill) throws Exception { - ServerName serverName = getServerHoldingRegion(util, hri); - crashRs(util, serverName, kill); - return serverName; - } - - public static ServerName getServerHoldingRegion(final HBaseTestingUtility util, - final HRegionInfo hri) throws Exception { - ServerName serverName = util.getMiniHBaseCluster().getServerHoldingRegion( - hri.getTable(), hri.getRegionName()); - ServerName amServerName = getMaster(util).getAssignmentManager().getRegionStates() - .getRegionServerOfRegion(hri); - - // Make sure AM and MiniCluster agrees on the Server holding the region - // and that the server is online. - assertEquals(amServerName, serverName); - assertEquals(true, getMaster(util).getServerManager().isServerOnline(serverName)); - return serverName; - } - - public static boolean isServerHoldingMeta(final HBaseTestingUtility util, - final ServerName serverName) throws Exception { - for (HRegionInfo hri: getMetaRegions(util)) { - if (serverName.equals(getServerHoldingRegion(util, hri))) { - return true; - } - } - return false; - } - - public static Set<HRegionInfo> getMetaRegions(final HBaseTestingUtility util) { - return getMaster(util).getAssignmentManager().getMetaRegionSet(); - } - - private static HMaster getMaster(final HBaseTestingUtility util) { - return util.getMiniHBaseCluster().getMaster(); - } -} \ No newline at end of file
