Author: stack
Date: Fri Sep 9 04:51:47 2011
New Revision: 1167002
URL: http://svn.apache.org/viewvc?rev=1167002&view=rev
Log:
HBASE-4313 Refactor TestHBaseFsck to make adding individual hbck tests easier
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
Modified: hbase/branches/0.90/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1167002&r1=1167001&r2=1167002&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Fri Sep 9 04:51:47 2011
@@ -42,6 +42,8 @@ Release 0.90.5 - Unreleased
META/ROOT regions (todd)
HBASE-4323 Add debug logging when AssignmentManager can't make a plan
for a region (todd)
+ HBASE-4313 Refactor TestHBaseFsck to make adding individual hbck tests
+ easier (Jonathan Hsieh)
Release 0.90.4 - August 10, 2011
BUG FIXES
Modified:
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java?rev=1167002&r1=1167001&r2=1167002&view=diff
==============================================================================
---
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
(original)
+++
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
Fri Sep 9 04:51:47 2011
@@ -19,110 +19,70 @@
*/
package org.apache.hadoop.hbase.util;
-import static org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
-import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import org.apache.zookeeper.KeeperException;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Random;
-
+/**
+ * This tests HBaseFsck's ability to detect reasons for inconsistent tables.
+ */
public class TestHBaseFsck {
-
final Log LOG = LogFactory.getLog(getClass());
- private final static HBaseTestingUtility TEST_UTIL =
- new HBaseTestingUtility();
+ private final static HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private final static Configuration conf = TEST_UTIL.getConfiguration();
- private final static byte[] TABLE = Bytes.toBytes("table");
private final static byte[] FAM = Bytes.toBytes("fam");
+ // for the instance, reset every test run
+ private HTable tbl;
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
}
- private List doFsck(boolean fix) throws Exception {
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ private List<ERROR_CODE> doFsck(boolean fix) throws Exception {
HBaseFsck fsck = new HBaseFsck(conf);
- fsck.displayFullReport(); // i.e. -details
+ fsck.displayFullReport(); // i.e. -details
fsck.setTimeLag(0);
fsck.setFixErrors(fix);
fsck.doWork();
return fsck.getErrors().getErrorList();
}
- private void assertNoErrors(List errs) throws Exception {
+ private void assertNoErrors(List<ERROR_CODE> errs) throws Exception {
assertEquals(0, errs.size());
}
- private void assertErrors(List errs, ERROR_CODE[] expectedErrors) {
+ private void assertErrors(List<ERROR_CODE> errs, ERROR_CODE[]
expectedErrors) {
assertEquals(Arrays.asList(expectedErrors), errs);
}
- @Test
- public void testHBaseFsck() throws Exception {
- assertNoErrors(doFsck(false));
-
- TEST_UTIL.createTable(TABLE, FAM);
-
- // We created 1 table, should be fine
- assertNoErrors(doFsck(false));
-
- // Now let's mess it up and change the assignment in .META. to
- // point to a different region server
- HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
- ResultScanner scanner = meta.getScanner(new Scan());
-
- resforloop:
- for (Result res : scanner) {
- long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
- HConstants.STARTCODE_QUALIFIER));
-
- for (JVMClusterUtil.RegionServerThread rs :
- TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
-
- HServerInfo hsi = rs.getRegionServer().getServerInfo();
-
- // When we find a diff RS, change the assignment and break
- if (startCode != hsi.getStartCode()) {
- Put put = new Put(res.getRow());
- put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
- Bytes.toBytes(hsi.getHostnamePort()));
- put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
- Bytes.toBytes(hsi.getStartCode()));
- meta.put(put);
- break resforloop;
- }
- }
- }
-
- // Try to fix the data
- assertErrors(doFsck(true), new ERROR_CODE[]{
- ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
- Thread.sleep(15000);
-
- // Should be fixed now
- assertNoErrors(doFsck(false));
-
- // comment needed - what is the purpose of this line
- new HTable(conf, TABLE).getScanner(new Scan());
- }
-
- private HRegionInfo createRegion(Configuration conf, final HTableDescriptor
- htd, byte[] startKey, byte[] endKey)
+ private HRegionInfo createRegion(Configuration conf,
+ final HTableDescriptor htd, byte[] startKey, byte[] endKey)
throws IOException {
HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
HRegionInfo hri = new HRegionInfo(htd, startKey, endKey);
@@ -133,67 +93,123 @@ public class TestHBaseFsck {
return hri;
}
- @Test
/**
- * Tests for inconsistencies in the META data (duplicate start keys, or
holes)
+ * Setup a clean table before we start mucking with it.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws KeeperException
*/
- public void testHBaseFsckMeta() throws Exception {
- assertNoErrors(doFsck(false));
+ void setupTable(String tablename) throws Exception {
+ byte[][] startKeys = new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"),
+ Bytes.toBytes("C") };
+ HTableDescriptor desc = new HTableDescriptor(tablename);
+ HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
+ desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
+ TEST_UTIL.getHBaseAdmin().createTable(desc, startKeys);
+ tbl = new HTable(TEST_UTIL.getConfiguration(), tablename);
+ }
- HTable tbl = TEST_UTIL.createTable(Bytes.toBytes("table2"), FAM);
+ /**
+ * delete table in preparation for next test
+ *
+ * @param tablename
+ * @throws IOException
+ */
+ void deleteTable(String tablename) throws IOException {
+ HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
+ byte[] tbytes = Bytes.toBytes(tablename);
+ admin.disableTable(tbytes);
+ admin.deleteTable(tbytes);
+ }
- HRegionInfo hriOrig = tbl.getRegionsInfo().keySet().iterator().next();
- HServerAddress rsAddressOrig = tbl.getRegionsInfo().get(hriOrig);
- byte[][] startKeys = new byte[][]{
- HConstants.EMPTY_BYTE_ARRAY,
- Bytes.toBytes("A"),
- Bytes.toBytes("B"),
- Bytes.toBytes("C")
- };
- TEST_UTIL.createMultiRegions(conf, tbl, FAM, startKeys);
- Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
- FileSystem fs = rootDir.getFileSystem(conf);
- Path p = new Path(rootDir + "/table2", hriOrig.getEncodedName());
- fs.delete(p, true);
-
- Thread.sleep(1 * 1000);
- ArrayList servers = new ArrayList();
- servers.add(rsAddressOrig);
- HBaseFsckRepair.fixDupeAssignment(conf, hriOrig, servers);
+
+ /**
+ * This creates a clean table and confirms that the table is clean.
+ */
+ @Test
+ public void testHBaseFsckClean() throws Exception {
+ assertNoErrors(doFsck(false));
+ String table = "tableClean";
+ try {
+ setupTable(table);
+
+ // We created 1 table, should be fine
+ assertNoErrors(doFsck(false));
+ } finally {
+ deleteTable(table);
+ }
+ }
- // We created 1 table, should be fine
+ /**
+ * This creates a bad table with regions that have a duplicate start key
+ */
+ @Test
+ public void testDupeStartKey() throws Exception {
assertNoErrors(doFsck(false));
+ String table = "tableDupeStartKey";
+ try {
+ setupTable(table);
+ assertNoErrors(doFsck(false));
+
+ // Now let's mess it up, by adding a region with a duplicate startkey
+ HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
+ Bytes.toBytes("A"), Bytes.toBytes("A2"));
+ TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
+ TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+ .waitForAssignment(hriDupe);
+
+ assertErrors(doFsck(false),
+ new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS });
+ } finally {
+ deleteTable(table);
+ }
+ }
- // Now let's mess it up, by adding a region with a duplicate startkey
- HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
- Bytes.toBytes("A"), Bytes.toBytes("A2"));
- TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
- TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
- .waitForAssignment(hriDupe);
- assertErrors(doFsck(false), new ERROR_CODE[]{ERROR_CODE.DUPE_STARTKEYS});
-
- // Mess it up by creating an overlap in the metadata
- HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
- Bytes.toBytes("A2"), Bytes.toBytes("B2"));
- TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
- TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
- .waitForAssignment(hriOverlap);
- assertErrors(doFsck(false), new ERROR_CODE[]{
- ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
- ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
-
- // Mess it up by leaving a hole in the meta data
- HRegionInfo hriHole = createRegion(conf, tbl.getTableDescriptor(),
- Bytes.toBytes("D"), Bytes.toBytes("E"));
- TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriHole);
- TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
- .waitForAssignment(hriHole);
-// assertError(doFsck(false), ERROR_CODE.OVERLAP_IN_REGION_CHAIN);
- assertErrors(doFsck(false), new ERROR_CODE[]{ ERROR_CODE.DUPE_STARTKEYS,
- ERROR_CODE.OVERLAP_IN_REGION_CHAIN, ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
- ERROR_CODE.HOLE_IN_REGION_CHAIN });
+ /**
+ * This creates a bad table where a start key contained in another region.
+ */
+ @Test
+ public void testCoveredStartKey() throws Exception {
+ String table = "tableCoveredStartKey";
+ try {
+ setupTable(table);
+
+ // Mess it up by creating an overlap in the metadata
+ HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
+ Bytes.toBytes("A2"), Bytes.toBytes("B2"));
+ TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
+ TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+ .waitForAssignment(hriOverlap);
+ assertErrors(doFsck(false), new ERROR_CODE[] {
+ ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
+ ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
+ } finally {
+ deleteTable(table);
+ }
+ }
+ /**
+ * This creates a bad table with a hole in meta.
+ */
+ @Test
+ public void testMetaHole() throws Exception {
+ String table = "tableMetaHole";
+ try {
+ setupTable(table);
+
+ // Mess it up by leaving a hole in the meta data
+ HRegionInfo hriHole = createRegion(conf, tbl.getTableDescriptor(),
+ Bytes.toBytes("D"), Bytes.toBytes("E"));
+ TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriHole);
+ TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
+ .waitForAssignment(hriHole);
+ assertErrors(doFsck(false),
+ new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN });
+ } finally {
+ deleteTable(table);
+ }
}
}
