Author: stack
Date: Thu Jun 16 18:28:59 2011
New Revision: 1136587
URL: http://svn.apache.org/viewvc?rev=1136587&view=rev
Log:
HBASE-3970 Address HMaster crash/failure half way through meta migration
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1136587&r1=1136586&r2=1136587&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Jun 16 18:28:59 2011
@@ -271,6 +271,8 @@ Release 0.91.0 - Unreleased
HBASE-3573 Move shutdown messaging OFF hearbeat; prereq for fix of
hbase-1502
HBASE-3071 Graceful decommissioning of a regionserver
+ HBASE-3970 Address HMaster crash/failure half way through meta migration
+ (Subbu M Iyer)
NEW FEATURES
HBASE-2001 Coprocessors: Colocate user code with regions (Mingjie Lai via
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1136587&r1=1136586&r2=1136587&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
Thu Jun 16 18:28:59 2011
@@ -246,6 +246,7 @@ public class MetaEditor {
public boolean visit(Result r) throws IOException {
if (r == null || r.isEmpty()) return true;
HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
+ if (hrfm == null) return true;
htds.add(hrfm.getTableDesc());
masterServices.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc());
HRegionInfo regionInfo = new HRegionInfo(hrfm);
@@ -267,10 +268,19 @@ public class MetaEditor {
public static HRegionInfo090x getHRegionInfoForMigration(
Result data) throws IOException {
+ HRegionInfo090x info = null;
byte [] bytes =
data.getValue(HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER);
if (bytes == null) return null;
- HRegionInfo090x info = Writables.getHRegionInfoForMigration(bytes);
+ try {
+ info = Writables.getHRegionInfoForMigration(bytes);
+ } catch(IOException ioe) {
+ if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
+ return null;
+ } else {
+ throw ioe;
+ }
+ }
LOG.info("Current INFO from scan results = " + info);
return info;
}
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1136587&r1=1136586&r2=1136587&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Thu Jun 16 18:28:59 2011
@@ -862,6 +862,22 @@ public class HBaseTestingUtility {
public int createMultiRegionsWithLegacyHRI(final Configuration c,
final HTableDescriptor htd,
+ final byte [] family, int numRegions)
+ throws IOException {
+ if (numRegions < 3) throw new IOException("Must create at least 3
regions");
+ byte [] startKey = Bytes.toBytes("aaaaa");
+ byte [] endKey = Bytes.toBytes("zzzzz");
+ byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
+ byte [][] regionStartKeys = new byte[splitKeys.length+1][];
+ for (int i=0;i<splitKeys.length;i++) {
+ regionStartKeys[i+1] = splitKeys[i];
+ }
+ regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
+ return createMultiRegionsWithLegacyHRI(c, htd, family, regionStartKeys);
+ }
+
+ public int createMultiRegionsWithLegacyHRI(final Configuration c,
+ final HTableDescriptor htd,
final byte[] columnFamily, byte [][] startKeys)
throws IOException {
Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
@@ -890,6 +906,22 @@ public class HBaseTestingUtility {
}
+ public int createMultiRegionsWithNewHRI(final Configuration c,
+ final HTableDescriptor htd,
+ final byte [] family, int numRegions)
+ throws IOException {
+ if (numRegions < 3) throw new IOException("Must create at least 3
regions");
+ byte [] startKey = Bytes.toBytes("aaaaa");
+ byte [] endKey = Bytes.toBytes("zzzzz");
+ byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
+ byte [][] regionStartKeys = new byte[splitKeys.length+1][];
+ for (int i=0;i<splitKeys.length;i++) {
+ regionStartKeys[i+1] = splitKeys[i];
+ }
+ regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
+ return createMultiRegionsWithNewHRI(c, htd, family, regionStartKeys);
+ }
+
public int createMultiRegionsWithNewHRI(final Configuration c, final
HTableDescriptor htd,
final byte[] columnFamily, byte [][] startKeys)
throws IOException {
Modified:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java?rev=1136587&r1=1136586&r2=1136587&view=diff
==============================================================================
---
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
(original)
+++
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
Thu Jun 16 18:28:59 2011
@@ -83,7 +83,7 @@ public class TestMetaMigration {
}
@Test
- public void testUpdatesOnMetaWithLegacyHRI() throws Exception {
+ public void testMetaMigration() throws Exception {
LOG.info("Starting testMetaWithLegacyHRI");
final byte[] FAMILY = Bytes.toBytes("family");
HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
@@ -103,6 +103,7 @@ public class TestMetaMigration {
List<HTableDescriptor> htds = MetaEditor.updateMetaWithNewRegionInfo(
TEST_UTIL.getHBaseCluster().getMaster());
+ MetaReader.fullScanMetaAndPrint(ct);
assertEquals(3, htds.size());
// Assert that the flag in ROOT is updated to reflect the correct status
boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated();
@@ -111,26 +112,45 @@ public class TestMetaMigration {
}
- //@Test
- public void dtestUpdatesOnMetaWithNewHRI() throws Exception {
- LOG.info("Starting testMetaWithLegacyHRI");
+ /**
+ * This test assumes a master crash/failure during the meta migration process
+ * and attempts to continue the meta migration process when a new master
takes over.
+ * When a master dies during the meta migration we will have some rows of
+ * META.CatalogFamily updated with new HRI, (i.e HRI with out HTD) and some
+ * still hanging with legacy HRI. (i.e HRI with HTD). When the backup
master/ or
+ * fresh start of master attempts the migration it will encouter some rows
of META
+ * already updated with new HRI and some still legacy. This test will
simulate this
+ * scenario and validates that the migration process can safely skip the
updated
+ * rows and migrate any pending rows at startup.
+ * @throws Exception
+ */
+ @Test
+ public void testMasterCrashDuringMetaMigration() throws Exception {
+ LOG.info("Starting testMasterCrashDuringMetaMigration");
final byte[] FAMILY = Bytes.toBytes("family");
- HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
+ HTableDescriptor htd = new
HTableDescriptor("testMasterCrashDuringMetaMigration");
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
htd.addFamily(hcd);
Configuration conf = TEST_UTIL.getConfiguration();
- TEST_UTIL.createMultiRegionsWithNewHRI(conf, htd, FAMILY,
- new byte[][]{
- HConstants.EMPTY_START_ROW,
- Bytes.toBytes("region_a"),
- Bytes.toBytes("region_b")});
+ // Create 10 New regions.
+ TEST_UTIL.createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10);
+ // Create 10 Legacy regions.
+ TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10);
+ CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker();
+ // just for this test set it to false.
+ MetaEditor.updateRootWithMetaMigrationStatus(ct, false);
+ //MetaReader.fullScanMetaAndPrint(ct);
+ LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI");
+
List<HTableDescriptor> htds = MetaEditor.updateMetaWithNewRegionInfo(
TEST_UTIL.getHBaseCluster().getMaster());
- assertEquals(3, htds.size());
- }
-
-
+ assertEquals(10, htds.size());
+ // Assert that the flag in ROOT is updated to reflect the correct status
+ boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated();
+ assertEquals(true, metaUpdated);
+ LOG.info("END testMetaWithLegacyHRI");
+ }
public static void assertEquals(int expected,
int actual) {
