Author: tedyu
Date: Fri Sep 9 05:25:24 2011
New Revision: 1167006
URL: http://svn.apache.org/viewvc?rev=1167006&view=rev
Log:
HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer)
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
Modified: hbase/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep 9 05:25:24 2011
@@ -255,7 +255,7 @@ Release 0.91.0 - Unreleased
TestMultiColumnScanner that caught it (Mikhail Bautin)
HBASE-4007 distributed log splitting can get indefinitely stuck
(Prakash Khemani)
-
+ HBASE-4301 META migration from 0.90 to trunk fails (Subbu Iyer)
IMPROVEMENTS
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
Fri Sep 9 05:25:24 2011
@@ -231,27 +231,23 @@ public class MetaEditor {
}
/**
- * Updates the region information for the specified region in META.
+ * Update the metamigrated flag in -ROOT-.
* @param catalogTracker
- * @param regionInfo region to be updated in META
* @throws IOException
*/
- public static void updateRegionInfo(CatalogTracker catalogTracker,
- HRegionInfo regionInfo)
- throws IOException {
- Put put = new Put(regionInfo.getRegionName());
- addRegionInfo(put, regionInfo);
- catalogTracker.waitForMetaServerConnectionDefault().put(
- CatalogTracker.META_REGION, put);
- LOG.info("Updated region " + regionInfo.getRegionNameAsString() + " in
META");
- }
-
- public static void updateRootWithMetaMigrationStatus(CatalogTracker
catalogTracker) throws IOException {
+ public static void updateRootWithMetaMigrationStatus(
+ CatalogTracker catalogTracker) throws IOException {
updateRootWithMetaMigrationStatus(catalogTracker, true);
}
- public static void updateRootWithMetaMigrationStatus(CatalogTracker
catalogTracker,
- boolean metaUpdated)
+ /**
+ * Update the metamigrated flag in -ROOT-.
+ * @param catalogTracker
+ * @param metaUpdated
+ * @throws IOException
+ */
+ public static void updateRootWithMetaMigrationStatus(
+ CatalogTracker catalogTracker, boolean metaUpdated)
throws IOException {
Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
addMetaUpdateStatus(put, metaUpdated);
@@ -260,6 +256,12 @@ public class MetaEditor {
LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
}
+ /**
+ * Update legacy META rows, removing HTD from HRI.
+ * @param masterServices
+ * @return
+ * @throws IOException
+ */
public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
final MasterServices masterServices)
throws IOException {
@@ -271,16 +273,11 @@ public class MetaEditor {
HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
if (hrfm == null) return true;
htds.add(hrfm.getTableDesc());
-
masterServices.getMasterFileSystem().createTableDescriptor(hrfm.getTableDesc());
- HRegionInfo regionInfo = new HRegionInfo(hrfm);
- LOG.debug(" MetaEditor.updatemeta RegionInfo = " +
regionInfo.toString()
- + " old HRI = " + hrfm.toString());
- Put put = new Put(regionInfo.getRegionName());
- put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
- Writables.getBytes(regionInfo));
-
masterServices.getCatalogTracker().waitForMetaServerConnectionDefault().put(
- CatalogTracker.META_REGION, put);
- LOG.info("Updated region " + regionInfo + " to META");
+ masterServices.getMasterFileSystem()
+ .createTableDescriptor(hrfm.getTableDesc());
+ updateHRI(masterServices.getCatalogTracker()
+ .waitForMetaServerConnectionDefault(),
+ hrfm, CatalogTracker.META_REGION);
return true;
}
};
@@ -289,6 +286,60 @@ public class MetaEditor {
return htds;
}
+ /**
+ * Migrate root and meta to newer version. This updates the META and ROOT
+ * and removes the HTD from HRI.
+ * @param masterServices
+ * @throws IOException
+ */
+ public static void migrateRootAndMeta(final MasterServices masterServices)
+ throws IOException {
+ updateRootWithNewRegionInfo(masterServices);
+ updateMetaWithNewRegionInfo(masterServices);
+ }
+
+ /**
+ * Update the ROOT with new HRI. (HRI with no HTD)
+ * @param masterServices
+ * @return
+ * @throws IOException
+ */
+ public static List<HTableDescriptor> updateRootWithNewRegionInfo(
+ final MasterServices masterServices)
+ throws IOException {
+ final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+ Visitor v = new Visitor() {
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
+ if (hrfm == null) return true;
+ htds.add(hrfm.getTableDesc());
+ masterServices.getMasterFileSystem().createTableDescriptor(
+ hrfm.getTableDesc());
+ updateHRI(masterServices.getCatalogTracker()
+ .waitForRootServerConnectionDefault(),
+ hrfm, CatalogTracker.ROOT_REGION);
+ return true;
+ }
+ };
+ MetaReader.fullScan(
+
masterServices.getCatalogTracker().waitForRootServerConnectionDefault(),
+ v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null);
+ return htds;
+ }
+
+ private static void updateHRI(HRegionInterface hRegionInterface,
+ HRegionInfo090x hRegionInfo090x, byte[]
regionName)
+ throws IOException {
+ HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
+ Put put = new Put(regionInfo.getRegionName());
+ put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(regionInfo));
+ hRegionInterface.put(regionName, put);
+ LOG.info("Updated region " + regionInfo + " to " +
Bytes.toString(regionName));
+ }
+
public static HRegionInfo090x getHRegionInfoForMigration(
Result data) throws IOException {
HRegionInfo090x info = null;
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
Fri Sep 9 05:25:24 2011
@@ -252,19 +252,8 @@ public class MetaReader {
throws IOException {
HRegionInterface metaServer =
catalogTracker.waitForMetaServerConnectionDefault();
- Scan scan = new Scan();
- if (startrow != null) scan.setStartRow(startrow);
- scan.addFamily(HConstants.CATALOG_FAMILY);
- long scannerid = metaServer.openScanner(
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
- try {
- Result data;
- while((data = metaServer.next(scannerid)) != null) {
- if (!data.isEmpty()) visitor.visit(data);
- }
- } finally {
- metaServer.close(scannerid);
- }
+ fullScan(metaServer, visitor,
+ HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow);
return;
}
@@ -622,7 +611,32 @@ public class MetaReader {
return regions;
}
-
+ /**
+ * Fully scan a given region, on a given server starting with given row.
+ * @param hRegionInterface region server
+ * @param visitor visitor
+ * @param regionName name of region
+ * @param startrow start row
+ * @throws IOException
+ */
+ public static void fullScan(HRegionInterface hRegionInterface,
+ Visitor visitor, final byte[] regionName,
+ byte[] startrow) throws IOException {
+ if (hRegionInterface == null) return;
+ Scan scan = new Scan();
+ if (startrow != null) scan.setStartRow(startrow);
+ scan.addFamily(HConstants.CATALOG_FAMILY);
+ long scannerid = hRegionInterface.openScanner(regionName, scan);
+ try {
+ Result data;
+ while((data = hRegionInterface.next(scannerid)) != null) {
+ if (!data.isEmpty()) visitor.visit(data);
+ }
+ } finally {
+ hRegionInterface.close(scannerid);
+ }
+ return;
+ }
/**
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Fri Sep 9 05:25:24 2011
@@ -914,7 +914,8 @@ public class HConnectionManager {
// possible we got a region of a different table...
if (!Bytes.equals(regionInfo.getTableName(), tableName)) {
throw new TableNotFoundException(
- "Table '" + Bytes.toString(tableName) + "' was not found.");
+ "Table '" + Bytes.toString(tableName) + "' was not found,
got: " +
+ Bytes.toString(regionInfo.getTableName()) + ".");
}
if (regionInfo.isSplit()) {
throw new RegionOfflineException("the only available region for" +
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Fri
Sep 9 05:25:24 2011
@@ -526,14 +526,15 @@ implements HMasterInterface, HMasterRegi
if (!isMetaHRIUpdated()) {
LOG.info("Meta has HRI with HTDs. Updating meta now.");
try {
- MetaEditor.updateMetaWithNewRegionInfo(this);
- LOG.info("Meta updated with new HRI.");
+ MetaEditor.migrateRootAndMeta(this);
+ LOG.info("ROOT and Meta updated with new HRI.");
return true;
} catch (IOException e) {
- throw new RuntimeException("Update Meta with nw HRI failed. Master
startup aborted.");
+ throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
+ "Master startup aborted.");
}
}
- LOG.info("Meta already up-to date with new HRI.");
+ LOG.info("ROOT/Meta already up-to date with new HRI.");
return true;
}
@@ -977,7 +978,6 @@ implements HMasterInterface, HMasterRegi
return hRegionInfos;
}
-
private static boolean isCatalogTable(final byte [] tableName) {
return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) ||
Bytes.equals(tableName, HConstants.META_TABLE_NAME);
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
Fri Sep 9 05:25:24 2011
@@ -327,9 +327,18 @@ public class MasterFileSystem {
if (!FSUtils.rootRegionExists(fs, rd)) {
bootstrap(rd, c);
}
+ createRootTableInfo(rd);
return rd;
}
+ private void createRootTableInfo(Path rd) throws IOException {
+ // Create ROOT tableInfo if required.
+ if (!FSUtils.tableInfoExists(fs, rd,
+ Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) {
+ FSUtils.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC,
this.conf);
+ }
+ }
+
private static void bootstrap(final Path rd, final Configuration c)
throws IOException {
LOG.info("BOOTSTRAP: creating ROOT and first META regions");
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
---
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
(original)
+++
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
Fri Sep 9 05:25:24 2011
@@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* Implementation of {@link TableDescriptors} that reads descriptors from the
@@ -39,6 +42,8 @@ import org.apache.hadoop.hbase.TableExis
* the filesystem or can be read and write.
*/
public class FSTableDescriptors implements TableDescriptors {
+
+ private static final Log LOG = LogFactory.getLog(FSTableDescriptors.class);
private final FileSystem fs;
private final Path rootdir;
private final boolean fsreadonly;
@@ -151,7 +156,14 @@ public class FSTableDescriptors implemen
Map<String, HTableDescriptor> htds = new TreeMap<String,
HTableDescriptor>();
List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
for (Path d: tableDirs) {
- HTableDescriptor htd = get(d.getName());
+ HTableDescriptor htd = null;
+ try {
+
+ htd = get(d.getName());
+ } catch (FileNotFoundException fnfe) {
+ // inability of retrieving one HTD shouldn't stop getting the remaining
+ LOG.warn("Trouble retrieving htd", fnfe);
+ }
if (htd == null) continue;
htds.put(d.getName(), htd);
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
URL:
http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=1167006&r1=1167005&r2=1167006&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java
(original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java Fri Sep
9 05:25:24 2011
@@ -490,6 +490,20 @@ public abstract class FSUtils {
return fs.exists(rootRegionDir);
}
+ /**
+ * Checks if .tableinfo exists for given table
+ *
+ * @param fs file system
+ * @param rootdir root directory of HBase installation
+ * @param tableName name of table
+ * @return true if exists
+ * @throws IOException
+ */
+ public static boolean tableInfoExists(FileSystem fs, Path rootdir,
+ String tableName) throws IOException {
+ Path tablePath = getTableInfoPath(rootdir, tableName);
+ return fs.exists(tablePath);
+ }
/**
* Compute HDFS blocks distribution of a given file, or a portion of the file
