Author: jimk
Date: Sat Jan 19 12:20:15 2008
New Revision: 613446
URL: http://svn.apache.org/viewvc?rev=613446&view=rev
Log:
HADOOP-2643 Make migration tool smarter.
Modified:
lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Sat Jan 19 12:20:15 2008
@@ -201,7 +201,8 @@
HMaster shutdown
HADOOP-2616 hbase not spliting when the total size of region reaches max
region size * 1.5
-
+ HADOOP-2643 Make migration tool smarter.
+
Release 0.15.1
Branch 0.15
Modified:
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
(original)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConstants.java
Sat Jan 19 12:20:15 2008
@@ -26,6 +26,14 @@
*/
public interface HConstants {
+ // For migration
+
+ /** name of version file */
+ static final String VERSION_FILE_NAME = "hbase.version";
+
+ /** version of file system */
+ static final String FILE_SYSTEM_VERSION = "0.1";
+
// Configuration parameters
// TODO: URL for hbase master like hdfs URLs with host and port.
Modified:
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
(original)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
Sat Jan 19 12:20:15 2008
@@ -889,6 +889,10 @@
// Make sure the root directory exists!
if(! fs.exists(rootdir)) {
fs.mkdirs(rootdir);
+ FSUtils.setVersion(fs, rootdir);
+ } else if (!FSUtils.checkVersion(fs, rootdir)) {
+ throw new IOException(
+ "file system not correct version. Run hbase.util.Migrate");
}
if (!fs.exists(rootRegionDir)) {
Modified:
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
(original)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java
Sat Jan 19 12:20:15 2008
@@ -19,12 +19,16 @@
*/
package org.apache.hadoop.hbase.util;
+import java.io.DataInputStream;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.dfs.DistributedFileSystem;
/**
@@ -71,4 +75,40 @@
}
return available;
}
+
+ /**
+ * Verifies current version of file system
+ *
+ * @param fs
+ * @param rootdir
+ * @return true if the current file system is the correct version
+ * @throws IOException
+ */
+ public static boolean checkVersion(FileSystem fs, Path rootdir) throws
IOException {
+ Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
+ boolean versionOk = false;
+ if (fs.exists(versionFile)) {
+ FSDataInputStream s =
+ fs.open(new Path(rootdir, HConstants.VERSION_FILE_NAME));
+ String version = DataInputStream.readUTF(s);
+ s.close();
+ versionOk = version.compareTo(HConstants.FILE_SYSTEM_VERSION) == 0;
+ }
+ return versionOk;
+ }
+
+ /**
+ * Sets version of file system
+ *
+ * @param fs
+ * @param rootdir
+ * @throws IOException
+ */
+ public static void setVersion(FileSystem fs, Path rootdir) throws
IOException {
+ FSDataOutputStream s =
+ fs.create(new Path(rootdir, HConstants.VERSION_FILE_NAME));
+ s.writeUTF(HConstants.FILE_SYSTEM_VERSION);
+ s.close();
+ }
+
}
Modified:
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
(original)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Migrate.java
Sat Jan 19 12:20:15 2008
@@ -136,7 +136,14 @@
Path rootdir = fs.makeQualified(new Path( // get path for instance
conf.get(HConstants.HBASE_DIR, HConstants.DEFAULT_HBASE_DIR)));
- // check for "extra" files
+ // See if there is a file system version file
+
+ if (FSUtils.checkVersion(fs, rootdir)) {
+ LOG.info("file system is at current level, no upgrade necessary");
+ return 0;
+ }
+
+ // check for "extra" files and for old upgradable regions
extraFiles(fs, rootdir);
@@ -155,10 +162,14 @@
// scan for left over regions
extraRegions(fs, rootdir);
+
+ // set file system version
+
+ FSUtils.setVersion(fs, rootdir);
return 0;
}
-
+
private void extraFiles(FileSystem fs, Path rootdir) throws IOException {
FileStatus[] stats = fs.listStatus(rootdir);
if (stats == null || stats.length == 0) {
@@ -174,6 +185,15 @@
} else {
String message = "unrecognized file " + name;
extraFile(otherFiles, message, fs, stats[i].getPath());
+ }
+ } else {
+ String regionName = name.substring(OLD_PREFIX.length());
+ try {
+ Integer.parseInt(regionName);
+
+ } catch (NumberFormatException e) {
+ extraFile(otherFiles, "old region format can not be converted: " +
+ name, fs, stats[i].getPath());
}
}
}
Modified:
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?rev=613446&r1=613445&r2=613446&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
(original)
+++
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
Sat Jan 19 12:20:15 2008
@@ -28,6 +28,8 @@
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
+import org.apache.hadoop.hbase.util.FSUtils;
+
/**
* This class creates a single process HBase cluster. One thread is created for
* each server.
@@ -138,6 +140,7 @@
try {
this.parentdir = new Path(conf.get(HBASE_DIR, DEFAULT_HBASE_DIR));
fs.mkdirs(parentdir);
+ FSUtils.setVersion(fs, parentdir);
this.hbaseCluster = new LocalHBaseCluster(this.conf, nRegionNodes);
this.hbaseCluster.startup();
} catch(IOException e) {
