Author: stack
Date: Tue Jul 21 21:18:52 2009
New Revision: 796542
URL: http://svn.apache.org/viewvc?rev=796542&view=rev
Log:
HBASE-1215 part 5 of migration; adds a migration test to unit suite with
verification that migration worked
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigration.java
Removed:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/MigrationTest.java
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=796542&r1=796541&r2=796542&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java
Tue Jul 21 21:18:52 2009
@@ -391,7 +391,8 @@
/** @return compression type being used for the column family */
@TOJSON
public Compression.Algorithm getCompression() {
- return Compression.Algorithm.valueOf(getValue(COMPRESSION));
+ String n = getValue(COMPRESSION);
+ return Compression.Algorithm.valueOf(n.toUpperCase());
}
/** @return maximum number of versions */
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=796542&r1=796541&r2=796542&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Tue Jul
21 21:18:52 2009
@@ -49,8 +49,8 @@
* Version 6 enables blockcaching on catalog tables.
* Version 7 introduces hfile -- hbase 0.19 to 0.20..
*/
- public static final String FILE_SYSTEM_VERSION = "6";
- // public static final String FILE_SYSTEM_VERSION = "7";
+ // public static final String FILE_SYSTEM_VERSION = "6";
+ public static final String FILE_SYSTEM_VERSION = "7";
// Configuration parameters
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java?rev=796542&r1=796541&r2=796542&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/Migrate.java Tue
Jul 21 21:18:52 2009
@@ -21,6 +21,8 @@
package org.apache.hadoop.hbase.util;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.cli.Options;
import org.apache.commons.logging.Log;
@@ -29,6 +31,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -40,6 +43,7 @@
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.migration.nineteen.io.BloomFilterMapFile;
import org.apache.hadoop.hbase.migration.nineteen.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -249,6 +253,7 @@
}
// TOOD: Verify all has been brought over from old to new layout.
final MetaUtils utils = new MetaUtils(this.conf);
+ final List<HRegionInfo> metas = new ArrayList<HRegionInfo>();
try {
rewriteHRegionInfo(utils.getRootRegion().getRegionInfo());
// Scan the root region
@@ -259,29 +264,53 @@
migrationNeeded = true;
return false;
}
+ metas.add(info);
rewriteHRegionInfo(utils.getRootRegion(), info);
return true;
}
});
- LOG.info("TODO: Note on make sure not using old hbase-default.xml");
- /*
- * hbase.master / hbase.master.hostname are obsolete, that's replaced by
-hbase.cluster.distributed. This config must be set to "true" to have a
-fully-distributed cluster and the server lines in zoo.cfg must not
-point to "localhost".
-
-The clients must have a valid zoo.cfg in their classpath since we
-don't provide the master address.
-
-hbase.master.dns.interface and hbase.master.dns.nameserver should be
-set to control the master's address (not mandatory).
- */
- LOG.info("TODO: Note on zookeeper config. before starting:");
+ // Scan meta.
+ for (HRegionInfo hri: metas) {
+ final HRegionInfo metahri = hri;
+ utils.scanMetaRegion(hri, new MetaUtils.ScannerListener() {
+ public boolean processRow(HRegionInfo info) throws IOException {
+ if (readOnly && !migrationNeeded) {
+ migrationNeeded = true;
+ return false;
+ }
+ rewriteHRegionInfo(utils.getMetaRegion(metahri), info);
+ return true;
+ }
+ });
+ }
+ cleanOldLogFiles(hbaseRootDir);
} finally {
utils.shutdown();
}
}
-
+
+ /*
+ * Remove old log files.
+ * @param fs
+ * @param hbaseRootDir
+ * @throws IOException
+ */
+ private void cleanOldLogFiles(final Path hbaseRootDir)
+ throws IOException {
+ FileStatus [] oldlogfiles = fs.listStatus(hbaseRootDir, new PathFilter () {
+ public boolean accept(Path p) {
+ return p.getName().startsWith("log_");
+ }
+ });
+ // Return if nothing to do.
+ if (oldlogfiles.length <= 0) return;
+ LOG.info("Removing " + oldlogfiles.length + " old logs file clutter");
+ for (int i = 0; i < oldlogfiles.length; i++) {
+ fs.delete(oldlogfiles[i].getPath(), true);
+ LOG.info("Deleted: " + oldlogfiles[i].getPath());
+ }
+ }
+
/*
* Rewrite all under hbase root dir.
* Presumes that {...@link FSUtils#isMajorCompactedPre020(FileSystem, Path)}
@@ -390,7 +419,6 @@
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(oldHri));
mr.put(put);
- LOG.info("Enabled blockcache on " + oldHri.getRegionNameAsString());
}
/*
@@ -399,26 +427,25 @@
*/
private boolean rewriteHRegionInfo(final HRegionInfo hri) {
boolean result = false;
+ // Set flush size at 32k if a catalog table.
+ int catalogMemStoreFlushSize = 32 * 1024;
+ if (hri.isMetaRegion() &&
+ hri.getTableDesc().getMemStoreFlushSize() != catalogMemStoreFlushSize)
{
+ hri.getTableDesc().setMemStoreFlushSize(catalogMemStoreFlushSize);
+ result = true;
+ }
HColumnDescriptor hcd =
hri.getTableDesc().getFamily(HConstants.CATALOG_FAMILY);
if (hcd == null) {
LOG.info("No info family in: " + hri.getRegionNameAsString());
return result;
}
- // Set blockcache enabled.
+ // Set block cache on all tables.
hcd.setBlockCacheEnabled(true);
+ // Set compression to none. Previous was 'none'. Needs to be upper-case.
+ // Any other compression we are turning off. Have user enable it.
+ hcd.setCompressionType(Algorithm.NONE);
return true;
-
- // TODO: Rewrite MEMCACHE_FLUSHSIZE as MEMSTORE_FLUSHSIZE â name has
changed.
- // TODO: Remove tableindexer 'index' attribute index from TableDescriptor
(See HBASE-1586)
- // TODO: TODO: Move of in-memory parameter from table to column family
(from HTD to HCD).
- // TODO: Purge isInMemory, etc., methods from HTD as part of migration.
-
-
- // TODO: Set the .META. and -ROOT- to flush at 16k? 32k?
- // TODO: Enable block cache on all tables
-
- // TODO: Clean up old region log files (HBASE-698)
}
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigration.java
URL:
http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigration.java?rev=796542&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigration.java
(added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/util/TestMigration.java
Tue Jul 21 21:18:52 2009
@@ -0,0 +1,253 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.MiniZooKeeperCluster;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * Runs migration of filesystem from hbase 0.19 to hbase 0.20.
+ * Not part of general test suite because takes time.
+ */
+public class TestMigration extends HBaseTestCase {
+ private static final Log LOG = LogFactory.getLog(TestMigration.class);
+
+ // Expected count of rows in migrated table.
+ private static final int EXPECTED_COUNT = 3;
+
+ /**
+ * Test migration.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void testMigration() throws IOException, InterruptedException {
+ Path rootdir = getUnitTestdir(getName());
+ Path hbasedir = loadTestData(fs, rootdir);
+ assertTrue(fs.exists(hbasedir));
+ Migrate migrator = new Migrate(this.conf);
+ Path qualified = fs.makeQualified(hbasedir);
+ String uri = qualified.toString();
+ this.conf.set("hbase.rootdir", uri);
+ int result = migrator.run(new String [] {"upgrade"});
+ assertEquals(0, result);
+ verify();
+ }
+
+ /*
+ * Load up test data.
+ * @param dfs
+ * @param rootDir
+ * @throws IOException
+ */
+ private Path loadTestData(final FileSystem dfs, final Path rootDir)
+ throws IOException {
+ String hbasedir = "hbase-0.19-two-small-tables";
+ InputStream is = this.getClass().getClassLoader().
+ getResourceAsStream("data/" + hbasedir + ".zip");
+ ZipInputStream zip = new ZipInputStream(is);
+ try {
+ unzip(zip, dfs, rootDir);
+ } finally {
+ zip.close();
+ }
+ return new Path(rootDir, hbasedir);
+ }
+
+ /*
+ * Verify can read the migrated table.
+ * @throws IOException
+ */
+ private void verify() throws IOException, InterruptedException {
+ // Delete any cached connections. Need to do this because connection was
+ // created earlier when no master was around. The fact that there was no
+ // master gets cached. Need to delete so we go get master afresh.
+ HConnectionManager.deleteConnectionInfo(conf, false);
+ LOG.info("Start a cluster against migrated FS");
+ // Up number of retries. Needed while cluster starts up. Its been set to 1
+ // above.
+ final int retries = 5;
+ this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER_KEY, retries);
+ // Note that this is done before we create the MiniHBaseCluster because we
+ // need to edit the config to add the ZooKeeper servers.
+ MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster();
+ int clientPort =
+ zooKeeperCluster.startup(new java.io.File(this.testDir.toString()));
+ conf.set("hbase.zookeeper.property.clientPort",
+ Integer.toString(clientPort));
+ MiniHBaseCluster cluster = new MiniHBaseCluster(this.conf, 1);
+ try {
+ HBaseAdmin hb = new HBaseAdmin(this.conf);
+ assertTrue(hb.isMasterRunning());
+ HTableDescriptor [] tables = hb.listTables();
+ assertEquals(2, tables.length);
+ boolean foundTable = false;
+ // Just look at table 'a'.
+ final String tablenameStr = "a";
+ final byte [] tablename = Bytes.toBytes(tablenameStr);
+ for (int i = 0; i < tables.length; i++) {
+ byte [] tableName = tables[i].getName();
+ if (Bytes.equals(tablename, tables[i].getName())) {
+ foundTable = true;
+ break;
+ }
+ }
+ assertTrue(foundTable);
+ LOG.info(tablenameStr + " exists. Now waiting till startcode " +
+ "changes before opening a scanner");
+ waitOnStartCodeChange(retries);
+ // Delete again so we go get it all fresh.
+ HConnectionManager.deleteConnectionInfo(conf, false);
+ HTable t = new HTable(this.conf, tablename);
+ int count = 0;
+ LOG.info("OPENING SCANNER");
+ Scan scan = new Scan();
+ ResultScanner s = t.getScanner(scan);
+ try {
+ for (Result r: s) {
+ if (r == null || r.size() == 0) {
+ break;
+ }
+ count++;
+ }
+ assertEquals(EXPECTED_COUNT, count);
+ } finally {
+ s.close();
+ }
+ } finally {
+ HConnectionManager.deleteConnectionInfo(conf, false);
+ cluster.shutdown();
+ try {
+ zooKeeperCluster.shutdown();
+ } catch (IOException e) {
+ LOG.warn("Shutting down ZooKeeper cluster", e);
+ }
+ }
+ }
+
+ /*
+ * Wait till the startcode changes before we put up a scanner. Otherwise
+ * we tend to hang, at least on hudson and I've had it time to time on
+ * my laptop. The hang is down in RPC Client doing its call. It
+ * never returns though the socket has a read timeout of 60 seconds by
+ * default. St.Ack
+ * @param retries How many retries to run.
+ * @throws IOException
+ */
+ private void waitOnStartCodeChange(final int retries) throws IOException {
+ HTable m = new HTable(this.conf, HConstants.META_TABLE_NAME);
+ // This is the start code that is in the old data.
+ long oldStartCode = 1199736332062L;
+ // This is the first row for the TestTable that is in the old data.
+ byte [] row = Bytes.toBytes("TestUpgrade,,1199736362468");
+ long pause = conf.getLong("hbase.client.pause", 5 * 1000);
+ long startcode = -1;
+ boolean changed = false;
+ for (int i = 0; i < retries; i++) {
+ Get get = new Get(row);
+ get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
+ Result res = m.get(get);
+ KeyValue [] kvs = res.raw();
+ if(kvs.length <= 0){
+ return;
+ }
+ byte [] value = kvs[0].getValue();
+ startcode = Bytes.toLong(value);
+ if (startcode != oldStartCode) {
+ changed = true;
+ break;
+ }
+ if ((i + 1) != retries) {
+ try {
+ Thread.sleep(pause);
+ } catch (InterruptedException e) {
+ // continue
+ }
+ }
+ }
+ // If after all attempts startcode has not changed, fail.
+ if (!changed) {
+ throw new IOException("Startcode didn't change after " + retries +
+ " attempts");
+ }
+ }
+
+ private void unzip(ZipInputStream zip, FileSystem dfs, Path rootDir)
+ throws IOException {
+ ZipEntry e = null;
+ while ((e = zip.getNextEntry()) != null) {
+ if (e.isDirectory()) {
+ dfs.mkdirs(new Path(rootDir, e.getName()));
+ } else {
+ FSDataOutputStream out = dfs.create(new Path(rootDir, e.getName()));
+ byte[] buffer = new byte[4096];
+ int len;
+ do {
+ len = zip.read(buffer);
+ if (len > 0) {
+ out.write(buffer, 0, len);
+ }
+ } while (len > 0);
+ out.close();
+ }
+ zip.closeEntry();
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private void listPaths(FileSystem filesystem, Path dir, int rootdirlength)
+ throws IOException {
+ FileStatus[] stats = filesystem.listStatus(dir);
+ if (stats == null || stats.length == 0) {
+ return;
+ }
+ for (int i = 0; i < stats.length; i++) {
+ String path = stats[i].getPath().toString();
+ if (stats[i].isDir()) {
+ System.out.println("d " + path);
+ listPaths(filesystem, stats[i].getPath(), rootdirlength);
+ } else {
+ System.out.println("f " + path + " size=" + stats[i].getLen());
+ }
+ }
+ }
+}
\ No newline at end of file
