Author: mbertozzi
Date: Wed Jul 31 11:11:09 2013
New Revision: 1508811
URL: http://svn.apache.org/r1508811
Log:
HBASE-9090 cleanup snapshot tests setup/teardown code
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
Wed Jul 31 11:11:09 2013
@@ -32,8 +32,8 @@ import org.apache.hadoop.hbase.LargeTest
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.MD5Hash;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -96,7 +96,7 @@ public class TestCloneSnapshotFromClient
snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
// create Table and disable it
- createTable(tableName, FAMILY);
+ SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
admin.disableTable(tableName);
// take an empty snapshot
@@ -106,7 +106,7 @@ public class TestCloneSnapshotFromClient
try {
// enable table and insert data
admin.enableTable(tableName);
- loadData(table, 500, FAMILY);
+ SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
snapshot0Rows = TEST_UTIL.countRows(table);
admin.disableTable(tableName);
@@ -115,7 +115,7 @@ public class TestCloneSnapshotFromClient
// enable table and insert more data
admin.enableTable(tableName);
- loadData(table, 500, FAMILY);
+ SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
snapshot1Rows = TEST_UTIL.countRows(table);
admin.disableTable(tableName);
@@ -134,13 +134,8 @@ public class TestCloneSnapshotFromClient
if (admin.tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
- admin.deleteSnapshot(snapshotName0);
- admin.deleteSnapshot(snapshotName1);
-
- // Ensure the archiver to be empty
- MasterFileSystem mfs =
TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
- mfs.getFileSystem().delete(
- new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+ SnapshotTestingUtils.deleteAllSnapshots(admin);
+ SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
}
@Test(expected=SnapshotDoesNotExistException.class)
@@ -162,10 +157,9 @@ public class TestCloneSnapshotFromClient
int snapshotRows) throws IOException, InterruptedException {
// create a new table from snapshot
admin.cloneSnapshot(snapshotName, tableName);
- verifyRowCount(tableName, snapshotRows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshotRows);
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
+ TEST_UTIL.deleteTable(tableName);
}
/**
@@ -176,7 +170,7 @@ public class TestCloneSnapshotFromClient
// Clone a table from the first snapshot
byte[] clonedTableName = Bytes.toBytes("clonedtb1-" +
System.currentTimeMillis());
admin.cloneSnapshot(snapshotName0, clonedTableName);
- verifyRowCount(clonedTableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName,
snapshot0Rows);
// Take a snapshot of this cloned table.
admin.disableTable(clonedTableName);
@@ -185,85 +179,45 @@ public class TestCloneSnapshotFromClient
// Clone the snapshot of the cloned table
byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" +
System.currentTimeMillis());
admin.cloneSnapshot(snapshotName2, clonedTableName2);
- verifyRowCount(clonedTableName2, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2,
snapshot0Rows);
admin.disableTable(clonedTableName2);
// Remove the original table
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
+ TEST_UTIL.deleteTable(tableName);
waitCleanerRun();
// Verify the first cloned table
admin.enableTable(clonedTableName);
- verifyRowCount(clonedTableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName,
snapshot0Rows);
// Verify the second cloned table
admin.enableTable(clonedTableName2);
- verifyRowCount(clonedTableName2, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2,
snapshot0Rows);
admin.disableTable(clonedTableName2);
// Delete the first cloned table
- admin.disableTable(clonedTableName);
- admin.deleteTable(clonedTableName);
+ TEST_UTIL.deleteTable(clonedTableName);
waitCleanerRun();
// Verify the second cloned table
admin.enableTable(clonedTableName2);
- verifyRowCount(clonedTableName2, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2,
snapshot0Rows);
// Clone a new table from cloned
byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" +
System.currentTimeMillis());
admin.cloneSnapshot(snapshotName2, clonedTableName3);
- verifyRowCount(clonedTableName3, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName3,
snapshot0Rows);
// Delete the cloned tables
- admin.disableTable(clonedTableName2);
- admin.deleteTable(clonedTableName2);
- admin.disableTable(clonedTableName3);
- admin.deleteTable(clonedTableName3);
+ TEST_UTIL.deleteTable(clonedTableName2);
+ TEST_UTIL.deleteTable(clonedTableName3);
admin.deleteSnapshot(snapshotName2);
}
// ==========================================================================
// Helpers
// ==========================================================================
- private void createTable(final byte[] tableName, final byte[]... families)
throws IOException {
- HTableDescriptor htd = new HTableDescriptor(tableName);
- for (byte[] family: families) {
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- htd.addFamily(hcd);
- }
- byte[][] splitKeys = new byte[16][];
- byte[] hex = Bytes.toBytes("0123456789abcdef");
- for (int i = 0; i < 16; ++i) {
- splitKeys[i] = new byte[] { hex[i] };
- }
- admin.createTable(htd, splitKeys);
- }
-
- public void loadData(final HTable table, int rows, byte[]... families)
throws IOException {
- byte[] qualifier = Bytes.toBytes("q");
- table.setAutoFlush(false);
- while (rows-- > 0) {
- byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()),
Bytes.toBytes(rows));
- byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
- Put put = new Put(key);
- put.setWriteToWAL(false);
- for (byte[] family: families) {
- put.add(family, qualifier, value);
- }
- table.put(put);
- }
- table.flushCommits();
- }
-
private void waitCleanerRun() throws InterruptedException {
TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
}
-
- private void verifyRowCount(final byte[] tableName, long expectedRows)
throws IOException {
- HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
- assertEquals(expectedRows, TEST_UTIL.countRows(table));
- table.close();
- }
}
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestRestoreSnapshotFromClient.java
Wed Jul 31 11:11:09 2013
@@ -35,9 +35,9 @@ import org.apache.hadoop.hbase.LargeTest
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.MD5Hash;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -100,7 +100,7 @@ public class TestRestoreSnapshotFromClie
snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
// create Table and disable it
- createTable(tableName, FAMILY);
+ SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
admin.disableTable(tableName);
// take an empty snapshot
@@ -110,7 +110,7 @@ public class TestRestoreSnapshotFromClie
try {
// enable table and insert data
admin.enableTable(tableName);
- loadData(table, 500, FAMILY);
+ SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
snapshot0Rows = TEST_UTIL.countRows(table);
admin.disableTable(tableName);
@@ -119,7 +119,7 @@ public class TestRestoreSnapshotFromClie
// enable table and insert more data
admin.enableTable(tableName);
- loadData(table, 500, FAMILY);
+ SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
snapshot1Rows = TEST_UTIL.countRows(table);
admin.disableTable(tableName);
@@ -135,43 +135,36 @@ public class TestRestoreSnapshotFromClie
@After
public void tearDown() throws Exception {
- if (admin.tableExists(tableName)) {
- TEST_UTIL.deleteTable(tableName);
- }
- admin.deleteSnapshot(snapshotName0);
- admin.deleteSnapshot(snapshotName1);
-
- // Ensure the archiver to be empty
- MasterFileSystem mfs =
TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
- mfs.getFileSystem().delete(
- new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+ TEST_UTIL.deleteTable(tableName);
+ SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
}
@Test
public void testRestoreSnapshot() throws IOException {
- verifyRowCount(tableName, snapshot1Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
// Restore from snapshot-0
admin.disableTable(tableName);
admin.restoreSnapshot(snapshotName0);
admin.enableTable(tableName);
- verifyRowCount(tableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
// Restore from emptySnapshot
admin.disableTable(tableName);
admin.restoreSnapshot(emptySnapshot);
admin.enableTable(tableName);
- verifyRowCount(tableName, 0);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
// Restore from snapshot-1
admin.disableTable(tableName);
admin.restoreSnapshot(snapshotName1);
admin.enableTable(tableName);
- verifyRowCount(tableName, snapshot1Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
}
@Test
- public void testRestoreSchemaChange() throws IOException {
+ public void testRestoreSchemaChange() throws Exception {
byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
@@ -183,7 +176,7 @@ public class TestRestoreSnapshotFromClie
assertEquals(2, table.getTableDescriptor().getFamilies().size());
HTableDescriptor htd = admin.getTableDescriptor(tableName);
assertEquals(2, htd.getFamilies().size());
- loadData(table, 500, TEST_FAMILY2);
+ SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, TEST_FAMILY2);
long snapshot2Rows = snapshot1Rows + 500;
assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
@@ -230,51 +223,20 @@ public class TestRestoreSnapshotFromClie
public void testRestoreSnapshotOfCloned() throws IOException,
InterruptedException {
byte[] clonedTableName = Bytes.toBytes("clonedtb-" +
System.currentTimeMillis());
admin.cloneSnapshot(snapshotName0, clonedTableName);
- verifyRowCount(clonedTableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName,
snapshot0Rows);
admin.disableTable(clonedTableName);
admin.snapshot(snapshotName2, clonedTableName);
admin.deleteTable(clonedTableName);
waitCleanerRun();
admin.cloneSnapshot(snapshotName2, clonedTableName);
- verifyRowCount(clonedTableName, snapshot0Rows);
- admin.disableTable(clonedTableName);
- admin.deleteTable(clonedTableName);
+ SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName,
snapshot0Rows);
+ TEST_UTIL.deleteTable(clonedTableName);
}
// ==========================================================================
// Helpers
// ==========================================================================
- private void createTable(final byte[] tableName, final byte[]... families)
throws IOException {
- HTableDescriptor htd = new HTableDescriptor(tableName);
- for (byte[] family: families) {
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- htd.addFamily(hcd);
- }
- byte[][] splitKeys = new byte[16][];
- byte[] hex = Bytes.toBytes("0123456789abcdef");
- for (int i = 0; i < 16; ++i) {
- splitKeys[i] = new byte[] { hex[i] };
- }
- admin.createTable(htd, splitKeys);
- }
-
- public void loadData(final HTable table, int rows, byte[]... families)
throws IOException {
- byte[] qualifier = Bytes.toBytes("q");
- table.setAutoFlush(false);
- while (rows-- > 0) {
- byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()),
Bytes.toBytes(rows));
- byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
- Put put = new Put(key);
- put.setWriteToWAL(false);
- for (byte[] family: families) {
- put.add(family, qualifier, value);
- }
- table.put(put);
- }
- table.flushCommits();
- }
-
private void waitCleanerRun() throws InterruptedException {
TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
}
@@ -290,10 +252,4 @@ public class TestRestoreSnapshotFromClie
}
return families;
}
-
- private void verifyRowCount(final byte[] tableName, long expectedRows)
throws IOException {
- HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
- assertEquals(expectedRows, TEST_UTIL.countRows(table));
- table.close();
- }
}
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java
Wed Jul 31 11:11:09 2013
@@ -101,12 +101,8 @@ public class TestSnapshotFromClient {
@After
public void tearDown() throws Exception {
UTIL.deleteTable(TABLE_NAME);
- // and cleanup the archive directory
- try {
- UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(),
".archive"), true);
- } catch (IOException e) {
- LOG.warn("Failure to delete archive directory", e);
- }
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
}
@AfterClass
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
Wed Jul 31 11:11:09 2013
@@ -133,22 +133,8 @@ public class TestSnapshotFromMaster {
@After
public void tearDown() throws Exception {
UTIL.deleteTable(TABLE_NAME);
-
- // delete the archive directory, if its exists
- if (fs.exists(archiveDir)) {
- if (!fs.delete(archiveDir, true)) {
- throw new IOException("Couldn't delete archive directory (" +
archiveDir
- + " for an unknown reason");
- }
- }
-
- // delete the snapshot directory, if its exists
- if (fs.exists(snapshots)) {
- if (!fs.delete(snapshots, true)) {
- throw new IOException("Couldn't delete snapshots directory (" +
snapshots
- + " for an unknown reason");
- }
- }
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
}
@AfterClass
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
Wed Jul 31 11:11:09 2013
@@ -34,19 +34,27 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.MD5Hash;
import org.junit.Assert;
/**
@@ -252,4 +260,102 @@ public class SnapshotTestingUtils {
}
return files;
}
+
+ // ==========================================================================
+ // Table Helpers
+ // ==========================================================================
+ public static void waitForTableToBeOnline(final HBaseTestingUtility util,
final byte[] tableName)
+ throws IOException, InterruptedException {
+ HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
+ List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
+ for (HRegion region : onlineRegions) {
+ region.waitForFlushesAndCompactions();
+ }
+ util.getHBaseAdmin().isTableAvailable(tableName);
+ }
+
+ public static void createTable(final HBaseTestingUtility util, final byte[]
tableName,
+ final byte[]... families) throws IOException, InterruptedException {
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ for (byte[] family: families) {
+ HColumnDescriptor hcd = new HColumnDescriptor(family);
+ htd.addFamily(hcd);
+ }
+ byte[][] splitKeys = new byte[14][];
+ byte[] hex = Bytes.toBytes("123456789abcde");
+ for (int i = 0; i < splitKeys.length; ++i) {
+ splitKeys[i] = new byte[] { hex[i] };
+ }
+ util.getHBaseAdmin().createTable(htd, splitKeys);
+ waitForTableToBeOnline(util, tableName);
+ assertEquals(15, util.getHBaseAdmin().getTableRegions(tableName).size());
+ }
+
+ public static void loadData(final HBaseTestingUtility util, final byte[]
tableName, int rows,
+ byte[]... families) throws IOException, InterruptedException {
+ loadData(util, new HTable(util.getConfiguration(), tableName), rows,
families);
+ }
+
+ public static void loadData(final HBaseTestingUtility util, final HTable
table, int rows,
+ byte[]... families) throws IOException, InterruptedException {
+ table.setAutoFlush(false);
+
+ // Ensure one row per region
+ assertTrue(rows >= 16);
+ for (byte k0: Bytes.toBytes("0123456789abcdef")) {
+ byte[] k = new byte[] { k0 };
+ byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
+ byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
+ putData(table, families, key, value);
+ rows--;
+ }
+
+ // Add other extra rows. more rows, more files
+ while (rows-- > 0) {
+ byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()),
Bytes.toBytes(rows));
+ byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
+ putData(table, families, key, value);
+ }
+ table.flushCommits();
+
+ waitForTableToBeOnline(util, table.getTableName());
+ }
+
+ private static void putData(final HTable table, final byte[][] families,
+ final byte[] key, final byte[] value) throws IOException {
+ byte[] q = Bytes.toBytes("q");
+ Put put = new Put(key);
+ put.setDurability(Durability.SKIP_WAL);
+ for (byte[] family: families) {
+ put.add(family, q, value);
+ }
+ table.put(put);
+ }
+
+ public static void deleteAllSnapshots(final HBaseAdmin admin)
+ throws IOException {
+ // Delete all the snapshots
+ for (SnapshotDescription snapshot: admin.listSnapshots()) {
+ admin.deleteSnapshot(snapshot.getName());
+ }
+ SnapshotTestingUtils.assertNoSnapshots(admin);
+ }
+
+ public static void deleteArchiveDirectory(final HBaseTestingUtility util)
+ throws IOException {
+ // Ensure the archiver to be empty
+ MasterFileSystem mfs =
util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ Path archiveDir = new Path(mfs.getRootDir(),
HConstants.HFILE_ARCHIVE_DIRECTORY);
+ mfs.getFileSystem().delete(archiveDir, true);
+ }
+
+ public static void verifyRowCount(final HBaseTestingUtility util, final
byte[] tableName,
+ long expectedRows) throws IOException {
+ HTable table = new HTable(util.getConfiguration(), tableName);
+ try {
+ assertEquals(expectedRows, util.countRows(table));
+ } finally {
+ table.close();
+ }
+ }
}
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
Wed Jul 31 11:11:09 2013
@@ -126,10 +126,9 @@ public class TestExportSnapshot {
@After
public void tearDown() throws Exception {
- admin.disableTable(tableName);
- admin.deleteSnapshot(snapshotName);
- admin.deleteSnapshot(emptySnapshotName);
- admin.deleteTable(tableName);
+ TEST_UTIL.deleteTable(tableName);
+ SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
admin.close();
}
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
Wed Jul 31 11:11:09 2013
@@ -112,12 +112,8 @@ public class TestFlushSnapshotFromClient
@After
public void tearDown() throws Exception {
UTIL.deleteTable(TABLE_NAME);
- // and cleanup the archive directory
- try {
- UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(),
".archive"), true);
- } catch (IOException e) {
- LOG.warn("Failure to delete archive directory", e);
- }
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
}
@AfterClass
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java?rev=1508811&r1=1508810&r2=1508811&view=diff
==============================================================================
---
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
(original)
+++
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java
Wed Jul 31 11:11:09 2013
@@ -18,8 +18,10 @@
package org.apache.hadoop.hbase.snapshot;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -56,9 +58,10 @@ import org.junit.experimental.categories
public class TestRestoreFlushSnapshotFromClient {
final Log LOG = LogFactory.getLog(getClass());
- private final static HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
+ private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
private final byte[] FAMILY = Bytes.toBytes("cf");
+ private static final byte[] TEST_QUAL = Bytes.toBytes("q");
private byte[] snapshotName0;
private byte[] snapshotName1;
@@ -70,22 +73,22 @@ public class TestRestoreFlushSnapshotFro
@BeforeClass
public static void setUpBeforeClass() throws Exception {
-
TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable",
true);
- TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
- TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
- TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
- TEST_UTIL.getConfiguration().setBoolean(
+ UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable",
true);
+ UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+ UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+ UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+ UTIL.getConfiguration().setBoolean(
"hbase.master.enabletable.roundrobin", true);
// Enable snapshot
-
TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED,
true);
+ UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED,
true);
- TEST_UTIL.startMiniCluster(3);
+ UTIL.startMiniCluster(3);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
+ UTIL.shutdownMiniCluster();
}
/**
@@ -95,7 +98,7 @@ public class TestRestoreFlushSnapshotFro
*/
@Before
public void setup() throws Exception {
- this.admin = TEST_UTIL.getHBaseAdmin();
+ this.admin = UTIL.getHBaseAdmin();
long tid = System.currentTimeMillis();
tableName = Bytes.toBytes("testtb-" + tid);
@@ -104,11 +107,11 @@ public class TestRestoreFlushSnapshotFro
snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
// create Table and disable it
- createTable(tableName, FAMILY);
- HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+ SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY);
+ HTable table = new HTable(UTIL.getConfiguration(), tableName);
try {
- loadData(table, 500, FAMILY);
- snapshot0Rows = TEST_UTIL.countRows(table);
+ SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
+ snapshot0Rows = UTIL.countRows(table);
LOG.info("=== before snapshot with 500 rows");
logFSTree();
@@ -120,8 +123,8 @@ public class TestRestoreFlushSnapshotFro
logFSTree();
// insert more data
- loadData(table, 500, FAMILY);
- snapshot1Rows = TEST_UTIL.countRows(table);
+ SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
+ snapshot1Rows = UTIL.countRows(table);
LOG.info("=== before snapshot with 1000 rows");
logFSTree();
@@ -137,14 +140,8 @@ public class TestRestoreFlushSnapshotFro
@After
public void tearDown() throws Exception {
- TEST_UTIL.deleteTable(tableName);
- admin.deleteSnapshot(snapshotName0);
- admin.deleteSnapshot(snapshotName1);
-
- // Ensure the archiver to be empty
- MasterFileSystem mfs =
TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
- mfs.getFileSystem().delete(
- new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
+ SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
+ SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
}
@Test
@@ -154,7 +151,7 @@ public class TestRestoreFlushSnapshotFro
@Test
public void testRestoreSnapshot() throws IOException {
- verifyRowCount(tableName, snapshot1Rows);
+ SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
// Restore from snapshot-0
admin.disableTable(tableName);
@@ -163,13 +160,13 @@ public class TestRestoreFlushSnapshotFro
admin.enableTable(tableName);
LOG.info("=== after restore with 500 row snapshot");
logFSTree();
- verifyRowCount(tableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows);
// Restore from snapshot-1
admin.disableTable(tableName);
admin.restoreSnapshot(snapshotName1);
admin.enableTable(tableName);
- verifyRowCount(tableName, snapshot1Rows);
+ SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
}
@Test(expected=SnapshotDoesNotExistException.class)
@@ -190,65 +187,29 @@ public class TestRestoreFlushSnapshotFro
int snapshotRows) throws IOException, InterruptedException {
// create a new table from snapshot
admin.cloneSnapshot(snapshotName, tableName);
- verifyRowCount(tableName, snapshotRows);
+ SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshotRows);
- TEST_UTIL.deleteTable(tableName);
+ UTIL.deleteTable(tableName);
}
@Test
public void testRestoreSnapshotOfCloned() throws IOException,
InterruptedException {
byte[] clonedTableName = Bytes.toBytes("clonedtb-" +
System.currentTimeMillis());
admin.cloneSnapshot(snapshotName0, clonedTableName);
- verifyRowCount(clonedTableName, snapshot0Rows);
+ SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
admin.snapshot(Bytes.toString(snapshotName2),
Bytes.toString(clonedTableName), SnapshotDescription.Type.FLUSH);
- TEST_UTIL.deleteTable(clonedTableName);
+ UTIL.deleteTable(clonedTableName);
admin.cloneSnapshot(snapshotName2, clonedTableName);
- verifyRowCount(clonedTableName, snapshot0Rows);
- TEST_UTIL.deleteTable(clonedTableName);
+ SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
+ UTIL.deleteTable(clonedTableName);
}
// ==========================================================================
// Helpers
// ==========================================================================
- private void createTable(final byte[] tableName, final byte[]... families)
throws IOException {
- HTableDescriptor htd = new HTableDescriptor(tableName);
- for (byte[] family: families) {
- HColumnDescriptor hcd = new HColumnDescriptor(family);
- htd.addFamily(hcd);
- }
- byte[][] splitKeys = new byte[16][];
- byte[] hex = Bytes.toBytes("0123456789abcdef");
- for (int i = 0; i < 16; ++i) {
- splitKeys[i] = new byte[] { hex[i] };
- }
- admin.createTable(htd, splitKeys);
- }
-
- public void loadData(final HTable table, int rows, byte[]... families)
throws IOException {
- byte[] qualifier = Bytes.toBytes("q");
- table.setAutoFlush(false);
- while (rows-- > 0) {
- byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()),
Bytes.toBytes(rows));
- byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
- Put put = new Put(key);
- put.setWriteToWAL(false);
- for (byte[] family: families) {
- put.add(family, qualifier, value);
- }
- table.put(put);
- }
- table.flushCommits();
- }
-
private void logFSTree() throws IOException {
- MasterFileSystem mfs =
TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ MasterFileSystem mfs =
UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
}
-
- private void verifyRowCount(final byte[] tableName, long expectedRows)
throws IOException {
- HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
- assertEquals(expectedRows, TEST_UTIL.countRows(table));
- table.close();
- }
}
