HBASE-16634 Speedup TestExportSnapshot
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e19632a1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e19632a1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e19632a1 Branch: refs/heads/hbase-12439 Commit: e19632a19abc52a983d444811852bf0f11cd0d0b Parents: 216e847 Author: Matteo Bertozzi <matteo.berto...@cloudera.com> Authored: Thu Sep 15 18:29:09 2016 -0700 Committer: Matteo Bertozzi <matteo.berto...@cloudera.com> Committed: Thu Sep 15 18:29:09 2016 -0700 ---------------------------------------------------------------------- .../hbase/snapshot/MobSnapshotTestingUtils.java | 15 ++++++++- .../hbase/snapshot/SnapshotTestingUtils.java | 34 ++++++++++++++++---- .../hbase/snapshot/TestExportSnapshot.java | 8 ++--- .../snapshot/TestFlushSnapshotFromClient.java | 8 ++--- .../hbase/snapshot/TestMobExportSnapshot.java | 8 ++--- .../snapshot/TestMobSecureExportSnapshot.java | 4 +-- .../snapshot/TestSecureExportSnapshot.java | 1 - 7 files changed, 54 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index 9026115..107f487 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -38,6 +38,20 @@ public class MobSnapshotTestingUtils { public static void createMobTable(final HBaseTestingUtility util, final TableName tableName, int regionReplication, final byte[]... families) throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(), + regionReplication, families); + } + + public static void createPreSplitMobTable(final HBaseTestingUtility util, + final TableName tableName, int nRegions, final byte[]... families) + throws IOException, InterruptedException { + createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions), + 1, families); + } + + private static void createMobTable(final HBaseTestingUtility util, + final TableName tableName, final byte[][] splitKeys, int regionReplication, + final byte[]... families) throws IOException, InterruptedException { HTableDescriptor htd = new HTableDescriptor(tableName); htd.setRegionReplication(regionReplication); for (byte[] family : families) { @@ -46,7 +60,6 @@ public class MobSnapshotTestingUtils { hcd.setMobThreshold(0L); htd.addFamily(hcd); } - byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys(); util.getHBaseAdmin().createTable(htd, splitKeys); SnapshotTestingUtils.waitForTableToBeOnline(util, tableName); assertEquals((splitKeys.length + 1) * regionReplication, util http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 770bb00..dfd00b3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -82,9 +82,10 @@ import com.google.protobuf.ServiceException; */ @InterfaceAudience.Private public final class SnapshotTestingUtils { - private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class); - private static byte[] KEYS = Bytes.toBytes("0123456789"); + + // default number of regions (and keys) given by getSplitKeys() and createTable() + private static byte[] KEYS = Bytes.toBytes("0123456"); private SnapshotTestingUtils() { // private constructor for utility class @@ -750,23 +751,32 @@ public final class SnapshotTestingUtils { } public static void createTable(final HBaseTestingUtility util, final TableName tableName, - int regionReplication, final byte[]... families) throws IOException, InterruptedException { + int regionReplication, int nRegions, final byte[]... families) + throws IOException, InterruptedException { HTableDescriptor htd = new HTableDescriptor(tableName); htd.setRegionReplication(regionReplication); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); htd.addFamily(hcd); } - byte[][] splitKeys = getSplitKeys(); + byte[][] splitKeys = getSplitKeys(nRegions); util.createTable(htd, splitKeys); assertEquals((splitKeys.length + 1) * regionReplication, util.getHBaseAdmin().getTableRegions(tableName).size()); } public static byte[][] getSplitKeys() { - byte[][] splitKeys = new byte[KEYS.length-2][]; + return getSplitKeys(KEYS.length); + } + + public static byte[][] getSplitKeys(int nRegions) { + nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1); + final byte[][] splitKeys = new byte[nRegions-1][]; + final int step = KEYS.length / nRegions; + int keyIndex = 1; for (int i = 0; i < splitKeys.length; ++i) { - splitKeys[i] = new byte[] { KEYS[i+1] }; + splitKeys[i] = new byte[] { KEYS[keyIndex] }; + keyIndex += step; } return splitKeys; } @@ -776,6 +786,16 @@ public final class SnapshotTestingUtils { createTable(util, tableName, 1, families); } + public static void createTable(final HBaseTestingUtility util, final TableName tableName, + final int regionReplication, final byte[]... families) throws IOException, InterruptedException { + createTable(util, tableName, regionReplication, KEYS.length, families); + } + + public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName, + final int nRegions, final byte[]... families) throws IOException, InterruptedException { + createTable(util, tableName, 1, nRegions, families); + } + public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows, byte[]... families) throws IOException, InterruptedException { BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName); @@ -861,6 +881,6 @@ public final class SnapshotTestingUtils { } } } - assert(set.size() == getSplitKeys().length + 1); + assertEquals(getSplitKeys().length + 1, set.size()); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index d0b32f8..ee0d0e2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -62,7 +62,7 @@ import org.junit.rules.TestRule; /** * Test Export Snapshot Tool */ -@Category({VerySlowRegionServerTests.class, MediumTests.class}) +@Category({VerySlowRegionServerTests.class, LargeTests.class}) public class TestExportSnapshot { @Rule public final TestRule timeout = CategoryBasedTimeout.builder(). withTimeout(this.getClass()).withLookingForStuckThread(true).build(); @@ -91,12 +91,10 @@ public class TestExportSnapshot { public static void setUpBeforeClass() throws Exception { setUpBaseConf(TEST_UTIL.getConfiguration()); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } @@ -127,7 +125,7 @@ public class TestExportSnapshot { } protected void createTable() throws Exception { - SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY); + SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); } @After http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 8efc2d7..a852547 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -325,8 +325,8 @@ public class TestFlushSnapshotFromClient { int numRegionsAfterMerge = numRegions - 2; admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(), regions.get(2).getEncodedNameAsBytes(), true); - admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(), - regions.get(6).getEncodedNameAsBytes(), true); + admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(), + regions.get(5).getEncodedNameAsBytes(), true); // Verify that there's one region less waitRegionsAfterMerge(numRegionsAfterMerge); @@ -366,8 +366,8 @@ public class TestFlushSnapshotFromClient { int numRegionsAfterMerge = numRegions - 2; admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(), regions.get(2).getEncodedNameAsBytes(), true); - admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(), - regions.get(6).getEncodedNameAsBytes(), true); + admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(), + regions.get(5).getEncodedNameAsBytes(), true); waitRegionsAfterMerge(numRegionsAfterMerge); assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size()); http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java index b6cb339..d755961 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java @@ -25,14 +25,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; /** * Test Export Snapshot Tool */ -@Category(MediumTests.class) +@Category({VerySlowRegionServerTests.class, LargeTests.class}) public class TestMobExportSnapshot extends TestExportSnapshot { private final Log LOG = LogFactory.getLog(getClass()); @@ -45,12 +46,11 @@ public class TestMobExportSnapshot extends TestExportSnapshot { public static void setUpBeforeClass() throws Exception { setUpBaseConf(TEST_UTIL.getConfiguration()); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); } @Override protected void createTable() throws Exception { - MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, 1, FAMILY); + MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java index 4a89db8..8154995 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.snapshot; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.access.AccessControlLists; @@ -31,7 +32,7 @@ import org.junit.experimental.categories.Category; /** * Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode. */ -@Category(LargeTests.class) +@Category({VerySlowRegionServerTests.class, LargeTests.class}) public class TestMobSecureExportSnapshot extends TestMobExportSnapshot { @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -45,7 +46,6 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot { SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); // Wait for the ACL table to become available TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java index cf55c4d..f335e4e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java @@ -51,7 +51,6 @@ public class TestSecureExportSnapshot extends TestExportSnapshot { SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration()); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); // Wait for the ACL table to become available TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
