Repository: bigtop Updated Branches: refs/heads/master 5e140da0f -> c432fd2b2
BIGTOP-1302. Update hbase smoke tests to escape Hbase's "event horizon" Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c432fd2b Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c432fd2b Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c432fd2b Branch: refs/heads/master Commit: c432fd2b204e7722142d9a0003396b31801e747c Parents: 5e140da Author: Konstantin Boudnik <[email protected]> Authored: Mon May 12 11:31:26 2014 -0700 Committer: Konstantin Boudnik <[email protected]> Committed: Mon May 12 11:31:26 2014 -0700 ---------------------------------------------------------------------- bigtop-tests/test-artifacts/hbase/pom.xml | 13 +++++- .../hbase/smoke/TestHFileOutputFormat.java | 2 +- .../itest/hbase/system/TestLoadAndVerify.java | 2 + .../hbase/system/TestMoveRootMetaRegions.java | 42 ++------------------ .../bigtop/itest/hbase/util/HBaseTestUtil.java | 18 ++++++--- .../test-execution/smokes/hbase/pom.xml | 20 ++++++---- pom.xml | 15 ++++++- 7 files changed, 55 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/bigtop-tests/test-artifacts/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/pom.xml b/bigtop-tests/test-artifacts/hbase/pom.xml index 8b20c8e..3d0027f 100644 --- a/bigtop-tests/test-artifacts/hbase/pom.xml +++ b/bigtop-tests/test-artifacts/hbase/pom.xml @@ -41,11 +41,20 @@ </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> <type>test-jar</type> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java index 37f7009..1a0ed46 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java @@ -106,7 +106,7 @@ public class TestHFileOutputFormat { Random random = new Random(); byte[][] ret = new byte[numKeys][]; for (int i = 0; i < numKeys; i++) { - ret[i] = PerformanceEvaluation.generateValue(random); + ret[i] = PerformanceEvaluation.generateData(random, 10); } return ret; } http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java index 6280e45..b15b20d 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java @@ -18,6 +18,7 @@ package org.apache.bigtop.itest.hbase.system; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -300,6 +301,7 @@ public class TestLoadAndVerify extends Configured implements Tool { HTableDescriptor htd = HBaseTestUtil.createTestTableDescriptor(TEST_NAME, TEST_FAMILY); HBaseAdmin admin = HBaseTestUtil.getAdmin(); + assertNotNull("HBaseAdmin shouldn't be null", admin); int numPreCreate = 40; admin.createTable(htd, Bytes.toBytes(0L), Bytes.toBytes(-1L), numPreCreate); http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java index 08adf5d..dcad25d 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.util.Bytes; @@ -76,10 +77,8 @@ public class TestMoveRootMetaRegions { private static HBaseAdmin admin; private static CatalogTracker ct; - private static String root_table = - Bytes.toStringBinary(HConstants.ROOT_TABLE_NAME); private static String meta_table = - Bytes.toStringBinary(HConstants.META_TABLE_NAME); + Bytes.toStringBinary(TableName.META_TABLE_NAME.getName()); @BeforeClass public static void setUp() throws Exception { @@ -98,8 +97,6 @@ public class TestMoveRootMetaRegions { ct = new CatalogTracker(admin.getConfiguration()); ct.start(); - Assert.assertTrue(admin.tableExists(HConstants.ROOT_TABLE_NAME)); - Assert.assertTrue(admin.isTableEnabled(HConstants.ROOT_TABLE_NAME)); Assert.assertTrue(admin.tableExists(HConstants.META_TABLE_NAME)); Assert.assertTrue(admin.isTableEnabled(HConstants.META_TABLE_NAME)); } @@ -110,33 +107,11 @@ public class TestMoveRootMetaRegions { ct.stop(); } - public static ServerName getRootAddress() throws Exception{ - ct.verifyRootRegionLocation(root_timeout_ms); - return ct.getRootLocation(); - } - public static ServerName getMetaAddress() throws Exception{ return ct.waitForMeta(meta_timeout_ms); } @Test - public void unloadRootRegionServer() throws Exception{ - ServerName root_address = getRootAddress(); - String cmd = unload_regionserver + root_address.getHostname(); - System.out.println("Unloading the region server hosting " + root_table); - System.out.println(cmd); - sh.exec(cmd); - - Thread.sleep(wait_after_move_ms); - getRootAddress(); - - cmd = load_regionserver + root_address.getHostname(); - System.out.println("Reloading the region server"); - sh.exec(cmd); - Thread.sleep(wait_after_move_ms); - } - - @Test public void unloadMetaRegionServer() throws Exception{ ServerName meta_address = getMetaAddress(); String cmd = unload_regionserver + meta_address.getHostname(); @@ -155,31 +130,20 @@ public class TestMoveRootMetaRegions { @Test public void testStopRootMetaRegionServers() throws Exception { - ServerName root_address = getRootAddress(); ServerName meta_address = getMetaAddress(); boolean same_server = false; - if (root_address.equals(meta_address)) { - same_server = true; - } - System.out.println(root_table + " server address: " + root_address); System.out.println(meta_table + " server address: " + meta_address); System.out.println("Stopping region server(s)"); - admin.stopRegionServer(root_address.getHostAndPort()); - if (!same_server) { - admin.stopRegionServer(meta_address.getHostAndPort()); - } + admin.stopRegionServer(meta_address.getHostAndPort()); Thread.sleep(wait_after_move_ms); - ServerName new_root_address = getRootAddress(); ServerName new_meta_address = getMetaAddress(); System.out.println(meta_table + " server address: " + new_meta_address); - System.out.println(root_table + " server address: " + new_root_address); Assert.assertThat(meta_address, not(equalTo(new_meta_address))); - Assert.assertThat(root_address, not(equalTo(new_root_address))); } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java index 8526754..fd2f398 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java @@ -35,8 +35,9 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; +import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -64,7 +65,13 @@ public class HBaseTestUtil { public static HBaseAdmin getAdmin() throws MasterNotRunningException, ZooKeeperConnectionException { - return new HBaseAdmin(HBaseConfiguration.create()); + HBaseAdmin hAdmin = null; + try { + hAdmin = new HBaseAdmin(HBaseConfiguration.create()); + } catch (IOException e) { + e.printStackTrace(); + } + return hAdmin; } public static FileSystem getClusterFileSystem() throws IOException { @@ -87,10 +94,9 @@ public class HBaseTestUtil { byte[] startKey, byte[] endKey, int numRows) throws IOException { HFile.WriterFactory wf = HFile.getWriterFactory(conf, new CacheConfig(conf)); - wf.withChecksumType(ChecksumType.CRC32); - wf.withBlockSize(BLOCKSIZE); - wf.withCompression(COMPRESSION); - wf.withComparator(KeyValue.KEY_COMPARATOR); + HFileContext hFileContext = new HFileContext(); + wf.withFileContext(hFileContext); + wf.withComparator(KeyValue.COMPARATOR); wf.withPath(fs, path); HFile.Writer writer = wf.create(); long now = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/bigtop-tests/test-execution/smokes/hbase/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/hbase/pom.xml b/bigtop-tests/test-execution/smokes/hbase/pom.xml index 46c9cec..3c82ba6 100644 --- a/bigtop-tests/test-execution/smokes/hbase/pom.xml +++ b/bigtop-tests/test-execution/smokes/hbase/pom.xml @@ -52,6 +52,11 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${hbase.version}</version> + </dependency> </dependencies> <build> @@ -69,7 +74,7 @@ <configuration> <file>${HBASE_HOME}/${hbase.jar}</file> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-client</artifactId> <version>${hbase.version}</version> <packaging>jar</packaging> </configuration> @@ -83,7 +88,7 @@ <configuration> <file>${HBASE_HOME}/${hbase.tests.jar}</file> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-client</artifactId> <version>${hbase.version}</version> <packaging>jar</packaging> <classifier>tests</classifier> @@ -135,7 +140,7 @@ <configuration> <systemPropertyVariables> <org.apache.bigtop.itest.hbasesmoke.TestHbasePigSmoke.extra_jars> - ${ZOOKEEPER_HOME}/${zookeeper.jar}:${HBASE_HOME}/${hbase.jar}:${HBASE_HOME}/lib/guava-r06.jar + ${ZOOKEEPER_HOME}/${zookeeper.jar}:${HBASE_HOME}/${hbase.jar}:${HBASE_HOME}/lib/guava-12.0.1.jar </org.apache.bigtop.itest.hbasesmoke.TestHbasePigSmoke.extra_jars> <loadmapper.num_to_write> 10 @@ -169,16 +174,17 @@ } project.properties['hbase.jar'] = org.apache.bigtop.itest.JarContent.getJarName( - "${HBASE_HOME}", 'hbase-[0-9.-]*.jar'); + "${HBASE_HOME}", 'hbase-client.*.jar'); + println ("HBASE JAR: ${project.properties['hbase.jar']}"); if ( project.properties['hbase.jar'] == null ) { - throw new IOException ("hbase*.jar isn't found"); + throw new IOException ("hbase-client.jar isn't found"); } project.properties['hbase.tests.jar'] = org.apache.bigtop.itest.JarContent.getJarName( - "${HBASE_HOME}", 'hbase-.*tests.jar'); + "${HBASE_HOME}", 'hbase-common.*tests.jar'); if ( project.properties['hbase.tests.jar'] == null ) { - throw new IOException ("hbase*tests.jar isn't found"); + throw new IOException ("hbase-common*tests.jar isn't found"); } } </source> http://git-wip-us.apache.org/repos/asf/bigtop/blob/c432fd2b/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 428f25e..2abace0 100644 --- a/pom.xml +++ b/pom.xml @@ -176,12 +176,23 @@ </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> <version>${hbase.version}</version> </dependency> <dependency> <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> + <artifactId>hbase-common</artifactId> + <version>${hbase.version}</version> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>${hbase.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> <version>${hbase.version}</version> <type>test-jar</type> </dependency>
