Updated Branches: refs/heads/trunk dc4a82102 -> 33a7a8141
SQOOP-725: Enable HBase tests in sqoop 1 (Hari Shreedharan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/33a7a814 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/33a7a814 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/33a7a814 Branch: refs/heads/trunk Commit: 33a7a81412d789371a235323ac15f875e1870894 Parents: dc4a821 Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Nov 29 08:29:44 2012 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Nov 29 08:29:44 2012 -0800 ---------------------------------------------------------------------- build.xml | 2 +- ivy/ivysettings.xml | 1 + .../com/cloudera/sqoop/hbase/HBaseTestCase.java | 108 +++++++++++---- .../com/cloudera/sqoop/hbase/TestHBaseImport.java | 18 --- .../cloudera/sqoop/hbase/TestHBaseQueryImport.java | 6 - 5 files changed, 81 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 12ba785..a0b6326 100644 --- a/build.xml +++ b/build.xml @@ -77,7 +77,7 @@ <equals arg1="${hadoopversion}" arg2="200" /> <then> <property name="hadoop.version" value="2.0.0-alpha" /> - <property name="hbase.version" value="0.92.0" /> + <property name="hbase.version" value="0.94.2" /> <property name="zookeeper.version" value="3.4.2" /> <property name="hadoop.version.full" value="2.0.0-alpha" /> </then> http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/ivy/ivysettings.xml ---------------------------------------------------------------------- diff --git a/ivy/ivysettings.xml b/ivy/ivysettings.xml index 251131c..c4cc561 100644 --- a/ivy/ivysettings.xml +++ b/ivy/ivysettings.xml @@ -66,6 +66,7 @@ under the License. <chain name="default" dual="true" checkmodified="true" changingPattern=".*SNAPSHOT"> + <resolver ref="fs"/> <resolver ref="apache-snapshot"/> <resolver ref="cloudera-releases"/> <resolver ref="cloudera-staging"/> http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java index 3f5899e..65ff87b 100644 --- a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java +++ b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java @@ -28,13 +28,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; + import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.VersionInfo; import org.junit.After; import org.junit.Before; @@ -42,6 +46,10 @@ import org.junit.Before; import com.cloudera.sqoop.testutil.CommonArgs; import com.cloudera.sqoop.testutil.HsqldbTestServer; import com.cloudera.sqoop.testutil.ImportJobTestCase; +import java.io.File; +import java.lang.reflect.Method; +import java.util.UUID; +import org.apache.commons.io.FileUtils; /** * Utility methods that facilitate HBase import tests. @@ -105,56 +113,94 @@ public abstract class HBaseTestCase extends ImportJobTestCase { return args.toArray(new String[0]); } - + // Starts a mini hbase cluster in this process. // Starts a mini hbase cluster in this process. private HBaseTestingUtility hbaseTestUtil; - - private void startMaster() throws Exception { - if (null == hbaseTestUtil) { - Configuration conf = new Configuration(); - conf = HBaseConfiguration.addHbaseResources(conf); - hbaseTestUtil = new HBaseTestingUtility(conf); - hbaseTestUtil.startMiniCluster(1); - } - } + private String workDir = createTempDir().getAbsolutePath(); + private MiniZooKeeperCluster zookeeperCluster; + private MiniHBaseCluster hbaseCluster; @Override @Before public void setUp() { - if (!isHadoop20()) { - return; - } - HBaseTestCase.recordTestBuildDataProperty(); try { - startMaster(); - } catch (Exception e) { - fail(e.toString()); + HBaseTestCase.recordTestBuildDataProperty(); + String hbaseDir = new File(workDir, "hbase").getAbsolutePath(); + String hbaseRoot = "file://" + hbaseDir; + Configuration hbaseConf = HBaseConfiguration.create(); + hbaseConf.set(HConstants.HBASE_DIR, hbaseRoot); + //Hbase 0.90 does not have HConstants.ZOOKEEPER_CLIENT_PORT + hbaseConf.setInt("hbase.zookeeper.property.clientPort", 21818); + hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "0.0.0.0"); + hbaseConf.setInt("hbase.master.info.port", -1); + hbaseConf.setInt("hbase.zookeeper.property.maxClientCnxns", 500); + String zookeeperDir = new File(workDir, "zk").getAbsolutePath(); + int zookeeperPort = 21818; + zookeeperCluster = new MiniZooKeeperCluster(); + Method m; + Class<?> zkParam[] = {Integer.TYPE}; + try { + m = MiniZooKeeperCluster.class.getDeclaredMethod("setDefaultClientPort", + zkParam); + } catch (NoSuchMethodException e) { + m = MiniZooKeeperCluster.class.getDeclaredMethod("setClientPort", + zkParam); + } + m.invoke(zookeeperCluster, new Object[]{new Integer(zookeeperPort)}); + zookeeperCluster.startup(new File(zookeeperDir)); + hbaseCluster = new MiniHBaseCluster(hbaseConf, 1); + HMaster master = hbaseCluster.getMaster(); + Object serverName = master.getServerName(); + + String hostAndPort; + if (serverName instanceof String) { + System.out.println("Server name is string, using HServerAddress."); + m = HMaster.class.getDeclaredMethod("getMasterAddress", + new Class<?>[]{}); + Class<?> clazz = Class.forName("org.apache.hadoop.hbase.HServerAddress"); + /* + * Call method to get server address + */ + Object serverAddr = clazz.cast(m.invoke(master, new Object[]{})); + //returns the address as hostname:port + hostAndPort = serverAddr.toString(); + } else { + System.out.println("ServerName is org.apache.hadoop.hbase.ServerName," + + "using getHostAndPort()"); + Class<?> clazz = Class.forName("org.apache.hadoop.hbase.ServerName"); + m = clazz.getDeclaredMethod("getHostAndPort", new Class<?>[]{}); + hostAndPort = m.invoke(serverName, new Object[]{}).toString(); + } + hbaseConf.set("hbase.master", hostAndPort); + hbaseTestUtil = new HBaseTestingUtility(hbaseConf); + hbaseTestUtil.setZkCluster(zookeeperCluster); + hbaseCluster.startMaster(); + super.setUp(); + } catch (Throwable e) { + throw new RuntimeException(e); } - super.setUp(); } - public void shutdown() throws Exception { LOG.info("In shutdown() method"); if (null != hbaseTestUtil) { LOG.info("Shutting down HBase cluster"); - hbaseTestUtil.shutdownMiniCluster(); - this.hbaseTestUtil = null; + hbaseCluster.shutdown(); + zookeeperCluster.shutdown(); + hbaseTestUtil = null; } + FileUtils.deleteDirectory(new File(workDir)); LOG.info("shutdown() method returning."); } @Override @After public void tearDown() { - if (!isHadoop20()) { - return; - } try { shutdown(); } catch (Exception e) { LOG.warn("Error shutting down HBase minicluster: " - + StringUtils.stringifyException(e)); + + StringUtils.stringifyException(e)); } HBaseTestCase.restoreTestBuidlDataProperty(); super.tearDown(); @@ -180,8 +226,12 @@ public abstract class HBaseTestCase extends ImportJobTestCase { table.close(); } } - - protected boolean isHadoop20() { - return VersionInfo.getVersion().startsWith("0.20"); + public static File createTempDir() { + File baseDir = new File(System.getProperty("java.io.tmpdir")); + File tempDir = new File(baseDir, UUID.randomUUID().toString()); + if (tempDir.mkdir()) { + return tempDir; + } + throw new IllegalStateException("Failed to create directory"); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java b/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java index 2108e28..d5e15e8 100644 --- a/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java +++ b/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java @@ -29,9 +29,6 @@ public class TestHBaseImport extends HBaseTestCase { @Test public void testBasicUsage() throws IOException { - if (!isHadoop20()) { - return; - } // Create the HBase table in Sqoop as we run the job. String [] argv = getArgv(true, "BasicUsage", "BasicColFam", true, null); String [] types = { "INT", "INT" }; @@ -43,9 +40,6 @@ public class TestHBaseImport extends HBaseTestCase { @Test public void testMissingTableFails() throws IOException { - if (!isHadoop20()) { - return; - } // Test that if the table doesn't exist, we fail unless we // explicitly create the table. String [] argv = getArgv(true, "MissingTable", "MissingFam", false, null); @@ -62,9 +56,6 @@ public class TestHBaseImport extends HBaseTestCase { @Test public void testOverwriteSucceeds() throws IOException { - if (!isHadoop20()) { - return; - } // Test that we can create a table and then import immediately // back on top of it without problem. String [] argv = getArgv(true, "OverwriteT", "OverwriteF", true, null); @@ -80,9 +71,6 @@ public class TestHBaseImport extends HBaseTestCase { @Test public void testStrings() throws IOException { - if (!isHadoop20()) { - return; - } String [] argv = getArgv(true, "stringT", "stringF", true, null); String [] types = { "INT", "VARCHAR(32)" }; String [] vals = { "0", "'abc'" }; @@ -93,9 +81,6 @@ public class TestHBaseImport extends HBaseTestCase { @Test public void testNulls() throws IOException { - if (!isHadoop20()) { - return; - } String [] argv = getArgv(true, "nullT", "nullF", true, null); String [] types = { "INT", "INT", "INT" }; String [] vals = { "0", "42", "null" }; @@ -111,9 +96,6 @@ public class TestHBaseImport extends HBaseTestCase { @Test public void testExitFailure() throws IOException { - if (!isHadoop20()) { - return; - } String [] types = { "INT", "INT", "INT" }; String [] vals = { "0", "42", "43" }; createTableWithColTypes(types, vals); http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java b/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java index 705dcae..74eff50 100644 --- a/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java +++ b/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java @@ -29,9 +29,6 @@ public class TestHBaseQueryImport extends HBaseTestCase { @Test public void testImportFromQuery() throws IOException { - if (!isHadoop20()) { - return; - } String [] types = { "INT", "INT", "INT" }; String [] vals = { "0", "42", "43" }; createTableWithColTypes(types, vals); @@ -50,9 +47,6 @@ public class TestHBaseQueryImport extends HBaseTestCase { @Test public void testExitFailure() throws IOException { - if (!isHadoop20()) { - return; - } String [] types = { "INT", "INT", "INT" }; String [] vals = { "0", "42", "43" }; createTableWithColTypes(types, vals);
