Updated Branches: refs/heads/master 59f069bc1 -> ed58bb5e4
HBase tests are no longer compile with HBase 0.94 (Konstantin Boudnik via rvs) Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ed58bb5e Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ed58bb5e Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ed58bb5e Branch: refs/heads/master Commit: ed58bb5e4da8a49a9fc836713330141a7d783917 Parents: 59f069b Author: Roman Shaposhnik <[email protected]> Authored: Sun Mar 24 17:45:09 2013 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Sun Mar 24 17:45:09 2013 -0700 ---------------------------------------------------------------------- .../itest/hbase/smoke/IncrementalPELoad.java | 5 ++- .../itest/hbase/smoke/TestHBaseImportExport.groovy | 4 +- .../itest/hbase/smoke/TestHFileOutputFormat.java | 2 +- .../apache/bigtop/itest/hbase/system/Putter.java | 7 ++++- .../apache/bigtop/itest/hbase/system/Scanner.java | 7 ++++- .../hbase/system/TestConcurrentScanAndPut.java | 17 ++++++++------- .../itest/hbase/system/TestLoadAndVerify.java | 2 +- .../bigtop/itest/hbase/util/HBaseTestUtil.java | 13 ++++++++--- bigtop-tests/test-artifacts/pom.xml | 2 +- 9 files changed, 36 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java index aaf7d82..d1cb391 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/IncrementalPELoad.java @@ -99,13 +99,14 @@ public class IncrementalPELoad extends Configured implements Tool { } public int run(String[] args) throws Exception { - Job job = new Job(getConf(), "testMRIncrementalLoad"); + Configuration conf = getConf(); + Job job = new Job(conf, "testMRIncrementalLoad"); job.setJarByClass(IncrementalPELoad.class); job.setInputFormatClass(NMapInputFormat.class); job.setMapperClass(IncrementalPELoad.RandomKVGeneratingMapper.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(KeyValue.class); - HTable table = new HTable(Bytes.toBytes(args[0])); + HTable table = new HTable(conf, Bytes.toBytes(args[0])); HFileOutputFormat.configureIncrementalLoad(job, table); FileOutputFormat.setOutputPath(job, new Path(args[1])); http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy index 2ea01db..66696d3 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHBaseImportExport.groovy @@ -68,13 +68,13 @@ class TestHBaseImportExport { HTableDescriptor htd_orig = HBaseTestUtil.createTestTableDescriptor("orig", TEST_FAMILY) admin.createTable(htd_orig) - orig = new String(htd_orig.getName()) + orig = htd_orig.getName() origTable = new HTable(conf, orig) HTableDescriptor htd_export = HBaseTestUtil.createTestTableDescriptor("export", TEST_FAMILY) admin.createTable(htd_export) - export = new String(htd_export.getName()) + export = htd_export.getName() exportTable = new HTable(conf, export) // Write some rows to the table. http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java index f444226..37f7009 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/smoke/TestHFileOutputFormat.java @@ -126,7 +126,7 @@ public class TestHFileOutputFormat { tbldesc.addFamily(coldesc1); tbldesc.addFamily(coldesc2); admin.createTable(tbldesc, splitKeys); - HTable table = new HTable(TABLE_NAME); + HTable table = new HTable(conf, TABLE_NAME); assertEquals("Should start with empty table", 0, HBaseTestUtil.countRows(table)); http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java index 953fc8d..0aaffb3 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Putter.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.Map; import java.util.NavigableMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -108,10 +110,11 @@ public class Putter { loops = Integer.parseInt(argv[i]); } } + Configuration conf = HBaseConfiguration.create(); - String tableName = argv[0]; + byte [] tableName = Bytes.toBytes(argv[0]); int val = Integer.parseInt(argv[1]); - HTable table = new HTable(tableName); + HTable table = new HTable(conf, tableName); for (int i = 0; i < loops; i++) { try { doScanAndPut(table, val, autoflush); http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java index 8f4433b..b6a7e2d 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/Scanner.java @@ -25,6 +25,8 @@ import java.util.NavigableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -106,8 +108,9 @@ public class Scanner { System.err.println(" <value>: a numeric value [0,500)"); System.exit(1); } + Configuration conf = HBaseConfiguration.create(); - String tableName = argv[0]; + byte [] tableName = Bytes.toBytes(argv[0]); int val = Integer.parseInt(argv[1]); int loops = 1; for (int i = 1; i < argv.length; i++) { @@ -117,7 +120,7 @@ public class Scanner { } } - HTable table = new HTable(tableName); + HTable table = new HTable(conf, tableName); int exitVal = 0; for (int i = 0; i < loops; i++) { try { http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java index 08e8781..0b1414d 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestConcurrentScanAndPut.java @@ -53,7 +53,7 @@ public class TestConcurrentScanAndPut { public static Shell putSh = new Shell( "/bin/bash -s" ); public static HBaseAdmin admin; - public static String tableName; + public static byte [] tableName; public static String putter_pid; public static int scannerLoops; @@ -75,7 +75,7 @@ public class TestConcurrentScanAndPut { } tableName = - new String(HBaseTestUtil.getTestTableName("concurrentScanAndPut")); + Bytes.toBytes(new String(HBaseTestUtil.getTestTableName("concurrentScanAndPut"))); HTableDescriptor htd = new HTableDescriptor(tableName); for (int i = 0; i < 10; i++) { htd.addFamily(new HColumnDescriptor("f" + i)); @@ -83,7 +83,7 @@ public class TestConcurrentScanAndPut { admin = new HBaseAdmin(conf); admin.createTable(htd); - HTable table = new HTable(tableName); + HTable table = new HTable(conf, tableName); ArrayList<Put> puts = new ArrayList<Put>(1000); Random rnd = new Random(); @@ -137,25 +137,26 @@ public class TestConcurrentScanAndPut { System.out.println("Killing putter process"); putSh.exec("kill -9 " + putter_pid); - System.out.println("Removing test table " + tableName); + System.out.println("Removing test table " + Bytes.toString(tableName)); admin.disableTable(tableName); admin.deleteTable(tableName); } @Test public void testConcurrentScanAndPut() { - System.out.println("Starting puts to test table " + tableName); + String tableNameStr = Bytes.toString(tableName); + System.out.println("Starting puts to test table " + tableNameStr); putSh.exec("(HBASE_CLASSPATH=. " + "hbase com.cloudera.itest.hbase.system.Putter " + - tableName + " 13 -l " + putterLoops + + tableNameStr + " 13 -l " + putterLoops + " > /dev/null 2>&1 & echo $! ) 2> /dev/null"); putter_pid = putSh.getOut().get(0); System.out.println("Starting concurrent scans of test table " + - tableName); + tableNameStr); scanSh.exec("HBASE_CLASSPATH=. hbase " + "com.cloudera.itest.hbase.system.Scanner " + - tableName + " 13 -l " + scannerLoops + " 2>/dev/null"); + tableNameStr + " 13 -l " + scannerLoops + " 2>/dev/null"); int splitRows = scanSh.getRet(); System.out.println("Split rows: " + splitRows); http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java index 3841bd5..3bf403e 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestLoadAndVerify.java @@ -121,7 +121,7 @@ public class TestLoadAndVerify extends Configured implements Tool { public void setup(Context context) throws IOException { conf = context.getConfiguration(); recordsToWrite = conf.getLong(NUM_TO_WRITE_KEY, NUM_TO_WRITE_DEFAULT); - String tableName = conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); + byte [] tableName = Bytes.toBytes(conf.get(TABLE_NAME_KEY, TABLE_NAME_DEFAULT)); numBackReferencesPerRow = conf.getInt(NUM_BACKREFS_KEY, NUM_BACKREFS_DEFAULT); table = new HTable(conf, tableName); table.setWriteBufferSize(4*1024*1024); http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java index d78b2e6..8526754 100644 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java +++ b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/util/HBaseTestUtil.java @@ -34,9 +34,11 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ChecksumType; public class HBaseTestUtil { @@ -84,10 +86,13 @@ public class HBaseTestUtil { byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows) throws IOException { - HFile.Writer writer = - HFile.getWriterFactory(conf).createWriter(fs, path, - BLOCKSIZE, COMPRESSION, - KeyValue.KEY_COMPARATOR); + HFile.WriterFactory wf = HFile.getWriterFactory(conf, new CacheConfig(conf)); + wf.withChecksumType(ChecksumType.CRC32); + wf.withBlockSize(BLOCKSIZE); + wf.withCompression(COMPRESSION); + wf.withComparator(KeyValue.KEY_COMPARATOR); + wf.withPath(fs, path); + HFile.Writer writer = wf.create(); long now = System.currentTimeMillis(); try { // subtract 2 since iterateOnSplits doesn't include boundary keys http://git-wip-us.apache.org/repos/asf/bigtop/blob/ed58bb5e/bigtop-tests/test-artifacts/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/pom.xml b/bigtop-tests/test-artifacts/pom.xml index 856e6e6..db9b524 100644 --- a/bigtop-tests/test-artifacts/pom.xml +++ b/bigtop-tests/test-artifacts/pom.xml @@ -54,7 +54,7 @@ <hadoop.version>2.0.0-SNAPSHOT</hadoop.version> <pig.version>0.9.2</pig.version> <sqoop.version>1.4.1-incubating</sqoop.version> - <hbase.version>0.92.1</hbase.version> + <hbase.version>0.94.5</hbase.version> <zookeeper.version>3.4.3</zookeeper.version> <giraph.version>0.2-SNAPSHOT</giraph.version> <solr.version>4.0.0-BETA</solr.version>
