Repository: hbase Updated Branches: refs/heads/branch-1 44f392a0c -> 173eba815
http://git-wip-us.apache.org/repos/asf/hbase/blob/173eba81/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index 18ba522..8ed8464 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -56,7 +56,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.PerformanceEvaluation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HRegionLocator; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionLocator; @@ -336,10 +335,9 @@ public class TestHFileOutputFormat { public void testJobConfiguration() throws Exception { Job job = new Job(util.getConfiguration()); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); - HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class); - RegionLocator regionLocator = Mockito.mock(RegionLocator.class); - setupMockStartKeys(regionLocator); - HFileOutputFormat2.configureIncrementalLoad(job, tableDescriptor, regionLocator); + HTable table = Mockito.mock(HTable.class); + setupMockStartKeys(table); + HFileOutputFormat.configureIncrementalLoad(job, table); assertEquals(job.getNumReduceTasks(), 4); } @@ -469,13 +467,12 @@ public class TestHFileOutputFormat { MutationSerialization.class.getName(), ResultSerialization.class.getName(), KeyValueSerialization.class.getName()); setupRandomGeneratorMapper(job); - HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), - table.getRegionLocator()); + HFileOutputFormat.configureIncrementalLoad(job, table); FileOutputFormat.setOutputPath(job, outDir); Assert.assertFalse( util.getTestFileSystem().exists(outDir)) ; - assertEquals(table.getRegionLocator().getAllRegionLocations().size(), job.getNumReduceTasks()); + assertEquals(table.getRegionLocations().size(), job.getNumReduceTasks()); assertTrue(job.waitForCompletion(true)); } @@ -772,14 +769,14 @@ public class TestHFileOutputFormat { return familyToDataBlockEncoding; } - private void setupMockStartKeys(RegionLocator regionLocator) throws IOException { + private void setupMockStartKeys(RegionLocator table) throws IOException { byte[][] mockKeys = new byte[][] { HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("aaa"), Bytes.toBytes("ggg"), Bytes.toBytes("zzz") }; - Mockito.doReturn(mockKeys).when(regionLocator).getStartKeys(); + Mockito.doReturn(mockKeys).when(table).getStartKeys(); } /** @@ -794,8 +791,7 @@ public class TestHFileOutputFormat { Path dir = util.getDataTestDir("testColumnFamilySettings"); // Setup table descriptor - Table table = Mockito.mock(Table.class); - RegionLocator regionLocator = Mockito.mock(RegionLocator.class); + HTable table = Mockito.mock(HTable.class); HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); Mockito.doReturn(htd).when(table).getTableDescriptor(); for (HColumnDescriptor hcd: this.util.generateColumnDescriptors()) { @@ -803,7 +799,7 @@ public class TestHFileOutputFormat { } // set up the table to return some mock keys - setupMockStartKeys(regionLocator); + setupMockStartKeys(table); try { // partial map red setup to get an operational writer for testing @@ -813,7 +809,7 @@ public class TestHFileOutputFormat { Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings")); setupRandomGeneratorMapper(job); - HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); + HFileOutputFormat.configureIncrementalLoad(job, table); FileOutputFormat.setOutputPath(job, dir); context = createTestTaskAttemptContext(job); HFileOutputFormat hof = new HFileOutputFormat(); http://git-wip-us.apache.org/repos/asf/hbase/blob/173eba81/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 67aacc1..2a780d4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -25,6 +25,15 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.Callable; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -44,9 +53,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.PerformanceEvaluation; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionLocator; @@ -79,16 +86,6 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.Callable; - /** * Simple test for {@link CellSortReducer} and {@link HFileOutputFormat2}. * Sets up and runs a mapreduce job that writes hfile output. @@ -134,7 +131,6 @@ public class TestHFileOutputFormat2 { valLength = conf.getInt(VALLEN_CONF, VALLEN_DEFAULT); } - @Override protected void map( NullWritable n1, NullWritable n2, Mapper<NullWritable, NullWritable, @@ -219,7 +215,7 @@ public class TestHFileOutputFormat2 { } private TaskAttemptContext createTestTaskAttemptContext(final Job job) - throws Exception { + throws IOException, Exception { HadoopShims hadoop = CompatibilitySingletonFactory.getInstance(HadoopShims.class); TaskAttemptContext context = hadoop.createTestTaskAttemptContext( job, "attempt_201402131733_0001_m_000000_0"); @@ -338,10 +334,9 @@ public class TestHFileOutputFormat2 { public void testJobConfiguration() throws Exception { Job job = new Job(util.getConfiguration()); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); - Table table = Mockito.mock(Table.class); - RegionLocator regionLocator = Mockito.mock(RegionLocator.class); - setupMockStartKeys(regionLocator); - HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); + HTable table = Mockito.mock(HTable.class); + setupMockStartKeys(table); + HFileOutputFormat2.configureIncrementalLoad(job, table, table); assertEquals(job.getNumReduceTasks(), 4); } @@ -374,10 +369,12 @@ public class TestHFileOutputFormat2 { util = new HBaseTestingUtility(); Configuration conf = util.getConfiguration(); byte[][] startKeys = generateRandomStartKeys(5); - util.startMiniCluster(); - try (HTable table = util.createTable(TABLE_NAME, FAMILIES); - Admin admin = table.getConnection().getAdmin()) { + HBaseAdmin admin = null; + try { + util.startMiniCluster(); Path testDir = util.getDataTestDirOnTestFS("testLocalMRIncrementalLoad"); + admin = new HBaseAdmin(conf); + HTable table = util.createTable(TABLE_NAME, FAMILIES); assertEquals("Should start with empty table", 0, util.countRows(table)); int numRegions = util.createMultiRegions( @@ -386,7 +383,7 @@ public class TestHFileOutputFormat2 { // Generate the bulk load files util.startMiniMapReduceCluster(); - runIncrementalPELoad(conf, table.getTableDescriptor(), table.getRegionLocator(), testDir); + runIncrementalPELoad(conf, table, testDir); // This doesn't write into the table, just makes files assertEquals("HFOF should not touch actual table", 0, util.countRows(table)); @@ -406,7 +403,7 @@ public class TestHFileOutputFormat2 { // handle the split case if (shouldChangeRegions) { LOG.info("Changing regions in table"); - admin.disableTable(table.getName()); + admin.disableTable(table.getTableName()); while(util.getMiniHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().isRegionsInTransition()) { Threads.sleep(200); @@ -415,9 +412,9 @@ public class TestHFileOutputFormat2 { byte[][] newStartKeys = generateRandomStartKeys(15); util.createMultiRegions( util.getConfiguration(), table, FAMILIES[0], newStartKeys); - admin.enableTable(table.getName()); - while (table.getRegionLocator().getAllRegionLocations().size() != 15 || - !admin.isTableAvailable(table.getName())) { + admin.enableTable(table.getTableName()); + while (table.getRegionLocations().size() != 15 || + !admin.isTableAvailable(table.getTableName())) { Thread.sleep(200); LOG.info("Waiting for new region assignment to happen"); } @@ -454,26 +451,27 @@ public class TestHFileOutputFormat2 { assertEquals("Data should remain after reopening of regions", tableDigestBefore, util.checksumRows(table)); } finally { + if (admin != null) admin.close(); util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } - private void runIncrementalPELoad(Configuration conf, HTableDescriptor tableDescriptor, - RegionLocator regionLocator, Path outDir) throws IOException, UnsupportedEncodingException, - InterruptedException, ClassNotFoundException { + private void runIncrementalPELoad( + Configuration conf, HTable table, Path outDir) + throws Exception { Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("runIncrementalPELoad")); job.getConfiguration().setStrings("io.serializations", conf.get("io.serializations"), MutationSerialization.class.getName(), ResultSerialization.class.getName(), KeyValueSerialization.class.getName()); setupRandomGeneratorMapper(job); - HFileOutputFormat2.configureIncrementalLoad(job, tableDescriptor, regionLocator); + HFileOutputFormat2.configureIncrementalLoad(job, table, table); FileOutputFormat.setOutputPath(job, outDir); assertFalse(util.getTestFileSystem().exists(outDir)) ; - assertEquals(regionLocator.getAllRegionLocations().size(), job.getNumReduceTasks()); + assertEquals(table.getRegionLocations().size(), job.getNumReduceTasks()); assertTrue(job.waitForCompletion(true)); } @@ -495,7 +493,7 @@ public class TestHFileOutputFormat2 { getMockColumnFamiliesForCompression(numCfs); Table table = Mockito.mock(HTable.class); setupMockColumnFamiliesForCompression(table, familyToCompression); - HFileOutputFormat2.configureCompression(conf, table.getTableDescriptor()); + HFileOutputFormat2.configureCompression(table, conf); // read back family specific compression setting from the configuration Map<byte[], Algorithm> retrievedFamilyToCompressionMap = HFileOutputFormat2 @@ -567,7 +565,7 @@ public class TestHFileOutputFormat2 { Table table = Mockito.mock(HTable.class); setupMockColumnFamiliesForBloomType(table, familyToBloomType); - HFileOutputFormat2.configureBloomType(table.getTableDescriptor(), conf); + HFileOutputFormat2.configureBloomType(table, conf); // read back family specific data block encoding settings from the // configuration @@ -638,7 +636,7 @@ public class TestHFileOutputFormat2 { Table table = Mockito.mock(HTable.class); setupMockColumnFamiliesForBlockSize(table, familyToBlockSize); - HFileOutputFormat2.configureBlockSize(table.getTableDescriptor(), conf); + HFileOutputFormat2.configureBlockSize(table, conf); // read back family specific data block encoding settings from the // configuration @@ -696,9 +694,10 @@ public class TestHFileOutputFormat2 { return familyToBlockSize; } - /** - * Test for {@link HFileOutputFormat2#configureDataBlockEncoding(HTableDescriptor, Configuration)} - * and {@link HFileOutputFormat2#createFamilyDataBlockEncodingMap(Configuration)}. + /** + * Test for {@link HFileOutputFormat2#configureDataBlockEncoding(org.apache.hadoop.hbase.client.Table, + * Configuration)} and {@link HFileOutputFormat2#createFamilyDataBlockEncodingMap + * (Configuration)}. * Tests that the compression map is correctly serialized into * and deserialized from configuration * @@ -713,8 +712,7 @@ public class TestHFileOutputFormat2 { Table table = Mockito.mock(HTable.class); setupMockColumnFamiliesForDataBlockEncoding(table, familyToDataBlockEncoding); - HTableDescriptor tableDescriptor = table.getTableDescriptor(); - HFileOutputFormat2.configureDataBlockEncoding(tableDescriptor, conf); + HFileOutputFormat2.configureDataBlockEncoding(table, conf); // read back family specific data block encoding settings from the // configuration @@ -793,8 +791,7 @@ public class TestHFileOutputFormat2 { Path dir = util.getDataTestDir("testColumnFamilySettings"); // Setup table descriptor - Table table = Mockito.mock(Table.class); - RegionLocator regionLocator = Mockito.mock(RegionLocator.class); + HTable table = Mockito.mock(HTable.class); HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); Mockito.doReturn(htd).when(table).getTableDescriptor(); for (HColumnDescriptor hcd: HBaseTestingUtility.generateColumnDescriptors()) { @@ -802,7 +799,7 @@ public class TestHFileOutputFormat2 { } // set up the table to return some mock keys - setupMockStartKeys(regionLocator); + setupMockStartKeys(table); try { // partial map red setup to get an operational writer for testing @@ -812,7 +809,7 @@ public class TestHFileOutputFormat2 { Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings")); setupRandomGeneratorMapper(job); - HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); + HFileOutputFormat2.configureIncrementalLoad(job, table, table); FileOutputFormat.setOutputPath(job, dir); context = createTestTaskAttemptContext(job); HFileOutputFormat2 hof = new HFileOutputFormat2(); @@ -893,10 +890,10 @@ public class TestHFileOutputFormat2 { conf.setInt("hbase.hstore.compaction.min", 2); generateRandomStartKeys(5); - util.startMiniCluster(); - try (Connection conn = ConnectionFactory.createConnection(); - Admin admin = conn.getAdmin()) { + try { + util.startMiniCluster(); final FileSystem fs = util.getDFSCluster().getFileSystem(); + HBaseAdmin admin = new HBaseAdmin(conf); HTable table = util.createTable(TABLE_NAME, FAMILIES); assertEquals("Should start with empty table", 0, util.countRows(table)); @@ -914,8 +911,7 @@ public class TestHFileOutputFormat2 { for (int i = 0; i < 2; i++) { Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i); - runIncrementalPELoad(conf, table.getTableDescriptor(), conn.getRegionLocator(TABLE_NAME), - testDir); + runIncrementalPELoad(conf, table, testDir); // Perform the actual load new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); } @@ -929,10 +925,9 @@ public class TestHFileOutputFormat2 { assertEquals(2, fs.listStatus(storePath).length); // minor compactions shouldn't get rid of the file - admin.compact(TABLE_NAME); + admin.compact(TABLE_NAME.getName()); try { quickPoll(new Callable<Boolean>() { - @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } @@ -943,9 +938,8 @@ public class TestHFileOutputFormat2 { } // a major compaction should work though - admin.majorCompact(TABLE_NAME); + admin.majorCompact(TABLE_NAME.getName()); quickPoll(new Callable<Boolean>() { - @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } @@ -963,12 +957,12 @@ public class TestHFileOutputFormat2 { conf.setInt("hbase.hstore.compaction.min", 2); generateRandomStartKeys(5); - util.startMiniCluster(); - try (Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin()){ + try { + util.startMiniCluster(); Path testDir = util.getDataTestDirOnTestFS("testExcludeMinorCompaction"); final FileSystem fs = util.getDFSCluster().getFileSystem(); - Table table = util.createTable(TABLE_NAME, FAMILIES); + HBaseAdmin admin = new HBaseAdmin(conf); + HTable table = util.createTable(TABLE_NAME, FAMILIES); assertEquals("Should start with empty table", 0, util.countRows(table)); // deep inspection: get the StoreFile dir @@ -982,10 +976,9 @@ public class TestHFileOutputFormat2 { Put p = new Put(Bytes.toBytes("test")); p.add(FAMILIES[0], Bytes.toBytes("1"), Bytes.toBytes("1")); table.put(p); - admin.flush(TABLE_NAME); + admin.flush(TABLE_NAME.getName()); assertEquals(1, util.countRows(table)); quickPoll(new Callable<Boolean>() { - @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } @@ -995,12 +988,10 @@ public class TestHFileOutputFormat2 { conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true); util.startMiniMapReduceCluster(); - - RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME); - runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir); + runIncrementalPELoad(conf, table, testDir); // Perform the actual load - new LoadIncrementalHFiles(conf).doBulkLoad(testDir, admin, table, regionLocator); + new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); // Ensure data shows up int expectedRows = NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT; @@ -1011,10 +1002,9 @@ public class TestHFileOutputFormat2 { assertEquals(2, fs.listStatus(storePath).length); // minor compactions shouldn't get rid of the file - admin.compact(TABLE_NAME); + admin.compact(TABLE_NAME.getName()); try { quickPoll(new Callable<Boolean>() { - @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } @@ -1025,9 +1015,8 @@ public class TestHFileOutputFormat2 { } // a major compaction should work though - admin.majorCompact(TABLE_NAME); + admin.majorCompact(TABLE_NAME.getName()); quickPoll(new Callable<Boolean>() { - @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } @@ -1059,22 +1048,18 @@ public class TestHFileOutputFormat2 { Configuration conf = HBaseConfiguration.create(); util = new HBaseTestingUtility(conf); if ("newtable".equals(args[0])) { - TableName tname = TableName.valueOf(args[1]); - try (HTable table = util.createTable(tname, FAMILIES); - Admin admin = table.getConnection().getAdmin()) { - admin.disableTable(tname); - byte[][] startKeys = generateRandomStartKeys(5); - util.createMultiRegions(conf, table, FAMILIES[0], startKeys); - admin.enableTable(tname); - } + byte[] tname = args[1].getBytes(); + HTable table = util.createTable(tname, FAMILIES); + HBaseAdmin admin = new HBaseAdmin(conf); + admin.disableTable(tname); + byte[][] startKeys = generateRandomStartKeys(5); + util.createMultiRegions(conf, table, FAMILIES[0], startKeys); + admin.enableTable(tname); } else if ("incremental".equals(args[0])) { TableName tname = TableName.valueOf(args[1]); - try(Connection c = ConnectionFactory.createConnection(conf); - Admin admin = c.getAdmin(); - RegionLocator regionLocator = c.getRegionLocator(tname)) { - Path outDir = new Path("incremental-out"); - runIncrementalPELoad(conf, admin.getTableDescriptor(tname), regionLocator, outDir); - } + HTable table = new HTable(conf, tname); + Path outDir = new Path("incremental-out"); + runIncrementalPELoad(conf, table, outDir); } else { throw new RuntimeException( "usage: TestHFileOutputFormat2 newtable | incremental"); http://git-wip-us.apache.org/repos/asf/hbase/blob/173eba81/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java index c4ac827..e7ee0ab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java @@ -274,8 +274,7 @@ public class TestLoadIncrementalHFilesSplitRecovery { try (Connection connection = ConnectionFactory.createConnection(this.util.getConfiguration())) { setupTable(connection, table, 10); LoadIncrementalHFiles lih = new LoadIncrementalHFiles(util.getConfiguration()) { - @Override - protected List<LoadQueueItem> tryAtomicRegionLoad(final Connection conn, + protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn, TableName tableName, final byte[] first, Collection<LoadQueueItem> lqis) throws IOException { int i = attmptedCalls.incrementAndGet(); @@ -349,8 +348,7 @@ public class TestLoadIncrementalHFilesSplitRecovery { // files to fail when attempt to atomically import. This is recoverable. final AtomicInteger attemptedCalls = new AtomicInteger(); LoadIncrementalHFiles lih2 = new LoadIncrementalHFiles(util.getConfiguration()) { - @Override - protected void bulkLoadPhase(final Table htable, final Connection conn, + protected void bulkLoadPhase(final Table htable, final HConnection conn, ExecutorService pool, Deque<LoadQueueItem> queue, final Multimap<ByteBuffer, LoadQueueItem> regionGroups) throws IOException { int i = attemptedCalls.incrementAndGet(); @@ -392,10 +390,9 @@ public class TestLoadIncrementalHFilesSplitRecovery { final AtomicInteger countedLqis= new AtomicInteger(); LoadIncrementalHFiles lih = new LoadIncrementalHFiles( util.getConfiguration()) { - @Override protected List<LoadQueueItem> groupOrSplit( Multimap<ByteBuffer, LoadQueueItem> regionGroups, - final LoadQueueItem item, final Table htable, + final LoadQueueItem item, final HTable htable, final Pair<byte[][], byte[][]> startEndKeys) throws IOException { List<LoadQueueItem> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys); if (lqis != null) { @@ -429,10 +426,9 @@ public class TestLoadIncrementalHFilesSplitRecovery { util.getConfiguration()) { int i = 0; - @Override protected List<LoadQueueItem> groupOrSplit( Multimap<ByteBuffer, LoadQueueItem> regionGroups, - final LoadQueueItem item, final Table table, + final LoadQueueItem item, final HTable table, final Pair<byte[][], byte[][]> startEndKeys) throws IOException { i++; http://git-wip-us.apache.org/repos/asf/hbase/blob/173eba81/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java index 3c6bb8f..d8f6b24 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java @@ -75,11 +75,10 @@ public class TestMultiTableInputFormat { TEST_UTIL.startMiniCluster(3); // create and fill table for (int i = 0; i < 3; i++) { - try (HTable table = - TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME + String.valueOf(i)), INPUT_FAMILY)) { - TEST_UTIL.createMultiRegions(TEST_UTIL.getConfiguration(), table, INPUT_FAMILY, 4); - TEST_UTIL.loadTable(table, INPUT_FAMILY, false); - } + HTable table = + TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME + String.valueOf(i)), INPUT_FAMILY); + TEST_UTIL.createMultiRegions(TEST_UTIL.getConfiguration(), table, INPUT_FAMILY, 4); + TEST_UTIL.loadTable(table, INPUT_FAMILY, false); } // start MR cluster TEST_UTIL.startMiniMapReduceCluster(); @@ -139,7 +138,6 @@ public class TestMultiTableInputFormat { private String first = null; private String last = null; - @Override protected void reduce(ImmutableBytesWritable key, Iterable<ImmutableBytesWritable> values, Context context) throws IOException, InterruptedException { @@ -155,7 +153,6 @@ public class TestMultiTableInputFormat { assertEquals(3, count); } - @Override protected void cleanup(Context context) throws IOException, InterruptedException { Configuration c = context.getConfiguration(); http://git-wip-us.apache.org/repos/asf/hbase/blob/173eba81/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java index 086cb80..1f0ab99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestServerCustomProtocol.java @@ -307,93 +307,91 @@ public class TestServerCustomProtocol { @Test public void testSingleMethod() throws Throwable { - try (HTable table = new HTable(util.getConfiguration(), TEST_TABLE)) { - RegionLocator locator = table.getRegionLocator(); - Map<byte [], String> results = table.coprocessorService(PingProtos.PingService.class, - null, ROW_A, - new Batch.Call<PingProtos.PingService, String>() { - @Override - public String call(PingProtos.PingService instance) throws IOException { - BlockingRpcCallback<PingProtos.PingResponse> rpcCallback = - new BlockingRpcCallback<PingProtos.PingResponse>(); - instance.ping(null, PingProtos.PingRequest.newBuilder().build(), rpcCallback); - return rpcCallback.get().getPong(); - } - }); - // Should have gotten results for 1 of the three regions only since we specified - // rows from 1 region - assertEquals(1, results.size()); - verifyRegionResults(locator, results, ROW_A); - - final String name = "NAME"; - results = hello(table, name, null, ROW_A); - // Should have gotten results for 1 of the three regions only since we specified - // rows from 1 region - assertEquals(1, results.size()); - verifyRegionResults(locator, results, "Hello, NAME", ROW_A); - } + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + Map<byte [], String> results = table.coprocessorService(PingProtos.PingService.class, + null, ROW_A, + new Batch.Call<PingProtos.PingService, String>() { + @Override + public String call(PingProtos.PingService instance) throws IOException { + BlockingRpcCallback<PingProtos.PingResponse> rpcCallback = + new BlockingRpcCallback<PingProtos.PingResponse>(); + instance.ping(null, PingProtos.PingRequest.newBuilder().build(), rpcCallback); + return rpcCallback.get().getPong(); + } + }); + // Should have gotten results for 1 of the three regions only since we specified + // rows from 1 region + assertEquals(1, results.size()); + verifyRegionResults(table, results, ROW_A); + + final String name = "NAME"; + results = hello(table, name, null, ROW_A); + // Should have gotten results for 1 of the three regions only since we specified + // rows from 1 region + assertEquals(1, results.size()); + verifyRegionResults(table, results, "Hello, NAME", ROW_A); + table.close(); } @Test public void testRowRange() throws Throwable { - try (HTable table = new HTable(util.getConfiguration(), TEST_TABLE)) { - RegionLocator locator = table.getRegionLocator(); - for (Entry<HRegionInfo, ServerName> e: table.getRegionLocations().entrySet()) { - LOG.info("Region " + e.getKey().getRegionNameAsString() + ", servername=" + e.getValue()); - } - // Here are what regions looked like on a run: - // - // test,,1355943549657.c65d4822d8bdecc033a96451f3a0f55d. - // test,bbb,1355943549661.110393b070dd1ed93441e0bc9b3ffb7e. - // test,ccc,1355943549665.c3d6d125141359cbbd2a43eaff3cdf74. - - Map<byte [], String> results = ping(table, null, ROW_A); - // Should contain first region only. - assertEquals(1, results.size()); - verifyRegionResults(locator, results, ROW_A); - - // Test start row + empty end - results = ping(table, ROW_BC, null); - assertEquals(2, results.size()); - // should contain last 2 regions - HRegionLocation loc = table.getRegionLocation(ROW_A, true); - assertNull("Should be missing region for row aaa (prior to start row)", - results.get(loc.getRegionInfo().getRegionName())); - verifyRegionResults(locator, results, ROW_B); - verifyRegionResults(locator, results, ROW_C); - - // test empty start + end - results = ping(table, null, ROW_BC); - // should contain the first 2 regions - assertEquals(2, results.size()); - verifyRegionResults(locator, results, ROW_A); - verifyRegionResults(locator, results, ROW_B); - loc = table.getRegionLocation(ROW_C, true); - assertNull("Should be missing region for row ccc (past stop row)", - results.get(loc.getRegionInfo().getRegionName())); - - // test explicit start + end - results = ping(table, ROW_AB, ROW_BC); - // should contain first 2 regions - assertEquals(2, results.size()); - verifyRegionResults(locator, results, ROW_A); - verifyRegionResults(locator, results, ROW_B); - loc = table.getRegionLocation(ROW_C, true); - assertNull("Should be missing region for row ccc (past stop row)", - results.get(loc.getRegionInfo().getRegionName())); - - // test single region - results = ping(table, ROW_B, ROW_BC); - // should only contain region bbb - assertEquals(1, results.size()); - verifyRegionResults(locator, results, ROW_B); - loc = table.getRegionLocation(ROW_A, true); - assertNull("Should be missing region for row aaa (prior to start)", - results.get(loc.getRegionInfo().getRegionName())); - loc = table.getRegionLocation(ROW_C, true); - assertNull("Should be missing region for row ccc (past stop row)", - results.get(loc.getRegionInfo().getRegionName())); + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + for (Entry<HRegionInfo, ServerName> e: table.getRegionLocations().entrySet()) { + LOG.info("Region " + e.getKey().getRegionNameAsString() + ", servername=" + e.getValue()); } + // Here are what regions looked like on a run: + // + // test,,1355943549657.c65d4822d8bdecc033a96451f3a0f55d. + // test,bbb,1355943549661.110393b070dd1ed93441e0bc9b3ffb7e. + // test,ccc,1355943549665.c3d6d125141359cbbd2a43eaff3cdf74. + + Map<byte [], String> results = ping(table, null, ROW_A); + // Should contain first region only. + assertEquals(1, results.size()); + verifyRegionResults(table, results, ROW_A); + + // Test start row + empty end + results = ping(table, ROW_BC, null); + assertEquals(2, results.size()); + // should contain last 2 regions + HRegionLocation loc = table.getRegionLocation(ROW_A, true); + assertNull("Should be missing region for row aaa (prior to start row)", + results.get(loc.getRegionInfo().getRegionName())); + verifyRegionResults(table, results, ROW_B); + verifyRegionResults(table, results, ROW_C); + + // test empty start + end + results = ping(table, null, ROW_BC); + // should contain the first 2 regions + assertEquals(2, results.size()); + verifyRegionResults(table, results, ROW_A); + verifyRegionResults(table, results, ROW_B); + loc = table.getRegionLocation(ROW_C, true); + assertNull("Should be missing region for row ccc (past stop row)", + results.get(loc.getRegionInfo().getRegionName())); + + // test explicit start + end + results = ping(table, ROW_AB, ROW_BC); + // should contain first 2 regions + assertEquals(2, results.size()); + verifyRegionResults(table, results, ROW_A); + verifyRegionResults(table, results, ROW_B); + loc = table.getRegionLocation(ROW_C, true); + assertNull("Should be missing region for row ccc (past stop row)", + results.get(loc.getRegionInfo().getRegionName())); + + // test single region + results = ping(table, ROW_B, ROW_BC); + // should only contain region bbb + assertEquals(1, results.size()); + verifyRegionResults(table, results, ROW_B); + loc = table.getRegionLocation(ROW_A, true); + assertNull("Should be missing region for row aaa (prior to start)", + results.get(loc.getRegionInfo().getRegionName())); + loc = table.getRegionLocation(ROW_C, true); + assertNull("Should be missing region for row ccc (past stop row)", + results.get(loc.getRegionInfo().getRegionName())); + table.close(); } private Map<byte [], String> ping(final Table table, final byte [] start, final byte [] end) @@ -416,46 +414,40 @@ public class TestServerCustomProtocol { @Test public void testCompoundCall() throws Throwable { - try (HTable table = new HTable(util.getConfiguration(), TEST_TABLE)) { - RegionLocator locator = table.getRegionLocator(); - Map<byte [], String> results = compoundOfHelloAndPing(table, ROW_A, ROW_C); - verifyRegionResults(locator, results, "Hello, pong", ROW_A); - verifyRegionResults(locator, results, "Hello, pong", ROW_B); - verifyRegionResults(locator, results, "Hello, pong", ROW_C); - } + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + Map<byte [], String> results = compoundOfHelloAndPing(table, ROW_A, ROW_C); + verifyRegionResults(table, results, "Hello, pong", ROW_A); + verifyRegionResults(table, results, "Hello, pong", ROW_B); + verifyRegionResults(table, results, "Hello, pong", ROW_C); + table.close(); } @Test public void testNullCall() throws Throwable { - try(HTable table = new HTable(util.getConfiguration(), TEST_TABLE)) { - RegionLocator locator = table.getRegionLocator(); - Map<byte[],String> results = hello(table, null, ROW_A, ROW_C); - verifyRegionResults(locator, results, "Who are you?", ROW_A); - verifyRegionResults(locator, results, "Who are you?", ROW_B); - verifyRegionResults(locator, results, "Who are you?", ROW_C); - } + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + Map<byte[],String> results = hello(table, null, ROW_A, ROW_C); + verifyRegionResults(table, results, "Who are you?", ROW_A); + verifyRegionResults(table, results, "Who are you?", ROW_B); + verifyRegionResults(table, results, "Who are you?", ROW_C); } @Test public void testNullReturn() throws Throwable { - try (HTable table = new HTable(util.getConfiguration(), TEST_TABLE)) { - RegionLocator locator = table.getRegionLocator(); - Map<byte[],String> results = hello(table, "nobody", ROW_A, ROW_C); - verifyRegionResults(locator, results, null, ROW_A); - verifyRegionResults(locator, results, null, ROW_B); - verifyRegionResults(locator, results, null, ROW_C); - } + HTable table = new HTable(util.getConfiguration(), TEST_TABLE); + Map<byte[],String> results = hello(table, "nobody", ROW_A, ROW_C); + verifyRegionResults(table, results, null, ROW_A); + verifyRegionResults(table, results, null, ROW_B); + verifyRegionResults(table, results, null, ROW_C); } @Test public void testEmptyReturnType() throws Throwable { - try (HTable table = new HTable(util.getConfiguration(), TEST_TABLE)) { - Map<byte[],String> results = noop(table, ROW_A, ROW_C); - assertEquals("Should have results from three regions", 3, results.size()); - // all results should be null - for (Object v : results.values()) { - assertNull(v); - } + Table table = new HTable(util.getConfiguration(), TEST_TABLE); + Map<byte[],String> results = noop(table, ROW_A, ROW_C); + assertEquals("Should have results from three regions", 3, results.size()); + // all results should be null + for (Object v : results.values()) { + assertNull(v); } } @@ -464,7 +456,7 @@ public class TestServerCustomProtocol { verifyRegionResults(table, results, "pong", row); } - private void verifyRegionResults(RegionLocator regionLocator, + private void verifyRegionResults(RegionLocator table, Map<byte[], String> results, String expected, byte[] row) throws Exception { for (Map.Entry<byte [], String> e: results.entrySet()) { @@ -472,7 +464,7 @@ public class TestServerCustomProtocol { ", result key=" + Bytes.toString(e.getKey()) + ", value=" + e.getValue()); } - HRegionLocation loc = regionLocator.getRegionLocation(row, true); + HRegionLocation loc = table.getRegionLocation(row, true); byte[] region = loc.getRegionInfo().getRegionName(); assertTrue("Results should contain region " + Bytes.toStringBinary(region) + " for row '" + Bytes.toStringBinary(row)+ "'",
