HBASE-14688 Cleanup MOB tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c91bfff5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c91bfff5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c91bfff5 Branch: refs/heads/hbase-12439 Commit: c91bfff5862fd38b3d301e3371d2f643d0c501ea Parents: e04e740 Author: Matteo Bertozzi <matteo.berto...@cloudera.com> Authored: Tue Oct 27 10:28:07 2015 -0700 Committer: Matteo Bertozzi <matteo.berto...@cloudera.com> Committed: Tue Oct 27 11:37:39 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/HBaseTestingUtility.java | 7 +- .../apache/hadoop/hbase/mob/MobTestUtil.java | 42 +++-- .../hadoop/hbase/mob/TestCachedMobFile.java | 31 ++-- .../hbase/mob/TestDefaultMobStoreFlusher.java | 181 ++++++------------- .../hbase/mob/TestExpiredMobFileCleaner.java | 8 +- .../hbase/mob/TestMobDataBlockEncoding.java | 19 +- .../apache/hadoop/hbase/mob/TestMobFile.java | 16 +- .../hbase/mob/compactions/TestMobCompactor.java | 28 +-- .../TestPartitionedMobCompactor.java | 95 +++------- .../hbase/regionserver/TestDeleteMobTable.java | 155 +++++++--------- .../regionserver/TestMobStoreCompaction.java | 31 ++-- .../hbase/regionserver/TestMobStoreScanner.java | 114 ++++-------- .../hadoop/hbase/util/BaseTestHBaseFsck.java | 5 +- 13 files changed, 265 insertions(+), 467 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index ab7b51c..eb1494e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -1845,7 +1845,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { // // ========================================================================== - + /** * Provide an existing table name to truncate. * Scans the table and issues a delete for each row read. @@ -2141,7 +2141,10 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * Return the number of rows in the given table. */ public int countRows(final Table table) throws IOException { - Scan scan = new Scan(); + return countRows(table, new Scan()); + } + + public int countRows(final Table table, final Scan scan) throws IOException { ResultScanner results = table.getScanner(scan); int count = 0; for (@SuppressWarnings("unused") Result res : results) { http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java index bc3354b..3e0e2d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java @@ -19,11 +19,16 @@ package org.apache.hadoop.hbase.mob; import java.io.IOException; +import java.util.List; import java.util.Random; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; @@ -73,15 +78,30 @@ public class MobTestUtil { /** * Compare two Cells only for their row family qualifier value */ - public static void assertCellEquals(Cell firstKeyValue, - Cell secondKeyValue) { - Assert.assertEquals(Bytes.toString(CellUtil.cloneRow(firstKeyValue)), - Bytes.toString(CellUtil.cloneRow(secondKeyValue))); - Assert.assertEquals(Bytes.toString(CellUtil.cloneFamily(firstKeyValue)), - Bytes.toString(CellUtil.cloneFamily(secondKeyValue))); - Assert.assertEquals(Bytes.toString(CellUtil.cloneQualifier(firstKeyValue)), - Bytes.toString(CellUtil.cloneQualifier(secondKeyValue))); - Assert.assertEquals(Bytes.toString(CellUtil.cloneValue(firstKeyValue)), - Bytes.toString(CellUtil.cloneValue(secondKeyValue))); - } + public static void assertCellEquals(Cell firstKeyValue, Cell secondKeyValue) { + Assert.assertArrayEquals(CellUtil.cloneRow(firstKeyValue), + CellUtil.cloneRow(secondKeyValue)); + Assert.assertArrayEquals(CellUtil.cloneFamily(firstKeyValue), + CellUtil.cloneFamily(secondKeyValue)); + Assert.assertArrayEquals(CellUtil.cloneQualifier(firstKeyValue), + CellUtil.cloneQualifier(secondKeyValue)); + Assert.assertArrayEquals(CellUtil.cloneValue(firstKeyValue), + CellUtil.cloneValue(secondKeyValue)); + } + + public static void assertCellsValue(Table table, Scan scan, + byte[] expectedValue, int expectedCount) throws IOException { + ResultScanner results = table.getScanner(scan); + int count = 0; + for (Result res : results) { + List<Cell> cells = res.listCells(); + for(Cell cell : cells) { + // Verify the value + Assert.assertArrayEquals(expectedValue, CellUtil.cloneValue(cell)); + count++; + } + } + results.close(); + Assert.assertEquals(expectedCount, count); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java index b38e7cb..4bb525d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -43,10 +44,9 @@ import org.junit.experimental.categories.Category; @Category(SmallTests.class) public class TestCachedMobFile extends TestCase{ static final Log LOG = LogFactory.getLog(TestCachedMobFile.class); - private Configuration conf = HBaseConfiguration.create(); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private Configuration conf = TEST_UTIL.getConfiguration(); private CacheConfig cacheConf = new CacheConfig(conf); - private static final String TABLE = "tableName"; - private static final String FAMILY = "familyName"; private static final String FAMILY1 = "familyName1"; private static final String FAMILY2 = "familyName2"; private static final long EXPECTED_REFERENCE_ZERO = 0; @@ -56,13 +56,11 @@ public class TestCachedMobFile extends TestCase{ @Test public void testOpenClose() throws Exception { String caseName = getName(); - FileSystem fs = FileSystem.get(conf); - Path testDir = FSUtils.getRootDir(conf); - Path outputDir = new Path(new Path(testDir, TABLE), - FAMILY); + Path testDir = TEST_UTIL.getDataTestDir(); + FileSystem fs = testDir.getFileSystem(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build(); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs) - .withOutputDir(outputDir).withFileContext(meta).build(); + .withOutputDir(testDir).withFileContext(meta).build(); MobTestUtil.writeStoreFile(writer, caseName); CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf); Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount()); @@ -79,17 +77,15 @@ public class TestCachedMobFile extends TestCase{ @Test public void testCompare() throws Exception { String caseName = getName(); - FileSystem fs = FileSystem.get(conf); - Path testDir = FSUtils.getRootDir(conf); - Path outputDir1 = new Path(new Path(testDir, TABLE), - FAMILY1); + Path testDir = TEST_UTIL.getDataTestDir(); + FileSystem fs = testDir.getFileSystem(conf); + Path outputDir1 = new Path(testDir, FAMILY1); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); StoreFile.Writer writer1 = new StoreFile.WriterBuilder(conf, cacheConf, fs) .withOutputDir(outputDir1).withFileContext(meta).build(); MobTestUtil.writeStoreFile(writer1, caseName); CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf); - Path outputDir2 = new Path(new Path(testDir, TABLE), - FAMILY2); + Path outputDir2 = new Path(testDir, FAMILY2); StoreFile.Writer writer2 = new StoreFile.WriterBuilder(conf, cacheConf, fs) .withOutputDir(outputDir2) .withFileContext(meta) @@ -105,12 +101,11 @@ public class TestCachedMobFile extends TestCase{ @Test public void testReadKeyValue() throws Exception { - FileSystem fs = FileSystem.get(conf); - Path testDir = FSUtils.getRootDir(conf); - Path outputDir = new Path(new Path(testDir, TABLE), "familyname"); + Path testDir = TEST_UTIL.getDataTestDir(); + FileSystem fs = testDir.getFileSystem(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs) - .withOutputDir(outputDir).withFileContext(meta).build(); + .withOutputDir(testDir).withFileContext(meta).build(); String caseName = getName(); MobTestUtil.writeStoreFile(writer, caseName); CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf); http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java index b91d4d1..94a2ed4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java @@ -18,7 +18,6 @@ */ package org.apache.hadoop.hbase.mob; -import java.io.IOException; import java.util.List; import org.apache.hadoop.hbase.Cell; @@ -26,9 +25,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -64,127 +61,67 @@ public class TestDefaultMobStoreFlusher { } @Test - public void testFlushNonMobFile() throws InterruptedException { - String TN = "testFlushNonMobFile"; - TableName tn = TableName.valueOf(TN); - Table table = null; - HBaseAdmin admin = null; - - try { - HTableDescriptor desc = new HTableDescriptor(tn); - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setMaxVersions(4); - desc.addFamily(hcd); - - admin = TEST_UTIL.getHBaseAdmin(); - admin.createTable(desc); - table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()) - .getTable(TableName.valueOf(TN)); - - //Put data - Put put0 = new Put(row1); - put0.addColumn(family, qf1, 1, value1); - table.put(put0); - - //Put more data - Put put1 = new Put(row2); - put1.addColumn(family, qf2, 1, value2); - table.put(put1); - - //Flush - admin.flush(tn); - - Scan scan = new Scan(); - scan.addColumn(family, qf1); - scan.setMaxVersions(4); - ResultScanner scanner = table.getScanner(scan); - - //Compare - Result result = scanner.next(); - int size = 0; - while (result != null) { - size++; - List<Cell> cells = result.getColumnCells(family, qf1); - // Verify the cell size - Assert.assertEquals(1, cells.size()); - // Verify the value - Assert.assertEquals(Bytes.toString(value1), - Bytes.toString(CellUtil.cloneValue(cells.get(0)))); - result = scanner.next(); - } - scanner.close(); - Assert.assertEquals(1, size); - admin.close(); - } catch (MasterNotRunningException e1) { - e1.printStackTrace(); - } catch (ZooKeeperConnectionException e2) { - e2.printStackTrace(); - } catch (IOException e3) { - e3.printStackTrace(); - } + public void testFlushNonMobFile() throws Exception { + TableName tn = TableName.valueOf("testFlushNonMobFile"); + HTableDescriptor desc = new HTableDescriptor(tn); + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setMaxVersions(4); + desc.addFamily(hcd); + + testFlushFile(desc); } @Test - public void testFlushMobFile() throws InterruptedException { - String TN = "testFlushMobFile"; - TableName tn = TableName.valueOf(TN); - Table table = null; - Admin admin = null; - - try { - HTableDescriptor desc = new HTableDescriptor(tn); - HColumnDescriptor hcd = new HColumnDescriptor(family); - hcd.setMobEnabled(true); - hcd.setMobThreshold(3L); - hcd.setMaxVersions(4); - desc.addFamily(hcd); - - Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - admin = c.getAdmin(); - admin.createTable(desc); - table = c.getTable(TableName.valueOf(TN)); - - //put data - Put put0 = new Put(row1); - put0.addColumn(family, qf1, 1, value1); - table.put(put0); - - //put more data - Put put1 = new Put(row2); - put1.addColumn(family, qf2, 1, value2); - table.put(put1); - - //flush - admin.flush(tn); - - //Scan - Scan scan = new Scan(); - scan.addColumn(family, qf1); - scan.setMaxVersions(4); - ResultScanner scanner = table.getScanner(scan); - - //Compare - Result result = scanner.next(); - int size = 0; - while (result != null) { - size++; - List<Cell> cells = result.getColumnCells(family, qf1); - // Verify the the cell size - Assert.assertEquals(1, cells.size()); - // Verify the value - Assert.assertEquals(Bytes.toString(value1), - Bytes.toString(CellUtil.cloneValue(cells.get(0)))); - result = scanner.next(); - } - scanner.close(); - Assert.assertEquals(1, size); - admin.close(); - } catch (MasterNotRunningException e1) { - e1.printStackTrace(); - } catch (ZooKeeperConnectionException e2) { - e2.printStackTrace(); - } catch (IOException e3) { - e3.printStackTrace(); - } + public void testFlushMobFile() throws Exception { + TableName tn = TableName.valueOf("testFlushMobFile"); + HTableDescriptor desc = new HTableDescriptor(tn); + HColumnDescriptor hcd = new HColumnDescriptor(family); + hcd.setMobEnabled(true); + hcd.setMobThreshold(3L); + hcd.setMaxVersions(4); + desc.addFamily(hcd); + + testFlushFile(desc); } + + private void testFlushFile(HTableDescriptor htd) throws Exception { + Table table = null; + try { + table = TEST_UTIL.createTable(htd, null); + + //put data + Put put0 = new Put(row1); + put0.addColumn(family, qf1, 1, value1); + table.put(put0); + + //put more data + Put put1 = new Put(row2); + put1.addColumn(family, qf2, 1, value2); + table.put(put1); + + //flush + TEST_UTIL.flush(htd.getTableName()); + + //Scan + Scan scan = new Scan(); + scan.addColumn(family, qf1); + scan.setMaxVersions(4); + ResultScanner scanner = table.getScanner(scan); + + //Compare + int size = 0; + for (Result result: scanner) { + size++; + List<Cell> cells = result.getColumnCells(family, qf1); + // Verify the cell size + Assert.assertEquals(1, cells.size()); + // Verify the value + Assert.assertArrayEquals(value1, CellUtil.cloneValue(cells.get(0))); + } + scanner.close(); + Assert.assertEquals(1, size); + } finally { + table.close(); + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java index dfaeca6..fff6f44 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestExpiredMobFileCleaner.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertEquals; import java.util.Random; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -124,7 +123,7 @@ public class TestExpiredMobFileCleaner { public void testCleaner() throws Exception { init(); - Path mobDirPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); + Path mobDirPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family); byte[] dummyData = makeDummyData(600); long ts = System.currentTimeMillis() - 3 * secondsOfDay() * 1000; // 3 days before @@ -158,11 +157,6 @@ public class TestExpiredMobFileCleaner { assertEquals("After cleanup without delay 2", secondFile, lastFile); } - private Path getMobFamilyPath(Configuration conf, TableName tableName, String familyName) { - Path p = new Path(MobUtils.getMobRegionPath(conf, tableName), familyName); - return p; - } - private int secondsOfDay() { return 24 * 3600; } http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java index 15aa7d4..8ad6f95 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java @@ -18,11 +18,8 @@ */ package org.apache.hadoop.hbase.mob; -import java.util.List; import java.util.Random; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; @@ -32,7 +29,6 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -117,19 +113,6 @@ public class TestMobDataBlockEncoding { Scan scan = new Scan(); scan.setMaxVersions(4); - - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - List<Cell> cells = res.listCells(); - for(Cell cell : cells) { - // Verify the value - Assert.assertEquals(Bytes.toString(value), - Bytes.toString(CellUtil.cloneValue(cell))); - count++; - } - } - results.close(); - Assert.assertEquals(3, count); + MobTestUtil.assertCellsValue(table, scan, value, 3); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java index d05da24..e2dced9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java @@ -47,17 +47,14 @@ public class TestMobFile extends TestCase { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private Configuration conf = TEST_UTIL.getConfiguration(); private CacheConfig cacheConf = new CacheConfig(conf); - private final String TABLE = "tableName"; - private final String FAMILY = "familyName"; @Test public void testReadKeyValue() throws Exception { - FileSystem fs = FileSystem.get(conf); - Path testDir = FSUtils.getRootDir(conf); - Path outputDir = new Path(new Path(testDir, TABLE), FAMILY); + Path testDir = TEST_UTIL.getDataTestDir(); + FileSystem fs = testDir.getFileSystem(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build(); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs) - .withOutputDir(outputDir) + .withOutputDir(testDir) .withFileContext(meta) .build(); String caseName = getName(); @@ -106,12 +103,11 @@ public class TestMobFile extends TestCase { @Test public void testGetScanner() throws Exception { - FileSystem fs = FileSystem.get(conf); - Path testDir = FSUtils.getRootDir(conf); - Path outputDir = new Path(new Path(testDir, TABLE), FAMILY); + Path testDir = TEST_UTIL.getDataTestDir(); + FileSystem fs = testDir.getFileSystem(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build(); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs) - .withOutputDir(outputDir) + .withOutputDir(testDir) .withFileContext(meta) .build(); MobTestUtil.writeStoreFile(writer, getName()); http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 6da7655..9922aff 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -413,8 +413,8 @@ public class TestMobCompactor { result = table.get(get); cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1)); // the ref name is the new file - Path mobFamilyPath = new Path( - MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString()); + Path mobFamilyPath = + MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, hcd1.getNameAsString()); List<Path> paths = new ArrayList<Path>(); if (fs.exists(mobFamilyPath)) { FileStatus[] files = fs.listStatus(mobFamilyPath); @@ -495,13 +495,7 @@ public class TestMobCompactor { Scan scan = new Scan(); // Do not retrieve the mob data when scanning scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - count++; - } - results.close(); - return count; + return TEST_UTIL.countRows(table, scan); } /** @@ -532,8 +526,7 @@ public class TestMobCompactor { */ private int countFiles(TableName tableName, boolean isMobFile, String familyName) throws IOException { - Path mobDirPath = MobUtils.getMobFamilyPath( - MobUtils.getMobRegionPath(conf, tableName), familyName); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); int count = 0; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); @@ -553,8 +546,7 @@ public class TestMobCompactor { } private boolean verifyEncryption(TableName tableName, String familyName) throws IOException { - Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName), - familyName); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); boolean hasFiles = false; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); @@ -579,8 +571,7 @@ public class TestMobCompactor { * @return the number of the HFileLink */ private int countHFileLinks(String familyName) throws IOException { - Path mobDirPath = MobUtils.getMobFamilyPath( - MobUtils.getMobRegionPath(conf, tableName), familyName); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); int count = 0; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); @@ -601,8 +592,7 @@ public class TestMobCompactor { * @return the number of files large than the size */ private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException { - Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName), - familyName); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); int count = 0; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); @@ -729,8 +719,8 @@ public class TestMobCompactor { // Do not retrieve the mob data when scanning scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); ResultScanner results = table.getScanner(scan); - Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), - tableName), familyName); + Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), + tableName, familyName); List<Path> actualFilePaths = new ArrayList<>(); List<Path> expectFilePaths = new ArrayList<>(); for (Result res : results) { http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java index aba63c6..6774ac3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java @@ -104,56 +104,27 @@ public class TestPartitionedMobCompactor { @Test public void testCompactionSelectWithAllFiles() throws Exception { - resetConf(); String tableName = "testCompactionSelectWithAllFiles"; - init(tableName); - int count = 10; - // create 10 mob files. - createStoreFiles(basePath, family, qf, count, Type.Put); - // create 10 del files - createStoreFiles(basePath, family, qf, count, Type.Delete); - listFiles(); - long mergeSize = MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD; - List<String> expectedStartKeys = new ArrayList<>(); - for(FileStatus file : mobFiles) { - if(file.getLen() < mergeSize) { - String fileName = file.getPath().getName(); - String startKey = fileName.substring(0, 32); - expectedStartKeys.add(startKey); - } - } - testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys); + testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD, + CompactionType.ALL_FILES, false); } @Test public void testCompactionSelectWithPartFiles() throws Exception { - resetConf(); String tableName = "testCompactionSelectWithPartFiles"; - init(tableName); - int count = 10; - // create 10 mob files. - createStoreFiles(basePath, family, qf, count, Type.Put); - // create 10 del files - createStoreFiles(basePath, family, qf, count, Type.Delete); - listFiles(); - long mergeSize = 4000; - List<String> expectedStartKeys = new ArrayList<>(); - for(FileStatus file : mobFiles) { - if(file.getLen() < 4000) { - String fileName = file.getPath().getName(); - String startKey = fileName.substring(0, 32); - expectedStartKeys.add(startKey); - } - } - // set the mob compaction mergeable threshold - conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); - testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys); + testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false); } @Test public void testCompactionSelectWithForceAllFiles() throws Exception { - resetConf(); String tableName = "testCompactionSelectWithForceAllFiles"; + testCompactionAtMergeSize(tableName, Long.MAX_VALUE, CompactionType.ALL_FILES, true); + } + + private void testCompactionAtMergeSize(final String tableName, + final long mergeSize, final CompactionType type, final boolean isForceAllFiles) + throws Exception { + resetConf(); init(tableName); int count = 10; // create 10 mob files. @@ -161,51 +132,41 @@ public class TestPartitionedMobCompactor { // create 10 del files createStoreFiles(basePath, family, qf, count, Type.Delete); listFiles(); - long mergeSize = 4000; List<String> expectedStartKeys = new ArrayList<>(); for(FileStatus file : mobFiles) { - String fileName = file.getPath().getName(); - String startKey = fileName.substring(0, 32); - expectedStartKeys.add(startKey); + if(file.getLen() < mergeSize) { + String fileName = file.getPath().getName(); + String startKey = fileName.substring(0, 32); + expectedStartKeys.add(startKey); + } } // set the mob compaction mergeable threshold conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize); - testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys); + testSelectFiles(tableName, type, isForceAllFiles, expectedStartKeys); } @Test public void testCompactDelFilesWithDefaultBatchSize() throws Exception { - resetConf(); String tableName = "testCompactDelFilesWithDefaultBatchSize"; - init(tableName); - // create 20 mob files. - createStoreFiles(basePath, family, qf, 20, Type.Put); - // create 13 del files - createStoreFiles(basePath, family, qf, 13, Type.Delete); - listFiles(); - testCompactDelFiles(tableName, 1, 13, false); + testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE, + MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); } @Test public void testCompactDelFilesWithSmallBatchSize() throws Exception { - resetConf(); String tableName = "testCompactDelFilesWithSmallBatchSize"; - init(tableName); - // create 20 mob files. - createStoreFiles(basePath, family, qf, 20, Type.Put); - // create 13 del files - createStoreFiles(basePath, family, qf, 13, Type.Delete); - listFiles(); - - // set the mob compaction batch size - conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 4); - testCompactDelFiles(tableName, 1, 13, false); + testCompactDelFilesAtBatchSize(tableName, 4, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT); } @Test public void testCompactDelFilesChangeMaxDelFileCount() throws Exception { - resetConf(); String tableName = "testCompactDelFilesWithSmallBatchSize"; + testCompactDelFilesAtBatchSize(tableName, 4, 2); + } + + private void testCompactDelFilesAtBatchSize(String tableName, int batchSize, + int delfileMaxCount) throws Exception { + resetConf(); init(tableName); // create 20 mob files. createStoreFiles(basePath, family, qf, 20, Type.Put); @@ -214,10 +175,10 @@ public class TestPartitionedMobCompactor { listFiles(); // set the max del file count - conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5); + conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, delfileMaxCount); // set the mob compaction batch size - conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 2); - testCompactDelFiles(tableName, 4, 13, false); + conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize); + testCompactDelFiles(tableName, 1, 13, false); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java index 6312653..779fdca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; @@ -74,132 +73,110 @@ public class TestDeleteMobTable { return mobVal; } - @Test - public void testDeleteMobTable() throws Exception { - byte[] tableName = Bytes.toBytes("testDeleteMobTable"); - TableName tn = TableName.valueOf(tableName); - HTableDescriptor htd = new HTableDescriptor(tn); + private HTableDescriptor createTableDescriptor(TableName tableName, boolean hasMob) { + HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); + if (hasMob) { + hcd.setMobEnabled(true); + hcd.setMobThreshold(0); + } htd.addFamily(hcd); - HBaseAdmin admin = null; - Table table = null; + return htd; + } + + private Table createTableWithOneFile(HTableDescriptor htd) throws IOException { + Table table = TEST_UTIL.createTable(htd, null); try { - admin = TEST_UTIL.getHBaseAdmin(); - admin.createTable(htd); - table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn); + // insert data byte[] value = generateMobValue(10); - byte[] row = Bytes.toBytes("row"); Put put = new Put(row); put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value); table.put(put); - admin.flush(tn); + // create an hfile + TEST_UTIL.getHBaseAdmin().flush(htd.getTableName()); + } catch (IOException e) { + table.close(); + throw e; + } + return table; + } + + @Test + public void testDeleteMobTable() throws Exception { + TableName tn = TableName.valueOf("testDeleteMobTable"); + HTableDescriptor htd = createTableDescriptor(tn, true); + HColumnDescriptor hcd = htd.getFamily(FAMILY); + String fileName = null; + Table table = createTableWithOneFile(htd); + try { // the mob file exists Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString())); - String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString()); + fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString()); Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName)); Assert.assertTrue(mobTableDirExist(tn)); - table.close(); - - admin.disableTable(tn); - admin.deleteTable(tn); - - Assert.assertFalse(admin.tableExists(tn)); - Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString())); - Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString())); - Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName)); - Assert.assertFalse(mobTableDirExist(tn)); } finally { - if (admin != null) { - admin.close(); - } + table.close(); + TEST_UTIL.deleteTable(tn); } + + Assert.assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(tn)); + Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString())); + Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString())); + Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName)); + Assert.assertFalse(mobTableDirExist(tn)); } @Test public void testDeleteNonMobTable() throws Exception { - byte[] tableName = Bytes.toBytes("testDeleteNonMobTable"); - TableName tn = TableName.valueOf(tableName); - HTableDescriptor htd = new HTableDescriptor(tn); - HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); - htd.addFamily(hcd); - HBaseAdmin admin = null; - Table table = null; - try { - admin = TEST_UTIL.getHBaseAdmin(); - admin.createTable(htd); - table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn); - byte[] value = generateMobValue(10); - - byte[] row = Bytes.toBytes("row"); - Put put = new Put(row); - put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value); - table.put(put); - - admin.flush(tn); - table.close(); + TableName tn = TableName.valueOf("testDeleteNonMobTable"); + HTableDescriptor htd = createTableDescriptor(tn, false); + HColumnDescriptor hcd = htd.getFamily(FAMILY); + Table table = createTableWithOneFile(htd); + try { // the mob file doesn't exist Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString())); Assert.assertFalse(mobTableDirExist(tn)); - - admin.disableTable(tn); - admin.deleteTable(tn); - - Assert.assertFalse(admin.tableExists(tn)); - Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString())); - Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString())); - Assert.assertFalse(mobTableDirExist(tn)); } finally { - if (admin != null) { - admin.close(); - } + table.close(); + TEST_UTIL.deleteTable(tn); } + + Assert.assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(tn)); + Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString())); + Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString())); + Assert.assertFalse(mobTableDirExist(tn)); } @Test public void testMobFamilyDelete() throws Exception { TableName tn = TableName.valueOf("testMobFamilyDelete"); - HTableDescriptor htd = new HTableDescriptor(tn); - HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); - hcd.setMobEnabled(true); - hcd.setMobThreshold(0); - htd.addFamily(hcd); + HTableDescriptor htd = createTableDescriptor(tn, true); + HColumnDescriptor hcd = htd.getFamily(FAMILY); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("family2"))); - HBaseAdmin admin = null; - Table table = null; + + Table table = createTableWithOneFile(htd); try { - admin = TEST_UTIL.getHBaseAdmin(); - admin.createTable(htd); - table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()).getTable(tn); - byte[] value = generateMobValue(10); - byte[] row = Bytes.toBytes("row"); - Put put = new Put(row); - put.addColumn(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value); - table.put(put); - admin.flush(tn); // the mob file exists Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString())); Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString())); String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString()); Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName)); Assert.assertTrue(mobTableDirExist(tn)); - admin.deleteColumnFamily(tn, FAMILY); + + TEST_UTIL.getHBaseAdmin().deleteColumnFamily(tn, FAMILY); + Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString())); Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString())); Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName)); - Assert.assertFalse(mobColumnFamilyDirExist(tn)); + Assert.assertFalse(mobColumnFamilyDirExist(tn, hcd.getNameAsString())); } finally { table.close(); - if (admin != null) { - admin.close(); - } TEST_UTIL.deleteTable(tn); } } @@ -209,9 +186,8 @@ public class TestDeleteMobTable { Path mobFileDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName); if (fs.exists(mobFileDir)) { return fs.listStatus(mobFileDir).length; - } else { - return 0; } + return 0; } private int countArchiveMobFiles(TableName tn, String familyName) @@ -221,9 +197,8 @@ public class TestDeleteMobTable { MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName); if (fs.exists(storePath)) { return fs.listStatus(storePath).length; - } else { - return 0; } + return 0; } private boolean mobTableDirExist(TableName tn) throws IOException { @@ -232,12 +207,9 @@ public class TestDeleteMobTable { return fs.exists(tableDir); } - private boolean mobColumnFamilyDirExist(TableName tn) throws IOException { + private boolean mobColumnFamilyDirExist(TableName tn, String familyName) throws IOException { FileSystem fs = TEST_UTIL.getTestFileSystem(); - Path tableDir = FSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn); - HRegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tn); - Path mobFamilyDir = new Path(tableDir, new Path(mobRegionInfo.getEncodedName(), - Bytes.toString(FAMILY))); + Path mobFamilyDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName); return fs.exists(mobFamilyDir); } @@ -256,8 +228,7 @@ public class TestDeleteMobTable { ResultScanner rs = table.getScanner(scan); Result r = rs.next(); Assert.assertNotNull(r); - byte[] value = r.getValue(FAMILY, QF); - String fileName = Bytes.toString(value, Bytes.SIZEOF_INT, value.length - Bytes.SIZEOF_INT); + String fileName = MobUtils.getMobFileName(r.getColumnLatestCell(FAMILY, QF)); Path filePath = new Path( MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName), fileName); FileSystem fs = TEST_UTIL.getTestFileSystem(); http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java index 7cac301..f9ffc88 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java @@ -278,8 +278,7 @@ public class TestMobStoreCompaction { } private int countMobFiles() throws IOException { - Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()), - hcd.getNameAsString()); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); if (fs.exists(mobDirPath)) { FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath); return files.length; @@ -289,8 +288,7 @@ public class TestMobStoreCompaction { private long countMobCellsInMetadata() throws IOException { long mobCellsCount = 0; - Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()), - hcd.getNameAsString()); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); @@ -357,9 +355,16 @@ public class TestMobStoreCompaction { private int countRows() throws IOException { Scan scan = new Scan(); - // Do not retrieve the mob data when scanning InternalScanner scanner = region.getScanner(scan); + try { + return countRows(scanner); + } finally { + scanner.close(); + } + } + private int countRows(InternalScanner scanner) throws IOException { + // Do not retrieve the mob data when scanning int scannedCount = 0; List<Cell> results = new ArrayList<Cell>(); boolean hasMore = true; @@ -368,8 +373,6 @@ public class TestMobStoreCompaction { scannedCount += results.size(); results.clear(); } - scanner.close(); - return scannedCount; } @@ -423,8 +426,7 @@ public class TestMobStoreCompaction { Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); - Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()), - hcd.getNameAsString()); + Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); List<StoreFile> sfs = new ArrayList<>(); int numDelfiles = 0; int size = 0; @@ -436,6 +438,7 @@ public class TestMobStoreCompaction { numDelfiles++; } } + List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, false, HConstants.LATEST_TIMESTAMP); Scan scan = new Scan(); @@ -446,12 +449,10 @@ public class TestMobStoreCompaction { CellComparator.COMPARATOR); StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null, scanners, 0L, HConstants.LATEST_TIMESTAMP); - List<Cell> results = new ArrayList<>(); - boolean hasMore = true; - while (hasMore) { - hasMore = scanner.next(results); - size += results.size(); - results.clear(); + try { + size += countRows(scanner); + } finally { + scanner.close(); } } // assert the number of the existing del files http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java index 8250b47..f3af63a4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.io.hfile.CorruptHFileException; import org.apache.hadoop.hbase.io.hfile.TestHFile; import org.apache.hadoop.hbase.mob.MobConstants; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -196,12 +197,12 @@ public class TestMobStoreScanner { table.put(put4); Result result = rs.next(); Cell cell = result.getColumnLatestCell(family, qf1); - Assert.assertEquals("value1", Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertArrayEquals(value1, CellUtil.cloneValue(cell)); admin.flush(tn); result = rs.next(); cell = result.getColumnLatestCell(family, qf1); - Assert.assertEquals("value2", Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertArrayEquals(value2, CellUtil.cloneValue(cell)); } @Test @@ -213,7 +214,7 @@ public class TestMobStoreScanner { get.setAttribute(MobConstants.EMPTY_VALUE_ON_MOBCELL_MISS, Bytes.toBytes(true)); Result result = table.get(get); Cell cell = result.getColumnLatestCell(family, qf1); - Assert.assertEquals(0, CellUtil.cloneValue(cell).length); + Assert.assertEquals(0, cell.getValueLength()); } @Test @@ -249,8 +250,7 @@ public class TestMobStoreScanner { private Path getFlushedMobFile(Configuration conf, FileSystem fs, TableName table, String family) throws IOException { - Path regionDir = MobUtils.getMobRegionPath(conf, table); - Path famDir = new Path(regionDir, family); + Path famDir = MobUtils.getMobFamilyPath(conf, table, family); FileStatus[] hfFss = fs.listStatus(famDir); for (FileStatus hfs : hfFss) { if (!hfs.isDirectory()) { @@ -262,44 +262,21 @@ public class TestMobStoreScanner { private void testGetFromFiles(boolean reversed) throws Exception { TableName tn = TableName.valueOf("testGetFromFiles" + reversed); - setUp(defaultThreshold, tn); - long ts1 = System.currentTimeMillis(); - long ts2 = ts1 + 1; - long ts3 = ts1 + 2; - byte [] value = generateMobValue((int)defaultThreshold+1); - - Put put1 = new Put(row1); - put1.addColumn(family, qf1, ts3, value); - put1.addColumn(family, qf2, ts2, value); - put1.addColumn(family, qf3, ts1, value); - table.put(put1); - - admin.flush(tn); - - Scan scan = new Scan(); - setScan(scan, reversed, false); - - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - List<Cell> cells = res.listCells(); - for(Cell cell : cells) { - // Verify the value - Assert.assertEquals(Bytes.toString(value), - Bytes.toString(CellUtil.cloneValue(cell))); - count++; - } - } - results.close(); - Assert.assertEquals(3, count); + testGet(tn, reversed, true); } private void testGetFromMemStore(boolean reversed) throws Exception { - setUp(defaultThreshold, TableName.valueOf("testGetFromMemStore" + reversed)); + TableName tn = TableName.valueOf("testGetFromMemStore" + reversed); + testGet(tn, reversed, false); + } + + private void testGet(TableName tableName, boolean reversed, boolean doFlush) + throws Exception { + setUp(defaultThreshold, tableName); long ts1 = System.currentTimeMillis(); long ts2 = ts1 + 1; long ts3 = ts1 + 2; - byte [] value = generateMobValue((int)defaultThreshold+1);; + byte [] value = generateMobValue((int)defaultThreshold+1); Put put1 = new Put(row1); put1.addColumn(family, qf1, ts3, value); @@ -307,22 +284,13 @@ public class TestMobStoreScanner { put1.addColumn(family, qf3, ts1, value); table.put(put1); + if (doFlush) { + admin.flush(tableName); + } + Scan scan = new Scan(); setScan(scan, reversed, false); - - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - List<Cell> cells = res.listCells(); - for(Cell cell : cells) { - // Verify the value - Assert.assertEquals(Bytes.toString(value), - Bytes.toString(CellUtil.cloneValue(cell))); - count++; - } - } - results.close(); - Assert.assertEquals(3, count); + MobTestUtil.assertCellsValue(table, scan, value, 3); } private void testGetReferences(boolean reversed) throws Exception { @@ -426,8 +394,8 @@ public class TestMobStoreScanner { // Get the files in the mob path Path mobFamilyPath; - mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tn), - hcd.getNameAsString()); + mobFamilyPath = MobUtils.getMobFamilyPath( + TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString()); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); FileStatus[] files = fs.listStatus(mobFamilyPath); @@ -458,19 +426,7 @@ public class TestMobStoreScanner { // Scan from archive Scan scan = new Scan(); setScan(scan, reversed, false); - ResultScanner results = table.getScanner(scan); - int count = 0; - for (Result res : results) { - List<Cell> cells = res.listCells(); - for(Cell cell : cells) { - // Verify the value - Assert.assertEquals(Bytes.toString(value), - Bytes.toString(CellUtil.cloneValue(cell))); - count++; - } - } - results.close(); - Assert.assertEquals(3, count); + MobTestUtil.assertCellsValue(table, scan, value, 3); } /** @@ -478,12 +434,9 @@ public class TestMobStoreScanner { */ private static void assertNotMobReference(Cell cell, byte[] row, byte[] family, byte[] value) throws IOException { - Assert.assertEquals(Bytes.toString(row), - Bytes.toString(CellUtil.cloneRow(cell))); - Assert.assertEquals(Bytes.toString(family), - Bytes.toString(CellUtil.cloneFamily(cell))); - Assert.assertTrue(Bytes.toString(value).equals( - Bytes.toString(CellUtil.cloneValue(cell)))); + Assert.assertArrayEquals(row, CellUtil.cloneRow(cell)); + Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell)); + Assert.assertArrayEquals(value, CellUtil.cloneValue(cell)); } /** @@ -491,20 +444,15 @@ public class TestMobStoreScanner { */ private static void assertIsMobReference(Cell cell, byte[] row, byte[] family, byte[] value, TableName tn) throws IOException { - Assert.assertEquals(Bytes.toString(row), - Bytes.toString(CellUtil.cloneRow(cell))); - Assert.assertEquals(Bytes.toString(family), - Bytes.toString(CellUtil.cloneFamily(cell))); - Assert.assertFalse(Bytes.toString(value).equals( - Bytes.toString(CellUtil.cloneValue(cell)))); + Assert.assertArrayEquals(row, CellUtil.cloneRow(cell)); + Assert.assertArrayEquals(family, CellUtil.cloneFamily(cell)); + Assert.assertFalse(Bytes.equals(value, CellUtil.cloneValue(cell))); byte[] referenceValue = CellUtil.cloneValue(cell); - String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT, - referenceValue.length - Bytes.SIZEOF_INT); + String fileName = MobUtils.getMobFileName(cell); int valLen = Bytes.toInt(referenceValue, 0, Bytes.SIZEOF_INT); Assert.assertEquals(value.length, valLen); - Path mobFamilyPath; - mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), - tn), hcd.getNameAsString()); + Path mobFamilyPath = MobUtils.getMobFamilyPath( + TEST_UTIL.getConfiguration(), tn, hcd.getNameAsString()); Path targetPath = new Path(mobFamilyPath, fileName); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); Assert.assertTrue(fs.exists(targetPath)); http://git-wip-us.apache.org/repos/asf/hbase/blob/c91bfff5/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java index f90a8c9..7459a7d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/BaseTestHBaseFsck.java @@ -131,7 +131,7 @@ public class BaseTestHBaseFsck { Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"), Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") }; - + /** * Create a new region in META. */ @@ -633,8 +633,7 @@ public class BaseTestHBaseFsck { * @throws IOException */ Path getFlushedMobFile(FileSystem fs, TableName table) throws IOException { - Path regionDir = MobUtils.getMobRegionPath(conf, table); - Path famDir = new Path(regionDir, FAM_STR); + Path famDir = MobUtils.getMobFamilyPath(conf, table, FAM_STR); // keep doing this until we get a legit hfile while (true) {