Repository: hbase Updated Branches: refs/heads/master 7adf59010 -> b723ce105
HBASE-21194 Add tests in TestCopyTable which exercises MOB feature Signed-off-by: tedyu <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b723ce10 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b723ce10 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b723ce10 Branch: refs/heads/master Commit: b723ce105174825a19b391e7d64ac2b32d0aad63 Parents: 7adf590 Author: Artem Ervits <[email protected]> Authored: Fri Oct 19 14:25:25 2018 -0400 Committer: tedyu <[email protected]> Committed: Sat Oct 20 08:10:27 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/mapreduce/TestCopyTable.java | 180 ++++++++++++++----- .../hadoop/hbase/HBaseTestingUtility.java | 6 +- .../apache/hadoop/hbase/mob/MobTestUtil.java | 13 ++ .../hbase/mob/compactions/TestMobCompactor.java | 39 ++-- 4 files changed, 164 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java ---------------------------------------------------------------------- diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java index 6f41671..ed6857d 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java @@ -30,15 +30,20 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.LauncherSecurityManager; import org.apache.hadoop.util.ToolRunner; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -100,7 +105,7 @@ public class TestCopyTable { if (bulkload) { code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + tableName2.getNameAsString(), - "--bulkload", tableName1.getNameAsString() }); + "--bulkload", tableName1.getNameAsString() }); } else { code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + tableName2.getNameAsString(), @@ -121,9 +126,74 @@ public class TestCopyTable { } } + private void doCopyTableTestWithMob(boolean bulkload) throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("mob"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + + ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); + + cfd.setMobEnabled(true); + cfd.setMobThreshold(5); + TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1) + .setColumnFamily(cfd.build()) + .build(); + TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2) + .setColumnFamily(cfd.build()) + .build(); + + try (Table t1 = TEST_UTIL.createTable(desc1, null); + Table t2 = TEST_UTIL.createTable(desc2, null);) { + + // put rows into the first table + for (int i = 0; i < 10; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + } + + CopyTable copy = new CopyTable(); + + int code; + if (bulkload) { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + "--bulkload", tableName1.getNameAsString() }); + } else { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + tableName1.getNameAsString() }); + } + assertEquals("copy job failed", 0, code); + + // verify the data was copied into table 2 + for (int i = 0; i < 10; i++) { + Get g = new Get(Bytes.toBytes("row" + i)); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + assertEquals("compare row values between two tables", + t1.getDescriptor().getValue("row" + i), + t2.getDescriptor().getValue("row" + i)); + } + + assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1), + MobTestUtil.countMobRows(t2)); + assertEquals("compare count of mob row values between two tables", + t1.getDescriptor().getValues().size(), + t2.getDescriptor().getValues().size()); + assertTrue("The mob row count is 0 but should be > 0", + MobTestUtil.countMobRows(t2) > 0); + + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } + } + /** * Simple end-to-end test - * @throws Exception */ @Test public void testCopyTable() throws Exception { @@ -138,56 +208,70 @@ public class TestCopyTable { doCopyTableTest(true); } + /** + * Simple end-to-end test on table with MOB + */ + @Test + public void testCopyTableWithMob() throws Exception { + doCopyTableTestWithMob(false); + } + + /** + * Simple end-to-end test with bulkload on table with MOB. + */ + @Test + public void testCopyTableWithBulkloadWithMob() throws Exception { + doCopyTableTestWithMob(true); + } + @Test public void testStartStopRow() throws Exception { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); - final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0"); - final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1"); - final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2"); - - Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); - Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); - - // put rows into the first table - Put p = new Put(ROW0); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - p = new Put(ROW1); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - p = new Put(ROW2); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - - CopyTable copy = new CopyTable(); - assertEquals( - 0, - ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), - copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1", - "--stoprow=\\x01row2", tableName1.getNameAsString() })); - - // verify the data was copied into table 2 - // row1 exist, row0, row2 do not exist - Get g = new Get(ROW1); - Result r = t2.get(g); - assertEquals(1, r.size()); - assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); - - g = new Get(ROW0); - r = t2.get(g); - assertEquals(0, r.size()); - - g = new Get(ROW2); - r = t2.get(g); - assertEquals(0, r.size()); - - t1.close(); - t2.close(); - TEST_UTIL.deleteTable(tableName1); - TEST_UTIL.deleteTable(tableName2); + final byte[] row0 = Bytes.toBytesBinary("\\x01row0"); + final byte[] row1 = Bytes.toBytesBinary("\\x01row1"); + final byte[] row2 = Bytes.toBytesBinary("\\x01row2"); + + try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { + + // put rows into the first table + Put p = new Put(row0); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + p = new Put(row1); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + p = new Put(row2); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + + CopyTable copy = new CopyTable(); + assertEquals(0, ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[]{"--new.name=" + tableName2, "--startrow=\\x01row1", + "--stoprow=\\x01row2", tableName1.getNameAsString()})); + + // verify the data was copied into table 2 + // row1 exist, row0, row2 do not exist + Get g = new Get(row1); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + + g = new Get(row0); + r = t2.get(g); + assertEquals(0, r.size()); + + g = new Get(row2); + r = t2.get(g); + assertEquals(0, r.size()); + + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } } /** @@ -215,8 +299,8 @@ public class TestCopyTable { long currentTime = System.currentTimeMillis(); String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells", - "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000), - "--versions=1", sourceTable.getNameAsString() }; + "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000), + "--versions=1", sourceTable.getNameAsString() }; assertNull(t2.get(new Get(ROW1)).getRow()); assertTrue(runCopy(args)); http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index ab9a799..695daea 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2310,12 +2310,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * Return the number of rows in the given table. + * @param table to count rows + * @return count of rows */ - public int countRows(final Table table) throws IOException { + public static int countRows(final Table table) throws IOException { return countRows(table, new Scan()); } - public int countRows(final Table table, final Scan scan) throws IOException { + public static int countRows(final Table table, final Scan scan) throws IOException { try (ResultScanner results = table.getScanner(scan)) { int count = 0; while (results.next() != null) { http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java index 76cb664..8972dba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java @@ -24,6 +24,7 @@ import java.util.Random; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -104,4 +105,16 @@ public class MobTestUtil { results.close(); Assert.assertEquals(expectedCount, count); } + + /** + * Gets the number of rows in the given table. + * @param table to get the scanner + * @return the number of rows + */ + public static int countMobRows(final Table table) throws IOException { + Scan scan = new Scan(); + // Do not retrieve the mob data when scanning + scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); + return HBaseTestingUtility.countRows(table, scan); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/b723ce10/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 8ed4fbb..86e07d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobFileName; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -295,7 +296,7 @@ public class TestMobCompactor { int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, @@ -305,7 +306,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -322,7 +323,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); // After the compaction, the files smaller than the mob compaction merge size @@ -445,7 +446,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -462,7 +463,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After first compaction: mob rows count", regionNum - * (rowNumPerRegion - delRowNum), countMobRows(table)); + * (rowNumPerRegion - delRowNum), MobTestUtil.countMobRows(table)); assertEquals("After first compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After first compaction: family1 mob file count", regionNum, @@ -482,7 +483,7 @@ public class TestMobCompactor { admin.enableTable(tableName); assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("After restoring snapshot: family1 mob file count", regionNum * count, @@ -500,7 +501,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("After second compaction: family1 mob file count", regionNum, @@ -550,7 +551,7 @@ public class TestMobCompactor { int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, @@ -559,7 +560,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -576,7 +577,7 @@ public class TestMobCompactor { waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After compaction: family1 mob file count", regionNum, @@ -764,18 +765,6 @@ public class TestMobCompactor { } /** - * Gets the number of rows in the given table. - * @param table to get the scanner - * @return the number of rows - */ - private int countMobRows(final Table table) throws IOException { - Scan scan = new Scan(); - // Do not retrieve the mob data when scanning - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - return TEST_UTIL.countRows(table, scan); - } - - /** * Gets the number of cells in the given table. * @param table to get the scanner * @return the number of cells @@ -923,13 +912,15 @@ public class TestMobCompactor { for (int i = 0; i < 1000; i ++) { Put put0 = new Put(Bytes.toBytes("r0" + i)); - put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151130Monday, Bytes.toBytes(mobValue0)); + put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), + tsFor20151130Monday, Bytes.toBytes(mobValue0)); pArray[i] = put0; } loadData(admin, bufMut, tableName, pArray); Put put06 = new Put(mobKey06); - put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151128Saturday, Bytes.toBytes(mobValue0)); + put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), + tsFor20151128Saturday, Bytes.toBytes(mobValue0)); loadData(admin, bufMut, tableName, new Put[] { put06 });
