HBASE-19895 Add keepDeletedCells option in ScanOptions for customizing scanInfo in pre-hooks (Ankit Singhal)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1125859 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1125859 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1125859 Branch: refs/heads/HBASE-19064 Commit: a11258599e7412ea4867ef850cd67bb9e7bd8e67 Parents: 99b9fff Author: tedyu <yuzhih...@gmail.com> Authored: Thu Feb 1 18:27:18 2018 -0800 Committer: tedyu <yuzhih...@gmail.com> Committed: Thu Feb 1 18:27:18 2018 -0800 ---------------------------------------------------------------------- .../regionserver/CustomizedScanInfoBuilder.java | 18 +++- .../hadoop/hbase/regionserver/ScanInfo.java | 9 +- .../hadoop/hbase/regionserver/ScanOptions.java | 5 ++ .../hbase/regionserver/TestCompaction.java | 2 +- .../TestDefaultCompactSelection.java | 3 +- .../hbase/regionserver/TestMajorCompaction.java | 88 +++++++++++++------- 6 files changed, 87 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java index c3d5e57..b791010 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.regionserver; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.yetus.audience.InterfaceAudience; /** @@ -31,6 +32,8 @@ public class CustomizedScanInfoBuilder implements ScanOptions { private Long ttl; + private KeepDeletedCells keepDeletedCells = null; + public CustomizedScanInfoBuilder(ScanInfo scanInfo) { this.scanInfo = scanInfo; } @@ -56,14 +59,25 @@ public class CustomizedScanInfoBuilder implements ScanOptions { } public ScanInfo build() { - if (maxVersions == null && ttl == null) { + if (maxVersions == null && ttl == null && keepDeletedCells == null) { return scanInfo; } - return scanInfo.customize(getMaxVersions(), getTTL()); + return scanInfo.customize(getMaxVersions(), getTTL(), getKeepDeletedCells()); } @Override public String toString() { return "ScanOptions [maxVersions=" + getMaxVersions() + ", TTL=" + getTTL() + "]"; } + + @Override + public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) { + this.keepDeletedCells = keepDeletedCells; + } + + @Override + public KeepDeletedCells getKeepDeletedCells() { + return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells(); + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java index 419afff..2fde311 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java @@ -171,10 +171,11 @@ public class ScanInfo { } /** - * Used for CP users for customizing max versions and ttl. + * Used for CP users for customizing max versions, ttl and keepDeletedCells. */ - ScanInfo customize(int maxVersions, long ttl) { - return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, ttl, comparator, - ttl, usePread, maxVersions, parallelSeekEnabled, ttl, newVersionBehavior); + ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) { + return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, timeToPurgeDeletes, + comparator, tableMaxRowSize, usePread, cellsPerTimeoutCheck, parallelSeekEnabled, + preadMaxBytes, newVersionBehavior); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java index 5a35d51..aca857a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -59,4 +60,8 @@ public interface ScanOptions { long getTTL(); void setTTL(long ttl); + + void setKeepDeletedCells(KeepDeletedCells keepDeletedCells); + + KeepDeletedCells getKeepDeletedCells(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index 75110fd..6fe76d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -218,7 +218,7 @@ public class TestCompaction { final int ttl = 1000; for (HStore store : this.r.stores.values()) { ScanInfo old = store.getScanInfo(); - ScanInfo si = old.customize(old.getMaxVersions(), ttl); + ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells()); store.setScanInfo(si); } Thread.sleep(ttl); http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java index 3c3bbd8..9c45a83 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java @@ -170,7 +170,8 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy { public void testCompactionEmptyHFile() throws IOException { // Set TTL ScanInfo oldScanInfo = store.getScanInfo(); - ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600); + ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600, + oldScanInfo.getKeepDeletedCells()); store.setScanInfo(newScanInfo); // Do not compact empty store file List<HStoreFile> candidates = sfCreate(0); http://git-wip-us.apache.org/repos/asf/hbase/blob/a1125859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java index e261886..ee717f9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; @@ -134,36 +135,20 @@ public class TestMajorCompaction { * Test that on a major compaction, if all cells are expired or deleted, then * we'll end up with no product. Make sure scanner over region returns * right answer in this case - and that it just basically works. - * @throws IOException + * @throws IOException exception encountered */ @Test public void testMajorCompactingToNoOutput() throws IOException { - createStoreFile(r); - for (int i = 0; i < compactionThreshold; i++) { - createStoreFile(r); - } - // Now delete everything. - InternalScanner s = r.getScanner(new Scan()); - do { - List<Cell> results = new ArrayList<>(); - boolean result = s.next(results); - r.delete(new Delete(CellUtil.cloneRow(results.get(0)))); - if (!result) break; - } while(true); - s.close(); - // Flush - r.flush(true); - // Major compact. - r.compact(true); - s = r.getScanner(new Scan()); - int counter = 0; - do { - List<Cell> results = new ArrayList<>(); - boolean result = s.next(results); - if (!result) break; - counter++; - } while(true); - assertEquals(0, counter); + testMajorCompactingWithDeletes(KeepDeletedCells.FALSE); + } + + /** + * Test that on a major compaction,Deleted cells are retained if keep deleted cells is set to true + * @throws IOException exception encountered + */ + @Test + public void testMajorCompactingWithKeepDeletedCells() throws IOException { + testMajorCompactingWithDeletes(KeepDeletedCells.TRUE); } /** @@ -298,7 +283,7 @@ public class TestMajorCompaction { final int ttl = 1000; for (HStore store : r.getStores()) { ScanInfo old = store.getScanInfo(); - ScanInfo si = old.customize(old.getMaxVersions(), ttl); + ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells()); store.setScanInfo(si); } Thread.sleep(1000); @@ -471,7 +456,9 @@ public class TestMajorCompaction { boolean result = s.next(results); assertTrue(!results.isEmpty()); r.delete(new Delete(CellUtil.cloneRow(results.get(0)))); - if (!result) break; + if (!result) { + break; + } } while (true); s.close(); // Flush @@ -485,10 +472,51 @@ public class TestMajorCompaction { do { List<Cell> results = new ArrayList<>(); boolean result = s.next(results); - if (!result) break; + if (!result) { + break; + } counter++; } while (true); s.close(); assertEquals(0, counter); } + + private void testMajorCompactingWithDeletes(KeepDeletedCells keepDeletedCells) + throws IOException { + createStoreFile(r); + for (int i = 0; i < compactionThreshold; i++) { + createStoreFile(r); + } + // Now delete everything. + InternalScanner s = r.getScanner(new Scan()); + int originalCount = 0; + do { + List<Cell> results = new ArrayList<>(); + boolean result = s.next(results); + r.delete(new Delete(CellUtil.cloneRow(results.get(0)))); + if (!result) break; + originalCount++; + } while (true); + s.close(); + // Flush + r.flush(true); + + for (HStore store : this.r.stores.values()) { + ScanInfo old = store.getScanInfo(); + ScanInfo si = old.customize(old.getMaxVersions(), old.getTtl(), keepDeletedCells); + store.setScanInfo(si); + } + // Major compact. + r.compact(true); + s = r.getScanner(new Scan().setRaw(true)); + int counter = 0; + do { + List<Cell> results = new ArrayList<>(); + boolean result = s.next(results); + if (!result) break; + counter++; + } while (true); + assertEquals(keepDeletedCells == KeepDeletedCells.TRUE ? originalCount : 0, counter); + + } }