HBASE-19696 Filter returning INCLUDE_AND_NEXT_COL doesn't skip remaining versions when scan has explicit columns (Ankit Singhal)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5a66eb97 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5a66eb97 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5a66eb97 Branch: refs/heads/HBASE-19397 Commit: 5a66eb978c7ab865dad70ce70690c0b6ca519d2a Parents: 8ae2a21 Author: tedyu <[email protected]> Authored: Sun Jan 7 07:50:00 2018 -0800 Committer: tedyu <[email protected]> Committed: Sun Jan 7 07:50:00 2018 -0800 ---------------------------------------------------------------------- .../querymatcher/ColumnTracker.java | 9 +++ .../querymatcher/ExplicitColumnTracker.java | 7 +-- .../querymatcher/UserScanQueryMatcher.java | 3 + .../apache/hadoop/hbase/filter/TestFilter.java | 62 +++++++++++++++++++- 4 files changed, 74 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/5a66eb97/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ColumnTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ColumnTracker.java index 15dea6b..dc210ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ColumnTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ColumnTracker.java @@ -126,4 +126,13 @@ public interface ColumnTracker extends ShipperListener { * @return <code>true</code> to early out based on timestamp. */ boolean isDone(long timestamp); + + /** + * This method is used to inform the column tracker that we are done with this column. We may get + * this information from external filters or timestamp range and we then need to indicate this + * information to tracker. It is currently implemented for ExplicitColumnTracker. + * @param cell + */ + default void doneWithColumn(Cell cell) { + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5a66eb97/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java index 099b5df..85394fd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java @@ -207,12 +207,7 @@ public class ExplicitColumnTracker implements ColumnTracker { return timestamp < oldestStamp; } - /** - * This method is used to inform the column tracker that we are done with this column. We may get - * this information from external filters or timestamp range and we then need to indicate this - * information to tracker. It is required only in case of ExplicitColumnTracker. - * @param cell - */ + @Override public void doneWithColumn(Cell cell) { while (this.column != null) { int compare = CellUtil.compareQualifiers(cell, column.getBuffer(), column.getOffset(), http://git-wip-us.apache.org/repos/asf/hbase/blob/5a66eb97/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/UserScanQueryMatcher.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/UserScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/UserScanQueryMatcher.java index 7501594..e88e3a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/UserScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/UserScanQueryMatcher.java @@ -212,6 +212,9 @@ public abstract class UserScanQueryMatcher extends ScanQueryMatcher { case INCLUDE_AND_NEXT_COL: if (matchCode == MatchCode.INCLUDE) { matchCode = MatchCode.INCLUDE_AND_SEEK_NEXT_COL; + // Update column tracker to next column, As we use the column hint from the tracker to seek + // to next cell + columns.doneWithColumn(cell); } break; case INCLUDE_AND_SEEK_NEXT_ROW: http://git-wip-us.apache.org/repos/asf/hbase/blob/5a66eb97/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java index 1f20552..ca2c88b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java @@ -142,7 +142,8 @@ public class TestFilter { @Before public void setUp() throws Exception { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestFilter")); - htd.addFamily(new HColumnDescriptor(FAMILIES[0])); + HColumnDescriptor family0 = new HColumnDescriptor(FAMILIES[0]).setVersions(100, 100); + htd.addFamily(family0); htd.addFamily(new HColumnDescriptor(FAMILIES[1])); htd.addFamily(new HColumnDescriptor(FAMILIES_1[0])); htd.addFamily(new HColumnDescriptor(FAMILIES_1[1])); @@ -1861,6 +1862,65 @@ public class TestFilter { } @Test + public void testLatestVersionFilterWithExplicitColumn() throws Exception { + // Add multiple versions + Put p = new Put(ROWS_ONE[0]); + p.setDurability(Durability.SKIP_WAL); + p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]); + this.region.put(p); + p = new Put(ROWS_ONE[0]); + p.setDurability(Durability.SKIP_WAL); + p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]); + this.region.put(p); + this.region.flush(true); + Scan s = new Scan(); + s.setFilter(new FilterBase() { + @Override + public ReturnCode filterCell(Cell c) throws IOException { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + }); + s.readVersions(100); + s.addColumn(FAMILIES[0], QUALIFIERS_ONE[0]); + s.addColumn(FAMILIES[0], QUALIFIERS_ONE[1]); + s.addColumn(FAMILIES[0], QUALIFIERS_ONE[2]); + s.addColumn(FAMILIES[0], QUALIFIERS_ONE[3]); + s.addColumn(FAMILIES[0], QUALIFIERS_TWO[0]); + s.addColumn(FAMILIES[0], QUALIFIERS_TWO[1]); + s.addColumn(FAMILIES[0], QUALIFIERS_TWO[2]); + s.addColumn(FAMILIES[0], QUALIFIERS_TWO[3]); + KeyValue[] kvs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), }; + verifyScanFull(s, kvs); + + } + + @Test public void testColumnPaginationFilter() throws Exception { // Test that the filter skips multiple column versions. Put p = new Put(ROWS_ONE[0]);
