HBASE-14782 FuzzyRowFilter skips valid rows (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ebe5801e Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ebe5801e Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ebe5801e Branch: refs/heads/hbase-12439 Commit: ebe5801e0081dcce7f3eb918b39161fcd2298087 Parents: 3aa3fae Author: chenheng <[email protected]> Authored: Thu Nov 19 10:56:18 2015 +0800 Committer: chenheng <[email protected]> Committed: Thu Nov 19 11:28:18 2015 +0800 ---------------------------------------------------------------------- .../hadoop/hbase/filter/FuzzyRowFilter.java | 27 ++++++++++- .../hadoop/hbase/filter/TestFuzzyRowFilter.java | 8 +-- .../filter/TestFuzzyRowFilterEndToEnd.java | 51 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/ebe5801e/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java index a9dd596..a4c6c3b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java @@ -154,6 +154,7 @@ public class FuzzyRowFilter extends FilterBase { } // NOT FOUND -> seek next using hint lastFoundIndex = -1; + return ReturnCode.SEEK_NEXT_USING_HINT; } @@ -572,7 +573,31 @@ public class FuzzyRowFilter extends FilterBase { } } - return result; + return reverse? result: trimTrailingZeroes(result, fuzzyKeyMeta, toInc); + } + + /** + * For forward scanner, next cell hint should not contain any trailing zeroes + * unless they are part of fuzzyKeyMeta + * hint = '\x01\x01\x01\x00\x00' + * will skip valid row '\x01\x01\x01' + * + * @param result + * @param fuzzyKeyMeta + * @param toInc - position of incremented byte + * @return trimmed version of result + */ + + private static byte[] trimTrailingZeroes(byte[] result, byte[] fuzzyKeyMeta, int toInc) { + int off = fuzzyKeyMeta.length >= result.length? result.length -1: + fuzzyKeyMeta.length -1; + for( ; off >= 0; off--){ + if(fuzzyKeyMeta[off] != 0) break; + } + if (off < toInc) off = toInc; + byte[] retValue = new byte[off+1]; + System.arraycopy(result, 0, retValue, 0, retValue.length); + return retValue; } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/ebe5801e/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java index d420959..0d48088 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java @@ -231,7 +231,7 @@ public class TestFuzzyRowFilter { new byte[]{0, 1, 2}, // fuzzy row new byte[]{0, -1, -1}, // mask new byte[]{1, 2, 1, 0, 1}, // current - new byte[]{2, 1, 2, 0, 0}); // expected next + new byte[]{2, 1, 2}); // expected next assertNext(false, new byte[]{0, 1, 2}, // fuzzy row @@ -243,13 +243,13 @@ public class TestFuzzyRowFilter { new byte[]{0, 1, 0, 2, 0}, // fuzzy row new byte[]{0, -1, 0, -1, 0}, // mask new byte[]{1, 0, 2, 0, 1}, // current - new byte[]{1, 1, 0, 2, 0}); // expected next + new byte[]{1, 1, 0, 2}); // expected next assertNext(false, new byte[]{1, 0, 1}, new byte[]{-1, 0, -1}, new byte[]{1, (byte) 128, 2, 0, 1}, - new byte[]{1, (byte) 129, 1, 0, 0}); + new byte[]{1, (byte) 129, 1}); assertNext(false, new byte[]{0, 1, 0, 1}, @@ -315,7 +315,7 @@ public class TestFuzzyRowFilter { new byte[]{1, 1, 0, 0}, new byte[]{-1, -1, 0, 0}, new byte[]{0, 1, 3, 2}, - new byte[]{1, 1, 0, 0}); + new byte[]{1, 1}); // No next for this one Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( http://git-wip-us.apache.org/repos/asf/hbase/blob/ebe5801e/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java index ba1d2a1..cc09042 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java @@ -110,6 +110,57 @@ public class TestFuzzyRowFilterEndToEnd { // Nothing to do. } + @Test + public void testHBASE14782() throws IOException + { + String cf = "f"; + String cq = "q"; + String table = "HBASE14872"; + + Table ht = + TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE); + // Load data + String[] rows = new String[]{ + "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00", + "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00", + "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00", + "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00", + "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01", + "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB", + }; + + String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX"; + + for(int i=0; i < rows.length; i++){ + Put p = new Put(Bytes.toBytesBinary(rows[i])); + p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes()); + ht.put(p); + } + + Put p = new Put(Bytes.toBytesBinary(badRow)); + p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes()); + ht.put(p); + + TEST_UTIL.flush(); + + List<Pair<byte[], byte[]>> data = new ArrayList<Pair<byte[], byte[]>>(); + byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044"); + byte[] mask = new byte[] { 1,0,0,0}; + data.add(new Pair<byte[], byte[]>(fuzzyKey, mask)); + FuzzyRowFilter filter = new FuzzyRowFilter(data); + + Scan scan = new Scan(); + scan.setFilter(filter); + + ResultScanner scanner = ht.getScanner(scan); + int total = 0; + while(scanner.next() != null){ + total++; + } + assertEquals(rows.length, total); + TEST_UTIL.deleteTable(TableName.valueOf(table)); + } + @Test public void testEndToEnd() throws Exception { String cf = "f";
