PHOENIX-2455 Partial results for a query when PHOENIX-2194 is applied
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ee813cee Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ee813cee Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ee813cee Branch: refs/heads/4.x-HBase-1.0 Commit: ee813cee428c6ec6027076a838865849ef44b77d Parents: 72f80bd Author: James Taylor <[email protected]> Authored: Sun Dec 13 14:32:29 2015 -0800 Committer: James Taylor <[email protected]> Committed: Mon Dec 14 11:52:19 2015 -0800 ---------------------------------------------------------------------- .../apache/phoenix/end2end/SkipScanQueryIT.java | 31 +++++++++ .../apache/phoenix/filter/SkipScanFilter.java | 3 +- .../java/org/apache/phoenix/query/KeyRange.java | 48 +++++++++----- .../java/org/apache/phoenix/util/ScanUtil.java | 2 +- .../phoenix/filter/SkipScanFilterTest.java | 66 +++++++++++++------- 5 files changed, 108 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java index 1937f65..2ade0a4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java @@ -110,6 +110,37 @@ public class SkipScanQueryIT extends BaseHBaseManagedTimeIT { } @Test + public void testSkipScanFilterQuery() throws Exception { + String createTableDDL = "CREATE TABLE test" + "(col1 VARCHAR," + "col2 VARCHAR," + "col3 VARCHAR," + + "col4 VARCHAR," + "CONSTRAINT pk " + "PRIMARY KEY (col1,col2,col3,col4))"; + String upsertQuery = "upsert into test values(?,?,?,?)"; + String query = "SELECT col1, col2, col3, col4 FROM test WHERE col1 IN ('a','e','f') AND col2 = 'b' AND col4 = '1' "; + String[] col1Values = { "a", "e.f", "f" }; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + createTestTable(getUrl(), createTableDDL); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.setAutoCommit(true); + try { + PreparedStatement statement = conn.prepareStatement(upsertQuery); + for (String col1Value : col1Values) { + statement.setString(1, col1Value); + statement.setString(2, "b"); + statement.setString(3, ""); + statement.setString(4, "1"); + statement.execute(); + } + ResultSet rs = conn.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals(rs.getString(1), "a"); + assertTrue(rs.next()); + assertEquals(rs.getString(1), "f"); + assertFalse(rs.next()); + } finally { + conn.close(); + } + } + + @Test public void testSelectAfterUpsertInQuery() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); initSelectAfterUpsertTable(conn); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index d1c8532..667b3d7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -580,8 +580,7 @@ public class SkipScanFilter extends FilterBase implements Writable { List<KeyRange> orClause = Lists.newArrayListWithExpectedSize(orLen); slots.add(orClause); for (int j=0; j<orLen; j++) { - KeyRange range = new KeyRange(); - range.readFields(in); + KeyRange range = KeyRange.read(in); orClause.add(range); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java index 6618aa5..f4bf793 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java @@ -119,15 +119,30 @@ public class KeyRange implements Writable { return getKeyRange(lowerRange, true, upperRange, false); } - // TODO: make non public and move to org.apache.phoenix.type soon - public static KeyRange getKeyRange(byte[] lowerRange, boolean lowerInclusive, + private static KeyRange getSingleton(byte[] lowerRange, boolean lowerInclusive, byte[] upperRange, boolean upperInclusive) { if (lowerRange == null || upperRange == null) { return EMPTY_RANGE; } - // Need to treat null differently for a point range - if (lowerRange.length == 0 && upperRange.length == 0 && lowerInclusive && upperInclusive) { - return IS_NULL_RANGE; + if (lowerRange.length == 0 && upperRange.length == 0) { + // Need singleton to represent NULL range so it gets treated differently + // than an unbound RANGE. + return lowerInclusive && upperInclusive ? IS_NULL_RANGE : EVERYTHING_RANGE; + } + if (lowerRange.length != 0 && upperRange.length != 0) { + int cmp = Bytes.compareTo(lowerRange, upperRange); + if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) { + return EMPTY_RANGE; + } + } + return null; + } + + public static KeyRange getKeyRange(byte[] lowerRange, boolean lowerInclusive, + byte[] upperRange, boolean upperInclusive) { + KeyRange range = getSingleton(lowerRange, lowerInclusive, upperRange, upperInclusive); + if (range != null) { + return range; } boolean unboundLower = false; boolean unboundUpper = false; @@ -142,20 +157,23 @@ public class KeyRange implements Writable { unboundUpper = true; } - if (unboundLower && unboundUpper) { - return EVERYTHING_RANGE; - } - if (!unboundLower && !unboundUpper) { - int cmp = Bytes.compareTo(lowerRange, upperRange); - if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) { - return EMPTY_RANGE; - } - } return new KeyRange(lowerRange, unboundLower ? false : lowerInclusive, upperRange, unboundUpper ? false : upperInclusive); } - public KeyRange() { + public static KeyRange read(DataInput input) throws IOException { + KeyRange range = new KeyRange(); + range.readFields(input); + // Translate to singleton after reading + KeyRange singletonRange = getSingleton(range.lowerRange, range.lowerInclusive, range.upperRange, range.upperInclusive); + if (singletonRange != null) { + return singletonRange; + } + // Otherwise, just keep the range we read + return range; + } + + private KeyRange() { this.lowerRange = DEGENERATE_KEY; this.lowerInclusive = false; this.upperRange = DEGENERATE_KEY; http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index 912dd03..b6a2c85 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -375,7 +375,7 @@ public class ScanUtil { * incrementing the key value itself, and thus bumping it up too much. */ boolean inclusiveUpper = range.isUpperInclusive() && bound == Bound.UPPER; - boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER; + boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER && range != KeyRange.EVERYTHING_RANGE; boolean exclusiveUpper = !range.isUpperInclusive() && bound == Bound.UPPER; // If we are setting the upper bound of using inclusive single key, we remember // to increment the key if we exit the loop after this iteration. http://git-wip-us.apache.org/repos/asf/phoenix/blob/ee813cee/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java index 7e68e25..898f778 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java @@ -23,19 +23,17 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import junit.framework.TestCase; - import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.schema.types.PChar; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.PDatum; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PChar; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,6 +43,8 @@ import org.junit.runners.Parameterized.Parameters; import com.google.common.base.Function; import com.google.common.collect.Lists; +import junit.framework.TestCase; + //reset() //filterAllRemaining() -> true indicates scan is over, false, keep going on. //filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also call @@ -97,7 +97,6 @@ public class SkipScanFilterTest extends TestCase { @Test public void test() throws IOException { - System.out.println("CNF: " + cnf + "\n" + "Expectations: " + expectations); for (Expectation expectation : expectations) { expectation.examine(skipper); } @@ -106,6 +105,39 @@ public class SkipScanFilterTest extends TestCase { @Parameters(name="{0} {1} {2}") public static Collection<Object> data() { List<Object> testCases = Lists.newArrayList(); + // Variable length tests + testCases.addAll( + foreach(new KeyRange[][]{{ + PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("a"), true, Bytes.toBytes("a"), true), + PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("e"), true, Bytes.toBytes("e"), true), + PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("f"), true, Bytes.toBytes("f"), true), + }, + { + PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("b"), true, Bytes.toBytes("b"), true), + }, + { + KeyRange.EVERYTHING_RANGE, + }, + { + PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("1"), true, Bytes.toBytes("1"), true), + }}, + new int[4], + new Include(ByteUtil.concat(Bytes.toBytes("a"),QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY, + QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("1") ) ), + new SeekNext(ByteUtil.concat(Bytes.toBytes("e.f"),QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY, + QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("1") ), + ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("b") )), + new Include(ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY, + QueryConstants.SEPARATOR_BYTE_ARRAY, + Bytes.toBytes("1") ) ) ) + ); + // Fixed length tests testCases.addAll( foreach(new KeyRange[][]{{ PChar.INSTANCE.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true), @@ -124,7 +156,6 @@ public class SkipScanFilterTest extends TestCase { PChar.INSTANCE.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false), }}, new int[]{3,2,2,2,2}, - //new SeekNext("abcABABABAB", "abdAAAAAAAA"), new SeekNext("defAAABABAB", "dzzAAAAAAAA"), new Finished("xyyABABABAB")) ); @@ -309,23 +340,6 @@ public class SkipScanFilterTest extends TestCase { new SeekNext("dzzAB250", "dzzAB701"), new Finished("zzzAA000")) ); -// TODO variable length columns -// testCases.addAll( -// foreach(new KeyRange[][]{{ -// Char.INSTANCE.getKeyRange(Bytes.toBytes("apple"), true, Bytes.toBytes("lemon"), true), -// Char.INSTANCE.getKeyRange(Bytes.toBytes("pear"), false, Bytes.toBytes("yam"), false), -// }, -// { -// Char.INSTANCE.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true), -// Char.INSTANCE.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false), -// Char.INSTANCE.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false), -// }, -// { -// Char.INSTANCE.getKeyRange(Bytes.toBytes("100"), true, Bytes.toBytes("250"), false), -// Char.INSTANCE.getKeyRange(Bytes.toBytes("700"), false, Bytes.toBytes("901"), false), -// }}, -// new int[]{3,3}) -// ); return testCases; } @@ -378,6 +392,10 @@ public class SkipScanFilterTest extends TestCase { this.rowkey = Bytes.toBytes(rowkey); } + public Include(byte[] rowkey) { + this.rowkey = rowkey; + } + @SuppressWarnings("deprecation") @Override public void examine(SkipScanFilter skipper) throws IOException { KeyValue kv = KeyValue.createFirstOnRow(rowkey);
