Repository: phoenix Updated Branches: refs/heads/master 08d9b98f3 -> c51dc12b0
PHOENIX-1211 Use skip scan when row value constructor uses leading row key columns (Kyle Buzsaki) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c51dc12b Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c51dc12b Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c51dc12b Branch: refs/heads/master Commit: c51dc12b07276ed318982b170446b6867fa12521 Parents: 08d9b98 Author: James Taylor <jamestay...@apache.org> Authored: Fri Sep 12 17:21:14 2014 -0700 Committer: James Taylor <jamestay...@apache.org> Committed: Fri Sep 12 17:23:47 2014 -0700 ---------------------------------------------------------------------- .../org/apache/phoenix/end2end/InListIT.java | 24 +++++--- .../phoenix/end2end/RowValueConstructorIT.java | 53 +++++++++++++++++ .../org/apache/phoenix/compile/ScanRanges.java | 6 +- .../apache/phoenix/compile/WhereCompiler.java | 4 +- .../apache/phoenix/compile/WhereOptimizer.java | 6 -- .../apache/phoenix/filter/SkipScanFilter.java | 55 +++++++++++------ .../org/apache/phoenix/schema/RowKeySchema.java | 62 ++++++++++++++++++-- .../java/org/apache/phoenix/util/ScanUtil.java | 39 +++++++++--- .../phoenix/compile/WhereOptimizerTest.java | 4 +- 9 files changed, 202 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java index c257ccb..fefa2e2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java @@ -147,6 +147,11 @@ public class InListIT extends BaseHBaseManagedTimeIT { private static final List<Boolean> TENANCIES = Arrays.asList(false, true); private static final List<PDataType> INTEGER_TYPES = Arrays.asList(PDataType.INTEGER, PDataType.LONG); private static final List<Integer> SALT_BUCKET_NUMBERS = Arrays.asList(0, 4); + + // we should be including the RANGE_SCAN hint here, but a bug with ParallelIterators causes tests to fail + // see the relevant JIRA here: https://issues.apache.org/jira/browse/PHOENIX-1251 + private static final List<String> HINTS = Arrays.asList("", "/*+ SKIP_SCAN */"); +// private static final List<String> HINTS = Arrays.asList("", "/*+ SKIP_SCAN */", "/*+ RANGE_SCAN */"); /** * Tests the given where clause against the given upserts by comparing against the list of @@ -176,14 +181,19 @@ public class InListIT extends BaseHBaseManagedTimeIT { } conn.commit(); - // perform the query - String sql = "SELECT nonPk FROM " + tableName + " " + whereClause; - ResultSet rs = conn.createStatement().executeQuery(sql); - for(String expected : expecteds) { - assertTrue(rs.next()); - assertEquals(expected, rs.getString(1)); + for(String hint : HINTS) { + String context = "where: " + whereClause + ", type: " + pkType + ", salt buckets: " + + saltBuckets + ", multitenant: " + isMultiTenant + ", hint: " + hint + ""; + + // perform the query + String sql = "SELECT " + hint + " nonPk FROM " + tableName + " " + whereClause; + ResultSet rs = conn.createStatement().executeQuery(sql); + for (String expected : expecteds) { + assertTrue("did not include result '" + expected + "' (" + context + ")", rs.next()); + assertEquals(context, expected, rs.getString(1)); + } + assertFalse(context, rs.next()); } - assertFalse(rs.next()); } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java index 041725c..bf3d9db 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RowValueConstructorIT.java @@ -53,6 +53,7 @@ import java.util.Properties; import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -1228,4 +1229,56 @@ public class RowValueConstructorIT extends BaseClientManagedTimeIT { assertEquals(4, rs.getInt(2)); assertFalse(rs.next()); } + + @Test + public void testForceSkipScan() throws Exception { + String tempTableWithCompositePK = "TEMP_TABLE_COMPOSITE_PK"; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + try { + conn.createStatement().execute("CREATE TABLE " + tempTableWithCompositePK + + " (col0 INTEGER NOT NULL, " + + " col1 INTEGER NOT NULL, " + + " col2 INTEGER NOT NULL, " + + " col3 INTEGER " + + " CONSTRAINT pk PRIMARY KEY (col0, col1, col2)) " + + " SALT_BUCKETS=4"); + + PreparedStatement upsertStmt = conn.prepareStatement( + "upsert into " + tempTableWithCompositePK + "(col0, col1, col2, col3) " + "values (?, ?, ?, ?)"); + for (int i = 0; i < 3; i++) { + upsertStmt.setInt(1, i + 1); + upsertStmt.setInt(2, i + 2); + upsertStmt.setInt(3, i + 3); + upsertStmt.setInt(4, i + 5); + upsertStmt.execute(); + } + conn.commit(); + + String query = "SELECT * FROM " + tempTableWithCompositePK + " WHERE (col0, col1) in ((2, 3), (3, 4), (4, 5))"; + PreparedStatement statement = conn.prepareStatement(query); + ResultSet rs = statement.executeQuery(); + assertTrue(rs.next()); + assertEquals(rs.getInt(1), 2); + assertEquals(rs.getInt(2), 3); + assertEquals(rs.getInt(3), 4); + assertEquals(rs.getInt(4), 6); + assertTrue(rs.next()); + assertEquals(rs.getInt(1), 3); + assertEquals(rs.getInt(2), 4); + assertEquals(rs.getInt(3), 5); + assertEquals(rs.getInt(4), 7); + + assertFalse(rs.next()); + + String plan = "CLIENT PARALLEL 4-WAY SKIP SCAN ON 12 KEYS OVER TEMP_TABLE_COMPOSITE_PK [0,2] - [3,4]\n" + + "CLIENT MERGE SORT"; + String explainQuery = "EXPLAIN " + query; + rs = conn.createStatement().executeQuery(explainQuery); + assertEquals(query, plan, QueryUtil.getExplainPlan(rs)); + } finally { + conn.close(); + } + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 1052601..dc8e0b3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -100,7 +100,7 @@ public class ScanRanges { this.slotSpan = slotSpan; this.schema = schema; if (schema != null && !ranges.isEmpty()) { - this.filter = new SkipScanFilter(this.ranges, schema); + this.filter = new SkipScanFilter(this.ranges, slotSpan, schema); } this.forceRangeScan = forceRangeScan; } @@ -152,7 +152,7 @@ public class ScanRanges { } private static boolean isPointLookup(RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan) { - if (ScanUtil.calculateSlotSpan(ranges, slotSpan) < schema.getMaxFields()) { + if (ScanUtil.getTotalSpan(ranges, slotSpan) < schema.getMaxFields()) { return false; } for (List<KeyRange> orRanges : ranges) { @@ -261,7 +261,7 @@ public class ScanRanges { } public int getPkColumnSpan() { - return this == ScanRanges.NOTHING ? 0 : ScanUtil.calculateSlotSpan(ranges, slotSpan); + return this == ScanRanges.NOTHING ? 0 : ScanUtil.getTotalSpan(ranges, slotSpan); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index 7bcb6d0..2e72f43 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -255,9 +255,7 @@ public class WhereCompiler { } ScanRanges scanRanges = context.getScanRanges(); - boolean forcedSkipScan = statement.getHint().hasHint(Hint.SKIP_SCAN); - boolean forcedRangeScan = statement.getHint().hasHint(Hint.RANGE_SCAN); - if (forcedSkipScan || (scanRanges.useSkipScanFilter() && !forcedRangeScan)) { + if (scanRanges.useSkipScanFilter()) { ScanUtil.andFilterAtBeginning(scan, scanRanges.getSkipScanFilter()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java index 5e03158..ab92a14 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java @@ -284,12 +284,6 @@ public class WhereOptimizer { // If we have fully qualified point keys with multi-column spans (i.e. RVC), // we can still use our skip scan. The ScanRanges.create() call will explode // out the keys. - if (hasMultiColumnSpan) { - forcedRangeScan |= pkPos < nPKColumns; - if (forcedRangeScan && removeFromExtractNodes != null) { - extractNodes.removeAll(removeFromExtractNodes); - } - } context.setScanRanges( ScanRanges.create(schema, cnf, Arrays.copyOf(slotSpan, cnf.size()), forcedRangeScan, nBuckets), minMaxRange); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index b9b091d..ccdbe4c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -87,21 +87,25 @@ public class SkipScanFilter extends FilterBase implements Writable { } public SkipScanFilter(List<List<KeyRange>> slots, RowKeySchema schema) { - init(slots, schema); + this(slots, ScanUtil.getDefaultSlotSpans(slots.size()), schema); + } + + public SkipScanFilter(List<List<KeyRange>> slots, int[] slotSpan, RowKeySchema schema) { + init(slots, slotSpan, schema); } public void setOffset(int offset) { this.offset = offset; } - private void init(List<List<KeyRange>> slots, RowKeySchema schema) { + private void init(List<List<KeyRange>> slots, int[] slotSpan, RowKeySchema schema) { for (List<KeyRange> ranges : slots) { if (ranges.isEmpty()) { throw new IllegalStateException(); } } this.slots = slots; - this.slotSpan = ScanUtil.getDefaultSlotSpans(slots.size()); + this.slotSpan = slotSpan; this.schema = schema; this.maxKeyLength = SchemaUtil.getMaxKeyLength(schema, slots); this.position = new int[slots.size()]; @@ -130,6 +134,8 @@ public class SkipScanFilter extends FilterBase implements Writable { } private void setNextCellHint(Cell kv) { + Cell previousCellHint = nextCellHint; + if (offset == 0) { nextCellHint = new KeyValue(startKey, 0, startKeyLength, null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); @@ -140,6 +146,10 @@ public class SkipScanFilter extends FilterBase implements Writable { nextCellHint = new KeyValue(nextKey, 0, nextKey.length, null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); } + + // we should either have no previous hint, or the next hint should always come after the previous hint + assert previousCellHint == null || KeyValue.COMPARATOR.compare(nextCellHint, previousCellHint) > 0 + : "next hint must come after previous hint (prev=" + previousCellHint + ", next=" + nextCellHint + ", kv=" + kv + ")"; } @Override @@ -158,7 +168,7 @@ public class SkipScanFilter extends FilterBase implements Writable { public SkipScanFilter intersect(byte[] lowerInclusiveKey, byte[] upperExclusiveKey) { List<List<KeyRange>> newSlots = Lists.newArrayListWithCapacity(slots.size()); if (intersect(lowerInclusiveKey, upperExclusiveKey, newSlots)) { - return new SkipScanFilter(newSlots, schema); + return new SkipScanFilter(newSlots, slotSpan, schema); } return null; } @@ -185,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements Writable { int lastSlot = slots.size()-1; if (!lowerUnbound) { // Find the position of the first slot of the lower range - schema.next(ptr, 0, schema.iterator(lowerInclusiveKey,ptr)); + schema.next(ptr, 0, schema.iterator(lowerInclusiveKey,ptr), slotSpan[0]); startPos = ScanUtil.searchClosestKeyRangeWithUpperHigherThanPtr(slots.get(0), ptr, 0); // Lower range is past last upper range of first slot, so cannot possibly be in range if (startPos >= slots.get(0).size()) { @@ -196,7 +206,7 @@ public class SkipScanFilter extends FilterBase implements Writable { int endPos = slots.get(0).size()-1; if (!upperUnbound) { // Find the position of the first slot of the upper range - schema.next(ptr, 0, schema.iterator(upperExclusiveKey,ptr)); + schema.next(ptr, 0, schema.iterator(upperExclusiveKey,ptr), slotSpan[0]); endPos = ScanUtil.searchClosestKeyRangeWithUpperHigherThanPtr(slots.get(0), ptr, startPos); // Upper range lower than first lower range of first slot, so cannot possibly be in range if (endPos == 0 && Bytes.compareTo(upperExclusiveKey, slots.get(0).get(0).getLowerRange()) <= 0) { @@ -321,7 +331,7 @@ public class SkipScanFilter extends FilterBase implements Writable { int earliestRangeIndex = nSlots-1; int minOffset = offset; int maxOffset = schema.iterator(currentKey, minOffset, length, ptr); - schema.next(ptr, i, maxOffset); + schema.next(ptr, ScanUtil.getRowKeyPosition(slotSpan, i), maxOffset, slotSpan[i]); while (true) { // Increment to the next range while the upper bound of our current slot is less than our current key while (position[i] < slots.get(i).size() && slots.get(i).get(position[i]).compareUpperToLowerBound(ptr) < 0) { @@ -360,7 +370,7 @@ public class SkipScanFilter extends FilterBase implements Writable { // the current key, so we'll end up incrementing the start key until it's bigger than the // current key. setStartKey(); - schema.reposition(ptr, i, j, minOffset, maxOffset); + schema.reposition(ptr, ScanUtil.getRowKeyPosition(slotSpan, i), ScanUtil.getRowKeyPosition(slotSpan, j), minOffset, maxOffset, slotSpan[j]); } else { int currentLength = setStartKey(ptr, minOffset, j+1); // From here on, we use startKey as our buffer (resetting minOffset and maxOffset) @@ -368,7 +378,7 @@ public class SkipScanFilter extends FilterBase implements Writable { // Reinitialize the iterator to be positioned at previous slot position minOffset = 0; maxOffset = startKeyLength; - schema.iterator(startKey, minOffset, maxOffset, ptr, j+1); + schema.iterator(startKey, minOffset, maxOffset, ptr, ScanUtil.getRowKeyPosition(slotSpan, j)+1); // Do nextKey after setting the accessor b/c otherwise the null byte may have // been incremented causing us not to find it ByteUtil.nextKey(startKey, currentLength); @@ -393,7 +403,7 @@ public class SkipScanFilter extends FilterBase implements Writable { } i++; // If we run out of slots in our key, it means we have a partial key. - if (schema.next(ptr, i, maxOffset) == null) { + if (schema.next(ptr, ScanUtil.getRowKeyPosition(slotSpan, i), maxOffset, slotSpan[i]) == null) { // If the rest of the slots are checking for IS NULL, then break because // that's the case (since we don't store trailing nulls). if (allTrailingNulls(i)) { @@ -483,28 +493,35 @@ public class SkipScanFilter extends FilterBase implements Writable { int andLen = in.readInt(); List<List<KeyRange>> slots = Lists.newArrayListWithExpectedSize(andLen); for (int i=0; i<andLen; i++) { - int orlen = in.readInt(); - List<KeyRange> orclause = Lists.newArrayListWithExpectedSize(orlen); - slots.add(orclause); - for (int j=0; j<orlen; j++) { + int orLen = in.readInt(); + List<KeyRange> orClause = Lists.newArrayListWithExpectedSize(orLen); + slots.add(orClause); + for (int j=0; j<orLen; j++) { KeyRange range = new KeyRange(); range.readFields(in); - orclause.add(range); + orClause.add(range); } } - this.init(slots, schema); + int[] slotSpan = new int[andLen]; + for (int i = 0; i < andLen; i++) { + slotSpan[i] = in.readInt(); + } + this.init(slots, slotSpan, schema); } @Override public void write(DataOutput out) throws IOException { schema.write(out); out.writeInt(slots.size()); - for (List<KeyRange> orclause : slots) { - out.writeInt(orclause.size()); - for (KeyRange range : orclause) { + for (List<KeyRange> orClause : slots) { + out.writeInt(orClause.size()); + for (KeyRange range : orClause) { range.write(out); } } + for (int span : slotSpan) { + out.writeInt(span); + } } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java index 4d98c69..510d11b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java @@ -71,6 +71,8 @@ public class RowKeySchema extends ValueSchema { return this.getMinNullable(); } + // "iterator" initialization methods that initialize a bytes ptr with a row key for further navigation + @edu.umd.cs.findbugs.annotations.SuppressWarnings( value="NP_BOOLEAN_RETURN_NULL", justification="Designed to return null.") @@ -105,11 +107,12 @@ public class RowKeySchema extends ValueSchema { public int iterator(ImmutableBytesWritable ptr) { return iterator(ptr.get(),ptr.getOffset(),ptr.getLength(), ptr); } - + + // navigation methods that "select" different chunks of the row key held in a bytes ptr + /** - * Move the bytes ptr to the next position relative to the current ptr - * @param ptr bytes pointer pointing to the value at the positional index - * provided. + * Move the bytes ptr to the next position in the row key relative to its current position + * @param ptr bytes pointer pointing to the value at the positional index provided. * @param position zero-based index of the next field in the value schema * @param maxOffset max possible offset value when iterating * @return true if a value was found and ptr was set, false if the value is null and ptr was not @@ -151,6 +154,23 @@ public class RowKeySchema extends ValueSchema { } return ptr.getLength() > 0; } + + /** + * Like {@link #next(org.apache.hadoop.hbase.io.ImmutableBytesWritable, int, int)}, but also + * includes the next {@code extraSpan} additional fields in the bytes ptr. + * This allows multiple fields to be treated as one concatenated whole. + * @param ptr bytes pointer pointing to the value at the positional index provided. + * @param position zero-based index of the next field in the value schema + * @param maxOffset max possible offset value when iterating + * @param extraSpan the number of extra fields to expand the ptr to contain + * @return true if a value was found and ptr was set, false if the value is null and ptr was not + * set, and null if the value is null and there are no more values + */ + public Boolean next(ImmutableBytesWritable ptr, int position, int maxOffset, int extraSpan) { + Boolean returnValue = next(ptr, position, maxOffset); + readExtraFields(ptr, position + 1, maxOffset, extraSpan); + return returnValue; + } @edu.umd.cs.findbugs.annotations.SuppressWarnings( value="NP_BOOLEAN_RETURN_NULL", @@ -238,4 +258,38 @@ public class RowKeySchema extends ValueSchema { return hasValue; } + + /** + * Like {@link #reposition(org.apache.hadoop.hbase.io.ImmutableBytesWritable, int, int, int, int)}, + * but also includes the next {@code extraSpan} additional fields in the bytes ptr. + * This allows multiple fields to be treated as one concatenated whole. + * @param extraSpan the number of extra fields to expand the ptr to contain. + */ + public Boolean reposition(ImmutableBytesWritable ptr, int oldPosition, int newPosition, int minOffset, int maxOffset, int extraSpan) { + Boolean returnValue = reposition(ptr, oldPosition, newPosition, minOffset, maxOffset); + readExtraFields(ptr, newPosition + 1, maxOffset, extraSpan); + return returnValue; + } + + /** + * Extends the boundaries of the {@code ptr} to contain the next {@code extraSpan} fields in the row key. + * @param ptr bytes pointer pointing to the value at the positional index provided. + * @param position row key position of the first extra key to read + * @param maxOffset the maximum offset into the bytes pointer to allow + * @param extraSpan the number of extra fields to expand the ptr to contain. + */ + private void readExtraFields(ImmutableBytesWritable ptr, int position, int maxOffset, int extraSpan) { + int initialOffset = ptr.getOffset(); + + for(int i = 0; i < extraSpan; i++) { + Boolean returnValue = next(ptr, position + i, maxOffset); + + if(returnValue == null) { + break; + } + } + + int finalLength = ptr.getOffset() - initialOffset + ptr.getLength(); + ptr.set(ptr.get(), initialOffset, finalLength); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index d8e7f1b..42b20fe 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -304,7 +304,7 @@ public class ScanUtil { // but the index for the field it represents in the schema // should be incremented by 1 + value in the current slotSpan index // slotSpan stores the number of columns beyond one that the range spans - int i = slotStartIndex, fieldIndex = slotStartIndex; + int i = slotStartIndex, fieldIndex = ScanUtil.getRowKeyPosition(slotSpan, slotStartIndex); for (i = slotStartIndex; i < slotEndIndex; i++) { // Build up the key by appending the bound of each key range // from the current position of each slot. @@ -519,12 +519,37 @@ public class ScanUtil { return new int[nSlots]; } - public static int calculateSlotSpan(List<List<KeyRange>> ranges, int[] slotSpan) { - int nSlots = ranges.size(); - int totalSlotSpan = nSlots; - for (int i = 0; i < nSlots; i++) { - totalSlotSpan += slotSpan[i]; + /** + * Finds the total number of row keys spanned by this ranges / slotSpan pair. + * This accounts for slots in the ranges that may span more than on row key. + * @param ranges the KeyRange slots paired with this slotSpan. corresponds to {@link ScanRanges#ranges} + * @param slotSpan the extra span per skip scan slot. corresponds to {@link ScanRanges#slotSpan} + * @return the total number of row keys spanned yb this ranges / slotSpan pair. + * @see #getRowKeyPosition(int[], int) + */ + public static int getTotalSpan(List<List<KeyRange>> ranges, int[] slotSpan) { + // finds the position at the "end" of the ranges, which is also the total span + return getRowKeyPosition(slotSpan, ranges.size()); + } + + /** + * Finds the position in the row key schema for a given position in the scan slots. + * For example, with a slotSpan of {0, 1, 0}, the slot at index 1 spans an extra column in the row key. This means + * that the slot at index 2 has a slot index of 2 but a row key index of 3. + * To calculate the "adjusted position" index, we simply add up the number of extra slots spanned and offset + * the slotPosition by that much. + * @param slotSpan the extra span per skip scan slot. corresponds to {@link ScanRanges#slotSpan} + * @param slotPosition the index of a slot in the SkipScan slots list. + * @return the equivalent row key position in the RowKeySchema + * @see #getTotalSpan(java.util.List, int[]) + */ + public static int getRowKeyPosition(int[] slotSpan, int slotPosition) { + int offset = 0; + + for(int i = 0; i < slotPosition; i++) { + offset += slotSpan[i]; } - return totalSlotSpan; + + return offset + slotPosition; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/c51dc12b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java index 2bfe381..bd19663 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java @@ -1565,7 +1565,7 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest { assertArrayEquals(HConstants.EMPTY_START_ROW, scan.getStartRow()); assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow()); } - + @Test public void testUsingRVCNonFullyQualifiedInClause() throws Exception { String firstOrgId = "000000000000001"; @@ -1577,7 +1577,7 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest { StatementContext context = compileStatement(query, binds); Scan scan = context.getScan(); Filter filter = scan.getFilter(); - assertTrue(filter instanceof RowKeyComparisonFilter); + assertTrue(filter instanceof SkipScanFilter); assertArrayEquals(ByteUtil.concat(PDataType.VARCHAR.toBytes(firstOrgId), PDataType.VARCHAR.toBytes(firstParentId)), scan.getStartRow()); assertArrayEquals(ByteUtil.nextKey(ByteUtil.concat(PDataType.VARCHAR.toBytes(secondOrgId), PDataType.VARCHAR.toBytes(secondParentId))), scan.getStopRow()); }