This is an automated email from the ASF dual-hosted git repository. tdsilva pushed a commit to branch phoenix-stats in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/phoenix-stats by this push: new 2487d9d Fix slotSpan appending to schema issue in scanRanges.getRowKeyRanges() new 4d4e9c8 Merge pull request #496 from dbwong/phoenix-stats 2487d9d is described below commit 2487d9d729b12f808266bf343b1e36d2cfcd8089 Author: Daniel Wong <daniel.w...@salesforce.com> AuthorDate: Mon Apr 29 18:58:23 2019 -0700 Fix slotSpan appending to schema issue in scanRanges.getRowKeyRanges() --- .../org/apache/phoenix/compile/ScanRanges.java | 18 ++- .../phoenix/compile/ScanRangesIntersectTest.java | 123 +++++++++++++++++++++ 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index c802678..1b695eb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -719,13 +719,23 @@ public class ScanRanges { int[] slotSpans = this.getSlotSpans(); - // If the ranges here do not qualify all the keys then those keys are unbound - if (newRanges.size() < schema.getMaxFields()) { + int fieldSpan = 0; + for(int i = 0; i < slotSpans.length; i++){ + //Account for a slot covering multiple fields + fieldSpan = fieldSpan + slotSpans[i] + 1; + } + + // If the spans here do not qualify all the keys then those keys are unbound + int maxFields = schema.getMaxFields(); + if (fieldSpan < maxFields) { int originalSize = newRanges.size(); - for (int i = 0; i < schema.getMaxFields() - originalSize; i++) { + for (int i = 0; i < maxFields - originalSize; i++) { newRanges.add(Lists.newArrayList(KeyRange.EVERYTHING_RANGE)); } - slotSpans = new int[schema.getMaxFields()]; + //extend slots + int slotsToAdd = (maxFields - fieldSpan); + int newSlotLength = slotSpans.length + slotsToAdd; + slotSpans = new int[newSlotLength]; System.arraycopy(this.getSlotSpans(), 0, slotSpans, 0, this.getSlotSpans().length); } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java index accc28e..df1bade 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java @@ -36,6 +36,9 @@ import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.schema.types.PSmallint; +import org.apache.phoenix.schema.types.PUnsignedTinyint; import org.apache.phoenix.schema.types.PVarchar; import org.junit.Test; @@ -112,6 +115,33 @@ public class ScanRangesIntersectTest { } }; + private static PDatum SIMPLE_TINYINT = new PDatum() { + @Override + public boolean isNullable() { + return false; + } + + @Override + public PDataType getDataType() { + return PUnsignedTinyint.INSTANCE; + } + + @Override + public Integer getMaxLength() { + return 1; + } + + @Override + public Integer getScale() { + return null; + } + + @Override + public SortOrder getSortOrder() { + return SortOrder.getDefault(); + } + }; + // Does not handle some edge conditions like empty string private String handleScanNextKey(String key) { char lastChar = key.charAt(key.length() - 1); @@ -485,6 +515,96 @@ public class ScanRangesIntersectTest { assertEquals(singleKeyToScanRange("BD"),rowKeyRanges.get(3).toString()); } + @Test + public void getRowKeyRangesMultipleFieldsSingleSlot() { + int rowKeySchemaFields = 3; + RowKeySchema schema = buildSimpleRowKeySchema(rowKeySchemaFields); + + int[] slotSpan = new int[1]; + slotSpan[0] = 2; + + KeyRange keyRange1 = KeyRange.getKeyRange(stringToByteArray("ABC")); + KeyRange keyRange2 = KeyRange.getKeyRange(stringToByteArray("DEF")); + List<List<KeyRange>> ranges = new ArrayList<>(); + ranges.add(Lists.newArrayList(keyRange1,keyRange2)); + + ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1); + + List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges(); + assertEquals(2, rowKeyRanges.size()); + assertEquals(singleKeyToScanRange("ABC"),rowKeyRanges.get(0).toString()); + assertEquals(singleKeyToScanRange("DEF"),rowKeyRanges.get(1).toString()); + } + + @Test + public void getRowKeyRangesMultipleFieldsFrontLoadedSlot() { + int rowKeySchemaFields = 3; + RowKeySchema schema = buildSimpleRowKeySchema(rowKeySchemaFields); + + int[] slotSpan = new int[2]; + slotSpan[0] = 1; + + KeyRange keyRange1 = KeyRange.getKeyRange(stringToByteArray("AB")); + List<List<KeyRange>> ranges = new ArrayList<>(); + ranges.add(Lists.newArrayList(keyRange1)); + + KeyRange keyRange3 = KeyRange.getKeyRange(stringToByteArray("C")); + ranges.add(Lists.newArrayList(keyRange3)); + + ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1); + + List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges(); + assertEquals(1, rowKeyRanges.size()); + assertEquals("[ABC - ABD)",rowKeyRanges.get(0).toString()); + } + + @Test + public void getRowKeyRangesMultipleFieldsBackLoadedSlot() { + int rowKeySchemaFields = 3; + RowKeySchema schema = buildSimpleRowKeySchema(rowKeySchemaFields); + + int[] slotSpan = new int[2]; + slotSpan[1] = 1; + + KeyRange keyRange1 = KeyRange.getKeyRange(stringToByteArray("A")); + List<List<KeyRange>> ranges = new ArrayList<>(); + ranges.add(Lists.newArrayList(keyRange1)); + + KeyRange keyRange3 = KeyRange.getKeyRange(stringToByteArray("BC")); + ranges.add(Lists.newArrayList(keyRange3)); + + ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1); + + List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges(); + assertEquals(1, rowKeyRanges.size()); + assertEquals("[ABC - ABD)",rowKeyRanges.get(0).toString()); + } + + @Test + public void getRowKeyRangesUpperBoundOverflows() { + int rowKeySchemaFields = 2; + RowKeySchema.RowKeySchemaBuilder builder = new RowKeySchema.RowKeySchemaBuilder(rowKeySchemaFields); + for(int i = 0; i < rowKeySchemaFields; i++) { + builder.addField(SIMPLE_TINYINT, SIMPLE_TINYINT.isNullable(), SIMPLE_TINYINT.getSortOrder()); + } + RowKeySchema schema = builder.build(); + + int[] slotSpan = new int[2]; + + KeyRange keyRange1 = KeyRange.getKeyRange(new byte[]{-1}); + List<List<KeyRange>> ranges = new ArrayList<>(); + ranges.add(Lists.newArrayList(keyRange1)); + + KeyRange keyRange3 = KeyRange.getKeyRange(new byte[]{-1}); + ranges.add(Lists.newArrayList(keyRange3)); + + ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1); + + List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges(); + assertEquals(1, rowKeyRanges.size()); + assertEquals("[\\xFF\\xFF - *)",rowKeyRanges.get(0).toString()); + } + private RowKeySchema buildSimpleRowKeySchema(int fields){ RowKeySchema.RowKeySchemaBuilder builder = new RowKeySchema.RowKeySchemaBuilder(fields); for(int i = 0; i < fields; i++) { @@ -494,6 +614,9 @@ public class ScanRangesIntersectTest { } private String singleKeyToScanRange(String key){ + //Appending 0 byte is next key for variable length asc fields + //This includes point gets for more than 1 key as ScanRanges treats the combined pk + // as a varbinary return String.format("[%s - %s\\x00)",key,key); } }