Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.0 f31a11989 -> 3f353d4ac
PHOENIX-3012 DistinctPrefixFilter logic fails with local indexes and salted tables. Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3f353d4a Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3f353d4a Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3f353d4a Branch: refs/heads/4.x-HBase-1.0 Commit: 3f353d4acfb5a33ed192203b039bf3928aa933b1 Parents: f31a119 Author: Lars Hofhansl <la...@apache.org> Authored: Tue Jun 21 15:59:19 2016 -0700 Committer: Lars Hofhansl <la...@apache.org> Committed: Tue Jun 21 16:11:55 2016 -0700 ---------------------------------------------------------------------- .../phoenix/end2end/DistinctPrefixFilterIT.java | 2 +- .../phoenix/filter/DistinctPrefixFilter.java | 55 ++++++++++++++------ .../java/org/apache/phoenix/util/ScanUtil.java | 4 ++ 3 files changed, 44 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f353d4a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java index d6640ae..203d51e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java @@ -60,7 +60,7 @@ public class DistinctPrefixFilterIT extends BaseHBaseManagedTimeTableReuseIT { createTestTable(getUrl(), ddl); conn.prepareStatement("CREATE INDEX " + testTableF + "_idx ON "+testTableF+"(col2) DISABLE_WAL=true").execute(); - conn.prepareStatement("CREATE INDEX " + testTableV + "_idx ON "+testTableV+"(col2) DISABLE_WAL=true").execute(); + conn.prepareStatement("CREATE LOCAL INDEX " + testTableV + "_idx ON "+testTableV+"(col2) DISABLE_WAL=true").execute(); conn.prepareStatement("CREATE SEQUENCE " + testSeq + " CACHE 1000").execute(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f353d4a/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java index a3e8a0a..a4cbe8a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java @@ -36,6 +36,7 @@ import org.apache.phoenix.util.ByteUtil; public class DistinctPrefixFilter extends FilterBase implements Writable { private static byte VERSION = 1; + private int offset; private RowKeySchema schema; private int prefixLengh; private boolean filterAll = false; @@ -49,12 +50,16 @@ public class DistinctPrefixFilter extends FilterBase implements Writable { this.prefixLengh = prefixLength; } + public void setOffset(int offset) { + this.offset = offset; + } + @Override public ReturnCode filterKeyValue(Cell v) throws IOException { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); // First determine the prefix based on the schema - int maxOffset = schema.iterator(v.getRowArray(), v.getRowOffset(), v.getRowLength(), ptr); + int maxOffset = schema.iterator(v.getRowArray(), v.getRowOffset()+offset, v.getRowLength()-offset, ptr); schema.next(ptr, 0, maxOffset, prefixLengh - 1); // now check whether we have seen this prefix before @@ -70,26 +75,44 @@ public class DistinctPrefixFilter extends FilterBase implements Writable { @Override public Cell getNextCellHint(Cell v) throws IOException { - ImmutableBytesWritable tmp; PDataType<?> type = schema.getField(prefixLengh-1).getDataType(); - if (reversed) { - // simply seek right before the first occurrence of the row - tmp = lastKey; + + ImmutableBytesWritable tmp; + // In the following we make sure we copy the key at most once + // Either because we have an offset, or when needed for nextKey + if (offset > 0) { + // make space to copy the missing offset, also 0-pad here if needed + // (since we're making a copy anyway) + byte[] tmpKey = new byte[offset + lastKey.getLength() + + (reversed || type.isFixedWidth() ? 0 : 1)]; + System.arraycopy(v.getRowArray(), v.getRowOffset(), tmpKey, 0, offset); + System.arraycopy(lastKey.get(), 0, tmpKey, offset, lastKey.getLength()); + tmp = new ImmutableBytesWritable(tmpKey); + if (!reversed) { + // calculate the next key, the above already 0-padded if needed + if (!ByteUtil.nextKey(tmp.get(), tmp.getOffset(), tmp.getLength())) { + filterAll = true; + } + } } else { - if (type.isFixedWidth()) { - // copy the bytes, since nextKey will modify in place - tmp = new ImmutableBytesWritable(lastKey.copyBytes()); + if (reversed) { + // simply seek right before the first occurrence of the row + tmp = lastKey; } else { - // pad with a 0x00 byte (makes a copy) - tmp = new ImmutableBytesWritable(lastKey); - ByteUtil.nullPad(tmp, tmp.getLength() + 1); - } - // calculate the next key - if (!ByteUtil.nextKey(tmp.get(), tmp.getOffset(), tmp.getLength())) { - filterAll = true; + if (type.isFixedWidth()) { + // copy the bytes, since nextKey will modify in place + tmp = new ImmutableBytesWritable(lastKey.copyBytes()); + } else { + // pad with a 0x00 byte (makes a copy) + tmp = new ImmutableBytesWritable(lastKey); + ByteUtil.nullPad(tmp, tmp.getLength() + 1); + } + // calculate the next key + if (!ByteUtil.nextKey(tmp.get(), tmp.getOffset(), tmp.getLength())) { + filterAll = true; + } } } - return KeyValue.createFirstOnRow(tmp.get(), tmp.getOffset(), tmp.getLength(), null, 0, 0, null, 0, 0); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f353d4a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index 7a3014b..711717a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -53,6 +53,7 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.execute.DescVarLengthFastByteComparisons; import org.apache.phoenix.filter.BooleanExpressionFilter; +import org.apache.phoenix.filter.DistinctPrefixFilter; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.KeyRange; @@ -693,6 +694,9 @@ public class ScanUtil { } else if (filter instanceof SkipScanFilter) { SkipScanFilter skipScanFilter = (SkipScanFilter)filter; skipScanFilter.setOffset(offset); + } else if (filter instanceof DistinctPrefixFilter) { + DistinctPrefixFilter prefixFilter = (DistinctPrefixFilter) filter; + prefixFilter.setOffset(offset); } }
