[1/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client
Repository: phoenix Updated Branches: refs/heads/master 0440aca51 - 3f829751d http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f829751/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index 27fe0f9..e84ca2a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -62,6 +62,7 @@ import org.apache.phoenix.parse.TableName; import org.apache.phoenix.parse.TableWildcardParseNode; import org.apache.phoenix.parse.WildcardParseNode; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.AmbiguousColumnException; import org.apache.phoenix.schema.ArgumentTypeMismatchException; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.ColumnNotFoundException; @@ -144,12 +145,21 @@ public class ProjectionCompiler { } ColumnRef ref = new ColumnRef(tableRef,i); String colName = ref.getColumn().getName().getString(); +String tableAlias = tableRef.getTableAlias(); if (resolveColumn) { -if (tableRef.getTableAlias() != null) { -ref = resolver.resolveColumn(null, tableRef.getTableAlias(), colName); -} else { -String schemaName = table.getSchemaName().getString(); -ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, table.getTableName().getString(), colName); +try { +if (tableAlias != null) { +ref = resolver.resolveColumn(null, tableAlias, colName); +} else { +String schemaName = table.getSchemaName().getString(); +ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, table.getTableName().getString(), colName); +} +} catch (AmbiguousColumnException e) { +if (column.getFamilyName() != null) { +ref = resolver.resolveColumn(tableAlias != null ? tableAlias : table.getTableName().getString(), column.getFamilyName().getString(), colName); +} else { +throw e; +} } } Expression expression = ref.newColumnExpression(); @@ -219,12 +229,21 @@ public class ProjectionCompiler { } } String colName = tableColumn.getName().getString(); +String tableAlias = tableRef.getTableAlias(); if (resolveColumn) { -if (tableRef.getTableAlias() != null) { -ref = resolver.resolveColumn(null, tableRef.getTableAlias(), indexColName); -} else { -String schemaName = index.getSchemaName().getString(); -ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, index.getTableName().getString(), indexColName); +try { +if (tableAlias != null) { +ref = resolver.resolveColumn(null, tableAlias, indexColName); +} else { +String schemaName = index.getSchemaName().getString(); +ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, index.getTableName().getString(), indexColName); +} +} catch (AmbiguousColumnException e) { +if (indexColumn.getFamilyName() != null) { +ref = resolver.resolveColumn(tableAlias != null ? tableAlias : index.getTableName().getString(), indexColumn.getFamilyName().getString(), indexColName); +} else { +throw e; +} } } Expression expression = ref.newColumnExpression(); @@ -238,11 +257,14 @@ public class ProjectionCompiler { } } -private static void projectTableColumnFamily(StatementContext context, String cfName, TableRef tableRef, ListExpression projectedExpressions, ListExpressionProjector projectedColumns) throws SQLException { +private static void projectTableColumnFamily(StatementContext context, String cfName, TableRef tableRef, boolean resolveColumn, ListExpression projectedExpressions, ListExpressionProjector projectedColumns) throws SQLException { PTable table = tableRef.getTable(); PColumnFamily pfamily = table.getColumnFamily(cfName); for (PColumn column : pfamily.getColumns()) {
[2/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client
PHOENIX-1489 Access column values positionally from client Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3f829751 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3f829751 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3f829751 Branch: refs/heads/master Commit: 3f829751dd5158c526e5c5bd3da61dde1c6e4194 Parents: 0440aca Author: maryannxue wei@intel.com Authored: Tue Mar 3 12:12:49 2015 -0500 Committer: maryannxue wei@intel.com Committed: Tue Mar 3 12:12:49 2015 -0500 -- .../org/apache/phoenix/end2end/HashJoinIT.java | 55 ++-- .../phoenix/end2end/HashJoinLocalIndexIT.java | 6 +- .../org/apache/phoenix/end2end/SubqueryIT.java | 30 +-- .../end2end/SubqueryUsingSortMergeJoinIT.java | 32 +-- .../index/GlobalIndexOptimizationIT.java| 22 +- .../phoenix/end2end/index/LocalIndexIT.java | 4 +- .../phoenix/compile/ExpressionCompiler.java | 2 +- .../apache/phoenix/compile/FromCompiler.java| 87 ++- .../apache/phoenix/compile/JoinCompiler.java| 258 --- .../apache/phoenix/compile/OrderByCompiler.java | 2 +- .../phoenix/compile/ProjectionCompiler.java | 58 +++-- .../apache/phoenix/compile/QueryCompiler.java | 115 + .../compile/TupleProjectionCompiler.java| 214 +++ .../apache/phoenix/compile/UpsertCompiler.java | 4 +- .../apache/phoenix/compile/WhereCompiler.java | 2 +- .../coprocessor/BaseScannerRegionObserver.java | 21 +- .../GroupedAggregateRegionObserver.java | 11 +- .../coprocessor/HashJoinRegionScanner.java | 72 +++--- .../phoenix/coprocessor/ScanRegionObserver.java | 9 +- .../UngroupedAggregateRegionObserver.java | 10 +- .../apache/phoenix/execute/TupleProjector.java | 18 +- .../org/apache/phoenix/join/HashJoinInfo.java | 25 +- .../apache/phoenix/optimize/QueryOptimizer.java | 19 +- .../org/apache/phoenix/schema/ColumnRef.java| 2 +- .../phoenix/schema/LocalIndexDataColumnRef.java | 9 +- .../apache/phoenix/schema/MetaDataClient.java | 2 +- .../org/apache/phoenix/schema/PTableType.java | 2 +- .../apache/phoenix/schema/ProjectedColumn.java | 59 + .../org/apache/phoenix/schema/TableRef.java | 8 +- .../phoenix/compile/WhereCompilerTest.java | 6 +- 30 files changed, 691 insertions(+), 473 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f829751/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java -- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java index e915b36..596e5e9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java @@ -219,7 +219,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { SERVER FILTER BY QUANTITY 5000\n + PARALLEL INNER-JOIN TABLE 1\n + CLIENT PARALLEL 1-WAY FULL SCAN OVER + JOIN_SUPPLIER_TABLE_DISPLAY_NAME + \n + -DYNAMIC SERVER FILTER BY \item_id\ IN (\O.item_id\), +DYNAMIC SERVER FILTER BY \I.item_id\ IN (\O.item_id\), /* * testSelfJoin() * SELECT i2.item_id, i1.name FROM joinItemTable i1 @@ -230,7 +230,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { PARALLEL INNER-JOIN TABLE 0\n + CLIENT PARALLEL 1-WAY FULL SCAN OVER + JOIN_ITEM_TABLE_DISPLAY_NAME + \n + SERVER FILTER BY FIRST KEY ONLY\n + -DYNAMIC SERVER FILTER BY \item_id\ IN (\I2.item_id\), +DYNAMIC SERVER FILTER BY \I1.item_id\ IN (\I2.item_id\), /* * testSelfJoin() * SELECT i1.name, i2.name FROM joinItemTable i1 @@ -242,7 +242,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { CLIENT MERGE SORT\n + PARALLEL INNER-JOIN TABLE 0\n + CLIENT PARALLEL 1-WAY FULL SCAN OVER + JOIN_ITEM_TABLE_DISPLAY_NAME + \n + -DYNAMIC SERVER FILTER BY \item_id\ IN (\I2.supplier_id\), +DYNAMIC SERVER FILTER BY \I1.item_id\ IN (\I2.supplier_id\), /* * testStarJoin() * SELECT order_id, c.name, i.name iname, quantity, o.date @@ -271,7 +271,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { CLIENT PARALLEL 1-WAY FULL SCAN OVER +
Jenkins build is back to normal : Phoenix | Master #603
See https://builds.apache.org/job/Phoenix-master/603/changes
Apache-Phoenix | 4.0 | Build Successful
4.0 branch build status Successful Source repository https://git-wip-us.apache.org/repos/asf/incubator-phoenix.git Compiled Artifacts https://builds.apache.org/job/Phoenix-4.0/lastSuccessfulBuild/artifact/ Test Report https://builds.apache.org/job/Phoenix-4.0/lastCompletedBuild/testReport/ Changes [maryannxue] PHOENIX-1489 Access column values positionally from client Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout
[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java new file mode 100644 index 000..29e14bf --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -0,0 +1,717 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.SortedMap; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Maps; + + + +public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest { +private static final byte[][] REGION_BOUNDARIES_MINIMAL = { + Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + +private static final byte[][] GUIDE_POSTS_MINIMAL = { + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + + +private static final byte[][] REGION_BOUNDARIES_ALL = { + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xAA\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x82\\xB4]\\xE7\\x80\\x00\\x00\\x00ER\\xFE#\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +
[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
Repository: phoenix Updated Branches: refs/heads/master 3f829751d - 49f06b331 http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java index a2b6115..fea1d91 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java @@ -26,13 +26,13 @@ import java.util.List; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.query.KeyRange; -import org.apache.phoenix.schema.types.PChar; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.PDatum; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PChar; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarchar; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -92,7 +92,8 @@ public class SkipScanFilterIntersectTest { @Parameters(name={0} {4}) public static CollectionObject data() { ListObject testCases = Lists.newArrayList(); -// Causes increment of slot 2 to increment slot 1 +// Both ranges in second slot are required b/c first slot contains range and upper/lower +// values differ in this slot position. testCases.addAll(foreach( new KeyRange[][] {{ PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), @@ -109,10 +110,70 @@ public class SkipScanFilterIntersectTest { new KeyRange[][] {{ PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), }, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), }, { PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), }})); +// Only second range in second slot is required b/c though first slot contains range, +// upper/lower values do not differ in this slot position. +testCases.addAll(foreach( +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}}, +new int[] {1,1,1}, +Bytes.toBytes(j3A), +Bytes.toBytes(j4C), +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}})); +// Test case exercising repositioning multiple times (initially to slot #2 and then again +// to slot #4). Because there's a range for slot #4 and the lower/upper values are different, +// all slot #5 ranges are part of the intersection. +testCases.addAll(foreach( +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(b), true), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(d), true, Bytes.toBytes(d), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(C), true, Bytes.toBytes(C), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(m), true, Bytes.toBytes(u), false), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(z), true, Bytes.toBytes(z), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(A), true,
[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49f06b33 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49f06b33 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49f06b33 Branch: refs/heads/master Commit: 49f06b331c2b364f176d77548d04a4eb9d15c5c9 Parents: 3f82975 Author: James Taylor jtay...@salesforce.com Authored: Tue Mar 3 12:07:56 2015 -0800 Committer: James Taylor jtay...@salesforce.com Committed: Tue Mar 3 12:07:56 2015 -0800 -- .../apache/phoenix/filter/SkipScanFilter.java | 31 +- .../org/apache/phoenix/schema/PTableImpl.java | 9 + .../phoenix/filter/SkipScanBigFilterTest.java | 717 +++ .../filter/SkipScanFilterIntersectTest.java | 69 +- 4 files changed, 821 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index 0aafdbb..1923856 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements Writable { Arrays.fill(position, 0); } -private boolean intersect(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, ListListKeyRange newSlots) { +private boolean intersect(final byte[] lowerInclusiveKey, final byte[] upperExclusiveKey, ListListKeyRange newSlots) { resetState(); boolean lowerUnbound = (lowerInclusiveKey.length == 0); int startPos = 0; @@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements Writable { } int[] lowerPosition = Arrays.copyOf(position, position.length); // Navigate to the upperExclusiveKey, but not past it +// TODO: We're including everything between the lowerPosition and end position, which is +// more than we need. We can optimize this by tracking whether each range in each slot position +// intersects. ReturnCode endCode = navigate(upperExclusiveKey, 0, upperExclusiveKey.length, Terminate.AT); if (endCode == ReturnCode.INCLUDE) { setStartKey(); @@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements Writable { position[i] = slots.get(i).size() - 1; } } +int prevRowKeyPos = -1; +ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable(); +ImmutableBytesWritable upperPtr = new ImmutableBytesWritable(); +schema.iterator(lowerInclusiveKey, lowerPtr); +schema.iterator(upperExclusiveKey, upperPtr); // Copy inclusive all positions for (int i = 0; i = lastSlot; i++) { ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], Math.min(position[i] + 1, slots.get(i).size())); @@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements Writable { if (newSlots != null) { newSlots.add(newRanges); } +// Must include all less-significant slot values if: +// 1) a more-significant slot was incremented if (position[i] lowerPosition[i]) { if (newSlots != null) { newSlots.addAll(slots.subList(i+1, slots.size())); } break; } +// 2) we're at a slot containing a range and the values differ between the lower and upper range, +//since less-significant slots may be lower after traversal than where they started. +if (!slots.get(i).get(position[i]).isSingleKey()) { +int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i); +// Position lowerPtr/upperPtr within lowerInclusiveKey/upperExclusiveKey at value for slot i +// The reposition method will do this incrementally, where we we're initially have prevRowKeyPos = -1. +schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, lowerInclusiveKey.length, slotSpan[i]); +schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, upperExclusiveKey.length, slotSpan[i]); +// If we have a range and the values differ, we must include all slots that are less significant. +// For example: [A-D][1,23], the lower/upper keys could be B5/C2, where the C is in range and the
[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
http://git-wip-us.apache.org/repos/asf/phoenix/blob/193622e5/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java new file mode 100644 index 000..29e14bf --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -0,0 +1,717 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.SortedMap; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Maps; + + + +public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest { +private static final byte[][] REGION_BOUNDARIES_MINIMAL = { + Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + +private static final byte[][] GUIDE_POSTS_MINIMAL = { + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + + +private static final byte[][] REGION_BOUNDARIES_ALL = { + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xAA\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x82\\xB4]\\xE7\\x80\\x00\\x00\\x00ER\\xFE#\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +
[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
Repository: phoenix Updated Branches: refs/heads/4.0 7f3183ae9 - 193622e53 http://git-wip-us.apache.org/repos/asf/phoenix/blob/193622e5/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java index a2b6115..fea1d91 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java @@ -26,13 +26,13 @@ import java.util.List; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.query.KeyRange; -import org.apache.phoenix.schema.types.PChar; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.PDatum; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PChar; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarchar; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -92,7 +92,8 @@ public class SkipScanFilterIntersectTest { @Parameters(name={0} {4}) public static CollectionObject data() { ListObject testCases = Lists.newArrayList(); -// Causes increment of slot 2 to increment slot 1 +// Both ranges in second slot are required b/c first slot contains range and upper/lower +// values differ in this slot position. testCases.addAll(foreach( new KeyRange[][] {{ PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), @@ -109,10 +110,70 @@ public class SkipScanFilterIntersectTest { new KeyRange[][] {{ PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), }, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), }, { PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), }})); +// Only second range in second slot is required b/c though first slot contains range, +// upper/lower values do not differ in this slot position. +testCases.addAll(foreach( +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}}, +new int[] {1,1,1}, +Bytes.toBytes(j3A), +Bytes.toBytes(j4C), +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}})); +// Test case exercising repositioning multiple times (initially to slot #2 and then again +// to slot #4). Because there's a range for slot #4 and the lower/upper values are different, +// all slot #5 ranges are part of the intersection. +testCases.addAll(foreach( +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(b), true), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(d), true, Bytes.toBytes(d), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(C), true, Bytes.toBytes(C), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(m), true, Bytes.toBytes(u), false), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(z), true, Bytes.toBytes(z), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(A), true,
[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/193622e5 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/193622e5 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/193622e5 Branch: refs/heads/4.0 Commit: 193622e53fa5f9271c3c08e7871bc18912cfe491 Parents: 7f3183a Author: James Taylor jtay...@salesforce.com Authored: Tue Mar 3 12:07:56 2015 -0800 Committer: James Taylor jtay...@salesforce.com Committed: Tue Mar 3 18:26:58 2015 -0800 -- .../apache/phoenix/filter/SkipScanFilter.java | 31 +- .../org/apache/phoenix/schema/PTableImpl.java | 9 + .../phoenix/filter/SkipScanBigFilterTest.java | 717 +++ .../filter/SkipScanFilterIntersectTest.java | 69 +- 4 files changed, 821 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/193622e5/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index 0aafdbb..1923856 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements Writable { Arrays.fill(position, 0); } -private boolean intersect(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, ListListKeyRange newSlots) { +private boolean intersect(final byte[] lowerInclusiveKey, final byte[] upperExclusiveKey, ListListKeyRange newSlots) { resetState(); boolean lowerUnbound = (lowerInclusiveKey.length == 0); int startPos = 0; @@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements Writable { } int[] lowerPosition = Arrays.copyOf(position, position.length); // Navigate to the upperExclusiveKey, but not past it +// TODO: We're including everything between the lowerPosition and end position, which is +// more than we need. We can optimize this by tracking whether each range in each slot position +// intersects. ReturnCode endCode = navigate(upperExclusiveKey, 0, upperExclusiveKey.length, Terminate.AT); if (endCode == ReturnCode.INCLUDE) { setStartKey(); @@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements Writable { position[i] = slots.get(i).size() - 1; } } +int prevRowKeyPos = -1; +ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable(); +ImmutableBytesWritable upperPtr = new ImmutableBytesWritable(); +schema.iterator(lowerInclusiveKey, lowerPtr); +schema.iterator(upperExclusiveKey, upperPtr); // Copy inclusive all positions for (int i = 0; i = lastSlot; i++) { ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], Math.min(position[i] + 1, slots.get(i).size())); @@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements Writable { if (newSlots != null) { newSlots.add(newRanges); } +// Must include all less-significant slot values if: +// 1) a more-significant slot was incremented if (position[i] lowerPosition[i]) { if (newSlots != null) { newSlots.addAll(slots.subList(i+1, slots.size())); } break; } +// 2) we're at a slot containing a range and the values differ between the lower and upper range, +//since less-significant slots may be lower after traversal than where they started. +if (!slots.get(i).get(position[i]).isSingleKey()) { +int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i); +// Position lowerPtr/upperPtr within lowerInclusiveKey/upperExclusiveKey at value for slot i +// The reposition method will do this incrementally, where we we're initially have prevRowKeyPos = -1. +schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, lowerInclusiveKey.length, slotSpan[i]); +schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, upperExclusiveKey.length, slotSpan[i]); +// If we have a range and the values differ, we must include all slots that are less significant. +// For example: [A-D][1,23], the lower/upper keys could be B5/C2, where the C is in range and the +
[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd22ccd9/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java new file mode 100644 index 000..44f6b49 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -0,0 +1,728 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.SortedMap; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Maps; + + + +public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest { +private static final byte[][] REGION_BOUNDARIES_MINIMAL = { + Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + +private static final byte[][] GUIDE_POSTS_MINIMAL = { + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + + +private static final byte[][] REGION_BOUNDARIES_ALL = { + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%), +
[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Conflicts: phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dd22ccd9 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dd22ccd9 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dd22ccd9 Branch: refs/heads/3.0 Commit: dd22ccd94a33fe987661c6953bbe61a6cf6f9e71 Parents: b467af4 Author: James Taylor jtay...@salesforce.com Authored: Tue Mar 3 12:07:56 2015 -0800 Committer: James Taylor jtay...@salesforce.com Committed: Tue Mar 3 18:45:13 2015 -0800 -- .../apache/phoenix/filter/SkipScanFilter.java | 31 +- .../org/apache/phoenix/schema/PTableImpl.java | 9 + .../phoenix/filter/SkipScanBigFilterTest.java | 728 +++ .../filter/SkipScanFilterIntersectTest.java | 63 +- 4 files changed, 829 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd22ccd9/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index ba2f180..a8c1243 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -179,7 +179,7 @@ public class SkipScanFilter extends FilterBase { Arrays.fill(position, 0); } -private boolean intersect(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, ListListKeyRange newSlots) { +private boolean intersect(final byte[] lowerInclusiveKey, final byte[] upperExclusiveKey, ListListKeyRange newSlots) { resetState(); boolean lowerUnbound = (lowerInclusiveKey.length == 0); int startPos = 0; @@ -246,6 +246,9 @@ public class SkipScanFilter extends FilterBase { } int[] lowerPosition = Arrays.copyOf(position, position.length); // Navigate to the upperExclusiveKey, but not past it +// TODO: We're including everything between the lowerPosition and end position, which is +// more than we need. We can optimize this by tracking whether each range in each slot position +// intersects. ReturnCode endCode = navigate(upperExclusiveKey, 0, upperExclusiveKey.length, Terminate.AT); if (endCode == ReturnCode.INCLUDE) { setStartKey(); @@ -270,6 +273,11 @@ public class SkipScanFilter extends FilterBase { position[i] = slots.get(i).size() - 1; } } +int prevRowKeyPos = -1; +ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable(); +ImmutableBytesWritable upperPtr = new ImmutableBytesWritable(); +schema.iterator(lowerInclusiveKey, lowerPtr); +schema.iterator(upperExclusiveKey, upperPtr); // Copy inclusive all positions for (int i = 0; i = lastSlot; i++) { ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], Math.min(position[i] + 1, slots.get(i).size())); @@ -279,12 +287,33 @@ public class SkipScanFilter extends FilterBase { if (newSlots != null) { newSlots.add(newRanges); } +// Must include all less-significant slot values if: +// 1) a more-significant slot was incremented if (position[i] lowerPosition[i]) { if (newSlots != null) { newSlots.addAll(slots.subList(i+1, slots.size())); } break; } +// 2) we're at a slot containing a range and the values differ between the lower and upper range, +//since less-significant slots may be lower after traversal than where they started. +if (!slots.get(i).get(position[i]).isSingleKey()) { +int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i); +// Position lowerPtr/upperPtr within lowerInclusiveKey/upperExclusiveKey at value for slot i +// The reposition method will do this incrementally, where we we're initially have prevRowKeyPos = -1. +schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, lowerInclusiveKey.length, slotSpan[i]); +schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, upperExclusiveKey.length, slotSpan[i]); +// If we have a range and the values differ, we must include all slots that are less significant. +// For example: [A-D][1,23], the lower/upper keys could be B5/C2, where the C
[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
Repository: phoenix Updated Branches: refs/heads/3.0 b467af4fd - dd22ccd94 http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd22ccd9/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java index 6d44818..7e35ae8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java @@ -90,7 +90,8 @@ public class SkipScanFilterIntersectTest { @Parameters(name={0} {4}) public static CollectionObject data() { ListObject testCases = Lists.newArrayList(); -// Causes increment of slot 2 to increment slot 1 +// Both ranges in second slot are required b/c first slot contains range and upper/lower +// values differ in this slot position. testCases.addAll(foreach( new KeyRange[][] {{ PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), @@ -107,10 +108,70 @@ public class SkipScanFilterIntersectTest { new KeyRange[][] {{ PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), }, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), PDataType.CHAR.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), }, { PDataType.CHAR.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), }})); +// Only second range in second slot is required b/c though first slot contains range, +// upper/lower values do not differ in this slot position. +testCases.addAll(foreach( +new KeyRange[][] {{ +PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), +PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), +PDataType.CHAR.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}}, +new int[] {1,1,1}, +Bytes.toBytes(j3A), +Bytes.toBytes(j4C), +new KeyRange[][] {{ +PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}})); +// Test case exercising repositioning multiple times (initially to slot #2 and then again +// to slot #4). Because there's a range for slot #4 and the lower/upper values are different, +// all slot #5 ranges are part of the intersection. +testCases.addAll(foreach( +new KeyRange[][] {{ +PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(b), true), +PDataType.CHAR.getKeyRange(Bytes.toBytes(d), true, Bytes.toBytes(d), true), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(C), true, Bytes.toBytes(C), true), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(m), true, Bytes.toBytes(u), false), +PDataType.CHAR.getKeyRange(Bytes.toBytes(z), true, Bytes.toBytes(z), true), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(A), true, Bytes.toBytes(A), true), +PDataType.CHAR.getKeyRange(Bytes.toBytes(D), true, Bytes.toBytes(D), true), +PDataType.CHAR.getKeyRange(Bytes.toBytes(M), true, Bytes.toBytes(M), true), +} +}, +new int[] {1,1,1,1,1}, +Bytes.toBytes(bkCpM), +Bytes.toBytes(bkCtD), +new KeyRange[][] {{ +PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(b), true), +}, { +PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +
Apache-Phoenix | Master | Build Successful
Master branch build status Successful Source repository https://git-wip-us.apache.org/repos/asf/phoenix.git Last Successful Compiled Artifacts https://builds.apache.org/job/Phoenix-master/lastSuccessfulBuild/artifact/ Last Complete Test Report https://builds.apache.org/job/Phoenix-master/lastCompletedBuild/testReport/ Changes [jtaylor] PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout
Apache-Phoenix | 3.0 | Hadoop1 | Build Successful
3.0 branch build status Successful Source repository https://git-wip-us.apache.org/repos/asf/phoenix.git Last Successful Compiled Artifacts https://builds.apache.org/job/Phoenix-3.0-hadoop1/lastSuccessfulBuild/artifact/ Last Complete Test Report https://builds.apache.org/job/Phoenix-3.0-hadoop1/lastCompletedBuild/testReport/ Changes [jtaylor] PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout
Apache-Phoenix | 4.0 | Build Successful
4.0 branch build status Successful Source repository https://git-wip-us.apache.org/repos/asf/incubator-phoenix.git Compiled Artifacts https://builds.apache.org/job/Phoenix-4.0/lastSuccessfulBuild/artifact/ Test Report https://builds.apache.org/job/Phoenix-4.0/lastCompletedBuild/testReport/ Changes [jtaylor] PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout
[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
http://git-wip-us.apache.org/repos/asf/phoenix/blob/90cd05e8/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java new file mode 100644 index 000..29e14bf --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -0,0 +1,717 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.SortedMap; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.end2end.Shadower; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Maps; + + + +public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest { +private static final byte[][] REGION_BOUNDARIES_MINIMAL = { + Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + +private static final byte[][] GUIDE_POSTS_MINIMAL = { + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +}; + + +private static final byte[][] REGION_BOUNDARIES_ALL = { + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%), + Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xAA\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x82\\xB4]\\xE7\\x80\\x00\\x00\\x00ER\\xFE#\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00), +
[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/90cd05e8 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/90cd05e8 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/90cd05e8 Branch: refs/heads/4.3 Commit: 90cd05e8f41d948aba733e2bd9933936df4720f4 Parents: caead2e Author: James Taylor jtay...@salesforce.com Authored: Tue Mar 3 12:07:56 2015 -0800 Committer: James Taylor jtay...@salesforce.com Committed: Tue Mar 3 18:31:27 2015 -0800 -- .../apache/phoenix/filter/SkipScanFilter.java | 31 +- .../org/apache/phoenix/schema/PTableImpl.java | 9 + .../phoenix/filter/SkipScanBigFilterTest.java | 717 +++ .../filter/SkipScanFilterIntersectTest.java | 69 +- 4 files changed, 821 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/90cd05e8/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java index 0aafdbb..1923856 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java @@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements Writable { Arrays.fill(position, 0); } -private boolean intersect(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, ListListKeyRange newSlots) { +private boolean intersect(final byte[] lowerInclusiveKey, final byte[] upperExclusiveKey, ListListKeyRange newSlots) { resetState(); boolean lowerUnbound = (lowerInclusiveKey.length == 0); int startPos = 0; @@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements Writable { } int[] lowerPosition = Arrays.copyOf(position, position.length); // Navigate to the upperExclusiveKey, but not past it +// TODO: We're including everything between the lowerPosition and end position, which is +// more than we need. We can optimize this by tracking whether each range in each slot position +// intersects. ReturnCode endCode = navigate(upperExclusiveKey, 0, upperExclusiveKey.length, Terminate.AT); if (endCode == ReturnCode.INCLUDE) { setStartKey(); @@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements Writable { position[i] = slots.get(i).size() - 1; } } +int prevRowKeyPos = -1; +ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable(); +ImmutableBytesWritable upperPtr = new ImmutableBytesWritable(); +schema.iterator(lowerInclusiveKey, lowerPtr); +schema.iterator(upperExclusiveKey, upperPtr); // Copy inclusive all positions for (int i = 0; i = lastSlot; i++) { ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], Math.min(position[i] + 1, slots.get(i).size())); @@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements Writable { if (newSlots != null) { newSlots.add(newRanges); } +// Must include all less-significant slot values if: +// 1) a more-significant slot was incremented if (position[i] lowerPosition[i]) { if (newSlots != null) { newSlots.addAll(slots.subList(i+1, slots.size())); } break; } +// 2) we're at a slot containing a range and the values differ between the lower and upper range, +//since less-significant slots may be lower after traversal than where they started. +if (!slots.get(i).get(position[i]).isSingleKey()) { +int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i); +// Position lowerPtr/upperPtr within lowerInclusiveKey/upperExclusiveKey at value for slot i +// The reposition method will do this incrementally, where we we're initially have prevRowKeyPos = -1. +schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, lowerInclusiveKey.length, slotSpan[i]); +schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, upperExclusiveKey.length, slotSpan[i]); +// If we have a range and the values differ, we must include all slots that are less significant. +// For example: [A-D][1,23], the lower/upper keys could be B5/C2, where the C is in range and the +
[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect
Repository: phoenix Updated Branches: refs/heads/4.3 caead2e4f - 90cd05e8f http://git-wip-us.apache.org/repos/asf/phoenix/blob/90cd05e8/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java -- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java index a2b6115..fea1d91 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java @@ -26,13 +26,13 @@ import java.util.List; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.query.KeyRange; -import org.apache.phoenix.schema.types.PChar; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.PDatum; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PChar; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarchar; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -92,7 +92,8 @@ public class SkipScanFilterIntersectTest { @Parameters(name={0} {4}) public static CollectionObject data() { ListObject testCases = Lists.newArrayList(); -// Causes increment of slot 2 to increment slot 1 +// Both ranges in second slot are required b/c first slot contains range and upper/lower +// values differ in this slot position. testCases.addAll(foreach( new KeyRange[][] {{ PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), @@ -109,10 +110,70 @@ public class SkipScanFilterIntersectTest { new KeyRange[][] {{ PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), }, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), }, { PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), }})); +// Only second range in second slot is required b/c though first slot contains range, +// upper/lower values do not differ in this slot position. +testCases.addAll(foreach( +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(e), false), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, Bytes.toBytes(1), true), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}}, +new int[] {1,1,1}, +Bytes.toBytes(j3A), +Bytes.toBytes(j4C), +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, Bytes.toBytes(4), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, Bytes.toBytes(B), true), +}})); +// Test case exercising repositioning multiple times (initially to slot #2 and then again +// to slot #4). Because there's a range for slot #4 and the lower/upper values are different, +// all slot #5 ranges are part of the intersection. +testCases.addAll(foreach( +new KeyRange[][] {{ +PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, Bytes.toBytes(b), true), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(d), true, Bytes.toBytes(d), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, Bytes.toBytes(m), false), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(C), true, Bytes.toBytes(C), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(m), true, Bytes.toBytes(u), false), +PChar.INSTANCE.getKeyRange(Bytes.toBytes(z), true, Bytes.toBytes(z), true), +}, { +PChar.INSTANCE.getKeyRange(Bytes.toBytes(A), true,
[1/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client
Repository: phoenix Updated Branches: refs/heads/4.0 11d84bdf9 - 7f3183ae9 http://git-wip-us.apache.org/repos/asf/phoenix/blob/7f3183ae/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java index 27fe0f9..e84ca2a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java @@ -62,6 +62,7 @@ import org.apache.phoenix.parse.TableName; import org.apache.phoenix.parse.TableWildcardParseNode; import org.apache.phoenix.parse.WildcardParseNode; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.AmbiguousColumnException; import org.apache.phoenix.schema.ArgumentTypeMismatchException; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.ColumnNotFoundException; @@ -144,12 +145,21 @@ public class ProjectionCompiler { } ColumnRef ref = new ColumnRef(tableRef,i); String colName = ref.getColumn().getName().getString(); +String tableAlias = tableRef.getTableAlias(); if (resolveColumn) { -if (tableRef.getTableAlias() != null) { -ref = resolver.resolveColumn(null, tableRef.getTableAlias(), colName); -} else { -String schemaName = table.getSchemaName().getString(); -ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, table.getTableName().getString(), colName); +try { +if (tableAlias != null) { +ref = resolver.resolveColumn(null, tableAlias, colName); +} else { +String schemaName = table.getSchemaName().getString(); +ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, table.getTableName().getString(), colName); +} +} catch (AmbiguousColumnException e) { +if (column.getFamilyName() != null) { +ref = resolver.resolveColumn(tableAlias != null ? tableAlias : table.getTableName().getString(), column.getFamilyName().getString(), colName); +} else { +throw e; +} } } Expression expression = ref.newColumnExpression(); @@ -219,12 +229,21 @@ public class ProjectionCompiler { } } String colName = tableColumn.getName().getString(); +String tableAlias = tableRef.getTableAlias(); if (resolveColumn) { -if (tableRef.getTableAlias() != null) { -ref = resolver.resolveColumn(null, tableRef.getTableAlias(), indexColName); -} else { -String schemaName = index.getSchemaName().getString(); -ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, index.getTableName().getString(), indexColName); +try { +if (tableAlias != null) { +ref = resolver.resolveColumn(null, tableAlias, indexColName); +} else { +String schemaName = index.getSchemaName().getString(); +ref = resolver.resolveColumn(schemaName.length() == 0 ? null : schemaName, index.getTableName().getString(), indexColName); +} +} catch (AmbiguousColumnException e) { +if (indexColumn.getFamilyName() != null) { +ref = resolver.resolveColumn(tableAlias != null ? tableAlias : index.getTableName().getString(), indexColumn.getFamilyName().getString(), indexColName); +} else { +throw e; +} } } Expression expression = ref.newColumnExpression(); @@ -238,11 +257,14 @@ public class ProjectionCompiler { } } -private static void projectTableColumnFamily(StatementContext context, String cfName, TableRef tableRef, ListExpression projectedExpressions, ListExpressionProjector projectedColumns) throws SQLException { +private static void projectTableColumnFamily(StatementContext context, String cfName, TableRef tableRef, boolean resolveColumn, ListExpression projectedExpressions, ListExpressionProjector projectedColumns) throws SQLException { PTable table = tableRef.getTable(); PColumnFamily pfamily = table.getColumnFamily(cfName); for (PColumn column : pfamily.getColumns()) {
[2/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client
PHOENIX-1489 Access column values positionally from client Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7f3183ae Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7f3183ae Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7f3183ae Branch: refs/heads/4.0 Commit: 7f3183ae9e2ae88cd820d676e4dcc2cfa3576dd7 Parents: 11d84bd Author: maryannxue wei@intel.com Authored: Tue Mar 3 12:40:07 2015 -0500 Committer: maryannxue wei@intel.com Committed: Tue Mar 3 12:40:07 2015 -0500 -- .../org/apache/phoenix/end2end/HashJoinIT.java | 55 ++-- .../phoenix/end2end/HashJoinLocalIndexIT.java | 6 +- .../org/apache/phoenix/end2end/SubqueryIT.java | 30 +-- .../end2end/SubqueryUsingSortMergeJoinIT.java | 32 +-- .../index/GlobalIndexOptimizationIT.java| 22 +- .../phoenix/end2end/index/LocalIndexIT.java | 4 +- .../phoenix/compile/ExpressionCompiler.java | 2 +- .../apache/phoenix/compile/FromCompiler.java| 87 ++- .../apache/phoenix/compile/JoinCompiler.java| 258 --- .../apache/phoenix/compile/OrderByCompiler.java | 2 +- .../phoenix/compile/ProjectionCompiler.java | 58 +++-- .../apache/phoenix/compile/QueryCompiler.java | 117 + .../compile/TupleProjectionCompiler.java| 214 +++ .../apache/phoenix/compile/UpsertCompiler.java | 4 +- .../apache/phoenix/compile/WhereCompiler.java | 2 +- .../coprocessor/BaseScannerRegionObserver.java | 21 +- .../GroupedAggregateRegionObserver.java | 11 +- .../coprocessor/HashJoinRegionScanner.java | 72 +++--- .../phoenix/coprocessor/ScanRegionObserver.java | 9 +- .../UngroupedAggregateRegionObserver.java | 10 +- .../apache/phoenix/execute/TupleProjector.java | 18 +- .../org/apache/phoenix/join/HashJoinInfo.java | 25 +- .../apache/phoenix/optimize/QueryOptimizer.java | 19 +- .../org/apache/phoenix/schema/ColumnRef.java| 2 +- .../phoenix/schema/LocalIndexDataColumnRef.java | 9 +- .../apache/phoenix/schema/MetaDataClient.java | 2 +- .../org/apache/phoenix/schema/PTableType.java | 2 +- .../apache/phoenix/schema/ProjectedColumn.java | 59 + .../org/apache/phoenix/schema/TableRef.java | 8 +- .../phoenix/compile/WhereCompilerTest.java | 6 +- 30 files changed, 692 insertions(+), 474 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/7f3183ae/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java -- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java index e915b36..596e5e9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java @@ -219,7 +219,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { SERVER FILTER BY QUANTITY 5000\n + PARALLEL INNER-JOIN TABLE 1\n + CLIENT PARALLEL 1-WAY FULL SCAN OVER + JOIN_SUPPLIER_TABLE_DISPLAY_NAME + \n + -DYNAMIC SERVER FILTER BY \item_id\ IN (\O.item_id\), +DYNAMIC SERVER FILTER BY \I.item_id\ IN (\O.item_id\), /* * testSelfJoin() * SELECT i2.item_id, i1.name FROM joinItemTable i1 @@ -230,7 +230,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { PARALLEL INNER-JOIN TABLE 0\n + CLIENT PARALLEL 1-WAY FULL SCAN OVER + JOIN_ITEM_TABLE_DISPLAY_NAME + \n + SERVER FILTER BY FIRST KEY ONLY\n + -DYNAMIC SERVER FILTER BY \item_id\ IN (\I2.item_id\), +DYNAMIC SERVER FILTER BY \I1.item_id\ IN (\I2.item_id\), /* * testSelfJoin() * SELECT i1.name, i2.name FROM joinItemTable i1 @@ -242,7 +242,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { CLIENT MERGE SORT\n + PARALLEL INNER-JOIN TABLE 0\n + CLIENT PARALLEL 1-WAY FULL SCAN OVER + JOIN_ITEM_TABLE_DISPLAY_NAME + \n + -DYNAMIC SERVER FILTER BY \item_id\ IN (\I2.supplier_id\), +DYNAMIC SERVER FILTER BY \I1.item_id\ IN (\I2.supplier_id\), /* * testStarJoin() * SELECT order_id, c.name, i.name iname, quantity, o.date @@ -271,7 +271,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT { CLIENT PARALLEL 1-WAY FULL SCAN OVER +