[1/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client

2015-03-03 Thread maryannxue
Repository: phoenix
Updated Branches:
  refs/heads/master 0440aca51 - 3f829751d


http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f829751/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index 27fe0f9..e84ca2a 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -62,6 +62,7 @@ import org.apache.phoenix.parse.TableName;
 import org.apache.phoenix.parse.TableWildcardParseNode;
 import org.apache.phoenix.parse.WildcardParseNode;
 import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.AmbiguousColumnException;
 import org.apache.phoenix.schema.ArgumentTypeMismatchException;
 import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
 import org.apache.phoenix.schema.ColumnNotFoundException;
@@ -144,12 +145,21 @@ public class ProjectionCompiler {
 }
 ColumnRef ref = new ColumnRef(tableRef,i);
 String colName = ref.getColumn().getName().getString();
+String tableAlias = tableRef.getTableAlias();
 if (resolveColumn) {
-if (tableRef.getTableAlias() != null) {
-ref = resolver.resolveColumn(null, 
tableRef.getTableAlias(), colName);
-} else {
-String schemaName = table.getSchemaName().getString();
-ref = resolver.resolveColumn(schemaName.length() == 0 ? 
null : schemaName, table.getTableName().getString(), colName);
+try {
+if (tableAlias != null) {
+ref = resolver.resolveColumn(null, tableAlias, 
colName);
+} else {
+String schemaName = table.getSchemaName().getString();
+ref = resolver.resolveColumn(schemaName.length() == 0 
? null : schemaName, table.getTableName().getString(), colName);
+}
+} catch (AmbiguousColumnException e) {
+if (column.getFamilyName() != null) {
+ref = resolver.resolveColumn(tableAlias != null ? 
tableAlias : table.getTableName().getString(), 
column.getFamilyName().getString(), colName);
+} else {
+throw e;
+}
 }
 }
 Expression expression = ref.newColumnExpression();
@@ -219,12 +229,21 @@ public class ProjectionCompiler {
 }
 }
 String colName = tableColumn.getName().getString();
+String tableAlias = tableRef.getTableAlias();
 if (resolveColumn) {
-if (tableRef.getTableAlias() != null) {
-ref = resolver.resolveColumn(null, 
tableRef.getTableAlias(), indexColName);
-} else {
-String schemaName = index.getSchemaName().getString();
-ref = resolver.resolveColumn(schemaName.length() == 0 ? 
null : schemaName, index.getTableName().getString(), indexColName);
+try {
+if (tableAlias != null) {
+ref = resolver.resolveColumn(null, tableAlias, 
indexColName);
+} else {
+String schemaName = index.getSchemaName().getString();
+ref = resolver.resolveColumn(schemaName.length() == 0 
? null : schemaName, index.getTableName().getString(), indexColName);
+}
+} catch (AmbiguousColumnException e) {
+if (indexColumn.getFamilyName() != null) {
+ref = resolver.resolveColumn(tableAlias != null ? 
tableAlias : index.getTableName().getString(), 
indexColumn.getFamilyName().getString(), indexColName);
+} else {
+throw e;
+}
 }
 }
 Expression expression = ref.newColumnExpression();
@@ -238,11 +257,14 @@ public class ProjectionCompiler {
 }
 }
 
-private static void projectTableColumnFamily(StatementContext context, 
String cfName, TableRef tableRef, ListExpression projectedExpressions, 
ListExpressionProjector projectedColumns) throws SQLException {
+private static void projectTableColumnFamily(StatementContext context, 
String cfName, TableRef tableRef, boolean resolveColumn, ListExpression 
projectedExpressions, ListExpressionProjector projectedColumns) throws 
SQLException {
 PTable table = tableRef.getTable();
 PColumnFamily pfamily = table.getColumnFamily(cfName);
 for (PColumn column : pfamily.getColumns()) {

[2/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client

2015-03-03 Thread maryannxue
PHOENIX-1489 Access column values positionally from client


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3f829751
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3f829751
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3f829751

Branch: refs/heads/master
Commit: 3f829751dd5158c526e5c5bd3da61dde1c6e4194
Parents: 0440aca
Author: maryannxue wei@intel.com
Authored: Tue Mar 3 12:12:49 2015 -0500
Committer: maryannxue wei@intel.com
Committed: Tue Mar 3 12:12:49 2015 -0500

--
 .../org/apache/phoenix/end2end/HashJoinIT.java  |  55 ++--
 .../phoenix/end2end/HashJoinLocalIndexIT.java   |   6 +-
 .../org/apache/phoenix/end2end/SubqueryIT.java  |  30 +--
 .../end2end/SubqueryUsingSortMergeJoinIT.java   |  32 +--
 .../index/GlobalIndexOptimizationIT.java|  22 +-
 .../phoenix/end2end/index/LocalIndexIT.java |   4 +-
 .../phoenix/compile/ExpressionCompiler.java |   2 +-
 .../apache/phoenix/compile/FromCompiler.java|  87 ++-
 .../apache/phoenix/compile/JoinCompiler.java| 258 ---
 .../apache/phoenix/compile/OrderByCompiler.java |   2 +-
 .../phoenix/compile/ProjectionCompiler.java |  58 +++--
 .../apache/phoenix/compile/QueryCompiler.java   | 115 +
 .../compile/TupleProjectionCompiler.java| 214 +++
 .../apache/phoenix/compile/UpsertCompiler.java  |   4 +-
 .../apache/phoenix/compile/WhereCompiler.java   |   2 +-
 .../coprocessor/BaseScannerRegionObserver.java  |  21 +-
 .../GroupedAggregateRegionObserver.java |  11 +-
 .../coprocessor/HashJoinRegionScanner.java  |  72 +++---
 .../phoenix/coprocessor/ScanRegionObserver.java |   9 +-
 .../UngroupedAggregateRegionObserver.java   |  10 +-
 .../apache/phoenix/execute/TupleProjector.java  |  18 +-
 .../org/apache/phoenix/join/HashJoinInfo.java   |  25 +-
 .../apache/phoenix/optimize/QueryOptimizer.java |  19 +-
 .../org/apache/phoenix/schema/ColumnRef.java|   2 +-
 .../phoenix/schema/LocalIndexDataColumnRef.java |   9 +-
 .../apache/phoenix/schema/MetaDataClient.java   |   2 +-
 .../org/apache/phoenix/schema/PTableType.java   |   2 +-
 .../apache/phoenix/schema/ProjectedColumn.java  |  59 +
 .../org/apache/phoenix/schema/TableRef.java |   8 +-
 .../phoenix/compile/WhereCompilerTest.java  |   6 +-
 30 files changed, 691 insertions(+), 473 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/3f829751/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
index e915b36..596e5e9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
@@ -219,7 +219,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 SERVER FILTER BY QUANTITY  5000\n +
 PARALLEL INNER-JOIN TABLE 1\n +
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 
JOIN_SUPPLIER_TABLE_DISPLAY_NAME + \n +
-DYNAMIC SERVER FILTER BY \item_id\ IN (\O.item_id\),
+DYNAMIC SERVER FILTER BY \I.item_id\ IN 
(\O.item_id\),
 /*
  * testSelfJoin()
  * SELECT i2.item_id, i1.name FROM joinItemTable i1 
@@ -230,7 +230,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 PARALLEL INNER-JOIN TABLE 0\n +
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 
JOIN_ITEM_TABLE_DISPLAY_NAME + \n +
 SERVER FILTER BY FIRST KEY ONLY\n +
-DYNAMIC SERVER FILTER BY \item_id\ IN (\I2.item_id\),
+DYNAMIC SERVER FILTER BY \I1.item_id\ IN 
(\I2.item_id\),
 /*
  * testSelfJoin()
  * SELECT i1.name, i2.name FROM joinItemTable i1 
@@ -242,7 +242,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 CLIENT MERGE SORT\n +
 PARALLEL INNER-JOIN TABLE 0\n +
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 
JOIN_ITEM_TABLE_DISPLAY_NAME + \n +
-DYNAMIC SERVER FILTER BY \item_id\ IN 
(\I2.supplier_id\),
+DYNAMIC SERVER FILTER BY \I1.item_id\ IN 
(\I2.supplier_id\),
 /*
  * testStarJoin()
  * SELECT order_id, c.name, i.name iname, quantity, o.date 
@@ -271,7 +271,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 

Jenkins build is back to normal : Phoenix | Master #603

2015-03-03 Thread Apache Jenkins Server
See https://builds.apache.org/job/Phoenix-master/603/changes



Apache-Phoenix | 4.0 | Build Successful

2015-03-03 Thread Apache Jenkins Server
4.0 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf/incubator-phoenix.git

Compiled Artifacts https://builds.apache.org/job/Phoenix-4.0/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-4.0/lastCompletedBuild/testReport/

Changes
[maryannxue] PHOENIX-1489 Access column values positionally from client



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout


[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
new file mode 100644
index 000..29e14bf
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -0,0 +1,717 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedMap;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+
+
+public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
+private static final byte[][] REGION_BOUNDARIES_MINIMAL = {
+
Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+private static final byte[][] GUIDE_POSTS_MINIMAL = {
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+
+private static final byte[][] REGION_BOUNDARIES_ALL = {
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xAA\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x82\\xB4]\\xE7\\x80\\x00\\x00\\x00ER\\xFE#\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+

[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
Repository: phoenix
Updated Branches:
  refs/heads/master 3f829751d - 49f06b331


http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
index a2b6115..fea1d91 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
@@ -26,13 +26,13 @@ import java.util.List;
 
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarchar;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -92,7 +92,8 @@ public class SkipScanFilterIntersectTest {
 @Parameters(name={0} {4})
 public static CollectionObject data() {
 ListObject testCases = Lists.newArrayList();
-// Causes increment of slot 2 to increment slot 1
+// Both ranges in second slot are required b/c first slot contains 
range and upper/lower
+// values differ in this slot position.
 testCases.addAll(foreach(
 new KeyRange[][] {{
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
@@ -109,10 +110,70 @@ public class SkipScanFilterIntersectTest {
 new KeyRange[][] {{
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
 }, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
 }, {
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
 }}));
+// Only second range in second slot is required b/c though first slot 
contains range,
+// upper/lower values do not differ in this slot position.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}},
+new int[] {1,1,1},
+Bytes.toBytes(j3A),
+Bytes.toBytes(j4C),
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}}));
+// Test case exercising repositioning multiple times (initially to 
slot #2 and then again
+// to slot #4). Because there's a range for slot #4 and the 
lower/upper values are different,
+// all slot #5 ranges are part of the intersection.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(b), true),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(d), true, 
Bytes.toBytes(d), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(C), true, 
Bytes.toBytes(C), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(m), true, 
Bytes.toBytes(u), false),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(z), true, 
Bytes.toBytes(z), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(A), true, 

[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49f06b33
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49f06b33
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49f06b33

Branch: refs/heads/master
Commit: 49f06b331c2b364f176d77548d04a4eb9d15c5c9
Parents: 3f82975
Author: James Taylor jtay...@salesforce.com
Authored: Tue Mar 3 12:07:56 2015 -0800
Committer: James Taylor jtay...@salesforce.com
Committed: Tue Mar 3 12:07:56 2015 -0800

--
 .../apache/phoenix/filter/SkipScanFilter.java   |  31 +-
 .../org/apache/phoenix/schema/PTableImpl.java   |   9 +
 .../phoenix/filter/SkipScanBigFilterTest.java   | 717 +++
 .../filter/SkipScanFilterIntersectTest.java |  69 +-
 4 files changed, 821 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/49f06b33/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index 0aafdbb..1923856 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 Arrays.fill(position, 0);
 }
 
-private boolean intersect(byte[] lowerInclusiveKey, byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
+private boolean intersect(final byte[] lowerInclusiveKey, final byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
 resetState();
 boolean lowerUnbound = (lowerInclusiveKey.length == 0);
 int startPos = 0;
@@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 }
 int[] lowerPosition = Arrays.copyOf(position, position.length);
 // Navigate to the upperExclusiveKey, but not past it
+// TODO: We're including everything between the lowerPosition and end 
position, which is
+// more than we need. We can optimize this by tracking whether each 
range in each slot position
+// intersects.
 ReturnCode endCode = navigate(upperExclusiveKey, 0, 
upperExclusiveKey.length, Terminate.AT);
 if (endCode == ReturnCode.INCLUDE) {
 setStartKey();
@@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 position[i] = slots.get(i).size() - 1;
 }
 }
+int prevRowKeyPos = -1;
+ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable();
+ImmutableBytesWritable upperPtr = new ImmutableBytesWritable();
+schema.iterator(lowerInclusiveKey, lowerPtr);
+schema.iterator(upperExclusiveKey, upperPtr);
 // Copy inclusive all positions 
 for (int i = 0; i = lastSlot; i++) {
 ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], 
Math.min(position[i] + 1, slots.get(i).size()));
@@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 if (newSlots != null) {
 newSlots.add(newRanges);
 }
+// Must include all less-significant slot values if:
+// 1) a more-significant slot was incremented
 if (position[i]  lowerPosition[i]) {
 if (newSlots != null) {
 newSlots.addAll(slots.subList(i+1, slots.size()));
 }
 break;
 }
+// 2) we're at a slot containing a range and the values differ 
between the lower and upper range,
+//since less-significant slots may be lower after traversal 
than where they started.
+if (!slots.get(i).get(position[i]).isSingleKey()) {
+int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i);
+// Position lowerPtr/upperPtr within 
lowerInclusiveKey/upperExclusiveKey at value for slot i
+// The reposition method will do this incrementally, where we 
we're initially have prevRowKeyPos = -1. 
+schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, 
lowerInclusiveKey.length, slotSpan[i]);
+schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, 
upperExclusiveKey.length, slotSpan[i]);
+// If we have a range and the values differ, we must include 
all slots that are less significant.
+// For example: [A-D][1,23], the lower/upper keys could be 
B5/C2, where the C is in range and the

[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
http://git-wip-us.apache.org/repos/asf/phoenix/blob/193622e5/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
new file mode 100644
index 000..29e14bf
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -0,0 +1,717 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedMap;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+
+
+public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
+private static final byte[][] REGION_BOUNDARIES_MINIMAL = {
+
Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+private static final byte[][] GUIDE_POSTS_MINIMAL = {
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+
+private static final byte[][] REGION_BOUNDARIES_ALL = {
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xAA\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x82\\xB4]\\xE7\\x80\\x00\\x00\\x00ER\\xFE#\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+

[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
Repository: phoenix
Updated Branches:
  refs/heads/4.0 7f3183ae9 - 193622e53


http://git-wip-us.apache.org/repos/asf/phoenix/blob/193622e5/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
index a2b6115..fea1d91 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
@@ -26,13 +26,13 @@ import java.util.List;
 
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarchar;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -92,7 +92,8 @@ public class SkipScanFilterIntersectTest {
 @Parameters(name={0} {4})
 public static CollectionObject data() {
 ListObject testCases = Lists.newArrayList();
-// Causes increment of slot 2 to increment slot 1
+// Both ranges in second slot are required b/c first slot contains 
range and upper/lower
+// values differ in this slot position.
 testCases.addAll(foreach(
 new KeyRange[][] {{
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
@@ -109,10 +110,70 @@ public class SkipScanFilterIntersectTest {
 new KeyRange[][] {{
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
 }, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
 }, {
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
 }}));
+// Only second range in second slot is required b/c though first slot 
contains range,
+// upper/lower values do not differ in this slot position.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}},
+new int[] {1,1,1},
+Bytes.toBytes(j3A),
+Bytes.toBytes(j4C),
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}}));
+// Test case exercising repositioning multiple times (initially to 
slot #2 and then again
+// to slot #4). Because there's a range for slot #4 and the 
lower/upper values are different,
+// all slot #5 ranges are part of the intersection.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(b), true),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(d), true, 
Bytes.toBytes(d), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(C), true, 
Bytes.toBytes(C), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(m), true, 
Bytes.toBytes(u), false),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(z), true, 
Bytes.toBytes(z), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(A), true, 

[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/193622e5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/193622e5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/193622e5

Branch: refs/heads/4.0
Commit: 193622e53fa5f9271c3c08e7871bc18912cfe491
Parents: 7f3183a
Author: James Taylor jtay...@salesforce.com
Authored: Tue Mar 3 12:07:56 2015 -0800
Committer: James Taylor jtay...@salesforce.com
Committed: Tue Mar 3 18:26:58 2015 -0800

--
 .../apache/phoenix/filter/SkipScanFilter.java   |  31 +-
 .../org/apache/phoenix/schema/PTableImpl.java   |   9 +
 .../phoenix/filter/SkipScanBigFilterTest.java   | 717 +++
 .../filter/SkipScanFilterIntersectTest.java |  69 +-
 4 files changed, 821 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/193622e5/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index 0aafdbb..1923856 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 Arrays.fill(position, 0);
 }
 
-private boolean intersect(byte[] lowerInclusiveKey, byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
+private boolean intersect(final byte[] lowerInclusiveKey, final byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
 resetState();
 boolean lowerUnbound = (lowerInclusiveKey.length == 0);
 int startPos = 0;
@@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 }
 int[] lowerPosition = Arrays.copyOf(position, position.length);
 // Navigate to the upperExclusiveKey, but not past it
+// TODO: We're including everything between the lowerPosition and end 
position, which is
+// more than we need. We can optimize this by tracking whether each 
range in each slot position
+// intersects.
 ReturnCode endCode = navigate(upperExclusiveKey, 0, 
upperExclusiveKey.length, Terminate.AT);
 if (endCode == ReturnCode.INCLUDE) {
 setStartKey();
@@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 position[i] = slots.get(i).size() - 1;
 }
 }
+int prevRowKeyPos = -1;
+ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable();
+ImmutableBytesWritable upperPtr = new ImmutableBytesWritable();
+schema.iterator(lowerInclusiveKey, lowerPtr);
+schema.iterator(upperExclusiveKey, upperPtr);
 // Copy inclusive all positions 
 for (int i = 0; i = lastSlot; i++) {
 ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], 
Math.min(position[i] + 1, slots.get(i).size()));
@@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 if (newSlots != null) {
 newSlots.add(newRanges);
 }
+// Must include all less-significant slot values if:
+// 1) a more-significant slot was incremented
 if (position[i]  lowerPosition[i]) {
 if (newSlots != null) {
 newSlots.addAll(slots.subList(i+1, slots.size()));
 }
 break;
 }
+// 2) we're at a slot containing a range and the values differ 
between the lower and upper range,
+//since less-significant slots may be lower after traversal 
than where they started.
+if (!slots.get(i).get(position[i]).isSingleKey()) {
+int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i);
+// Position lowerPtr/upperPtr within 
lowerInclusiveKey/upperExclusiveKey at value for slot i
+// The reposition method will do this incrementally, where we 
we're initially have prevRowKeyPos = -1. 
+schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, 
lowerInclusiveKey.length, slotSpan[i]);
+schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, 
upperExclusiveKey.length, slotSpan[i]);
+// If we have a range and the values differ, we must include 
all slots that are less significant.
+// For example: [A-D][1,23], the lower/upper keys could be 
B5/C2, where the C is in range and the
+   

[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd22ccd9/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
new file mode 100644
index 000..44f6b49
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -0,0 +1,728 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedMap;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+
+
+public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
+private static final byte[][] REGION_BOUNDARIES_MINIMAL = {
+
Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+private static final byte[][] GUIDE_POSTS_MINIMAL = {
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+
+private static final byte[][] REGION_BOUNDARIES_ALL = {
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%),
+

[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

Conflicts:

phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dd22ccd9
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dd22ccd9
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dd22ccd9

Branch: refs/heads/3.0
Commit: dd22ccd94a33fe987661c6953bbe61a6cf6f9e71
Parents: b467af4
Author: James Taylor jtay...@salesforce.com
Authored: Tue Mar 3 12:07:56 2015 -0800
Committer: James Taylor jtay...@salesforce.com
Committed: Tue Mar 3 18:45:13 2015 -0800

--
 .../apache/phoenix/filter/SkipScanFilter.java   |  31 +-
 .../org/apache/phoenix/schema/PTableImpl.java   |   9 +
 .../phoenix/filter/SkipScanBigFilterTest.java   | 728 +++
 .../filter/SkipScanFilterIntersectTest.java |  63 +-
 4 files changed, 829 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd22ccd9/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index ba2f180..a8c1243 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -179,7 +179,7 @@ public class SkipScanFilter extends FilterBase {
 Arrays.fill(position, 0);
 }
 
-private boolean intersect(byte[] lowerInclusiveKey, byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
+private boolean intersect(final byte[] lowerInclusiveKey, final byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
 resetState();
 boolean lowerUnbound = (lowerInclusiveKey.length == 0);
 int startPos = 0;
@@ -246,6 +246,9 @@ public class SkipScanFilter extends FilterBase {
 }
 int[] lowerPosition = Arrays.copyOf(position, position.length);
 // Navigate to the upperExclusiveKey, but not past it
+// TODO: We're including everything between the lowerPosition and end 
position, which is
+// more than we need. We can optimize this by tracking whether each 
range in each slot position
+// intersects.
 ReturnCode endCode = navigate(upperExclusiveKey, 0, 
upperExclusiveKey.length, Terminate.AT);
 if (endCode == ReturnCode.INCLUDE) {
 setStartKey();
@@ -270,6 +273,11 @@ public class SkipScanFilter extends FilterBase {
 position[i] = slots.get(i).size() - 1;
 }
 }
+int prevRowKeyPos = -1;
+ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable();
+ImmutableBytesWritable upperPtr = new ImmutableBytesWritable();
+schema.iterator(lowerInclusiveKey, lowerPtr);
+schema.iterator(upperExclusiveKey, upperPtr);
 // Copy inclusive all positions 
 for (int i = 0; i = lastSlot; i++) {
 ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], 
Math.min(position[i] + 1, slots.get(i).size()));
@@ -279,12 +287,33 @@ public class SkipScanFilter extends FilterBase {
 if (newSlots != null) {
 newSlots.add(newRanges);
 }
+// Must include all less-significant slot values if:
+// 1) a more-significant slot was incremented
 if (position[i]  lowerPosition[i]) {
 if (newSlots != null) {
 newSlots.addAll(slots.subList(i+1, slots.size()));
 }
 break;
 }
+// 2) we're at a slot containing a range and the values differ 
between the lower and upper range,
+//since less-significant slots may be lower after traversal 
than where they started.
+if (!slots.get(i).get(position[i]).isSingleKey()) {
+int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i);
+// Position lowerPtr/upperPtr within 
lowerInclusiveKey/upperExclusiveKey at value for slot i
+// The reposition method will do this incrementally, where we 
we're initially have prevRowKeyPos = -1. 
+schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, 
lowerInclusiveKey.length, slotSpan[i]);
+schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, 
upperExclusiveKey.length, slotSpan[i]);
+// If we have a range and the values differ, we must include 
all slots that are less significant.
+// For example: [A-D][1,23], the lower/upper keys could be 
B5/C2, where the C 

[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
Repository: phoenix
Updated Branches:
  refs/heads/3.0 b467af4fd - dd22ccd94


http://git-wip-us.apache.org/repos/asf/phoenix/blob/dd22ccd9/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
index 6d44818..7e35ae8 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
@@ -90,7 +90,8 @@ public class SkipScanFilterIntersectTest {
 @Parameters(name={0} {4})
 public static CollectionObject data() {
 ListObject testCases = Lists.newArrayList();
-// Causes increment of slot 2 to increment slot 1
+// Both ranges in second slot are required b/c first slot contains 
range and upper/lower
+// values differ in this slot position.
 testCases.addAll(foreach(
 new KeyRange[][] {{
 PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
@@ -107,10 +108,70 @@ public class SkipScanFilterIntersectTest {
 new KeyRange[][] {{
 PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
 }, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
 PDataType.CHAR.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
 }, {
 PDataType.CHAR.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
 }}));
+// Only second range in second slot is required b/c though first slot 
contains range,
+// upper/lower values do not differ in this slot position.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
+PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
+PDataType.CHAR.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}},
+new int[] {1,1,1},
+Bytes.toBytes(j3A),
+Bytes.toBytes(j4C),
+new KeyRange[][] {{
+PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}}));
+// Test case exercising repositioning multiple times (initially to 
slot #2 and then again
+// to slot #4). Because there's a range for slot #4 and the 
lower/upper values are different,
+// all slot #5 ranges are part of the intersection.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(b), true),
+PDataType.CHAR.getKeyRange(Bytes.toBytes(d), true, 
Bytes.toBytes(d), true),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(C), true, 
Bytes.toBytes(C), true),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(m), true, 
Bytes.toBytes(u), false),
+PDataType.CHAR.getKeyRange(Bytes.toBytes(z), true, 
Bytes.toBytes(z), true),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(A), true, 
Bytes.toBytes(A), true),
+PDataType.CHAR.getKeyRange(Bytes.toBytes(D), true, 
Bytes.toBytes(D), true),
+PDataType.CHAR.getKeyRange(Bytes.toBytes(M), true, 
Bytes.toBytes(M), true),
+}
+},
+new int[] {1,1,1,1,1},
+Bytes.toBytes(bkCpM),
+Bytes.toBytes(bkCtD),
+new KeyRange[][] {{
+PDataType.CHAR.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(b), true),
+}, {
+PDataType.CHAR.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+  

Apache-Phoenix | Master | Build Successful

2015-03-03 Thread Apache Jenkins Server
Master branch build status Successful
Source repository https://git-wip-us.apache.org/repos/asf/phoenix.git

Last Successful Compiled Artifacts https://builds.apache.org/job/Phoenix-master/lastSuccessfulBuild/artifact/

Last Complete Test Report https://builds.apache.org/job/Phoenix-master/lastCompletedBuild/testReport/

Changes
[jtaylor] PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout


Apache-Phoenix | 3.0 | Hadoop1 | Build Successful

2015-03-03 Thread Apache Jenkins Server
3.0 branch build status Successful
Source repository https://git-wip-us.apache.org/repos/asf/phoenix.git

Last Successful Compiled Artifacts https://builds.apache.org/job/Phoenix-3.0-hadoop1/lastSuccessfulBuild/artifact/

Last Complete Test Report https://builds.apache.org/job/Phoenix-3.0-hadoop1/lastCompletedBuild/testReport/

Changes
[jtaylor] PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout


Apache-Phoenix | 4.0 | Build Successful

2015-03-03 Thread Apache Jenkins Server
4.0 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf/incubator-phoenix.git

Compiled Artifacts https://builds.apache.org/job/Phoenix-4.0/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-4.0/lastCompletedBuild/testReport/

Changes
[jtaylor] PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout


[2/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
http://git-wip-us.apache.org/repos/asf/phoenix/blob/90cd05e8/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
new file mode 100644
index 000..29e14bf
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java
@@ -0,0 +1,717 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedMap;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+
+
+public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest {
+private static final byte[][] REGION_BOUNDARIES_MINIMAL = {
+
Bytes.toBytesBinary(\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+private static final byte[][] GUIDE_POSTS_MINIMAL = {
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x00\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x07#[j\\x80\\x00\\x00\\x00Y\\x08u\\xF3\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x08\\x80\\x00)\\xE4\\x80\\x00\\x0E\\x17\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x00\\xD3U\\x88\\xFF\\x80\\x00\\x00\\x00\\x84\\xBFJ\\xE0\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+};
+
+
+private static final byte[][] REGION_BOUNDARIES_ALL = {
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D^\\x80\\x03t\\xC5DESKTOP\\x00\\x80\\x00\\x00\\x01y3\\xF7P\\x80\\x00\\x00\\x00B\\xE7\\xF6F\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x00\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xB3\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00oI\\x17B\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00)\\xE4\\x80\\x00\\x0E#\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\xFA._\\xE2\\x80\\x00\\x00\\x00\\x98\\xFE2\\xF5\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0Da\\x80\\x00\\x00\\x01MOBILE\\x00\\x80\\x00\\x00\\x01`1%),
+
Bytes.toBytesBinary(\\x01\\x80\\x00b\\xB9\\x80\\x00\\x0D\\xAA\\x80\\x00\\x00\\x01DESKTOP\\x00\\x80\\x00\\x00\\x01\\x82\\xB4]\\xE7\\x80\\x00\\x00\\x00ER\\xFE#\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x00\\x00\\x00\\x00\\x00\\x00\\x00),
+

[3/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/90cd05e8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/90cd05e8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/90cd05e8

Branch: refs/heads/4.3
Commit: 90cd05e8f41d948aba733e2bd9933936df4720f4
Parents: caead2e
Author: James Taylor jtay...@salesforce.com
Authored: Tue Mar 3 12:07:56 2015 -0800
Committer: James Taylor jtay...@salesforce.com
Committed: Tue Mar 3 18:31:27 2015 -0800

--
 .../apache/phoenix/filter/SkipScanFilter.java   |  31 +-
 .../org/apache/phoenix/schema/PTableImpl.java   |   9 +
 .../phoenix/filter/SkipScanBigFilterTest.java   | 717 +++
 .../filter/SkipScanFilterIntersectTest.java |  69 +-
 4 files changed, 821 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/90cd05e8/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index 0aafdbb..1923856 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -195,7 +195,7 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 Arrays.fill(position, 0);
 }
 
-private boolean intersect(byte[] lowerInclusiveKey, byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
+private boolean intersect(final byte[] lowerInclusiveKey, final byte[] 
upperExclusiveKey, ListListKeyRange newSlots) {
 resetState();
 boolean lowerUnbound = (lowerInclusiveKey.length == 0);
 int startPos = 0;
@@ -262,6 +262,9 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 }
 int[] lowerPosition = Arrays.copyOf(position, position.length);
 // Navigate to the upperExclusiveKey, but not past it
+// TODO: We're including everything between the lowerPosition and end 
position, which is
+// more than we need. We can optimize this by tracking whether each 
range in each slot position
+// intersects.
 ReturnCode endCode = navigate(upperExclusiveKey, 0, 
upperExclusiveKey.length, Terminate.AT);
 if (endCode == ReturnCode.INCLUDE) {
 setStartKey();
@@ -286,6 +289,11 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 position[i] = slots.get(i).size() - 1;
 }
 }
+int prevRowKeyPos = -1;
+ImmutableBytesWritable lowerPtr = new ImmutableBytesWritable();
+ImmutableBytesWritable upperPtr = new ImmutableBytesWritable();
+schema.iterator(lowerInclusiveKey, lowerPtr);
+schema.iterator(upperExclusiveKey, upperPtr);
 // Copy inclusive all positions 
 for (int i = 0; i = lastSlot; i++) {
 ListKeyRange newRanges = slots.get(i).subList(lowerPosition[i], 
Math.min(position[i] + 1, slots.get(i).size()));
@@ -295,12 +303,33 @@ public class SkipScanFilter extends FilterBase implements 
Writable {
 if (newSlots != null) {
 newSlots.add(newRanges);
 }
+// Must include all less-significant slot values if:
+// 1) a more-significant slot was incremented
 if (position[i]  lowerPosition[i]) {
 if (newSlots != null) {
 newSlots.addAll(slots.subList(i+1, slots.size()));
 }
 break;
 }
+// 2) we're at a slot containing a range and the values differ 
between the lower and upper range,
+//since less-significant slots may be lower after traversal 
than where they started.
+if (!slots.get(i).get(position[i]).isSingleKey()) {
+int rowKeyPos = ScanUtil.getRowKeyPosition(slotSpan, i);
+// Position lowerPtr/upperPtr within 
lowerInclusiveKey/upperExclusiveKey at value for slot i
+// The reposition method will do this incrementally, where we 
we're initially have prevRowKeyPos = -1. 
+schema.reposition(lowerPtr, prevRowKeyPos, rowKeyPos, 0, 
lowerInclusiveKey.length, slotSpan[i]);
+schema.reposition(upperPtr, prevRowKeyPos, rowKeyPos, 0, 
upperExclusiveKey.length, slotSpan[i]);
+// If we have a range and the values differ, we must include 
all slots that are less significant.
+// For example: [A-D][1,23], the lower/upper keys could be 
B5/C2, where the C is in range and the
+   

[1/3] phoenix git commit: PHOENIX-1690 IndexOutOfBoundsException during SkipScanFilter interesect

2015-03-03 Thread jamestaylor
Repository: phoenix
Updated Branches:
  refs/heads/4.3 caead2e4f - 90cd05e8f


http://git-wip-us.apache.org/repos/asf/phoenix/blob/90cd05e8/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
index a2b6115..fea1d91 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterIntersectTest.java
@@ -26,13 +26,13 @@ import java.util.List;
 
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarchar;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -92,7 +92,8 @@ public class SkipScanFilterIntersectTest {
 @Parameters(name={0} {4})
 public static CollectionObject data() {
 ListObject testCases = Lists.newArrayList();
-// Causes increment of slot 2 to increment slot 1
+// Both ranges in second slot are required b/c first slot contains 
range and upper/lower
+// values differ in this slot position.
 testCases.addAll(foreach(
 new KeyRange[][] {{
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
@@ -109,10 +110,70 @@ public class SkipScanFilterIntersectTest {
 new KeyRange[][] {{
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
 }, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
 }, {
 PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
 }}));
+// Only second range in second slot is required b/c though first slot 
contains range,
+// upper/lower values do not differ in this slot position.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(e), false),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(1), true, 
Bytes.toBytes(1), true),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}},
+new int[] {1,1,1},
+Bytes.toBytes(j3A),
+Bytes.toBytes(j4C),
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(2), true, 
Bytes.toBytes(4), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(B), true, 
Bytes.toBytes(B), true),
+}}));
+// Test case exercising repositioning multiple times (initially to 
slot #2 and then again
+// to slot #4). Because there's a range for slot #4 and the 
lower/upper values are different,
+// all slot #5 ranges are part of the intersection.
+testCases.addAll(foreach(
+new KeyRange[][] {{
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(b), true, 
Bytes.toBytes(b), true),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(d), true, 
Bytes.toBytes(d), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(j), true, 
Bytes.toBytes(m), false),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(C), true, 
Bytes.toBytes(C), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(m), true, 
Bytes.toBytes(u), false),
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(z), true, 
Bytes.toBytes(z), true),
+}, {
+PChar.INSTANCE.getKeyRange(Bytes.toBytes(A), true, 

[1/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client

2015-03-03 Thread maryannxue
Repository: phoenix
Updated Branches:
  refs/heads/4.0 11d84bdf9 - 7f3183ae9


http://git-wip-us.apache.org/repos/asf/phoenix/blob/7f3183ae/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index 27fe0f9..e84ca2a 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -62,6 +62,7 @@ import org.apache.phoenix.parse.TableName;
 import org.apache.phoenix.parse.TableWildcardParseNode;
 import org.apache.phoenix.parse.WildcardParseNode;
 import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.AmbiguousColumnException;
 import org.apache.phoenix.schema.ArgumentTypeMismatchException;
 import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
 import org.apache.phoenix.schema.ColumnNotFoundException;
@@ -144,12 +145,21 @@ public class ProjectionCompiler {
 }
 ColumnRef ref = new ColumnRef(tableRef,i);
 String colName = ref.getColumn().getName().getString();
+String tableAlias = tableRef.getTableAlias();
 if (resolveColumn) {
-if (tableRef.getTableAlias() != null) {
-ref = resolver.resolveColumn(null, 
tableRef.getTableAlias(), colName);
-} else {
-String schemaName = table.getSchemaName().getString();
-ref = resolver.resolveColumn(schemaName.length() == 0 ? 
null : schemaName, table.getTableName().getString(), colName);
+try {
+if (tableAlias != null) {
+ref = resolver.resolveColumn(null, tableAlias, 
colName);
+} else {
+String schemaName = table.getSchemaName().getString();
+ref = resolver.resolveColumn(schemaName.length() == 0 
? null : schemaName, table.getTableName().getString(), colName);
+}
+} catch (AmbiguousColumnException e) {
+if (column.getFamilyName() != null) {
+ref = resolver.resolveColumn(tableAlias != null ? 
tableAlias : table.getTableName().getString(), 
column.getFamilyName().getString(), colName);
+} else {
+throw e;
+}
 }
 }
 Expression expression = ref.newColumnExpression();
@@ -219,12 +229,21 @@ public class ProjectionCompiler {
 }
 }
 String colName = tableColumn.getName().getString();
+String tableAlias = tableRef.getTableAlias();
 if (resolveColumn) {
-if (tableRef.getTableAlias() != null) {
-ref = resolver.resolveColumn(null, 
tableRef.getTableAlias(), indexColName);
-} else {
-String schemaName = index.getSchemaName().getString();
-ref = resolver.resolveColumn(schemaName.length() == 0 ? 
null : schemaName, index.getTableName().getString(), indexColName);
+try {
+if (tableAlias != null) {
+ref = resolver.resolveColumn(null, tableAlias, 
indexColName);
+} else {
+String schemaName = index.getSchemaName().getString();
+ref = resolver.resolveColumn(schemaName.length() == 0 
? null : schemaName, index.getTableName().getString(), indexColName);
+}
+} catch (AmbiguousColumnException e) {
+if (indexColumn.getFamilyName() != null) {
+ref = resolver.resolveColumn(tableAlias != null ? 
tableAlias : index.getTableName().getString(), 
indexColumn.getFamilyName().getString(), indexColName);
+} else {
+throw e;
+}
 }
 }
 Expression expression = ref.newColumnExpression();
@@ -238,11 +257,14 @@ public class ProjectionCompiler {
 }
 }
 
-private static void projectTableColumnFamily(StatementContext context, 
String cfName, TableRef tableRef, ListExpression projectedExpressions, 
ListExpressionProjector projectedColumns) throws SQLException {
+private static void projectTableColumnFamily(StatementContext context, 
String cfName, TableRef tableRef, boolean resolveColumn, ListExpression 
projectedExpressions, ListExpressionProjector projectedColumns) throws 
SQLException {
 PTable table = tableRef.getTable();
 PColumnFamily pfamily = table.getColumnFamily(cfName);
 for (PColumn column : pfamily.getColumns()) {
 

[2/2] phoenix git commit: PHOENIX-1489 Access column values positionally from client

2015-03-03 Thread maryannxue
PHOENIX-1489 Access column values positionally from client


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7f3183ae
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7f3183ae
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7f3183ae

Branch: refs/heads/4.0
Commit: 7f3183ae9e2ae88cd820d676e4dcc2cfa3576dd7
Parents: 11d84bd
Author: maryannxue wei@intel.com
Authored: Tue Mar 3 12:40:07 2015 -0500
Committer: maryannxue wei@intel.com
Committed: Tue Mar 3 12:40:07 2015 -0500

--
 .../org/apache/phoenix/end2end/HashJoinIT.java  |  55 ++--
 .../phoenix/end2end/HashJoinLocalIndexIT.java   |   6 +-
 .../org/apache/phoenix/end2end/SubqueryIT.java  |  30 +--
 .../end2end/SubqueryUsingSortMergeJoinIT.java   |  32 +--
 .../index/GlobalIndexOptimizationIT.java|  22 +-
 .../phoenix/end2end/index/LocalIndexIT.java |   4 +-
 .../phoenix/compile/ExpressionCompiler.java |   2 +-
 .../apache/phoenix/compile/FromCompiler.java|  87 ++-
 .../apache/phoenix/compile/JoinCompiler.java| 258 ---
 .../apache/phoenix/compile/OrderByCompiler.java |   2 +-
 .../phoenix/compile/ProjectionCompiler.java |  58 +++--
 .../apache/phoenix/compile/QueryCompiler.java   | 117 +
 .../compile/TupleProjectionCompiler.java| 214 +++
 .../apache/phoenix/compile/UpsertCompiler.java  |   4 +-
 .../apache/phoenix/compile/WhereCompiler.java   |   2 +-
 .../coprocessor/BaseScannerRegionObserver.java  |  21 +-
 .../GroupedAggregateRegionObserver.java |  11 +-
 .../coprocessor/HashJoinRegionScanner.java  |  72 +++---
 .../phoenix/coprocessor/ScanRegionObserver.java |   9 +-
 .../UngroupedAggregateRegionObserver.java   |  10 +-
 .../apache/phoenix/execute/TupleProjector.java  |  18 +-
 .../org/apache/phoenix/join/HashJoinInfo.java   |  25 +-
 .../apache/phoenix/optimize/QueryOptimizer.java |  19 +-
 .../org/apache/phoenix/schema/ColumnRef.java|   2 +-
 .../phoenix/schema/LocalIndexDataColumnRef.java |   9 +-
 .../apache/phoenix/schema/MetaDataClient.java   |   2 +-
 .../org/apache/phoenix/schema/PTableType.java   |   2 +-
 .../apache/phoenix/schema/ProjectedColumn.java  |  59 +
 .../org/apache/phoenix/schema/TableRef.java |   8 +-
 .../phoenix/compile/WhereCompilerTest.java  |   6 +-
 30 files changed, 692 insertions(+), 474 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/7f3183ae/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
index e915b36..596e5e9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
@@ -219,7 +219,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 SERVER FILTER BY QUANTITY  5000\n +
 PARALLEL INNER-JOIN TABLE 1\n +
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 
JOIN_SUPPLIER_TABLE_DISPLAY_NAME + \n +
-DYNAMIC SERVER FILTER BY \item_id\ IN (\O.item_id\),
+DYNAMIC SERVER FILTER BY \I.item_id\ IN 
(\O.item_id\),
 /*
  * testSelfJoin()
  * SELECT i2.item_id, i1.name FROM joinItemTable i1 
@@ -230,7 +230,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 PARALLEL INNER-JOIN TABLE 0\n +
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 
JOIN_ITEM_TABLE_DISPLAY_NAME + \n +
 SERVER FILTER BY FIRST KEY ONLY\n +
-DYNAMIC SERVER FILTER BY \item_id\ IN (\I2.item_id\),
+DYNAMIC SERVER FILTER BY \I1.item_id\ IN 
(\I2.item_id\),
 /*
  * testSelfJoin()
  * SELECT i1.name, i2.name FROM joinItemTable i1 
@@ -242,7 +242,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 CLIENT MERGE SORT\n +
 PARALLEL INNER-JOIN TABLE 0\n +
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  + 
JOIN_ITEM_TABLE_DISPLAY_NAME + \n +
-DYNAMIC SERVER FILTER BY \item_id\ IN 
(\I2.supplier_id\),
+DYNAMIC SERVER FILTER BY \I1.item_id\ IN 
(\I2.supplier_id\),
 /*
  * testStarJoin()
  * SELECT order_id, c.name, i.name iname, quantity, o.date 
@@ -271,7 +271,7 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
 CLIENT PARALLEL 1-WAY FULL SCAN OVER  +