Build failed in Jenkins: Phoenix-4.x-HBase-1.1 #22

2016-05-12 Thread Apache Jenkins Server 
See 

Changes:

[jamestaylor] PHOENIX-2818 Optimize ORDERED group by

[jamestaylor] PHOENIX-2876 Using aggregation function in ORDER BY (Sergey 
Soldatov)

[jamestaylor] PHOENIX-2888 Aggregations for similar IN list expressions doesn't 
work

--
[...truncated 2069 lines...]
Caused by: java.io.IOException: Shutting down
Caused by: java.lang.RuntimeException: Master not initialized after 20ms 
seconds

Running org.apache.phoenix.end2end.index.ImmutableIndexWithStatsIT
Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.004 sec <<< 
FAILURE! - in org.apache.phoenix.end2end.index.ImmutableIndexWithStatsIT
org.apache.phoenix.end2end.index.ImmutableIndexWithStatsIT  Time elapsed: 0.003 
sec  <<< ERROR!
java.lang.RuntimeException: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.ImmutableIndexWithStatsIT.doSetup(ImmutableIndexWithStatsIT.java:52)
Caused by: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.ImmutableIndexWithStatsIT.doSetup(ImmutableIndexWithStatsIT.java:52)
Caused by: java.lang.RuntimeException: Master not initialized after 20ms 
seconds
at 
org.apache.phoenix.end2end.index.ImmutableIndexWithStatsIT.doSetup(ImmutableIndexWithStatsIT.java:52)

Running org.apache.phoenix.end2end.index.MutableIndexFailureIT
Running org.apache.phoenix.end2end.index.MutableIndexReplicationIT
Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.006 sec <<< 
FAILURE! - in org.apache.phoenix.end2end.index.MutableIndexReplicationIT
org.apache.phoenix.end2end.index.MutableIndexReplicationIT  Time elapsed: 0.005 
sec  <<< ERROR!
java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.MutableIndexReplicationIT.setupConfigsAndStartCluster(MutableIndexReplicationIT.java:170)
at 
org.apache.phoenix.end2end.index.MutableIndexReplicationIT.setUpBeforeClass(MutableIndexReplicationIT.java:108)
Caused by: java.lang.RuntimeException: Master not initialized after 20ms 
seconds
at 
org.apache.phoenix.end2end.index.MutableIndexReplicationIT.setupConfigsAndStartCluster(MutableIndexReplicationIT.java:170)
at 
org.apache.phoenix.end2end.index.MutableIndexReplicationIT.setUpBeforeClass(MutableIndexReplicationIT.java:108)

Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.004 sec <<< 
FAILURE! - in org.apache.phoenix.end2end.index.MutableIndexFailureIT
org.apache.phoenix.end2end.index.MutableIndexFailureIT  Time elapsed: 0.003 sec 
 <<< ERROR!
java.lang.RuntimeException: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.MutableIndexFailureIT.doSetup(MutableIndexFailureIT.java:113)
Caused by: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.MutableIndexFailureIT.doSetup(MutableIndexFailureIT.java:113)
Caused by: java.lang.RuntimeException: Master not initialized after 20ms 
seconds
at 
org.apache.phoenix.end2end.index.MutableIndexFailureIT.doSetup(MutableIndexFailureIT.java:113)

Running org.apache.phoenix.hbase.index.FailForUnsupportedHBaseVersionsIT
Running org.apache.phoenix.end2end.index.ReadOnlyIndexFailureIT
Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.006 sec <<< 
FAILURE! - in org.apache.phoenix.end2end.index.ReadOnlyIndexFailureIT
org.apache.phoenix.end2end.index.ReadOnlyIndexFailureIT  Time elapsed: 0.005 
sec  <<< ERROR!
java.lang.RuntimeException: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.ReadOnlyIndexFailureIT.doSetup(ReadOnlyIndexFailureIT.java:116)
Caused by: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.ReadOnlyIndexFailureIT.doSetup(ReadOnlyIndexFailureIT.java:116)
Caused by: java.lang.RuntimeException: Master not initialized after 20ms 
seconds
at 
org.apache.phoenix.end2end.index.ReadOnlyIndexFailureIT.doSetup(ReadOnlyIndexFailureIT.java:116)

Running org.apache.phoenix.end2end.index.txn.TxWriteFailureIT
Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.006 sec <<< 
FAILURE! - in org.apache.phoenix.end2end.index.txn.TxWriteFailureIT
org.apache.phoenix.end2end.index.txn.TxWriteFailureIT  Time elapsed: 0.005 sec  
<<< ERROR!
java.lang.RuntimeException: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.txn.TxWriteFailureIT.doSetup(TxWriteFailureIT.java:86)
Caused by: java.io.IOException: Shutting down
at 
org.apache.phoenix.end2end.index.txn.TxWriteFailureIT.doSetup(TxWriteFailureIT.java:86)
Caused by: java.lang.RuntimeException: Master not initialized after 20ms 
seconds
at 
org.apache.phoenix.end2end.index.txn.TxWriteFailureIT.doSetup(TxWriteFailureIT.java:86)

Running org.apache.phoenix.execute.PartialCommitIT
Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.004 sec <<< 
FAILURE! - in

Build failed in Jenkins: Phoenix-4.x-HBase-1.0 #471

2016-05-12 Thread Apache Jenkins Server 
See 

Changes:

[jamestaylor] PHOENIX-2818 Optimize ORDERED group by

[jamestaylor] PHOENIX-2876 Using aggregation function in ORDER BY (Sergey 
Soldatov)

[jamestaylor] PHOENIX-2888 Aggregations for similar IN list expressions doesn't 
work

--
[...truncated 744 lines...]
Running org.apache.phoenix.end2end.QueryMoreIT
Running org.apache.phoenix.end2end.PowerFunctionEnd2EndIT
Tests run: 9, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.279 sec - in 
org.apache.phoenix.end2end.PrimitiveTypeIT
Running org.apache.phoenix.end2end.RTrimFunctionIT
Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.877 sec - in 
org.apache.phoenix.end2end.PowerFunctionEnd2EndIT
Running org.apache.phoenix.end2end.ReadOnlyIT
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 1.704 sec - in 
org.apache.phoenix.end2end.RTrimFunctionIT
Tests run: 26, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 25.047 sec - 
in org.apache.phoenix.end2end.ArithmeticQueryIT
Running org.apache.phoenix.end2end.RegexpSplitFunctionIT
Running org.apache.phoenix.end2end.ReverseFunctionIT
Tests run: 11, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.374 sec - in 
org.apache.phoenix.end2end.RegexpSplitFunctionIT
Running org.apache.phoenix.end2end.RoundFloorCeilFunctionsEnd2EndIT
Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 5.736 sec - in 
org.apache.phoenix.end2end.ReadOnlyIT
Tests run: 5, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 4.137 sec - in 
org.apache.phoenix.end2end.ReverseFunctionIT
Tests run: 11, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 9.573 sec - in 
org.apache.phoenix.end2end.NthValueFunctionIT
Running org.apache.phoenix.end2end.SerialIteratorsIT
Running org.apache.phoenix.end2end.ServerExceptionIT
Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.956 sec - in 
org.apache.phoenix.end2end.ServerExceptionIT
Tests run: 33, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 4.934 sec - in 
org.apache.phoenix.end2end.RoundFloorCeilFunctionsEnd2EndIT
Running org.apache.phoenix.end2end.SignFunctionEnd2EndIT
Running org.apache.phoenix.end2end.StatementHintsIT
Running org.apache.phoenix.end2end.SortOrderIT
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.646 sec - in 
org.apache.phoenix.end2end.SignFunctionEnd2EndIT
Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 2.707 sec - in 
org.apache.phoenix.end2end.SerialIteratorsIT
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 1.2 sec - in 
org.apache.phoenix.end2end.StatementHintsIT
Running org.apache.phoenix.end2end.StringToArrayFunctionIT
Running org.apache.phoenix.end2end.TimezoneOffsetFunctionIT
Running org.apache.phoenix.end2end.StoreNullsIT
Tests run: 6, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 17.31 sec - in 
org.apache.phoenix.end2end.QueryMoreIT
Running org.apache.phoenix.end2end.ToCharFunctionIT
Tests run: 22, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 6.353 sec - in 
org.apache.phoenix.end2end.StringToArrayFunctionIT
Tests run: 7, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 6.338 sec - in 
org.apache.phoenix.end2end.TimezoneOffsetFunctionIT
Running org.apache.phoenix.end2end.UpsertSelectAutoCommitIT
Running org.apache.phoenix.end2end.salted.SaltedTableUpsertSelectIT
Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 5.149 sec - in 
org.apache.phoenix.end2end.UpsertSelectAutoCommitIT
Tests run: 11, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 7.496 sec - in 
org.apache.phoenix.end2end.ToCharFunctionIT
Running org.apache.phoenix.end2end.salted.SaltedTableVarLengthRowKeyIT
Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 13.456 sec - in 
org.apache.phoenix.end2end.StoreNullsIT
Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.974 sec - in 
org.apache.phoenix.end2end.salted.SaltedTableVarLengthRowKeyIT
Running org.apache.phoenix.iterate.PhoenixQueryTimeoutIT
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.161 sec - in 
org.apache.phoenix.iterate.PhoenixQueryTimeoutIT
Tests run: 6, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 16.235 sec - in 
org.apache.phoenix.end2end.salted.SaltedTableUpsertSelectIT
Tests run: 45, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 45.323 sec - 
in org.apache.phoenix.end2end.SortOrderIT

Results :

Tests run: 335, Failures: 0, Errors: 0, Skipped: 0

[INFO] 
[INFO] --- maven-failsafe-plugin:2.19.1:integration-test 
(NeedTheirOwnClusterTests) @ phoenix-core ---

---
 T E S T S
---
Running 
org.apache.hadoop.hbase.regionserver.wal.WALReplayWithIndexWritesAndCompressedWALIT
Running 
org.apache.hadoop.hbase.regionserver.wal.WALReplayWithIndexWritesAndUncompressedWALInHBase_094_9_IT
Running

Apache-Phoenix | Master | Build Successful

2016-05-12 Thread Apache Jenkins Server 
Master branch build status Successful
Source repository https://git-wip-us.apache.org/repos/asf?p=phoenix.git;a=shortlog;h=refs/heads/master

Last Successful Compiled Artifacts https://builds.apache.org/job/Phoenix-master/lastSuccessfulBuild/artifact/

Last Complete Test Report https://builds.apache.org/job/Phoenix-master/lastCompletedBuild/testReport/

Changes
[jamestaylor] PHOENIX-2818 Optimize ORDERED group by

[jamestaylor] PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)

[jamestaylor] PHOENIX-2888 Aggregations for similar IN list expressions doesn't work



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout

[2/3] phoenix git commit: PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bddc75ff
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bddc75ff
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bddc75ff

Branch: refs/heads/4.x-HBase-0.98
Commit: bddc75ffbd29059cbf41f7ec8e508d0104fd02f8
Parents: d6815f7
Author: James Taylor 
Authored: Thu May 12 14:32:34 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:21:36 2016 -0700

--
 .../phoenix/compile/AggregationManager.java | 60 
 .../apache/phoenix/compile/DeleteCompiler.java  |  1 +
 .../apache/phoenix/compile/PostDDLCompiler.java |  1 +
 .../phoenix/compile/ProjectionCompiler.java | 53 +
 .../apache/phoenix/compile/QueryCompiler.java   |  1 +
 .../apache/phoenix/compile/UpsertCompiler.java  |  6 +-
 .../phoenix/compile/QueryCompilerTest.java  | 16 +-
 7 files changed, 86 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/bddc75ff/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
index ee2497b..c8e672e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
@@ -17,7 +17,21 @@
  */
 package org.apache.phoenix.compile;
 
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.aggregator.ClientAggregators;
+import org.apache.phoenix.expression.aggregator.ServerAggregators;
+import org.apache.phoenix.expression.function.SingleAggregateFunction;
+import org.apache.phoenix.expression.visitor.SingleAggregateFunctionVisitor;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 /**
  * 
@@ -52,4 +66,50 @@ public class AggregationManager {
 public void setAggregators(ClientAggregators clientAggregator) {
 this.aggregators = clientAggregator;
 }
+/**
+ * Compiles projection by:
+ * 1) Adding RowCount aggregate function if not present when limiting 
rows. We need this
+ *to track how many rows have been scanned.
+ * 2) Reordering aggregation functions (by putting fixed length aggregates 
first) to
+ *optimize the positional access of the aggregated value.
+ */
+public void compile(StatementContext context, GroupByCompiler.GroupBy 
groupBy) throws
+SQLException {
+final Set aggFuncSet = 
Sets.newHashSetWithExpectedSize(context.getExpressionManager().getExpressionCount());
+
+Iterator expressions = 
context.getExpressionManager().getExpressions();
+while (expressions.hasNext()) {
+Expression expression = expressions.next();
+expression.accept(new SingleAggregateFunctionVisitor() {
+@Override
+public Iterator visitEnter(SingleAggregateFunction 
function) {
+aggFuncSet.add(function);
+return Iterators.emptyIterator();
+}
+});
+}
+if (aggFuncSet.isEmpty() && groupBy.isEmpty()) {
+return;
+}
+List aggFuncs = new 
ArrayList(aggFuncSet);
+Collections.sort(aggFuncs, SingleAggregateFunction.SCHEMA_COMPARATOR);
+
+int minNullableIndex = getMinNullableIndex(aggFuncs,groupBy.isEmpty());
+context.getScan().setAttribute(BaseScannerRegionObserver.AGGREGATORS, 
ServerAggregators.serialize(aggFuncs, minNullableIndex));
+ClientAggregators clientAggregators = new ClientAggregators(aggFuncs, 
minNullableIndex);
+context.getAggregationManager().setAggregators(clientAggregators);
+}
+
+private static int getMinNullableIndex(List 
aggFuncs, boolean isUngroupedAggregation) {
+int minNullableIndex = aggFuncs.size();
+for (int i = 0; i < aggFuncs.size(); i++) {
+SingleAggregateFunction aggFunc = aggFuncs.get(i);
+if (isUngroupedAggregation ? aggFunc.getAggregator().isNullable() 
: aggFunc.getAggregatorExpression().isNullable()) {
+minNullableIndex = i;
+break;
+}
+}
+return minNullableIndex;
+}
+
 }

[1/3] phoenix git commit: PHOENIX-2818 Optimize ORDERED group by

2016-05-12 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-0.98 119c5f906 -> b9a857472


PHOENIX-2818 Optimize ORDERED group by


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d6815f75
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d6815f75
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d6815f75

Branch: refs/heads/4.x-HBase-0.98
Commit: d6815f757cde7994a371ffb2965539f329bd6aa1
Parents: 119c5f9
Author: James Taylor 
Authored: Thu May 12 14:29:29 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:21:29 2016 -0700

--
 .../apache/phoenix/end2end/GroupByCaseIT.java   |  92 +
 .../apache/phoenix/execute/AggregatePlan.java   |   3 +-
 .../iterate/AggregatingResultIterator.java  |   4 +-
 .../BaseGroupedAggregatingResultIterator.java   |   3 +-
 .../DistinctAggregatingResultIterator.java  |  15 +-
 .../FilterAggregatingResultIterator.java|   8 +-
 .../OrderedAggregatingResultIterator.java   |   5 +-
 .../RowKeyOrderedAggregateResultIterator.java   | 190 +++
 .../iterate/AggregateResultScannerTest.java | 109 +--
 .../iterate/ConcatResultIteratorTest.java   |  31 +--
 .../iterate/MaterializedResultIterators.java|  66 +++
 ...owKeyOrderedAggregateResultIteratorTest.java | 149 +++
 .../java/org/apache/phoenix/util/TestUtil.java  |  70 +++
 13 files changed, 596 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6815f75/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 44f43b7..b0524da 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -32,6 +32,9 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
 
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
 import org.junit.Test;
@@ -343,4 +346,93 @@ public class GroupByCaseIT extends BaseHBaseManagedTimeIT {
 "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", 
QueryUtil.getExplainPlan(rs));
 }
 
+@Test
+public void testSumGroupByOrderPreservingDesc() throws Exception {
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+Connection conn = DriverManager.getConnection(getUrl(), props);
+PreparedStatement stmt = conn.prepareStatement("CREATE TABLE 
GROUP_BY_DESC (k1 char(1) not null, k2 integer not null, constraint pk primary 
key (k1,k2)) split on (?,?,?)");
+stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.execute();
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('b', 
5)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
4)");
+conn.commit();
+String query = "SELECT k1,sum(k2) FROM GROUP_BY_DESC GROUP BY k1 ORDER 
BY k1 DESC";
+ResultSet rs = conn.createStatement().executeQuery(query);
+assertTrue(rs.next());
+assertEquals("n", rs.getString(1));
+assertEquals(10, rs.getInt(2));
+assertTrue(rs.next());

[3/3] phoenix git commit: PHOENIX-2888 Aggregations for similar IN list expressions doesn't work correctly (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2888 Aggregations for similar IN list expressions doesn't work 
correctly (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b9a85747
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b9a85747
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b9a85747

Branch: refs/heads/4.x-HBase-0.98
Commit: b9a857472b6e99203cf3424b76c921f218af7456
Parents: bddc75f
Author: James Taylor 
Authored: Thu May 12 14:34:47 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:21:42 2016 -0700

--
 .../java/org/apache/phoenix/expression/InListExpression.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/b9a85747/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
index a4a9353..a977f1f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
@@ -175,7 +175,7 @@ public class InListExpression extends BaseSingleExpression {
 public int hashCode() {
 final int prime = 31;
 int result = 1;
-result = prime * result + values.hashCode();
+result = prime * result + children.hashCode() + values.hashCode();
 return result;
 }
 
@@ -185,7 +185,7 @@ public class InListExpression extends BaseSingleExpression {
 if (obj == null) return false;
 if (getClass() != obj.getClass()) return false;
 InListExpression other = (InListExpression)obj;
-if (!values.equals(other.values)) return false;
+if (!children.equals(other.children) || !values.equals(other.values)) 
return false;
 return true;
 }

[1/3] phoenix git commit: PHOENIX-2818 Optimize ORDERED group by

2016-05-12 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-1.0 c1e1b1f23 -> 9d0a664de


PHOENIX-2818 Optimize ORDERED group by


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2da0ef88
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2da0ef88
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2da0ef88

Branch: refs/heads/4.x-HBase-1.0
Commit: 2da0ef889463d746b4c9cc7189392120d1c28719
Parents: c1e1b1f
Author: James Taylor 
Authored: Thu May 12 14:29:29 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:17:53 2016 -0700

--
 .../apache/phoenix/end2end/GroupByCaseIT.java   |  92 +
 .../apache/phoenix/execute/AggregatePlan.java   |   3 +-
 .../iterate/AggregatingResultIterator.java  |   4 +-
 .../BaseGroupedAggregatingResultIterator.java   |   3 +-
 .../DistinctAggregatingResultIterator.java  |  15 +-
 .../FilterAggregatingResultIterator.java|   8 +-
 .../OrderedAggregatingResultIterator.java   |   5 +-
 .../RowKeyOrderedAggregateResultIterator.java   | 190 +++
 .../iterate/AggregateResultScannerTest.java | 109 +--
 .../iterate/ConcatResultIteratorTest.java   |  31 +--
 .../iterate/MaterializedResultIterators.java|  66 +++
 ...owKeyOrderedAggregateResultIteratorTest.java | 149 +++
 .../java/org/apache/phoenix/util/TestUtil.java  |  70 +++
 13 files changed, 596 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/2da0ef88/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 44f43b7..b0524da 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -32,6 +32,9 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
 
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
 import org.junit.Test;
@@ -343,4 +346,93 @@ public class GroupByCaseIT extends BaseHBaseManagedTimeIT {
 "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", 
QueryUtil.getExplainPlan(rs));
 }
 
+@Test
+public void testSumGroupByOrderPreservingDesc() throws Exception {
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+Connection conn = DriverManager.getConnection(getUrl(), props);
+PreparedStatement stmt = conn.prepareStatement("CREATE TABLE 
GROUP_BY_DESC (k1 char(1) not null, k2 integer not null, constraint pk primary 
key (k1,k2)) split on (?,?,?)");
+stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.execute();
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('b', 
5)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
4)");
+conn.commit();
+String query = "SELECT k1,sum(k2) FROM GROUP_BY_DESC GROUP BY k1 ORDER 
BY k1 DESC";
+ResultSet rs = conn.createStatement().executeQuery(query);
+assertTrue(rs.next());
+assertEquals("n", rs.getString(1));
+assertEquals(10, rs.getInt(2));
+assertTrue(rs.next());
+

[2/3] phoenix git commit: PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e71d8584
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e71d8584
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e71d8584

Branch: refs/heads/4.x-HBase-1.0
Commit: e71d85844e3b894d6bc9bcaa4af46ce55a7d6ef4
Parents: 2da0ef8
Author: James Taylor 
Authored: Thu May 12 14:32:34 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:18:04 2016 -0700

--
 .../phoenix/compile/AggregationManager.java | 60 
 .../apache/phoenix/compile/DeleteCompiler.java  |  1 +
 .../apache/phoenix/compile/PostDDLCompiler.java |  1 +
 .../phoenix/compile/ProjectionCompiler.java | 53 +
 .../apache/phoenix/compile/QueryCompiler.java   |  1 +
 .../apache/phoenix/compile/UpsertCompiler.java  |  6 +-
 .../phoenix/compile/QueryCompilerTest.java  | 16 +-
 7 files changed, 86 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/e71d8584/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
index ee2497b..c8e672e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
@@ -17,7 +17,21 @@
  */
 package org.apache.phoenix.compile;
 
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.aggregator.ClientAggregators;
+import org.apache.phoenix.expression.aggregator.ServerAggregators;
+import org.apache.phoenix.expression.function.SingleAggregateFunction;
+import org.apache.phoenix.expression.visitor.SingleAggregateFunctionVisitor;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 /**
  * 
@@ -52,4 +66,50 @@ public class AggregationManager {
 public void setAggregators(ClientAggregators clientAggregator) {
 this.aggregators = clientAggregator;
 }
+/**
+ * Compiles projection by:
+ * 1) Adding RowCount aggregate function if not present when limiting 
rows. We need this
+ *to track how many rows have been scanned.
+ * 2) Reordering aggregation functions (by putting fixed length aggregates 
first) to
+ *optimize the positional access of the aggregated value.
+ */
+public void compile(StatementContext context, GroupByCompiler.GroupBy 
groupBy) throws
+SQLException {
+final Set aggFuncSet = 
Sets.newHashSetWithExpectedSize(context.getExpressionManager().getExpressionCount());
+
+Iterator expressions = 
context.getExpressionManager().getExpressions();
+while (expressions.hasNext()) {
+Expression expression = expressions.next();
+expression.accept(new SingleAggregateFunctionVisitor() {
+@Override
+public Iterator visitEnter(SingleAggregateFunction 
function) {
+aggFuncSet.add(function);
+return Iterators.emptyIterator();
+}
+});
+}
+if (aggFuncSet.isEmpty() && groupBy.isEmpty()) {
+return;
+}
+List aggFuncs = new 
ArrayList(aggFuncSet);
+Collections.sort(aggFuncs, SingleAggregateFunction.SCHEMA_COMPARATOR);
+
+int minNullableIndex = getMinNullableIndex(aggFuncs,groupBy.isEmpty());
+context.getScan().setAttribute(BaseScannerRegionObserver.AGGREGATORS, 
ServerAggregators.serialize(aggFuncs, minNullableIndex));
+ClientAggregators clientAggregators = new ClientAggregators(aggFuncs, 
minNullableIndex);
+context.getAggregationManager().setAggregators(clientAggregators);
+}
+
+private static int getMinNullableIndex(List 
aggFuncs, boolean isUngroupedAggregation) {
+int minNullableIndex = aggFuncs.size();
+for (int i = 0; i < aggFuncs.size(); i++) {
+SingleAggregateFunction aggFunc = aggFuncs.get(i);
+if (isUngroupedAggregation ? aggFunc.getAggregator().isNullable() 
: aggFunc.getAggregatorExpression().isNullable()) {
+minNullableIndex = i;
+break;
+}
+}
+return minNullableIndex;
+}
+
 }

[3/3] phoenix git commit: PHOENIX-2888 Aggregations for similar IN list expressions doesn't work correctly (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2888 Aggregations for similar IN list expressions doesn't work 
correctly (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7cad42cb
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7cad42cb
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7cad42cb

Branch: refs/heads/4.x-HBase-1.1
Commit: 7cad42cb1916cd963d923416aaeab94c666a90dc
Parents: 8ec0bac
Author: James Taylor 
Authored: Thu May 12 14:34:47 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:11:51 2016 -0700

--
 .../java/org/apache/phoenix/expression/InListExpression.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/7cad42cb/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
index a4a9353..a977f1f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
@@ -175,7 +175,7 @@ public class InListExpression extends BaseSingleExpression {
 public int hashCode() {
 final int prime = 31;
 int result = 1;
-result = prime * result + values.hashCode();
+result = prime * result + children.hashCode() + values.hashCode();
 return result;
 }
 
@@ -185,7 +185,7 @@ public class InListExpression extends BaseSingleExpression {
 if (obj == null) return false;
 if (getClass() != obj.getClass()) return false;
 InListExpression other = (InListExpression)obj;
-if (!values.equals(other.values)) return false;
+if (!children.equals(other.children) || !values.equals(other.values)) 
return false;
 return true;
 }

[2/3] phoenix git commit: PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8ec0bacc
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8ec0bacc
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8ec0bacc

Branch: refs/heads/4.x-HBase-1.1
Commit: 8ec0baccc8f7bfe81ad086de61aade8c0e3529ff
Parents: d56d75a
Author: James Taylor 
Authored: Thu May 12 14:32:34 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:11:38 2016 -0700

--
 .../phoenix/compile/AggregationManager.java | 60 
 .../apache/phoenix/compile/DeleteCompiler.java  |  1 +
 .../apache/phoenix/compile/PostDDLCompiler.java |  1 +
 .../phoenix/compile/ProjectionCompiler.java | 53 +
 .../apache/phoenix/compile/QueryCompiler.java   |  1 +
 .../apache/phoenix/compile/UpsertCompiler.java  |  6 +-
 .../phoenix/compile/QueryCompilerTest.java  | 16 +-
 7 files changed, 86 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8ec0bacc/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
index ee2497b..c8e672e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
@@ -17,7 +17,21 @@
  */
 package org.apache.phoenix.compile;
 
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.aggregator.ClientAggregators;
+import org.apache.phoenix.expression.aggregator.ServerAggregators;
+import org.apache.phoenix.expression.function.SingleAggregateFunction;
+import org.apache.phoenix.expression.visitor.SingleAggregateFunctionVisitor;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 /**
  * 
@@ -52,4 +66,50 @@ public class AggregationManager {
 public void setAggregators(ClientAggregators clientAggregator) {
 this.aggregators = clientAggregator;
 }
+/**
+ * Compiles projection by:
+ * 1) Adding RowCount aggregate function if not present when limiting 
rows. We need this
+ *to track how many rows have been scanned.
+ * 2) Reordering aggregation functions (by putting fixed length aggregates 
first) to
+ *optimize the positional access of the aggregated value.
+ */
+public void compile(StatementContext context, GroupByCompiler.GroupBy 
groupBy) throws
+SQLException {
+final Set aggFuncSet = 
Sets.newHashSetWithExpectedSize(context.getExpressionManager().getExpressionCount());
+
+Iterator expressions = 
context.getExpressionManager().getExpressions();
+while (expressions.hasNext()) {
+Expression expression = expressions.next();
+expression.accept(new SingleAggregateFunctionVisitor() {
+@Override
+public Iterator visitEnter(SingleAggregateFunction 
function) {
+aggFuncSet.add(function);
+return Iterators.emptyIterator();
+}
+});
+}
+if (aggFuncSet.isEmpty() && groupBy.isEmpty()) {
+return;
+}
+List aggFuncs = new 
ArrayList(aggFuncSet);
+Collections.sort(aggFuncs, SingleAggregateFunction.SCHEMA_COMPARATOR);
+
+int minNullableIndex = getMinNullableIndex(aggFuncs,groupBy.isEmpty());
+context.getScan().setAttribute(BaseScannerRegionObserver.AGGREGATORS, 
ServerAggregators.serialize(aggFuncs, minNullableIndex));
+ClientAggregators clientAggregators = new ClientAggregators(aggFuncs, 
minNullableIndex);
+context.getAggregationManager().setAggregators(clientAggregators);
+}
+
+private static int getMinNullableIndex(List 
aggFuncs, boolean isUngroupedAggregation) {
+int minNullableIndex = aggFuncs.size();
+for (int i = 0; i < aggFuncs.size(); i++) {
+SingleAggregateFunction aggFunc = aggFuncs.get(i);
+if (isUngroupedAggregation ? aggFunc.getAggregator().isNullable() 
: aggFunc.getAggregatorExpression().isNullable()) {
+minNullableIndex = i;
+break;
+}
+}
+return minNullableIndex;
+}
+
 }

[1/3] phoenix git commit: PHOENIX-2818 Optimize ORDERED group by

2016-05-12 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-1.1 f458b82a0 -> 7cad42cb1


PHOENIX-2818 Optimize ORDERED group by


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d56d75a7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d56d75a7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d56d75a7

Branch: refs/heads/4.x-HBase-1.1
Commit: d56d75a7a41a2c9b25ce4e1a3d3b115c6de6d9f6
Parents: f458b82
Author: James Taylor 
Authored: Thu May 12 14:29:29 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 17:11:21 2016 -0700

--
 .../apache/phoenix/end2end/GroupByCaseIT.java   |  92 +
 .../apache/phoenix/execute/AggregatePlan.java   |   3 +-
 .../iterate/AggregatingResultIterator.java  |   4 +-
 .../BaseGroupedAggregatingResultIterator.java   |   3 +-
 .../DistinctAggregatingResultIterator.java  |  15 +-
 .../FilterAggregatingResultIterator.java|   8 +-
 .../OrderedAggregatingResultIterator.java   |   5 +-
 .../RowKeyOrderedAggregateResultIterator.java   | 190 +++
 .../iterate/AggregateResultScannerTest.java | 109 +--
 .../iterate/ConcatResultIteratorTest.java   |  31 +--
 .../iterate/MaterializedResultIterators.java|  66 +++
 ...owKeyOrderedAggregateResultIteratorTest.java | 149 +++
 .../java/org/apache/phoenix/util/TestUtil.java  |  70 +++
 13 files changed, 596 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d56d75a7/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 44f43b7..b0524da 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -32,6 +32,9 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
 
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
 import org.junit.Test;
@@ -343,4 +346,93 @@ public class GroupByCaseIT extends BaseHBaseManagedTimeIT {
 "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", 
QueryUtil.getExplainPlan(rs));
 }
 
+@Test
+public void testSumGroupByOrderPreservingDesc() throws Exception {
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+Connection conn = DriverManager.getConnection(getUrl(), props);
+PreparedStatement stmt = conn.prepareStatement("CREATE TABLE 
GROUP_BY_DESC (k1 char(1) not null, k2 integer not null, constraint pk primary 
key (k1,k2)) split on (?,?,?)");
+stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.execute();
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('b', 
5)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
4)");
+conn.commit();
+String query = "SELECT k1,sum(k2) FROM GROUP_BY_DESC GROUP BY k1 ORDER 
BY k1 DESC";
+ResultSet rs = conn.createStatement().executeQuery(query);
+assertTrue(rs.next());
+assertEquals("n", rs.getString(1));
+assertEquals(10, rs.getInt(2));
+assertTrue(rs.next());
+

[2/3] phoenix git commit: PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2876 Using aggregation function in ORDER BY (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/af8d3b65
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/af8d3b65
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/af8d3b65

Branch: refs/heads/master
Commit: af8d3b65c84fd57b91b99ff36de2194149c5a94e
Parents: d414505
Author: James Taylor 
Authored: Thu May 12 14:32:34 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 14:32:34 2016 -0700

--
 .../phoenix/compile/AggregationManager.java | 60 
 .../apache/phoenix/compile/DeleteCompiler.java  |  1 +
 .../apache/phoenix/compile/PostDDLCompiler.java |  1 +
 .../phoenix/compile/ProjectionCompiler.java | 53 +
 .../apache/phoenix/compile/QueryCompiler.java   |  1 +
 .../apache/phoenix/compile/UpsertCompiler.java  |  6 +-
 .../phoenix/compile/QueryCompilerTest.java  | 16 +-
 7 files changed, 86 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/af8d3b65/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
index ee2497b..c8e672e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/AggregationManager.java
@@ -17,7 +17,21 @@
  */
 package org.apache.phoenix.compile;
 
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.aggregator.ClientAggregators;
+import org.apache.phoenix.expression.aggregator.ServerAggregators;
+import org.apache.phoenix.expression.function.SingleAggregateFunction;
+import org.apache.phoenix.expression.visitor.SingleAggregateFunctionVisitor;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 /**
  * 
@@ -52,4 +66,50 @@ public class AggregationManager {
 public void setAggregators(ClientAggregators clientAggregator) {
 this.aggregators = clientAggregator;
 }
+/**
+ * Compiles projection by:
+ * 1) Adding RowCount aggregate function if not present when limiting 
rows. We need this
+ *to track how many rows have been scanned.
+ * 2) Reordering aggregation functions (by putting fixed length aggregates 
first) to
+ *optimize the positional access of the aggregated value.
+ */
+public void compile(StatementContext context, GroupByCompiler.GroupBy 
groupBy) throws
+SQLException {
+final Set aggFuncSet = 
Sets.newHashSetWithExpectedSize(context.getExpressionManager().getExpressionCount());
+
+Iterator expressions = 
context.getExpressionManager().getExpressions();
+while (expressions.hasNext()) {
+Expression expression = expressions.next();
+expression.accept(new SingleAggregateFunctionVisitor() {
+@Override
+public Iterator visitEnter(SingleAggregateFunction 
function) {
+aggFuncSet.add(function);
+return Iterators.emptyIterator();
+}
+});
+}
+if (aggFuncSet.isEmpty() && groupBy.isEmpty()) {
+return;
+}
+List aggFuncs = new 
ArrayList(aggFuncSet);
+Collections.sort(aggFuncs, SingleAggregateFunction.SCHEMA_COMPARATOR);
+
+int minNullableIndex = getMinNullableIndex(aggFuncs,groupBy.isEmpty());
+context.getScan().setAttribute(BaseScannerRegionObserver.AGGREGATORS, 
ServerAggregators.serialize(aggFuncs, minNullableIndex));
+ClientAggregators clientAggregators = new ClientAggregators(aggFuncs, 
minNullableIndex);
+context.getAggregationManager().setAggregators(clientAggregators);
+}
+
+private static int getMinNullableIndex(List 
aggFuncs, boolean isUngroupedAggregation) {
+int minNullableIndex = aggFuncs.size();
+for (int i = 0; i < aggFuncs.size(); i++) {
+SingleAggregateFunction aggFunc = aggFuncs.get(i);
+if (isUngroupedAggregation ? aggFunc.getAggregator().isNullable() 
: aggFunc.getAggregatorExpression().isNullable()) {
+minNullableIndex = i;
+break;
+}
+}
+return minNullableIndex;
+}
+
 }

[3/3] phoenix git commit: PHOENIX-2888 Aggregations for similar IN list expressions doesn't work correctly (Sergey Soldatov)

2016-05-12 Thread jamestaylor 
PHOENIX-2888 Aggregations for similar IN list expressions doesn't work 
correctly (Sergey Soldatov)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ae14e38c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ae14e38c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ae14e38c

Branch: refs/heads/master
Commit: ae14e38ccbed601dfbda9ec0dbb66bc88d63bae5
Parents: af8d3b6
Author: James Taylor 
Authored: Thu May 12 14:34:47 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 14:34:47 2016 -0700

--
 .../java/org/apache/phoenix/expression/InListExpression.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ae14e38c/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
index a4a9353..a977f1f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/expression/InListExpression.java
@@ -175,7 +175,7 @@ public class InListExpression extends BaseSingleExpression {
 public int hashCode() {
 final int prime = 31;
 int result = 1;
-result = prime * result + values.hashCode();
+result = prime * result + children.hashCode() + values.hashCode();
 return result;
 }
 
@@ -185,7 +185,7 @@ public class InListExpression extends BaseSingleExpression {
 if (obj == null) return false;
 if (getClass() != obj.getClass()) return false;
 InListExpression other = (InListExpression)obj;
-if (!values.equals(other.values)) return false;
+if (!children.equals(other.children) || !values.equals(other.values)) 
return false;
 return true;
 }

[1/3] phoenix git commit: PHOENIX-2818 Optimize ORDERED group by

2016-05-12 Thread jamestaylor 
Repository: phoenix
Updated Branches:
  refs/heads/master d97eb4967 -> ae14e38cc


PHOENIX-2818 Optimize ORDERED group by


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d414505d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d414505d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d414505d

Branch: refs/heads/master
Commit: d414505df8afeba553734db688547c6a4e9e90d9
Parents: d97eb49
Author: James Taylor 
Authored: Thu May 12 14:29:29 2016 -0700
Committer: James Taylor 
Committed: Thu May 12 14:29:29 2016 -0700

--
 .../apache/phoenix/end2end/GroupByCaseIT.java   |  92 +
 .../apache/phoenix/execute/AggregatePlan.java   |   3 +-
 .../iterate/AggregatingResultIterator.java  |   4 +-
 .../BaseGroupedAggregatingResultIterator.java   |   3 +-
 .../DistinctAggregatingResultIterator.java  |  15 +-
 .../FilterAggregatingResultIterator.java|   8 +-
 .../OrderedAggregatingResultIterator.java   |   5 +-
 .../RowKeyOrderedAggregateResultIterator.java   | 190 +++
 .../iterate/AggregateResultScannerTest.java | 109 +--
 .../iterate/ConcatResultIteratorTest.java   |  31 +--
 .../iterate/MaterializedResultIterators.java|  66 +++
 ...owKeyOrderedAggregateResultIteratorTest.java | 149 +++
 .../java/org/apache/phoenix/util/TestUtil.java  |  70 +++
 13 files changed, 596 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d414505d/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 44f43b7..b0524da 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -32,6 +32,9 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Properties;
 
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
 import org.junit.Test;
@@ -343,4 +346,93 @@ public class GroupByCaseIT extends BaseHBaseManagedTimeIT {
 "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", 
QueryUtil.getExplainPlan(rs));
 }
 
+@Test
+public void testSumGroupByOrderPreservingDesc() throws Exception {
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+Connection conn = DriverManager.getConnection(getUrl(), props);
+PreparedStatement stmt = conn.prepareStatement("CREATE TABLE 
GROUP_BY_DESC (k1 char(1) not null, k2 integer not null, constraint pk primary 
key (k1,k2)) split on (?,?,?)");
+stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), 
PInteger.INSTANCE.toBytes(3)));
+stmt.execute();
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('a', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('b', 
5)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('j', 
4)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
1)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
2)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
3)");
+conn.createStatement().execute("UPSERT INTO GROUP_BY_DESC VALUES('n', 
4)");
+conn.commit();
+String query = "SELECT k1,sum(k2) FROM GROUP_BY_DESC GROUP BY k1 ORDER 
BY k1 DESC";
+ResultSet rs = conn.createStatement().executeQuery(query);
+assertTrue(rs.next());
+assertEquals("n", rs.getString(1));
+assertEquals(10, rs.getInt(2));
+assertTrue(rs.next());
+

[2/3] phoenix git commit: PHOENIX-2628 Ensure split when iterating through results handled correctly(Rajeshbabu)

2016-05-12 Thread rajeshbabu 
http://git-wip-us.apache.org/repos/asf/phoenix/blob/119c5f90/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 96ee543..a391f05 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
@@ -84,7 +85,6 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 public static final String LOCAL_INDEX_JOIN_SCHEMA = 
"_LocalIndexJoinSchema";
 public static final String DATA_TABLE_COLUMNS_TO_JOIN = 
"_DataTableColumnsToJoin";
 public static final String VIEW_CONSTANTS = "_ViewConstants";
-public static final String STARTKEY_OFFSET = "_StartKeyOffset";
 public static final String EXPECTED_UPPER_REGION_KEY = 
"_ExpectedUpperRegionKey";
 public static final String REVERSE_SCAN = "_ReverseScan";
 public static final String ANALYZE_TABLE = "_ANALYZETABLE";
@@ -99,6 +99,8 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 public static final String SCAN_ACTUAL_START_ROW = "_ScanActualStartRow";
 public static final String IGNORE_NEWER_MUTATIONS = 
"_IGNORE_NEWER_MUTATIONS";
 public final static String SCAN_OFFSET = "_RowOffset";
+public static final String SCAN_START_ROW_SUFFIX = "_ScanStartRowSuffix";
+public static final String SCAN_STOP_ROW_SUFFIX = "_ScanStopRowSuffix";
 
 /**
  * Attribute name used to pass custom annotations in Scans and Mutations 
(later). Custom annotations
@@ -134,7 +136,9 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 byte[] upperExclusiveRegionKey = region.getEndKey();
 boolean isStaleRegionBoundaries;
 if (isLocalIndex) {
-byte[] expectedUpperRegionKey = 
scan.getAttribute(EXPECTED_UPPER_REGION_KEY);
+byte[] expectedUpperRegionKey =
+scan.getAttribute(EXPECTED_UPPER_REGION_KEY) == null ? 
scan.getStopRow() : scan
+.getAttribute(EXPECTED_UPPER_REGION_KEY);
 isStaleRegionBoundaries = expectedUpperRegionKey != null &&
 Bytes.compareTo(upperExclusiveRegionKey, 
expectedUpperRegionKey) != 0;
 } else {
@@ -146,6 +150,9 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 Exception cause = new 
StaleRegionBoundaryCacheException(region.getRegionInfo().getTable().getNameAsString());
 throw new DoNotRetryIOException(cause.getMessage(), cause);
 }
+if(isLocalIndex) {
+ScanUtil.setupLocalIndexScan(scan, lowerInclusiveRegionKey, 
upperExclusiveRegionKey);
+}
 }
 
 abstract protected boolean isRegionObserverFor(Scan scan);
@@ -165,7 +172,10 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 scan.setTimeRange(timeRange.getMin(), Bytes.toLong(txnScn));
 }
 if (isRegionObserverFor(scan)) {
-if (! skipRegionBoundaryCheck(scan)) {
+// For local indexes, we need to throw if out of region as we'll 
get inconsistent
+// results otherwise while in other cases, it may just mean out 
client-side data
+// on region boundaries is out of date and can safely be ignored.
+if (!skipRegionBoundaryCheck(scan) || ScanUtil.isLocalIndex(scan)) 
{
 throwIfScanOutOfRegion(scan, c.getEnvironment().getRegion());
 }
 // Muck with the start/stop row of the scan and set as reversed at 
the
@@ -226,6 +236,13 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 }
 }
 } catch (Throwable t) {
+// If the exception is NotServingRegionException then throw it as
+// StaleRegionBoundaryCacheException to handle it by phoenix 
client other wise hbase
+// client may recreate scans with wrong region boundaries.
+if(t instanceof NotServingRegionException) {
+Exception cause = new 
StaleRegionBoundaryCacheException(c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString());
+throw new

[3/3] phoenix git commit: PHOENIX-2628 Ensure split when iterating through results handled correctly(Rajeshbabu)

2016-05-12 Thread rajeshbabu 
PHOENIX-2628 Ensure split when iterating through results handled 
correctly(Rajeshbabu)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/119c5f90
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/119c5f90
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/119c5f90

Branch: refs/heads/4.x-HBase-0.98
Commit: 119c5f906af4b615865b5ed9e78c6eeda6f86646
Parents: c22fa09
Author: Rajeshbabu Chintaguntla 
Authored: Thu May 12 21:27:12 2016 +0530
Committer: Rajeshbabu Chintaguntla 
Committed: Thu May 12 21:27:12 2016 +0530

--
 .../org/apache/phoenix/end2end/BaseViewIT.java  |   5 +
 .../phoenix/end2end/index/LocalIndexIT.java |  61 ++-
 .../phoenix/end2end/index/MutableIndexIT.java   | 247 +++-
 .../DelayedTableResultIteratorFactory.java  |   9 +-
 .../iterate/MockParallelIteratorFactory.java|   3 +-
 .../regionserver/IndexHalfStoreFileReader.java  | 382 +--
 .../IndexHalfStoreFileReaderGenerator.java  | 123 +++---
 .../hbase/regionserver/LocalIndexSplitter.java  |  38 --
 .../LocalIndexStoreFileScanner.java | 253 
 .../phoenix/compile/ListJarsQueryPlan.java  |   4 +
 .../MutatingParallelIteratorFactory.java|   2 +-
 .../org/apache/phoenix/compile/QueryPlan.java   |   2 +
 .../org/apache/phoenix/compile/ScanRanges.java  |  11 +-
 .../apache/phoenix/compile/TraceQueryPlan.java  |   5 +
 .../coprocessor/BaseScannerRegionObserver.java  |  93 -
 .../GroupedAggregateRegionObserver.java |  16 +-
 .../UngroupedAggregateRegionObserver.java   |   8 +-
 .../apache/phoenix/execute/AggregatePlan.java   |  21 +-
 .../apache/phoenix/execute/BaseQueryPlan.java   |  20 +-
 .../phoenix/execute/ClientAggregatePlan.java|   6 +
 .../apache/phoenix/execute/ClientScanPlan.java  |   7 +-
 .../apache/phoenix/execute/CorrelatePlan.java   |   8 +-
 .../phoenix/execute/DegenerateQueryPlan.java|   2 +-
 .../apache/phoenix/execute/HashJoinPlan.java|   8 +-
 .../execute/LiteralResultIterationPlan.java |   2 +-
 .../org/apache/phoenix/execute/ScanPlan.java|   9 +-
 .../phoenix/execute/SortMergeJoinPlan.java  |   7 +-
 .../phoenix/execute/TupleProjectionPlan.java|   8 +-
 .../org/apache/phoenix/execute/UnionPlan.java   |   6 +-
 .../apache/phoenix/execute/UnnestArrayPlan.java |   8 +-
 .../phoenix/iterate/BaseResultIterators.java| 112 --
 .../phoenix/iterate/ChunkedResultIterator.java  |  50 +--
 .../DefaultTableResultIteratorFactory.java  |   5 +-
 .../iterate/ParallelIteratorFactory.java|   5 +-
 .../phoenix/iterate/ParallelIterators.java  |  14 +-
 .../apache/phoenix/iterate/SerialIterators.java |  10 +-
 .../phoenix/iterate/SpoolingResultIterator.java |   3 +-
 .../phoenix/iterate/TableResultIterator.java|  59 ++-
 .../iterate/TableResultIteratorFactory.java |   3 +-
 .../apache/phoenix/jdbc/PhoenixStatement.java   |   5 +
 .../phoenix/mapreduce/PhoenixRecordReader.java  |   5 +-
 .../java/org/apache/phoenix/util/ScanUtil.java  |  64 
 .../java/org/apache/phoenix/query/BaseTest.java |   2 +-
 .../query/ParallelIteratorsSplitTest.java   |   9 +-
 .../hive/mapreduce/PhoenixRecordReader.java |   7 +-
 45 files changed, 1088 insertions(+), 639 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/119c5f90/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
index 7e7175f..65f1f93 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
@@ -33,6 +33,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
@@ -45,6 +46,7 @@ import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.BeforeClass;
@@ -98,6 +100,9 @@ public abstract class BaseViewIT extends 
BaseOwnClusterHBaseManagedTimeIT {
 if (saltBuckets == null) {
 try (Connection conn = DriverManager.getConnection(getUrl())) {
 HTableInterface htable =

Apache-Phoenix | 4.x-HBase-1.0 | Build Successful

2016-05-12 Thread Apache Jenkins Server 
4.x-HBase-1.0 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf?p=phoenix.git;a=shortlog;h=refs/heads/4.x-HBase-1.0

Compiled Artifacts https://builds.apache.org/job/Phoenix-4.x-HBase-1.0/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-4.x-HBase-1.0/lastCompletedBuild/testReport/

Changes
[rajeshbabu] PHOENIX-2628 Ensure split when iterating through results handled

[rajeshbabu] PHOENIX-2628 Ensure split when iterating through results handled



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout

[1/3] phoenix git commit: PHOENIX-2628 Ensure split when iterating through results handled correctly(Rajeshbabu)

2016-05-12 Thread rajeshbabu 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-1.0 f7c6c5d8d -> c1e1b1f23


http://git-wip-us.apache.org/repos/asf/phoenix/blob/e79416e0/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
index b60cd7e..94d1fc8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
@@ -25,6 +25,7 @@ import java.util.List;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.phoenix.compile.GroupByCompiler.GroupBy;
 import org.apache.phoenix.compile.OrderByCompiler.OrderBy;
+import org.apache.phoenix.compile.QueryPlan;
 import org.apache.phoenix.compile.RowProjector;
 import org.apache.phoenix.compile.StatementContext;
 import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
@@ -126,7 +127,7 @@ public class AggregatePlan extends BaseQueryPlan {
 this.services = services;
 }
 @Override
-public PeekingResultIterator newIterator(StatementContext context, 
ResultIterator scanner, Scan scan, String tableName) throws SQLException {
+public PeekingResultIterator newIterator(StatementContext context, 
ResultIterator scanner, Scan scan, String tableName, QueryPlan plan) throws 
SQLException {
 Expression expression = RowKeyExpression.INSTANCE;
 OrderByExpression orderByExpression = new 
OrderByExpression(expression, false, true);
 int threshold = 
services.getProps().getInt(QueryServices.SPOOL_THRESHOLD_BYTES_ATTRIB, 
QueryServicesOptions.DEFAULT_SPOOL_THRESHOLD_BYTES);
@@ -143,9 +144,9 @@ public class AggregatePlan extends BaseQueryPlan {
 this.outerFactory = outerFactory;
 }
 @Override
-public PeekingResultIterator newIterator(StatementContext context, 
ResultIterator scanner, Scan scan, String tableName) throws SQLException {
-PeekingResultIterator iterator = innerFactory.newIterator(context, 
scanner, scan, tableName);
-return outerFactory.newIterator(context, iterator, scan, 
tableName);
+public PeekingResultIterator newIterator(StatementContext context, 
ResultIterator scanner, Scan scan, String tableName, QueryPlan plan) throws 
SQLException {
+PeekingResultIterator iterator = innerFactory.newIterator(context, 
scanner, scan, tableName, plan);
+return outerFactory.newIterator(context, iterator, scan, 
tableName, plan);
 }
 }
 
@@ -169,12 +170,12 @@ public class AggregatePlan extends BaseQueryPlan {
 }
 
 @Override
-protected ResultIterator newIterator(ParallelScanGrouper scanGrouper) 
throws SQLException {
+protected ResultIterator newIterator(ParallelScanGrouper scanGrouper, Scan 
scan) throws SQLException {
 if (groupBy.isEmpty()) {
-
UngroupedAggregateRegionObserver.serializeIntoScan(context.getScan());
+UngroupedAggregateRegionObserver.serializeIntoScan(scan);
 } else {
 // Set attribute with serialized expressions for coprocessor
-
GroupedAggregateRegionObserver.serializeIntoScan(context.getScan(), 
groupBy.getScanAttribName(), groupBy.getKeyExpressions());
+GroupedAggregateRegionObserver.serializeIntoScan(scan, 
groupBy.getScanAttribName(), groupBy.getKeyExpressions());
 if (limit != null && orderBy.getOrderByExpressions().isEmpty() && 
having == null
 && (  (   statement.isDistinct() && ! 
statement.isAggregate() )
 || ( ! statement.isDistinct() && (   
context.getAggregationManager().isEmpty()
@@ -200,7 +201,7 @@ public class AggregatePlan extends BaseQueryPlan {
  *order, so we can early exit, even when aggregate 
functions are used, as
  *the rows in the group are contiguous.
  */
-
context.getScan().setAttribute(BaseScannerRegionObserver.GROUP_BY_LIMIT,
+scan.setAttribute(BaseScannerRegionObserver.GROUP_BY_LIMIT,
 PInteger.INSTANCE.toBytes(limit + (offset == null ? 0 
: offset)));
 }
 }
@@ -211,8 +212,8 @@ public class AggregatePlan extends BaseQueryPlan {
 logger.warn("This query cannot be executed serially. Ignoring the 
hint");
 }
 BaseResultIterators iterators = hasSerialHint && canBeExecutedSerially
-? new SerialIterators(this, null, null, 
wrapParallelIteratorFactory(), scanGrouper)
-: new ParallelIterators(this, null, 
wrapParallelIteratorFactory());
+? new SerialIterators(this, null, null, 
wrapParallelIteratorFactory(), scanGrouper, scan)
+: new