[3/5] git commit: PHOENIX-1315 Optimize query for Pig loader

2014-10-05 Thread jamestaylor
PHOENIX-1315 Optimize query for Pig loader


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5df8d1ec
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5df8d1ec
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5df8d1ec

Branch: refs/heads/4.0
Commit: 5df8d1ec4a2d755365738f9e0e6e8310bf96d83e
Parents: 8840af6
Author: James Taylor jtay...@salesforce.com
Authored: Sun Oct 5 09:53:14 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Sun Oct 5 09:53:14 2014 -0700

--
 .../phoenix/end2end/EvaluationOfORIT.java   |  9 +--
 .../apache/phoenix/end2end/ReverseScanIT.java   |  4 +-
 ...ipRangeParallelIteratorRegionSplitterIT.java |  5 ++
 .../index/balancer/IndexLoadBalancerIT.java | 13 +++--
 .../org/apache/phoenix/compile/QueryPlan.java   |  3 +
 .../apache/phoenix/execute/AggregatePlan.java   |  6 ++
 .../phoenix/execute/DegenerateQueryPlan.java| 12 +++-
 .../apache/phoenix/execute/HashJoinPlan.java|  5 ++
 .../org/apache/phoenix/execute/ScanPlan.java|  8 +++
 .../phoenix/iterate/ConcatResultIterator.java   | 29 ++
 .../iterate/LookAheadResultIterator.java| 21 +++
 .../phoenix/iterate/ParallelIterators.java  | 39 +++--
 .../apache/phoenix/jdbc/PhoenixStatement.java   |  6 ++
 .../phoenix/pig/PhoenixHBaseLoaderIT.java   | 24 
 .../phoenix/pig/hadoop/PhoenixInputFormat.java  | 13 +++--
 .../phoenix/pig/hadoop/PhoenixInputSplit.java   | 60 +++-
 .../phoenix/pig/hadoop/PhoenixRecordReader.java | 25 
 17 files changed, 196 insertions(+), 86 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/5df8d1ec/phoenix-core/src/it/java/org/apache/phoenix/end2end/EvaluationOfORIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/EvaluationOfORIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/EvaluationOfORIT.java
index 052ff43..0e59542 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/EvaluationOfORIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/EvaluationOfORIT.java
@@ -28,21 +28,22 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Properties;
 
+import org.apache.phoenix.util.PropertiesUtil;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-@Category(BaseHBaseManagedTimeIT.class)
+@Category(HBaseManagedTimeTest.class)
 public class EvaluationOfORIT extends BaseHBaseManagedTimeIT{

@Test
public void testPKOrNotPKInOREvaluation() throws SQLException {
-   Properties props = new Properties(TEST_PROPERTIES);
+   Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props); 

conn.setAutoCommit(false);

-String create = CREATE TABLE DIE ( ID INTEGER NOT NULL PRIMARY 
KEY,NAME VARCHAR(50) NOT NULL);
+String create = CREATE TABLE DIE ( ID INTEGER NOT NULL PRIMARY 
KEY,NAME VARCHAR(50));
 PreparedStatement createStmt = conn.prepareStatement(create);
-createStmt.executeUpdate();
+createStmt.execute();
 PreparedStatement stmt = conn.prepareStatement(
 upsert into  +
 DIE VALUES (?, ?));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5df8d1ec/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java
index f7409a9..f738773 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ReverseScanIT.java
@@ -47,8 +47,8 @@ import org.junit.experimental.categories.Category;
 
 import com.google.common.collect.Maps;
 
-@Category(HBaseManagedTimeTest.class)
-public class ReverseScanIT extends BaseClientManagedTimeIT {
+@Category(ClientManagedTimeTest.class)
+public class ReverseScanIT extends BaseHBaseManagedTimeIT {
 @BeforeClass
 @Shadower(classBeingShadowed = BaseClientManagedTimeIT.class)
 public static void doSetup() throws Exception {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/5df8d1ec/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java
 

[5/5] git commit: PHOENIX-1257 Upserted data seen by SELECT in UPSERT SELECT execution (Lars Hofhansl)

2014-10-05 Thread jamestaylor
PHOENIX-1257 Upserted data seen by SELECT in UPSERT SELECT execution (Lars 
Hofhansl)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e49e8dcf
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e49e8dcf
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e49e8dcf

Branch: refs/heads/4.0
Commit: e49e8dcfbed740e13515c0b9aaf79db602059fd4
Parents: c9101f8
Author: James Taylor jtay...@salesforce.com
Authored: Sun Oct 5 13:26:52 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Sun Oct 5 18:11:37 2014 -0700

--
 .../phoenix/end2end/CoalesceFunctionIT.java | 67 
 .../apache/phoenix/end2end/ReverseScanIT.java   |  2 +-
 ...ipRangeParallelIteratorRegionSplitterIT.java |  3 +-
 .../end2end/TenantSpecificTablesDDLIT.java  |  2 +-
 .../phoenix/end2end/ToCharFunctionIT.java   |  4 +-
 .../phoenix/end2end/ToNumberFunctionIT.java |  4 +-
 .../end2end/UpsertSelectAutoCommitIT.java   | 23 +++
 .../salted/SaltedTableVarLengthRowKeyIT.java|  8 +--
 .../apache/phoenix/compile/FromCompiler.java| 32 +++---
 .../apache/phoenix/compile/UpsertCompiler.java  | 19 ++
 .../apache/phoenix/execute/BaseQueryPlan.java   |  6 --
 11 files changed, 104 insertions(+), 66 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/e49e8dcf/phoenix-core/src/it/java/org/apache/phoenix/end2end/CoalesceFunctionIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CoalesceFunctionIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CoalesceFunctionIT.java
index 57599e6..45fcb48 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CoalesceFunctionIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CoalesceFunctionIT.java
@@ -67,7 +67,7 @@ public class CoalesceFunctionIT extends 
BaseHBaseManagedTimeIT {
 public void coalesceWithSumExplicitLong() throws Exception {
 Connection conn = DriverManager.getConnection(getUrl());
 
-String ddl = CREATE TABLE IF NOT EXISTS TEST_COALESCE(
+String ddl = CREATE TABLE TEST_COALESCE(
 + ID BIGINT NOT NULL, 
 + COUNT BIGINT 
 + CONSTRAINT pk PRIMARY KEY(ID));
@@ -91,7 +91,7 @@ public class CoalesceFunctionIT extends 
BaseHBaseManagedTimeIT {
 public void coalesceWithSumImplicitLong() throws Exception {
 Connection conn = DriverManager.getConnection(getUrl());
 
-String ddl = CREATE TABLE IF NOT EXISTS TEST_COALESCE(
+String ddl = CREATE TABLE TEST_COALESCE(
 + ID BIGINT NOT NULL, 
 + COUNT BIGINT 
 + CONSTRAINT pk PRIMARY KEY(ID));
@@ -115,7 +115,7 @@ public class CoalesceFunctionIT extends 
BaseHBaseManagedTimeIT {
 public void coalesceWithSecondParamAsExpression() throws Exception {
 Connection conn = DriverManager.getConnection(getUrl());
 
-String ddl = CREATE TABLE IF NOT EXISTS TEST_COALESCE(
+String ddl = CREATE TABLE TEST_COALESCE(
 + ID BIGINT NOT NULL, 
 + COUNT BIGINT 
 + CONSTRAINT pk PRIMARY KEY(ID));
@@ -139,7 +139,7 @@ public class CoalesceFunctionIT extends 
BaseHBaseManagedTimeIT {
 public void nonTypedSecondParameterLong() throws Exception {
 Connection conn = DriverManager.getConnection(getUrl());
 
-String ddl = CREATE TABLE IF NOT EXISTS TEST_COALESCE(
+String ddl = CREATE TABLE TEST_COALESCE(
 + ID BIGINT NOT NULL, 
 + COUNT BIGINT  //first parameter to coalesce
 + CONSTRAINT pk PRIMARY KEY(ID));
@@ -163,47 +163,32 @@ public class CoalesceFunctionIT extends 
BaseHBaseManagedTimeIT {
 public void nonTypedSecondParameterUnsignedDataTypes() throws Exception {
 Connection conn = DriverManager.getConnection(getUrl());
 
-String[] dataTypes = {
-UNSIGNED_INT,
-UNSIGNED_LONG,
-UNSIGNED_TINYINT,
-UNSIGNED_SMALLINT,
-UNSIGNED_FLOAT,
-UNSIGNED_DOUBLE,
-UNSIGNED_TIME,
-UNSIGNED_DATE,
-UNSIGNED_TIMESTAMP
-};
-
-for (String dataType : dataTypes) {
-
-String ddl = CREATE TABLE IF NOT EXISTS TEST_COALESCE(
-+ ID BIGINT NOT NULL, 
-+ COUNT  + dataType //first parameter to coalesce
-+ CONSTRAINT pk PRIMARY KEY(ID));
-conn.createStatement().execute(ddl);
-
-conn.createStatement().execute(UPSERT INTO TEST_COALESCE(ID, 
COUNT) VALUES(2, null));
-conn.commit();
-
-

[1/5] git commit: PHOENIX-1320 Update stats atomically

2014-10-05 Thread jamestaylor
Repository: phoenix
Updated Branches:
  refs/heads/4.0 e0996ade7 - e49e8dcfb


PHOENIX-1320 Update stats atomically


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8c054b2a
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8c054b2a
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8c054b2a

Branch: refs/heads/4.0
Commit: 8c054b2ae9f59c9430d75b0bf1b1c269714db395
Parents: e0996ad
Author: James Taylor jtay...@salesforce.com
Authored: Sat Oct 4 16:54:36 2014 -0700
Committer: James Taylor jtay...@salesforce.com
Committed: Sat Oct 4 16:54:36 2014 -0700

--
 .../UngroupedAggregateRegionObserver.java   | 58 ---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  3 +-
 .../query/ConnectionQueryServicesImpl.java  | 44 ---
 .../apache/phoenix/query/QueryConstants.java|  5 +-
 .../schema/stat/StatisticsCollector.java| 66 +
 .../phoenix/schema/stat/StatisticsScanner.java  | 34 ++---
 .../phoenix/schema/stat/StatisticsTable.java| 78 
 .../phoenix/schema/stat/StatisticsUtils.java|  8 --
 .../org/apache/phoenix/util/SchemaUtil.java |  5 ++
 9 files changed, 166 insertions(+), 135 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8c054b2a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 0bf2710..4ddb322 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -34,8 +34,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HConstants;
@@ -83,7 +81,6 @@ import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.stat.StatisticsCollector;
-import org.apache.phoenix.schema.stat.StatisticsTable;
 import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.IndexUtil;
@@ -116,8 +113,6 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver{
 public static final String EMPTY_CF = EmptyCF;
 private static final Logger logger = 
LoggerFactory.getLogger(UngroupedAggregateRegionObserver.class);
 private KeyValueBuilder kvBuilder;
-private static final Log LOG = 
LogFactory.getLog(UngroupedAggregateRegionObserver.class);
-private StatisticsTable statsTable = null;
 
 @Override
 public void start(CoprocessorEnvironment e) throws IOException {
@@ -125,8 +120,6 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver{
 // Can't use ClientKeyValueBuilder on server-side because the memstore 
expects to
 // be able to get a single backing buffer for a KeyValue.
 this.kvBuilder = GenericKeyValueBuilder.INSTANCE;
-String name = 
((RegionCoprocessorEnvironment)e).getRegion().getTableDesc().getTableName().getNameAsString();
-this.statsTable = 
StatisticsTable.getStatisticsTableForCoprocessor(e.getConfiguration(), name);
 }
 
 private static void commitBatch(HRegion region, ListMutation mutations, 
byte[] indexUUID) throws IOException {
@@ -161,12 +154,11 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver{
 @Override
 protected RegionScanner doPostScannerOpen(final 
ObserverContextRegionCoprocessorEnvironment c, final Scan scan, final 
RegionScanner s) throws IOException {
 int offset = 0;
-boolean isAnalyze = false;
 HRegion region = c.getEnvironment().getRegion();
 StatisticsCollector stats = null;
-if(ScanUtil.isAnalyzeTable(scan)  statsTable != null) {
-stats = new StatisticsCollector(statsTable, 
c.getEnvironment().getConfiguration());
-isAnalyze = true;
+if(ScanUtil.isAnalyzeTable(scan)) {
+// Let this throw, as this scan is being done for the sole purpose 
of collecting stats
+stats = new StatisticsCollector(c.getEnvironment(), 
region.getRegionInfo().getTable().getNameAsString());
 }
 if (ScanUtil.isLocalIndex(scan)) {
 

Apache-Phoenix | 4.0 | Hadoop1 | Build Successful

2014-10-05 Thread Apache Jenkins Server
4.0 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf/incubator-phoenix.git

Compiled Artifacts https://builds.apache.org/job/Phoenix-4.0-hadoop1/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-4.0-hadoop1/lastCompletedBuild/testReport/

Changes
[jtaylor] PHOENIX-1320 Update stats atomically

[jtaylor] PHOENIX-1323 Use utility method to get HTableInterface for scans from coprocessor

[jtaylor] PHOENIX-1315 Optimize query for Pig loader

[jtaylor] PHOENIX-1325 Pass in instead of calculate if we've crossed a region boundary in ScanRanges intersect methods

[jtaylor] PHOENIX-1257 Upserted data seen by SELECT in UPSERT SELECT execution (Lars Hofhansl)