git commit: PHOENIX-539 Chunked loading of parallel scanning
Repository: phoenix Updated Branches: refs/heads/master a61a1a48d -> 61c948b73 PHOENIX-539 Chunked loading of parallel scanning Instead of spooling all data in a table to disk, load it in chunks as needed, potentially spooling large quantities of data to disk. Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/61c948b7 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/61c948b7 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/61c948b7 Branch: refs/heads/master Commit: 61c948b73f27c8d563561f5d3ef779742646a450 Parents: a61a1a4 Author: Gabriel Reid Authored: Tue Jun 10 07:36:37 2014 +0200 Committer: Gabriel Reid Committed: Tue Jul 8 21:28:04 2014 +0200 -- .../org/apache/phoenix/end2end/QueryIT.java | 11 +- .../phoenix/compile/StatementContext.java | 23 +++ .../org/apache/phoenix/execute/ScanPlan.java| 43 ++-- .../phoenix/iterate/ChunkedResultIterator.java | 196 +++ .../phoenix/iterate/ParallelIterators.java | 7 +- .../org/apache/phoenix/join/HashJoinInfo.java | 48 +++-- .../org/apache/phoenix/query/QueryServices.java | 5 + .../phoenix/query/QueryServicesOptions.java | 3 + 8 files changed, 298 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/61c948b7/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java -- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java index 867eb2a..f453853 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java @@ -861,7 +861,7 @@ public class QueryIT extends BaseClientManagedTimeIT { try { PreparedStatement statement = conn.prepareStatement(query); statement.setString(1, tenantId); -statement.setString(2,ROW4); +statement.setString(2, ROW4); ResultSet rs = statement.executeQuery(); assertTrue(rs.next()); assertEquals(A_VALUE, rs.getString(1)); @@ -879,17 +879,17 @@ public class QueryIT extends BaseClientManagedTimeIT { byte[] tableName = Bytes.toBytes(ATABLE_NAME); admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); -HTable htable = (HTable)conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); +HTable htable = (HTable) conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); htable.clearRegionCache(); int nRegions = htable.getRegionLocations().size(); -admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char)('3' + nextRunCount()))+ ts))); // vary split point with test run +admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char) ('3' + nextRunCount())) + ts))); // vary split point with test run int retryCount = 0; do { Thread.sleep(2000); retryCount++; //htable.clearRegionCache(); } while (retryCount < 10 && htable.getRegionLocations().size() == nRegions); -assertNotEquals(nRegions,htable.getRegionLocations().size()); +assertNotEquals(nRegions, htable.getRegionLocations().size()); statement.setString(1, tenantId); rs = statement.executeQuery(); @@ -906,9 +906,10 @@ public class QueryIT extends BaseClientManagedTimeIT { assertEquals(E_VALUE, rs.getString(2)); assertEquals(1, rs.getLong(3)); assertFalse(rs.next()); - } finally { +if (admin != null) { admin.close(); +} conn.close(); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/61c948b7/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java index 4c907d6..06d5f89 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -91,6 +91,29 @@ public class StatementContext { this.whereConditionColumns = new ArrayList>(); } +/** + * Copy constructor where an altered scan can be set. + * +
git commit: PHOENIX-539 Chunked loading of parallel scanning
Repository: phoenix Updated Branches: refs/heads/4.0 8496d76ac -> 2625274eb PHOENIX-539 Chunked loading of parallel scanning Instead of spooling all data in a table to disk, load it in chunks as needed, potentially spooling large quantities of data to disk. Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/2625274e Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/2625274e Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/2625274e Branch: refs/heads/4.0 Commit: 2625274ebcba7aa250e6064ec66538e074737604 Parents: 8496d76 Author: Gabriel Reid Authored: Tue Jun 10 07:36:37 2014 +0200 Committer: Gabriel Reid Committed: Tue Jul 8 21:28:43 2014 +0200 -- .../org/apache/phoenix/end2end/QueryIT.java | 11 +- .../phoenix/compile/StatementContext.java | 23 +++ .../org/apache/phoenix/execute/ScanPlan.java| 43 ++-- .../phoenix/iterate/ChunkedResultIterator.java | 196 +++ .../phoenix/iterate/ParallelIterators.java | 7 +- .../org/apache/phoenix/join/HashJoinInfo.java | 48 +++-- .../org/apache/phoenix/query/QueryServices.java | 5 + .../phoenix/query/QueryServicesOptions.java | 3 + 8 files changed, 298 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/2625274e/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java -- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java index 867eb2a..f453853 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java @@ -861,7 +861,7 @@ public class QueryIT extends BaseClientManagedTimeIT { try { PreparedStatement statement = conn.prepareStatement(query); statement.setString(1, tenantId); -statement.setString(2,ROW4); +statement.setString(2, ROW4); ResultSet rs = statement.executeQuery(); assertTrue(rs.next()); assertEquals(A_VALUE, rs.getString(1)); @@ -879,17 +879,17 @@ public class QueryIT extends BaseClientManagedTimeIT { byte[] tableName = Bytes.toBytes(ATABLE_NAME); admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); -HTable htable = (HTable)conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); +HTable htable = (HTable) conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); htable.clearRegionCache(); int nRegions = htable.getRegionLocations().size(); -admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char)('3' + nextRunCount()))+ ts))); // vary split point with test run +admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char) ('3' + nextRunCount())) + ts))); // vary split point with test run int retryCount = 0; do { Thread.sleep(2000); retryCount++; //htable.clearRegionCache(); } while (retryCount < 10 && htable.getRegionLocations().size() == nRegions); -assertNotEquals(nRegions,htable.getRegionLocations().size()); +assertNotEquals(nRegions, htable.getRegionLocations().size()); statement.setString(1, tenantId); rs = statement.executeQuery(); @@ -906,9 +906,10 @@ public class QueryIT extends BaseClientManagedTimeIT { assertEquals(E_VALUE, rs.getString(2)); assertEquals(1, rs.getLong(3)); assertFalse(rs.next()); - } finally { +if (admin != null) { admin.close(); +} conn.close(); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/2625274e/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java index 4c907d6..06d5f89 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -91,6 +91,29 @@ public class StatementContext { this.whereConditionColumns = new ArrayList>(); } +/** + * Copy constructor where an altered scan can be set. + * + *
git commit: PHOENIX-539 Chunked loading of parallel scanning
Repository: phoenix Updated Branches: refs/heads/3.0 c46364ed1 -> ec1b76ab2 PHOENIX-539 Chunked loading of parallel scanning Instead of spooling all data in a table to disk, load it in chunks as needed, potentially spooling large quantities of data to disk. Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ec1b76ab Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ec1b76ab Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ec1b76ab Branch: refs/heads/3.0 Commit: ec1b76ab28880d548c8061a6dcb80a6fe0d97ed6 Parents: c46364e Author: Gabriel Reid Authored: Tue Jun 10 07:36:37 2014 +0200 Committer: Gabriel Reid Committed: Tue Jul 8 20:37:30 2014 +0200 -- .../org/apache/phoenix/end2end/QueryIT.java | 11 +- .../phoenix/compile/StatementContext.java | 23 +++ .../org/apache/phoenix/execute/ScanPlan.java| 43 ++-- .../phoenix/iterate/ChunkedResultIterator.java | 196 +++ .../phoenix/iterate/ParallelIterators.java | 7 +- .../org/apache/phoenix/join/HashJoinInfo.java | 48 +++-- .../org/apache/phoenix/query/QueryServices.java | 5 + .../phoenix/query/QueryServicesOptions.java | 3 + 8 files changed, 298 insertions(+), 38 deletions(-) -- http://git-wip-us.apache.org/repos/asf/phoenix/blob/ec1b76ab/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java -- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java index 867eb2a..f453853 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java @@ -861,7 +861,7 @@ public class QueryIT extends BaseClientManagedTimeIT { try { PreparedStatement statement = conn.prepareStatement(query); statement.setString(1, tenantId); -statement.setString(2,ROW4); +statement.setString(2, ROW4); ResultSet rs = statement.executeQuery(); assertTrue(rs.next()); assertEquals(A_VALUE, rs.getString(1)); @@ -879,17 +879,17 @@ public class QueryIT extends BaseClientManagedTimeIT { byte[] tableName = Bytes.toBytes(ATABLE_NAME); admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); -HTable htable = (HTable)conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); +HTable htable = (HTable) conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); htable.clearRegionCache(); int nRegions = htable.getRegionLocations().size(); -admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char)('3' + nextRunCount()))+ ts))); // vary split point with test run +admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char) ('3' + nextRunCount())) + ts))); // vary split point with test run int retryCount = 0; do { Thread.sleep(2000); retryCount++; //htable.clearRegionCache(); } while (retryCount < 10 && htable.getRegionLocations().size() == nRegions); -assertNotEquals(nRegions,htable.getRegionLocations().size()); +assertNotEquals(nRegions, htable.getRegionLocations().size()); statement.setString(1, tenantId); rs = statement.executeQuery(); @@ -906,9 +906,10 @@ public class QueryIT extends BaseClientManagedTimeIT { assertEquals(E_VALUE, rs.getString(2)); assertEquals(1, rs.getLong(3)); assertFalse(rs.next()); - } finally { +if (admin != null) { admin.close(); +} conn.close(); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ec1b76ab/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java -- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java index 4c907d6..06d5f89 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -91,6 +91,29 @@ public class StatementContext { this.whereConditionColumns = new ArrayList>(); } +/** + * Copy constructor where an altered scan can be set. + * + *