Repository: phoenix Updated Branches: refs/heads/4.x-HBase-0.98 f47bcd13e -> c25243199
PHOENIX-3040 Don't use guideposts for executing queries serially simplification Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c2524319 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c2524319 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c2524319 Branch: refs/heads/4.x-HBase-0.98 Commit: c2524319912d649b7999c919c3be799ea61cd9e8 Parents: f47bcd1 Author: James Taylor <[email protected]> Authored: Sun Jul 10 10:43:25 2016 +0200 Committer: James Taylor <[email protected]> Committed: Sun Jul 10 15:16:16 2016 +0200 ---------------------------------------------------------------------- .../phoenix/compile/ListJarsQueryPlan.java | 5 -- .../org/apache/phoenix/compile/QueryPlan.java | 2 - .../apache/phoenix/compile/TraceQueryPlan.java | 5 -- .../apache/phoenix/execute/AggregatePlan.java | 5 -- .../phoenix/execute/DelegateQueryPlan.java | 5 -- .../execute/LiteralResultIterationPlan.java | 6 --- .../org/apache/phoenix/execute/ScanPlan.java | 48 +++----------------- .../phoenix/execute/SortMergeJoinPlan.java | 6 --- .../org/apache/phoenix/execute/UnionPlan.java | 9 ---- .../phoenix/iterate/BaseResultIterators.java | 9 ++-- .../apache/phoenix/iterate/SerialIterators.java | 8 ++++ .../apache/phoenix/jdbc/PhoenixStatement.java | 5 -- .../org/apache/phoenix/query/QueryServices.java | 5 +- .../phoenix/query/QueryServicesOptions.java | 2 +- .../query/ParallelIteratorsSplitTest.java | 5 -- 15 files changed, 22 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java index 9bffad9..2df0671 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java @@ -251,9 +251,4 @@ public class ListJarsQueryPlan implements QueryPlan { public Operation getOperation() { return stmt.getUpdateOperation(); } - - @Override - public boolean isSerial() { - return true; - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java index f403e34..ea77d79 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java @@ -86,6 +86,4 @@ public interface QueryPlan extends StatementPlan { * @throws SQLException */ public boolean useRoundRobinIterator() throws SQLException; - - public boolean isSerial(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java index aad3188..ed5cda9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java @@ -265,9 +265,4 @@ public class TraceQueryPlan implements QueryPlan { public boolean useRoundRobinIterator() { return false; } - - @Override - public boolean isSerial() { - return true; - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java index c439618..136379d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java @@ -265,9 +265,4 @@ public class AggregatePlan extends BaseQueryPlan { public boolean useRoundRobinIterator() throws SQLException { return false; } - - @Override - public boolean isSerial() { - return isSerial; - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java index f282aea..8f0d224 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java @@ -124,9 +124,4 @@ public abstract class DelegateQueryPlan implements QueryPlan { public Integer getOffset() { return delegate.getOffset(); } - - @Override - public boolean isSerial() { - return delegate.isSerial(); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java index 38cb65e..7b95cf9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java @@ -110,10 +110,4 @@ public class LiteralResultIterationPlan extends BaseQueryPlan { return scanner; } - - @Override - public boolean isSerial() { - return true; - } - } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java index 34354f3..7f735b7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java @@ -60,8 +60,6 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.SaltingUtil; import org.apache.phoenix.schema.TableRef; -import org.apache.phoenix.schema.stats.GuidePostsInfo; -import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; @@ -135,41 +133,15 @@ public class ScanPlan extends BaseQueryPlan { * the amount of data we need to scan is less than the threshold. */ return false; - } else if (perScanLimit != null && scan.getFilter() == null) { - /* - * In presence of a limit and in absence of a filter, we are not relying on guide post info to - * see if we are beyond a threshold. - */ - float factor = - services.getProps().getFloat(QueryServices.NONFILTERED_AND_LIMITED_QUERY_SERIAL_THRESHOLD, - QueryServicesOptions.DEFAULT_NONFILTERED_LIMITED_QUERY_SERIAL_THRESHOLD); - return Float.compare(estRowSize * perScanLimit, factor * regionSize) < 0; - } - long scn = context.getConnection().getSCN() == null ? Long.MAX_VALUE : context.getConnection().getSCN(); - PTableStats tableStats = context.getConnection().getQueryServices().getTableStats(table.getName().getBytes(), scn); - GuidePostsInfo gpsInfo = tableStats.getGuidePosts().get(SchemaUtil.getEmptyColumnFamily(table)); - long threshold; - if (gpsInfo == null || gpsInfo.getGuidePostsCount() == 0) { - threshold = regionSize; - } else { - long totByteSize = 0; - long totRowCount = 0; - for (long byteCount : gpsInfo.getByteCounts()) { - totByteSize += byteCount; - } - for (long rowCount : gpsInfo.getRowCounts()) { - totRowCount += rowCount; - } - estRowSize = totByteSize / totRowCount; - threshold = 2 - * services.getProps().getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES); - } - long thresholdToUse = services.getProps().getLong(QueryServices.FILTERED_OR_NONLIMITED_QUERY_SERIAL_THRESHOLD, - threshold); - return (perScanLimit * estRowSize < thresholdToUse); + } + float factor = + services.getProps().getFloat(QueryServices.LIMITED_QUERY_SERIAL_THRESHOLD, + QueryServicesOptions.DEFAULT_LIMITED_QUERY_SERIAL_THRESHOLD); + long threshold = (long)(factor * regionSize); + return (perScanLimit * estRowSize < threshold); } + @SuppressWarnings("deprecation") private static ParallelIteratorFactory buildResultIteratorFactory(StatementContext context, FilterableStatement statement, TableRef tableRef, OrderBy orderBy, Integer limit,Integer offset, boolean allowPageFilter) throws SQLException { @@ -292,10 +264,4 @@ public class ScanPlan extends BaseQueryPlan { public boolean useRoundRobinIterator() throws SQLException { return ScanUtil.isRoundRobinPossible(orderBy, context); } - - @Override - public boolean isSerial() { - return isSerial; - } - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java index a8ad3eb..f4ff289 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java @@ -671,10 +671,4 @@ public class SortMergeJoinPlan implements QueryPlan { public Set<TableRef> getSourceRefs() { return tableRefs; } - - @Override - public boolean isSerial() { - return false; - } - } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java index f60f81f..cf95b5b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java @@ -231,13 +231,4 @@ public class UnionPlan implements QueryPlan { } return sources; } - - @Override - public boolean isSerial() { - boolean isSerial = true; - for (QueryPlan plan : getPlans()) { - isSerial &= plan.isSerial(); - } - return isSerial; - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 523b419..6f053bf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -76,7 +76,6 @@ import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; @@ -141,14 +140,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result return plan.getTableRef().getTable(); } - private boolean useStats() { + protected boolean useStats() { /* * Don't use guide posts: * 1) If we're collecting stats, as in this case we need to scan entire * regions worth of data to track where to put the guide posts. * 2) If the query is going to be executed serially. */ - if (ScanUtil.isAnalyzeTable(scan) || plan.isSerial()) { + if (ScanUtil.isAnalyzeTable(scan)) { return false; } return true; @@ -664,8 +663,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result regionIndex++; } if (scanRanges.isPointLookup()) { - this.estimatedRows = 1L; - this.estimatedSize = SchemaUtil.estimateRowSize(table); + this.estimatedRows = Long.valueOf(scanRanges.getPointLookupCount()); + this.estimatedSize = this.estimatedRows * SchemaUtil.estimateRowSize(table); } else if (hasGuidePosts) { this.estimatedRows = estimatedRows; this.estimatedSize = estimatedSize; http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java index 0d0c5d3..d8f7f40 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java @@ -114,6 +114,14 @@ public class SerialIterators extends BaseResultIterators { } } + /** + * No need to use stats when executing serially + */ + @Override + protected boolean useStats() { + return false; + } + @Override protected String getName() { return NAME; http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index a2ffae0..5e0f53c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -570,11 +570,6 @@ public class PhoenixStatement implements Statement, SQLCloseable { public boolean useRoundRobinIterator() throws SQLException { return false; } - - @Override - public boolean isSerial() { - return true; - } }; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index c234902..6df3d1e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -216,9 +216,8 @@ public interface QueryServices extends SQLCloseable { // time to wait before running second index population upsert select (so that any pending batches of rows on region server are also written to index) public static final String INDEX_POPULATION_SLEEP_TIME = "phoenix.index.population.wait.time"; public static final String LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB = "phoenix.client.localIndexUpgrade"; - public static final String NONFILTERED_AND_LIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.nonfiltered.limited.query.serial.threshold"; - public static final String FILTERED_OR_NONLIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.filtered.limited.query.serial.threshold"; - + public static final String LIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.limited.query.serial.threshold"; + /** * Get executor service used for parallel scans */ http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 2b77417..6c49fbb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -245,7 +245,7 @@ public class QueryServicesOptions { (3 * DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD) / 4; public static final int DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE = 10; public static final boolean DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE = true; - public static final float DEFAULT_NONFILTERED_LIMITED_QUERY_SERIAL_THRESHOLD = 0.2f; + public static final float DEFAULT_LIMITED_QUERY_SERIAL_THRESHOLD = 0.2f; @SuppressWarnings("serial") public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() { http://git-wip-us.apache.org/repos/asf/phoenix/blob/c2524319/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java index 8c65937..05fbf81 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java @@ -471,11 +471,6 @@ public class ParallelIteratorsSplitTest extends BaseConnectionlessQueryTest { public boolean useRoundRobinIterator() { return false; } - - @Override - public boolean isSerial() { - return true; - } }, null, new SpoolingResultIterator.SpoolingResultIteratorFactory(context.getConnection().getQueryServices()), context.getScan(), false); List<KeyRange> keyRanges = parallelIterators.getSplits();
