This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7876-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit b58f4c39b298f48be342306420f26aba3b0b24fc Author: Andrew Purtell <[email protected]> AuthorDate: Thu Jun 11 15:49:05 2026 -0700 PHOENIX-7899 Emit plan level estimates only once in EXPLAIN (#2520) Co-authored-by: Claude Opus 4.8[1m] <[email protected]> --- .../org/apache/phoenix/compile/DeleteCompiler.java | 3 + .../phoenix/compile/ExplainPlanAttributes.java | 116 ++++++++++++++------- .../org/apache/phoenix/compile/UpsertCompiler.java | 3 + .../org/apache/phoenix/execute/BaseQueryPlan.java | 4 +- .../phoenix/execute/ClientAggregatePlan.java | 1 + .../org/apache/phoenix/execute/ClientScanPlan.java | 1 + .../org/apache/phoenix/execute/HashJoinPlan.java | 1 + .../apache/phoenix/execute/SortMergeJoinPlan.java | 1 + .../java/org/apache/phoenix/execute/UnionPlan.java | 1 + .../phoenix/iterate/BaseResultIterators.java | 4 +- .../org/apache/phoenix/iterate/ExplainTable.java | 18 ++++ .../org/apache/phoenix/jdbc/PhoenixStatement.java | 38 ++++--- .../end2end/ExplainPlanWithStatsEnabledIT.java | 61 +++++++++++ .../phoenix/schema/stats/BaseStatsCollectorIT.java | 32 +++--- .../query/explain/ExplainJsonNormalizer.java | 9 ++ .../phoenix/query/explain/ExplainPlanTest.java | 17 ++- .../phoenix/query/explain/ExplainPlanTestUtil.java | 16 +++ 17 files changed, 254 insertions(+), 72 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java index dc78cae33b..5acc76a12e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java @@ -803,6 +803,7 @@ public class DeleteCompiler { new ExplainPlanAttributesBuilder().setAbstractExplainPlan("DELETE SINGLE ROW"); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); } return new ExplainPlan(Collections.singletonList("DELETE SINGLE ROW"), builder.build()); } @@ -989,6 +990,7 @@ public class DeleteCompiler { planSteps.addAll(queryPlanSteps); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); } return new ExplainPlan(planSteps, newBuilder.build()); } @@ -1127,6 +1129,7 @@ public class DeleteCompiler { planSteps.addAll(queryPlanSteps); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index db729b014a..9b9dd8dd8c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -36,11 +36,12 @@ import org.apache.phoenix.schema.PColumn; * Strings containing entire plan. */ @JsonPropertyOrder({ "tenantId", "viewName", "viewBaseName", "cdcScopes", "txnProvider", "rewrites", - "abstractExplainPlan", "hint", "explainScanType", "consistency", "tableName", "keyRanges", - "indexName", "indexKind", "indexRule", "indexRejected", "saltBuckets", "regionsPlanned", - "scanTimeRangeMin", "scanTimeRangeMax", "splitsChunk", "useRoundRobinIterator", "samplingRate", - "hexStringRVCOffset", "iteratorTypeAndScanSize", "estimatedRows", "estimatedSizeInBytes", - "serverWhereFilter", "serverDistinctFilter", "serverMergeColumns", "serverArrayElementProjection", + "estimatedRows", "estimatedSizeInBytes", "estimateInfoTs", "abstractExplainPlan", "hint", + "explainScanType", "consistency", "tableName", "keyRanges", "indexName", "indexKind", "indexRule", + "indexRejected", "saltBuckets", "regionsPlanned", "scanTimeRangeMin", "scanTimeRangeMax", + "splitsChunk", "useRoundRobinIterator", "samplingRate", "hexStringRVCOffset", + "iteratorTypeAndScanSize", "scanEstimatedRows", "scanEstimatedSizeInBytes", "serverWhereFilter", + "serverDistinctFilter", "serverMergeColumns", "serverArrayElementProjection", "serverFirstKeyOnlyProjection", "serverEmptyColumnOnlyProjection", "serverAggregate", "serverGroupByLimit", "serverSortedBy", "serverOffset", "serverRowLimit", "clientFilterBy", "clientAggregate", "clientDistinctFilter", "clientAfterAggregate", "clientSortAlgo", @@ -57,6 +58,10 @@ public class ExplainPlanAttributes { private final String cdcScopes; private final String txnProvider; private final List<String> rewrites; + // Plan-total estimates. + private final Long estimatedRows; + private final Long estimatedSizeInBytes; + private final Long estimateInfoTs; // Plan identity and scan-level metadata private final String abstractExplainPlan; @@ -78,8 +83,9 @@ public class ExplainPlanAttributes { private final Double samplingRate; private final String hexStringRVCOffset; private final String iteratorTypeAndScanSize; - private final Long estimatedRows; - private final Long estimatedSizeInBytes; + // Per-scan estimates (populated on each plan level from stats). + private final Long scanEstimatedRows; + private final Long scanEstimatedSizeInBytes; // Server-side operations private final String serverWhereFilter; @@ -132,6 +138,9 @@ public class ExplainPlanAttributes { this.cdcScopes = null; this.txnProvider = null; this.rewrites = null; + this.estimatedRows = null; + this.estimatedSizeInBytes = null; + this.estimateInfoTs = null; this.abstractExplainPlan = null; this.hint = null; this.explainScanType = null; @@ -151,8 +160,8 @@ public class ExplainPlanAttributes { this.samplingRate = null; this.hexStringRVCOffset = null; this.iteratorTypeAndScanSize = null; - this.estimatedRows = null; - this.estimatedSizeInBytes = null; + this.scanEstimatedRows = null; + this.scanEstimatedSizeInBytes = null; this.serverWhereFilter = null; this.serverDistinctFilter = null; this.serverMergeColumns = null; @@ -188,13 +197,14 @@ public class ExplainPlanAttributes { } public ExplainPlanAttributes(String tenantId, String viewName, String viewBaseName, - String cdcScopes, String txnProvider, List<String> rewrites, String abstractExplainPlan, - Hint hint, String explainScanType, Consistency consistency, String tableName, String keyRanges, + String cdcScopes, String txnProvider, List<String> rewrites, Long estimatedRows, + Long estimatedSizeInBytes, Long estimateInfoTs, String abstractExplainPlan, Hint hint, + String explainScanType, Consistency consistency, String tableName, String keyRanges, String indexName, String indexKind, String indexRule, List<RejectedIndexEntry> indexRejected, Integer saltBuckets, Integer regionsPlanned, Long scanTimeRangeMin, Long scanTimeRangeMax, Integer splitsChunk, boolean useRoundRobinIterator, Double samplingRate, - String hexStringRVCOffset, String iteratorTypeAndScanSize, Long estimatedRows, - Long estimatedSizeInBytes, String serverWhereFilter, String serverDistinctFilter, + String hexStringRVCOffset, String iteratorTypeAndScanSize, Long scanEstimatedRows, + Long scanEstimatedSizeInBytes, String serverWhereFilter, String serverDistinctFilter, Set<PColumn> serverMergeColumns, boolean serverArrayElementProjection, boolean serverFirstKeyOnlyProjection, boolean serverEmptyColumnOnlyProjection, String serverAggregate, Integer serverGroupByLimit, String serverSortedBy, Integer serverOffset, @@ -214,6 +224,9 @@ public class ExplainPlanAttributes { this.rewrites = (rewrites == null || rewrites.isEmpty()) ? null : Collections.unmodifiableList(new ArrayList<>(rewrites)); + this.estimatedRows = estimatedRows; + this.estimatedSizeInBytes = estimatedSizeInBytes; + this.estimateInfoTs = estimateInfoTs; this.abstractExplainPlan = abstractExplainPlan; this.hint = hint; this.explainScanType = explainScanType; @@ -235,8 +248,8 @@ public class ExplainPlanAttributes { this.samplingRate = samplingRate; this.hexStringRVCOffset = hexStringRVCOffset; this.iteratorTypeAndScanSize = iteratorTypeAndScanSize; - this.estimatedRows = estimatedRows; - this.estimatedSizeInBytes = estimatedSizeInBytes; + this.scanEstimatedRows = scanEstimatedRows; + this.scanEstimatedSizeInBytes = scanEstimatedSizeInBytes; this.serverWhereFilter = serverWhereFilter; this.serverDistinctFilter = serverDistinctFilter; this.serverMergeColumns = serverMergeColumns; @@ -373,6 +386,14 @@ public class ExplainPlanAttributes { return iteratorTypeAndScanSize; } + public Long getScanEstimatedRows() { + return scanEstimatedRows; + } + + public Long getScanEstimatedSizeInBytes() { + return scanEstimatedSizeInBytes; + } + public Long getEstimatedRows() { return estimatedRows; } @@ -381,6 +402,10 @@ public class ExplainPlanAttributes { return estimatedSizeInBytes; } + public Long getEstimateInfoTs() { + return estimateInfoTs; + } + public String getServerWhereFilter() { return serverWhereFilter; } @@ -522,6 +547,9 @@ public class ExplainPlanAttributes { private String cdcScopes; private String txnProvider; private List<String> rewrites; + private Long estimatedRows; + private Long estimatedSizeInBytes; + private Long estimateInfoTs; private String abstractExplainPlan; private HintNode.Hint hint; private String explainScanType; @@ -541,8 +569,8 @@ public class ExplainPlanAttributes { private Double samplingRate; private String hexStringRVCOffset; private String iteratorTypeAndScanSize; - private Long estimatedRows; - private Long estimatedSizeInBytes; + private Long scanEstimatedRows; + private Long scanEstimatedSizeInBytes; private String serverWhereFilter; private String serverDistinctFilter; private Set<PColumn> serverMergeColumns; @@ -588,6 +616,9 @@ public class ExplainPlanAttributes { this.txnProvider = explainPlanAttributes.getTxnProvider(); List<String> srcRewrites = explainPlanAttributes.getRewrites(); this.rewrites = srcRewrites == null ? null : new ArrayList<>(srcRewrites); + this.estimatedRows = explainPlanAttributes.getEstimatedRows(); + this.estimatedSizeInBytes = explainPlanAttributes.getEstimatedSizeInBytes(); + this.estimateInfoTs = explainPlanAttributes.getEstimateInfoTs(); this.abstractExplainPlan = explainPlanAttributes.getAbstractExplainPlan(); this.hint = explainPlanAttributes.getHint(); this.explainScanType = explainPlanAttributes.getExplainScanType(); @@ -608,8 +639,8 @@ public class ExplainPlanAttributes { this.samplingRate = explainPlanAttributes.getSamplingRate(); this.hexStringRVCOffset = explainPlanAttributes.getHexStringRVCOffset(); this.iteratorTypeAndScanSize = explainPlanAttributes.getIteratorTypeAndScanSize(); - this.estimatedRows = explainPlanAttributes.getEstimatedRows(); - this.estimatedSizeInBytes = explainPlanAttributes.getEstimatedSizeInBytes(); + this.scanEstimatedRows = explainPlanAttributes.getScanEstimatedRows(); + this.scanEstimatedSizeInBytes = explainPlanAttributes.getScanEstimatedSizeInBytes(); this.serverWhereFilter = explainPlanAttributes.getServerWhereFilter(); this.serverDistinctFilter = explainPlanAttributes.getServerDistinctFilter(); this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns(); @@ -684,6 +715,21 @@ public class ExplainPlanAttributes { return this; } + public ExplainPlanAttributesBuilder setEstimatedRows(Long estimatedRows) { + this.estimatedRows = estimatedRows; + return this; + } + + public ExplainPlanAttributesBuilder setEstimatedSizeInBytes(Long estimatedSizeInBytes) { + this.estimatedSizeInBytes = estimatedSizeInBytes; + return this; + } + + public ExplainPlanAttributesBuilder setEstimateInfoTs(Long estimateInfoTs) { + this.estimateInfoTs = estimateInfoTs; + return this; + } + public ExplainPlanAttributesBuilder setAbstractExplainPlan(String abstractExplainPlan) { this.abstractExplainPlan = abstractExplainPlan; return this; @@ -779,13 +825,13 @@ public class ExplainPlanAttributes { return this; } - public ExplainPlanAttributesBuilder setEstimatedRows(Long estimatedRows) { - this.estimatedRows = estimatedRows; + public ExplainPlanAttributesBuilder setScanEstimatedRows(Long scanEstimatedRows) { + this.scanEstimatedRows = scanEstimatedRows; return this; } - public ExplainPlanAttributesBuilder setEstimatedSizeInBytes(Long estimatedSizeInBytes) { - this.estimatedSizeInBytes = estimatedSizeInBytes; + public ExplainPlanAttributesBuilder setScanEstimatedSizeInBytes(Long scanEstimatedSizeInBytes) { + this.scanEstimatedSizeInBytes = scanEstimatedSizeInBytes; return this; } @@ -965,18 +1011,18 @@ public class ExplainPlanAttributes { public ExplainPlanAttributes build() { return new ExplainPlanAttributes(tenantId, viewName, viewBaseName, cdcScopes, txnProvider, - rewrites, abstractExplainPlan, hint, explainScanType, consistency, tableName, keyRanges, - indexName, indexKind, indexRule, indexRejected, saltBuckets, regionsPlanned, - scanTimeRangeMin, scanTimeRangeMax, splitsChunk, useRoundRobinIterator, samplingRate, - hexStringRVCOffset, iteratorTypeAndScanSize, estimatedRows, estimatedSizeInBytes, - serverWhereFilter, serverDistinctFilter, serverMergeColumns, serverArrayElementProjection, - serverFirstKeyOnlyProjection, serverEmptyColumnOnlyProjection, serverAggregate, - serverGroupByLimit, serverSortedBy, serverOffset, serverRowLimit, clientFilterBy, - clientAggregate, clientDistinctFilter, clientAfterAggregate, clientSortAlgo, clientSortedBy, - clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, clientSteps, - lhsJoinQueryExplainPlan, rhsJoinQueryExplainPlan, subPlans, dynamicServerFilter, - afterJoinFilter, joinScannerLimit, sortMergeSkipMerge, regionLocations, - regionLocationsTotalSize, numRegionLocationLookups); + rewrites, estimatedRows, estimatedSizeInBytes, estimateInfoTs, abstractExplainPlan, hint, + explainScanType, consistency, tableName, keyRanges, indexName, indexKind, indexRule, + indexRejected, saltBuckets, regionsPlanned, scanTimeRangeMin, scanTimeRangeMax, splitsChunk, + useRoundRobinIterator, samplingRate, hexStringRVCOffset, iteratorTypeAndScanSize, + scanEstimatedRows, scanEstimatedSizeInBytes, serverWhereFilter, serverDistinctFilter, + serverMergeColumns, serverArrayElementProjection, serverFirstKeyOnlyProjection, + serverEmptyColumnOnlyProjection, serverAggregate, serverGroupByLimit, serverSortedBy, + serverOffset, serverRowLimit, clientFilterBy, clientAggregate, clientDistinctFilter, + clientAfterAggregate, clientSortAlgo, clientSortedBy, clientOffset, clientRowLimit, + clientSequenceCount, clientCursorName, clientSteps, lhsJoinQueryExplainPlan, + rhsJoinQueryExplainPlan, subPlans, dynamicServerFilter, afterJoinFilter, joinScannerLimit, + sortMergeSkipMerge, regionLocations, regionLocationsTotalSize, numRegionLocationLookups); } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java index 8a26cb8c8c..f5838a936e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java @@ -1256,6 +1256,7 @@ public class UpsertCompiler { planSteps.addAll(queryPlanSteps); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); } return new ExplainPlan(planSteps, newBuilder.build()); } @@ -1445,6 +1446,7 @@ public class UpsertCompiler { ExplainPlanAttributesBuilder builder = new ExplainPlanAttributesBuilder(ExplainPlanAttributes.getDefaultExplainPlan()); ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); return new ExplainPlan(planSteps, builder.build()); } return new ExplainPlan(planSteps); @@ -1573,6 +1575,7 @@ public class UpsertCompiler { planSteps.addAll(queryPlanSteps); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index d6839d6fa9..b955437b83 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -542,7 +542,8 @@ public abstract class BaseQueryPlan implements QueryPlan { return planSteps; } - private Pair<List<String>, ExplainPlanAttributes> getPlanStepsV2(ResultIterator iterator) { + private Pair<List<String>, ExplainPlanAttributes> getPlanStepsV2(ResultIterator iterator) + throws SQLException { List<String> planSteps = Lists.newArrayListWithExpectedSize(5); ExplainPlanAttributesBuilder builder = new ExplainPlanAttributesBuilder(); iterator.explain(planSteps, builder); @@ -553,6 +554,7 @@ public abstract class BaseQueryPlan implements QueryPlan { } if (context.isRoot()) { ExplainTable.populateTopOfPlanAttributes(builder, context, getTableRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); } return Pair.of(planSteps, builder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java index c0dc5ffe68..4ecdb69117 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java @@ -311,6 +311,7 @@ public class ClientAggregatePlan extends ClientProcessingPlan { if (context.isRoot()) { ExplainTable.populateTopOfPlanAttributes(newBuilder, context, getTableRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java index ae155b3a12..ec9f15d801 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java @@ -168,6 +168,7 @@ public class ClientScanPlan extends ClientProcessingPlan { if (context.isRoot()) { ExplainTable.populateTopOfPlanAttributes(newBuilder, context, getTableRef()); + ExplainTable.populateTopOfPlanEstimates(newBuilder, this); } return new ExplainPlan(planSteps, newBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java index 5a92f4adca..9995c2f06c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java @@ -363,6 +363,7 @@ public class HashJoinPlan extends DelegateQueryPlan { } if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTableRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); } return new ExplainPlan(planSteps, builder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java index e3ebae1779..3c769cf7bb 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java @@ -220,6 +220,7 @@ public class SortMergeJoinPlan implements QueryPlan { rootBuilder.setRhsJoinQueryExplainPlan(rhsPlanAttributes); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(rootBuilder, getContext(), getTableRef()); + ExplainTable.populateTopOfPlanEstimates(rootBuilder, this); } return new ExplainPlan(steps, rootBuilder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java index a704750319..0ff5496839 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/UnionPlan.java @@ -245,6 +245,7 @@ public class UnionPlan implements QueryPlan { addUnionTailLines(steps, builder); if (getContext().isRoot()) { ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTableRef()); + ExplainTable.populateTopOfPlanEstimates(builder, this); } return new ExplainPlan(steps, builder.build()); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 97f1dbf0b7..87956b1141 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -1850,8 +1850,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result buf.append(estimatedRows).append(" ROWS "); buf.append(estimatedSize).append(" BYTES "); if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setEstimatedRows(estimatedRows); - explainPlanAttributesBuilder.setEstimatedSizeInBytes(estimatedSize); + explainPlanAttributesBuilder.setScanEstimatedRows(estimatedRows); + explainPlanAttributesBuilder.setScanEstimatedSizeInBytes(estimatedSize); } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index ca08b984fd..28318ea2c6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.iterate; +import java.sql.SQLException; import java.text.Format; import java.util.ArrayList; import java.util.Arrays; @@ -41,6 +42,7 @@ import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.ScanRanges; import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.compile.StatementPlan; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.filter.BooleanExpressionFilter; import org.apache.phoenix.filter.DistinctPrefixFilter; @@ -227,6 +229,22 @@ public abstract class ExplainTable { } } + /** + * Populate the plan-total estimate attributes on the supplied builder from the given plan. Should + * be invoked only for a root plan. + * @param builder the attributes builder to populate (no-op when null) + * @param plan the plan to read estimates from (no-op when null) + */ + public static void populateTopOfPlanEstimates(ExplainPlanAttributesBuilder builder, + StatementPlan plan) throws SQLException { + if (builder == null || plan == null) { + return; + } + builder.setEstimatedRows(plan.getEstimatedRowsToScan()); + builder.setEstimatedSizeInBytes(plan.getEstimatedBytesToScan()); + builder.setEstimateInfoTs(plan.getEstimateInfoTimestamp()); + } + private static int collectAppliedRewrites(StatementContext context, Set<String> out) { int flattenCount = context.getDerivedTableFlattenCount(); out.addAll(context.getAppliedRewrites()); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 173d8ef29b..0b982288a4 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -1018,29 +1018,37 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable Long estimatedBytesToScan = plan.getEstimatedBytesToScan(); Long estimatedRowsToScan = plan.getEstimatedRowsToScan(); Long estimateInfoTimestamp = plan.getEstimateInfoTimestamp(); + // The three estimate columns are plan totals, not per-step values. Emit them only on the + // top-of-plan. Subsequent rows carry just the plan step column. + boolean firstRow = true; for (String planStep : planSteps) { byte[] row = PVarchar.INSTANCE.toBytes(planStep); - List<Cell> cells = Lists.newArrayListWithCapacity(3); + // The top-of-plan row carries the plan step plus up to three estimate cells. Every other + // row carries only the plan step. + List<Cell> cells = Lists.newArrayListWithCapacity(firstRow ? 4 : 1); cells.add(PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, EXPLAIN_PLAN_COLUMN, MetaDataProtocol.MIN_TABLE_TIMESTAMP, ByteUtil.EMPTY_BYTE_ARRAY)); - if (estimatedBytesToScan != null) { - cells.add( - PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, EXPLAIN_PLAN_BYTES_ESTIMATE, - MetaDataProtocol.MIN_TABLE_TIMESTAMP, PLong.INSTANCE.toBytes(estimatedBytesToScan))); - } - if (estimatedRowsToScan != null) { - cells.add( - PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, EXPLAIN_PLAN_ROWS_ESTIMATE, - MetaDataProtocol.MIN_TABLE_TIMESTAMP, PLong.INSTANCE.toBytes(estimatedRowsToScan))); - } - if (estimateInfoTimestamp != null) { - cells.add( - PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, EXPLAIN_PLAN_ESTIMATE_INFO_TS, - MetaDataProtocol.MIN_TABLE_TIMESTAMP, PLong.INSTANCE.toBytes(estimateInfoTimestamp))); + if (firstRow) { + if (estimatedBytesToScan != null) { + cells.add(PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, + EXPLAIN_PLAN_BYTES_ESTIMATE, MetaDataProtocol.MIN_TABLE_TIMESTAMP, + PLong.INSTANCE.toBytes(estimatedBytesToScan))); + } + if (estimatedRowsToScan != null) { + cells.add( + PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, EXPLAIN_PLAN_ROWS_ESTIMATE, + MetaDataProtocol.MIN_TABLE_TIMESTAMP, PLong.INSTANCE.toBytes(estimatedRowsToScan))); + } + if (estimateInfoTimestamp != null) { + cells.add(PhoenixKeyValueUtil.newKeyValue(row, EXPLAIN_PLAN_FAMILY, + EXPLAIN_PLAN_ESTIMATE_INFO_TS, MetaDataProtocol.MIN_TABLE_TIMESTAMP, + PLong.INSTANCE.toBytes(estimateInfoTimestamp))); + } } Collections.sort(cells, CellComparator.getInstance()); Tuple tuple = new MultiKeyValueTuple(cells); tuples.add(tuple); + firstRow = false; } final Long estimatedBytes = estimatedBytesToScan; final Long estimatedRows = estimatedRowsToScan; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java index 4767ac34ba..1c9a5a4bad 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java @@ -21,6 +21,7 @@ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FO import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -34,7 +35,10 @@ import java.util.List; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; @@ -412,6 +416,63 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT { return new Estimate(estimatedRows, estimatedBytes, estimateInfoTs); } + @Test + public void testTopOfPlanEstimatesMatchStats() throws Exception { + String sql = "SELECT * FROM " + tableB + " WHERE k >= ?"; + List<Object> binds = Lists.newArrayList(); + binds.add(99); + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixPreparedStatement stmt = + conn.prepareStatement(sql).unwrap(PhoenixPreparedStatement.class); + stmt.setInt(1, 99); + QueryPlan plan = stmt.optimizeQuery(); + Long planRows = plan.getEstimatedRowsToScan(); + Long planBytes = plan.getEstimatedBytesToScan(); + Long planTs = plan.getEstimateInfoTimestamp(); + assertEquals((Long) 10L, planRows); + assertEquals((Long) 634L, planBytes); + assertTrue(planTs > 0); + + // First-row EXPLAIN cells must equal the stats-driven values. + Estimate info = getByteRowEstimates(conn, sql, binds); + assertEquals(planRows, info.estimatedRows); + assertEquals(planBytes, info.estimatedBytes); + assertEquals(planTs, info.estimateInfoTs); + + // Top-of-plan attributes must equal the stats-driven values. + ExplainPlanAttributes attrs = plan.getExplainPlan().getPlanStepsAsAttributes(); + assertEquals(planRows, attrs.getEstimatedRows()); + assertEquals(planBytes, attrs.getEstimatedSizeInBytes()); + assertEquals(planTs, attrs.getEstimateInfoTs()); + } + } + + @Test + public void testEstimateCellsOnlyOnFirstRow() throws Exception { + // A query whose plan emits a SERVER FILTER BY step in addition to the scan line produces a + // multi row EXPLAIN result set with stats populated estimates at the top-of-plan. + String sql = "SELECT * FROM " + tableB + " WHERE k >= 99 AND c1.a > c2.b"; + try (Connection conn = DriverManager.getConnection(getUrl()); + PreparedStatement stmt = conn.prepareStatement("EXPLAIN " + sql); + ResultSet rs = stmt.executeQuery()) { + assertTrue("expected at least one EXPLAIN row", rs.next()); + assertNotNull(rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN)); + assertNotNull(rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_BYTES_READ_COLUMN)); + assertNotNull(rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN)); + int extraRows = 0; + while (rs.next()) { + extraRows++; + rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_ROWS_READ_COLUMN); + assertTrue("EST_ROWS_READ must be NULL on row " + (extraRows + 1), rs.wasNull()); + rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATED_BYTES_READ_COLUMN); + assertTrue("EST_BYTES_READ must be NULL on row " + (extraRows + 1), rs.wasNull()); + rs.getObject(PhoenixRuntime.EXPLAIN_PLAN_ESTIMATE_INFO_TS_COLUMN); + assertTrue("EST_INFO_TS must be NULL on row " + (extraRows + 1), rs.wasNull()); + } + assertTrue("expected multi-step plan", extraRows >= 1); + } + } + @Test public void testSettingUseStatsForParallelizationProperty() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl())) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java index 0b731fff6e..f6771376b4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/BaseStatsCollectorIT.java @@ -254,8 +254,8 @@ public abstract class BaseStatsCollectorIT extends BaseTest { conn.createStatement() .execute("CREATE TABLE " + fullTableName + " ( k CHAR(1) PRIMARY KEY )" + tableDDLOptions); collectStatistics(conn, fullTableName); - assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(1).estimatedRows(0L) - .estimatedBytes(20L).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(1).scanEstimatedRows(0L) + .scanEstimatedBytes(20L).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") .table(physicalTableName).serverProjectionFilter(columnEncoded); conn.close(); } @@ -276,22 +276,22 @@ public abstract class BaseStatsCollectorIT extends BaseTest { assertPlan(conn, "SELECT v2 FROM " + fullTableName + " WHERE v2='foo'") .splitsChunk(columnEncoded && !mutable ? 4 : 3) - .estimatedRows(columnEncoded && !mutable ? 1L : 0L) - .estimatedBytes(columnEncoded && !mutable ? 38L : 20L).iteratorType("PARALLEL 3-WAY") + .scanEstimatedRows(columnEncoded && !mutable ? 1L : 0L) + .scanEstimatedBytes(columnEncoded && !mutable ? 38L : 20L).iteratorType("PARALLEL 3-WAY") .scanType("FULL SCAN").table(physicalTableName) .serverWhereFilter("SERVER FILTER BY B.V2 = 'foo'").clientSortAlgo("CLIENT MERGE SORT"); long estimatedSizeInBytes = columnEncoded ? 28 : TransactionFactory.Provider.OMID.name().equals(transactionProvider) ? 38 : 34; - assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(4).estimatedRows(1L) - .estimatedBytes(estimatedSizeInBytes).iteratorType("PARALLEL 3-WAY").scanType("FULL SCAN") + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(4).scanEstimatedRows(1L) + .scanEstimatedBytes(estimatedSizeInBytes).iteratorType("PARALLEL 3-WAY").scanType("FULL SCAN") .table(physicalTableName).serverWhereFilter(null).clientSortAlgo("CLIENT MERGE SORT"); assertPlan(conn, "SELECT * FROM " + fullTableName + " WHERE k = 'a'").splitsChunk(1) - .estimatedRows(1L).estimatedBytes(columnEncoded ? 204L : 202L).iteratorType("PARALLEL 1-WAY") - .scanType("POINT LOOKUP ON 1 KEY").table(physicalTableName).serverWhereFilter(null) - .clientSortAlgo("CLIENT MERGE SORT"); + .scanEstimatedRows(1L).scanEstimatedBytes(columnEncoded ? 204L : 202L) + .iteratorType("PARALLEL 1-WAY").scanType("POINT LOOKUP ON 1 KEY").table(physicalTableName) + .serverWhereFilter(null).clientSortAlgo("CLIENT MERGE SORT"); conn.close(); } @@ -309,7 +309,8 @@ public abstract class BaseStatsCollectorIT extends BaseTest { Array array; conn = upsertValues(props, fullTableName); collectStatistics(conn, fullTableName); - Long rows1 = assertPlan(conn, "SELECT k FROM " + fullTableName).attributes().getEstimatedRows(); + Long rows1 = + assertPlan(conn, "SELECT k FROM " + fullTableName).attributes().getScanEstimatedRows(); stmt = upsertStmt(conn, fullTableName); stmt.setString(1, "z"); s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" }; @@ -321,7 +322,8 @@ public abstract class BaseStatsCollectorIT extends BaseTest { stmt.execute(); conn.commit(); collectStatistics(conn, fullTableName); - Long rows2 = assertPlan(conn, "SELECT k FROM " + fullTableName).attributes().getEstimatedRows(); + Long rows2 = + assertPlan(conn, "SELECT k FROM " + fullTableName).attributes().getScanEstimatedRows(); assertNotEquals(rows1, rows2); conn.close(); } @@ -573,8 +575,8 @@ public abstract class BaseStatsCollectorIT extends BaseTest { : (TransactionFactory.Provider.OMID.name().equals(transactionProvider)) ? 25044 : 12420; - assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(26).estimatedRows(25L) - .estimatedBytes(sizeInBytes).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(26).scanEstimatedRows(25L) + .scanEstimatedBytes(sizeInBytes).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") .table(physicalTableName); ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices(); @@ -636,8 +638,8 @@ public abstract class BaseStatsCollectorIT extends BaseTest { assertEquals(0, rs.getLong(1)); assertFalse(rs.next()); - assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(1).estimatedRows(null) - .estimatedBytes(null).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") + assertPlan(conn, "SELECT * FROM " + fullTableName).splitsChunk(1).scanEstimatedRows(null) + .scanEstimatedBytes(null).iteratorType("PARALLEL 1-WAY").scanType("FULL SCAN") .table(physicalTableName); } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java index eb96912f95..e07eb68261 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java @@ -66,12 +66,21 @@ public final class ExplainJsonNormalizer { if (obj.has("regionsPlanned")) { obj.set("regionsPlanned", NullNode.getInstance()); } + if (obj.has("scanEstimatedRows")) { + obj.set("scanEstimatedRows", NullNode.getInstance()); + } + if (obj.has("scanEstimatedSizeInBytes")) { + obj.set("scanEstimatedSizeInBytes", NullNode.getInstance()); + } if (obj.has("estimatedRows")) { obj.set("estimatedRows", NullNode.getInstance()); } if (obj.has("estimatedSizeInBytes")) { obj.set("estimatedSizeInBytes", NullNode.getInstance()); } + if (obj.has("estimateInfoTs")) { + obj.set("estimateInfoTs", NullNode.getInstance()); + } JsonNode iter = obj.get("iteratorTypeAndScanSize"); if (iter != null && iter.isTextual()) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java index 09ece953c6..cd4503bf3f 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java @@ -1167,15 +1167,21 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { ObjectNode root = mapper.createObjectNode(); root.put("iteratorTypeAndScanSize", "PARALLEL 16-WAY"); root.put("splitsChunk", 4); - root.put("estimatedRows", 1234L); - root.put("estimatedSizeInBytes", 9876L); + root.put("scanEstimatedRows", 1234L); + root.put("scanEstimatedSizeInBytes", 9876L); + root.put("estimatedRows", 4321L); + root.put("estimatedSizeInBytes", 6789L); + root.put("estimateInfoTs", 1748880000000L); root.put("numRegionLocationLookups", 7); root.set("regionLocations", mapper.createArrayNode().add("anything")); new ExplainJsonNormalizer().normalize(root); assertEquals("PARALLEL <N>-WAY", root.get("iteratorTypeAndScanSize").asText()); assertTrue(root.get("splitsChunk").isNull()); + assertTrue(root.get("scanEstimatedRows").isNull()); + assertTrue(root.get("scanEstimatedSizeInBytes").isNull()); assertTrue(root.get("estimatedRows").isNull()); assertTrue(root.get("estimatedSizeInBytes").isNull()); + assertTrue(root.get("estimateInfoTs").isNull()); assertTrue(root.get("regionLocations").isNull()); assertEquals(0, root.get("numRegionLocationLookups").asInt()); } @@ -1371,10 +1377,13 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { n.putNull("cdcScopes"); n.putNull("txnProvider"); n.putNull("rewrites"); - n.putNull("abstractExplainPlan"); - n.putNull("splitsChunk"); n.putNull("estimatedRows"); n.putNull("estimatedSizeInBytes"); + n.putNull("estimateInfoTs"); + n.putNull("abstractExplainPlan"); + n.putNull("splitsChunk"); + n.putNull("scanEstimatedRows"); + n.putNull("scanEstimatedSizeInBytes"); n.putNull("iteratorTypeAndScanSize"); n.putNull("samplingRate"); n.put("useRoundRobinIterator", false); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java index cac67da165..62e32025f0 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java @@ -451,6 +451,17 @@ public final class ExplainPlanTestUtil { return this; } + public ExplainPlanAssert scanEstimatedRows(Long expected) { + assertEquals(at("scanEstimatedRows"), expected, attributes.getScanEstimatedRows()); + return this; + } + + public ExplainPlanAssert scanEstimatedBytes(Long expected) { + assertEquals(at("scanEstimatedSizeInBytes"), expected, + attributes.getScanEstimatedSizeInBytes()); + return this; + } + public ExplainPlanAssert estimatedRows(Long expected) { assertEquals(at("estimatedRows"), expected, attributes.getEstimatedRows()); return this; @@ -461,6 +472,11 @@ public final class ExplainPlanTestUtil { return this; } + public ExplainPlanAssert estimateInfoTs(Long expected) { + assertEquals(at("estimateInfoTs"), expected, attributes.getEstimateInfoTs()); + return this; + } + public ExplainPlanAssert splitsChunk(Integer expected) { assertEquals(at("splitsChunk"), expected, attributes.getSplitsChunk()); return this;
