This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7876-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 708f69069da9535fa936221476035a6e5c78d976 Author: Andrew Purtell <[email protected]> AuthorDate: Fri Jun 12 23:10:16 2026 -0700 PHOENIX-7918 Implement EXPLAIN VERBOSE disclosures (#2526) Co-authored-by: Claude Opus 4.8[1m] <[email protected]> --- .../phoenix/compile/ExplainPlanAttributes.java | 168 +++++++++++++++- .../org/apache/phoenix/compile/HavingCompiler.java | 13 ++ .../org/apache/phoenix/compile/JoinCompiler.java | 6 + .../org/apache/phoenix/compile/QueryCompiler.java | 4 + .../apache/phoenix/compile/RVCOffsetCompiler.java | 3 + .../apache/phoenix/compile/StatementContext.java | 113 +++++++++++ .../apache/phoenix/compile/SubqueryRewriter.java | 27 ++- .../org/apache/phoenix/compile/WhereCompiler.java | 21 ++ .../org/apache/phoenix/execute/AggregatePlan.java | 2 +- .../phoenix/execute/ClientAggregatePlan.java | 25 ++- .../org/apache/phoenix/execute/ClientScanPlan.java | 23 ++- .../org/apache/phoenix/execute/HashJoinPlan.java | 2 +- .../phoenix/execute/TupleProjectionPlan.java | 24 ++- .../phoenix/iterate/BaseResultIterators.java | 5 + .../org/apache/phoenix/iterate/ExplainTable.java | 215 ++++++++++++++++++++- .../iterate/FilterAggregatingResultIterator.java | 28 ++- .../phoenix/iterate/FilterResultIterator.java | 27 ++- .../apache/phoenix/optimize/QueryOptimizer.java | 21 +- .../org/apache/phoenix/parse/ExplainOptions.java | 1 + .../java/org/apache/phoenix/schema/PTableImpl.java | 2 + .../query/explain/ExplainJsonNormalizer.java | 28 +++ .../phoenix/query/explain/ExplainPlanTest.java | 209 +++++++++++++++++++- .../phoenix/query/explain/ExplainPlanTestUtil.java | 174 +++++++++++++++++ 23 files changed, 1088 insertions(+), 53 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index 6a6f7c0ea8..24a74a771b 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -45,14 +45,14 @@ import org.apache.phoenix.schema.PColumn; "saltBuckets", "regionsPlanned", "scanTimeRangeMin", "scanTimeRangeMax", "splitsChunk", "useRoundRobinIterator", "samplingRate", "hexStringRVCOffset", "iteratorTypeAndScanSize", "scanEstimatedRows", "scanEstimatedSizeInBytes", "serverWhereFilter", "serverDistinctFilter", - "serverMergeColumns", "serverParsedProjections", "serverFirstKeyOnlyProjection", - "serverEmptyColumnOnlyProjection", "serverAggregate", "serverGroupByLimit", "serverSortedBy", - "serverOffset", "serverRowLimit", "clientFilterBy", "clientAggregate", "clientDistinctFilter", - "clientAfterAggregate", "clientSortAlgo", "clientSortedBy", "clientOffset", "clientRowLimit", - "clientSequenceCount", "clientCursorName", "clientSteps", "lhsJoinQueryExplainPlan", - "rhsJoinQueryExplainPlan", "subPlans", "dynamicServerFilter", "afterJoinFilter", - "joinScannerLimit", "sortMergeSkipMerge", "regionLocations", "regionLocationsTotalSize", - "numRegionLocationLookups" }) + "serverMergeColumns", "serverParsedProjections", "serverProject", "serverFilters", "ignoredHints", + "serverFirstKeyOnlyProjection", "serverEmptyColumnOnlyProjection", "serverAggregate", + "serverGroupByLimit", "serverSortedBy", "serverOffset", "serverRowLimit", "clientFilterBy", + "clientFilters", "clientAggregate", "clientDistinctFilter", "clientAfterAggregate", + "clientSortAlgo", "clientSortedBy", "clientOffset", "clientRowLimit", "clientSequenceCount", + "clientCursorName", "clientSteps", "lhsJoinQueryExplainPlan", "rhsJoinQueryExplainPlan", + "subPlans", "dynamicServerFilter", "afterJoinFilter", "joinScannerLimit", "sortMergeSkipMerge", + "regionLocations", "regionLocationsTotalSize", "numRegionLocationLookups" }) public class ExplainPlanAttributes { // Top-of-plan disclosures (populated only on the root plan) @@ -100,6 +100,9 @@ public class ExplainPlanAttributes { private final String serverDistinctFilter; private final Set<PColumn> serverMergeColumns; private final Map<String, List<String>> serverParsedProjections; + private final List<String> serverProject; + private final List<ExplainFilter> serverFilters; + private final Map<String, String> ignoredHints; private final boolean serverFirstKeyOnlyProjection; private final boolean serverEmptyColumnOnlyProjection; private final String serverAggregate; @@ -110,6 +113,7 @@ public class ExplainPlanAttributes { // Client-side operations private final String clientFilterBy; + private final List<ExplainFilter> clientFilters; private final String clientAggregate; private final String clientDistinctFilter; private final String clientAfterAggregate; @@ -177,6 +181,9 @@ public class ExplainPlanAttributes { this.serverDistinctFilter = null; this.serverMergeColumns = null; this.serverParsedProjections = null; + this.serverProject = null; + this.serverFilters = null; + this.ignoredHints = null; this.serverFirstKeyOnlyProjection = false; this.serverEmptyColumnOnlyProjection = false; this.serverAggregate = null; @@ -185,6 +192,7 @@ public class ExplainPlanAttributes { this.serverOffset = null; this.serverRowLimit = null; this.clientFilterBy = null; + this.clientFilters = null; this.clientAggregate = null; this.clientDistinctFilter = null; this.clientAfterAggregate = null; @@ -227,7 +235,9 @@ public class ExplainPlanAttributes { ExplainPlanAttributes rhsJoinQueryExplainPlan, List<ExplainPlanAttributes> subPlans, String dynamicServerFilter, String afterJoinFilter, Long joinScannerLimit, boolean sortMergeSkipMerge, List<HRegionLocation> regionLocations, - Integer regionLocationsTotalSize, int numRegionLocationLookups) { + Integer regionLocationsTotalSize, int numRegionLocationLookups, List<String> serverProject, + List<ExplainFilter> serverFilters, Map<String, String> ignoredHints, + List<ExplainFilter> clientFilters) { this.tenantId = tenantId; this.viewName = viewName; this.viewBaseName = viewBaseName; @@ -279,6 +289,9 @@ public class ExplainPlanAttributes { this.serverOffset = serverOffset; this.serverRowLimit = serverRowLimit; this.clientFilterBy = clientFilterBy; + this.clientFilters = (clientFilters == null || clientFilters.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(clientFilters)); this.clientAggregate = clientAggregate; this.clientDistinctFilter = clientDistinctFilter; this.clientAfterAggregate = clientAfterAggregate; @@ -301,6 +314,15 @@ public class ExplainPlanAttributes { this.regionLocations = regionLocations; this.regionLocationsTotalSize = regionLocationsTotalSize; this.numRegionLocationLookups = numRegionLocationLookups; + this.serverProject = (serverProject == null || serverProject.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(serverProject)); + this.serverFilters = (serverFilters == null || serverFilters.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(serverFilters)); + this.ignoredHints = (ignoredHints == null || ignoredHints.isEmpty()) + ? null + : Collections.unmodifiableMap(new LinkedHashMap<>(ignoredHints)); } public String getTenantId() { @@ -464,6 +486,18 @@ public class ExplainPlanAttributes { return Collections.unmodifiableMap(copy); } + public List<String> getServerProject() { + return serverProject; + } + + public List<ExplainFilter> getServerFilters() { + return serverFilters; + } + + public Map<String, String> getIgnoredHints() { + return ignoredHints; + } + public boolean isServerFirstKeyOnlyProjection() { return serverFirstKeyOnlyProjection; } @@ -496,6 +530,10 @@ public class ExplainPlanAttributes { return clientFilterBy; } + public List<ExplainFilter> getClientFilters() { + return clientFilters; + } + public String getClientAggregate() { return clientAggregate; } @@ -581,6 +619,59 @@ public class ExplainPlanAttributes { return EXPLAIN_PLAN_INSTANCE; } + /** A single VERBOSE-mode filter predicate. */ + @JsonPropertyOrder({ "expr", "origin", "pathTestSubtag" }) + public static class ExplainFilter { + private final String expr; + private final List<String> origin; + private final String pathTestSubtag; + + public ExplainFilter(String expr, List<String> origin, String pathTestSubtag) { + this.expr = expr; + this.origin = (origin == null || origin.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(origin)); + this.pathTestSubtag = pathTestSubtag; + } + + public String getExpr() { + return expr; + } + + public List<String> getOrigin() { + return origin; + } + + public String getPathTestSubtag() { + return pathTestSubtag; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ExplainFilter)) { + return false; + } + ExplainFilter that = (ExplainFilter) o; + return java.util.Objects.equals(expr, that.expr) + && java.util.Objects.equals(origin, that.origin) + && java.util.Objects.equals(pathTestSubtag, that.pathTestSubtag); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(expr, origin, pathTestSubtag); + } + + @Override + public String toString() { + return "ExplainFilter{expr=" + expr + ", origin=" + origin + ", pathTestSubtag=" + + pathTestSubtag + "}"; + } + } + public static class ExplainPlanAttributesBuilder { private String tenantId; private String viewName; @@ -619,6 +710,9 @@ public class ExplainPlanAttributes { private String serverDistinctFilter; private Set<PColumn> serverMergeColumns; private Map<String, List<String>> serverParsedProjections; + private List<String> serverProject; + private List<ExplainFilter> serverFilters; + private Map<String, String> ignoredHints; private boolean serverFirstKeyOnlyProjection; private boolean serverEmptyColumnOnlyProjection; private String serverAggregate; @@ -627,6 +721,7 @@ public class ExplainPlanAttributes { private Integer serverOffset; private Long serverRowLimit; private String clientFilterBy; + private List<ExplainFilter> clientFilters; private String clientAggregate; private String clientDistinctFilter; private String clientAfterAggregate; @@ -697,6 +792,12 @@ public class ExplainPlanAttributes { explainPlanAttributes.getServerParsedProjections(); this.serverParsedProjections = srcServerParsedProjections == null ? null : new LinkedHashMap<>(srcServerParsedProjections); + List<String> srcServerProject = explainPlanAttributes.getServerProject(); + this.serverProject = srcServerProject == null ? null : new ArrayList<>(srcServerProject); + List<ExplainFilter> srcServerFilters = explainPlanAttributes.getServerFilters(); + this.serverFilters = srcServerFilters == null ? null : new ArrayList<>(srcServerFilters); + Map<String, String> srcIgnoredHints = explainPlanAttributes.getIgnoredHints(); + this.ignoredHints = srcIgnoredHints == null ? null : new LinkedHashMap<>(srcIgnoredHints); this.serverFirstKeyOnlyProjection = explainPlanAttributes.isServerFirstKeyOnlyProjection(); this.serverEmptyColumnOnlyProjection = explainPlanAttributes.isServerEmptyColumnOnlyProjection(); @@ -706,6 +807,8 @@ public class ExplainPlanAttributes { this.serverOffset = explainPlanAttributes.getServerOffset(); this.serverRowLimit = explainPlanAttributes.getServerRowLimit(); this.clientFilterBy = explainPlanAttributes.getClientFilterBy(); + List<ExplainFilter> srcClientFilters = explainPlanAttributes.getClientFilters(); + this.clientFilters = srcClientFilters == null ? null : new ArrayList<>(srcClientFilters); this.clientAggregate = explainPlanAttributes.getClientAggregate(); this.clientDistinctFilter = explainPlanAttributes.getClientDistinctFilter(); this.clientAfterAggregate = explainPlanAttributes.getClientAfterAggregate(); @@ -935,6 +1038,37 @@ public class ExplainPlanAttributes { return this; } + public ExplainPlanAttributesBuilder setServerProject(List<String> serverProject) { + this.serverProject = serverProject == null ? null : new ArrayList<>(serverProject); + return this; + } + + public ExplainPlanAttributesBuilder setServerFilters(List<ExplainFilter> serverFilters) { + this.serverFilters = serverFilters == null ? null : new ArrayList<>(serverFilters); + return this; + } + + public ExplainPlanAttributesBuilder addServerFilter(ExplainFilter serverFilter) { + if (this.serverFilters == null) { + this.serverFilters = new ArrayList<>(); + } + this.serverFilters.add(serverFilter); + return this; + } + + public ExplainPlanAttributesBuilder setIgnoredHints(Map<String, String> ignoredHints) { + this.ignoredHints = ignoredHints == null ? null : new LinkedHashMap<>(ignoredHints); + return this; + } + + public ExplainPlanAttributesBuilder addIgnoredHint(String hint, String reason) { + if (this.ignoredHints == null) { + this.ignoredHints = new LinkedHashMap<>(); + } + this.ignoredHints.put(hint, reason); + return this; + } + public ExplainPlanAttributesBuilder setServerFirstKeyOnlyProjection(boolean serverFirstKeyOnlyProjection) { this.serverFirstKeyOnlyProjection = serverFirstKeyOnlyProjection; @@ -977,6 +1111,19 @@ public class ExplainPlanAttributes { return this; } + public ExplainPlanAttributesBuilder setClientFilters(List<ExplainFilter> clientFilters) { + this.clientFilters = clientFilters == null ? null : new ArrayList<>(clientFilters); + return this; + } + + public ExplainPlanAttributesBuilder addClientFilter(ExplainFilter clientFilter) { + if (this.clientFilters == null) { + this.clientFilters = new ArrayList<>(); + } + this.clientFilters.add(clientFilter); + return this; + } + public ExplainPlanAttributesBuilder setClientAggregate(String clientAggregate) { this.clientAggregate = clientAggregate; return this; @@ -1102,7 +1249,8 @@ public class ExplainPlanAttributes { clientSortedBy, clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, clientSteps, lhsJoinQueryExplainPlan, rhsJoinQueryExplainPlan, subPlans, dynamicServerFilter, afterJoinFilter, joinScannerLimit, sortMergeSkipMerge, regionLocations, - regionLocationsTotalSize, numRegionLocationLookups); + regionLocationsTotalSize, numRegionLocationLookups, serverProject, serverFilters, + ignoredHints, clientFilters); } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java index 0d989a8957..16f77a8b6c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/HavingCompiler.java @@ -64,6 +64,14 @@ public class HavingCompiler { throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_AGGREGATE_IN_HAVING_CLAUSE).build() .buildException(); } + // Tag the residual HAVING predicate(s) with their origin for VERBOSE attribution. + if (expression instanceof org.apache.phoenix.expression.AndExpression) { + for (Expression child : expression.getChildren()) { + context.tagPredicate(child, "HAVING"); + } + } else { + context.tagPredicate(expression, "HAVING"); + } return expression; } @@ -77,6 +85,11 @@ public class HavingCompiler { having.accept(visitor); if (!visitor.getMoveToWhereClauseExpressions().isEmpty()) { context.addAppliedRewrite("HAVING PREDICATE AS WHERE"); + // Record the parse nodes lifted from HAVING into WHERE so VERBOSE predicate attribution can + // distinguish them. + for (ParseNode lifted : visitor.getMoveToWhereClauseExpressions()) { + context.addLiftedHavingNode(lifted); + } } statement = SelectStatementRewriter.moveFromHavingToWhereClause(statement, visitor.getMoveToWhereClauseExpressions()); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index c408d37259..5beab671c8 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -675,6 +675,12 @@ public class JoinCompiler { if (right.getDataType() != toType || right.getSortOrder() != toSortOrder) { right = CoerceExpression.create(right, toType, toSortOrder, right.getMaxLength()); } + // Tag the compiled ON predicates with their origin for VERBOSE attribution. + String decorrelatedAlias = lhsCtx.getDecorrelatedSubqueryAlias(condition); + String onOrigin = + decorrelatedAlias == null ? "JOIN ON" : "decorrelated from " + decorrelatedAlias; + lhsCtx.tagPredicate(left, onOrigin); + rhsCtx.tagPredicate(right, onOrigin); compiled.add(new Pair<Expression, Expression>(left, right)); } // TODO PHOENIX-4618: diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java index 4bdbd9dbab..5ba9ec29f1 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java @@ -320,6 +320,10 @@ public class QueryCompiler { JoinTable joinTable = JoinCompiler.compile(statement, select, context.getResolver(), context); return compileJoinQuery(context, joinTable, false, false, null); } else { + // A USE_SORT_MERGE_JOIN hint on a query without any join is ignored. + if (select.getHint().hasHint(Hint.USE_SORT_MERGE_JOIN)) { + context.recordIgnoredHint(Hint.USE_SORT_MERGE_JOIN, "no join in query"); + } return compileSingleQuery(context, select, false, true); } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java index a7c124f256..abb3270017 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/RVCOffsetCompiler.java @@ -180,6 +180,9 @@ public class RVCOffsetCompiler { throw new RowValueConstructorOffsetInternalErrorException("RVC Offset unexpected failure."); } + // Tag the RVC offset predicate with its origin for VERBOSE attribution. + context.tagPredicate(whereExpression, "RVC OFFSET"); + Expression expression; try { expression = WhereOptimizer.pushKeyExpressionsToScan(context, originalHints, whereExpression, diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java index 0c89d1c82c..17cd40305a 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -24,7 +24,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Deque; +import java.util.EnumMap; +import java.util.IdentityHashMap; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,6 +45,7 @@ import org.apache.phoenix.monitoring.ScanMetricsHolder; import org.apache.phoenix.monitoring.SlowestScanMetricsQueue; import org.apache.phoenix.monitoring.TopNTreeMultiMap; import org.apache.phoenix.parse.ExplainOptions; +import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.query.QueryConstants; @@ -114,6 +118,10 @@ public class StatementContext { private Map<String, List<Expression>> serverParsedProjections; private StatementContext parentContext; private ExplainOptions explainOptions = ExplainOptions.DEFAULT; + private Map<Expression, Set<String>> predicateOrigins; + private Set<ParseNode> liftedHavingNodes; + private Map<ParseNode, String> decorrelatedSubqueryAlias; + private Map<Hint, String> ignoredHints; public StatementContext(PhoenixStatement statement) { this(statement, new Scan()); @@ -159,6 +167,10 @@ public class StatementContext { this.serverParsedProjections = context.serverParsedProjections; this.parentContext = context.parentContext; this.explainOptions = context.explainOptions; + this.predicateOrigins = context.predicateOrigins; + this.liftedHavingNodes = context.liftedHavingNodes; + this.decorrelatedSubqueryAlias = context.decorrelatedSubqueryAlias; + this.ignoredHints = context.ignoredHints; } /** @@ -229,6 +241,10 @@ public class StatementContext { this.partialIndexCheckedSet = Sets.newHashSet(); this.serverParsedProjections = null; this.parentContext = null; + this.predicateOrigins = new IdentityHashMap<>(); + this.liftedHavingNodes = Collections.newSetFromMap(new IdentityHashMap<>()); + this.decorrelatedSubqueryAlias = new IdentityHashMap<>(); + this.ignoredHints = new EnumMap<>(Hint.class); } /** @@ -523,6 +539,10 @@ public class StatementContext { this.functionalIndexNames = source.functionalIndexNames; this.partialIndexCheckedSet = source.partialIndexCheckedSet; this.serverParsedProjections = source.serverParsedProjections; + this.predicateOrigins = source.predicateOrigins; + this.liftedHavingNodes = source.liftedHavingNodes; + this.decorrelatedSubqueryAlias = source.decorrelatedSubqueryAlias; + this.ignoredHints = source.ignoredHints; } public void incrementDerivedTableFlattenCount() { @@ -622,6 +642,99 @@ public class StatementContext { this.explainOptions = explainOptions == null ? ExplainOptions.DEFAULT : explainOptions; } + /** Returns true if the {@code EXPLAIN} statement requested {@code VERBOSE} output. */ + public boolean isVerbose() { + return explainOptions != null && explainOptions.isVerbose(); + } + + /** + * Tag a predicate {@link Expression} with the given origin (e.g. {@code "WHERE"}, + * {@code "JOIN ON"}). Tags accumulate as a set so a predicate fused from multiple sources carries + * every contributing origin. Keyed by object identity, since Phoenix rewrites and re-instantiates + * expressions during compilation. Diagnostic only; never affects the compiled plan. + */ + public void tagPredicate(Expression expression, String origin) { + if (expression == null || origin == null) { + return; + } + predicateOrigins.computeIfAbsent(expression, k -> new LinkedHashSet<>()).add(origin); + } + + /** + * Propagate the accumulated origin tags of each source expression onto a freshly minted + * destination expression. Used by expression rewriters/clone visitors so identity-keyed tags + * survive node replacement. + */ + public void unionTags(Expression dst, Iterable<? extends Expression> srcs) { + if (dst == null || srcs == null) { + return; + } + for (Expression src : srcs) { + Set<String> tags = predicateOrigins.get(src); + if (tags != null && !tags.isEmpty()) { + predicateOrigins.computeIfAbsent(dst, k -> new LinkedHashSet<>()).addAll(tags); + } + } + } + + /** Returns the predicate origin tags accumulated during compilation. Identity keyed. */ + public Map<Expression, Set<String>> getPredicateOrigins() { + return predicateOrigins; + } + + /** Returns the origin tags recorded for {@code expression}, or an empty set if none. */ + public Set<String> getPredicateOrigins(Expression expression) { + Set<String> tags = predicateOrigins.get(expression); + return tags == null ? Collections.emptySet() : tags; + } + + /** Records a parse node lifted from HAVING into the WHERE clause (identity keyed). */ + public void addLiftedHavingNode(ParseNode node) { + if (node != null) { + liftedHavingNodes.add(node); + } + } + + /** Returns true if {@code node} was lifted from HAVING into the WHERE clause. */ + public boolean isLiftedHavingNode(ParseNode node) { + return liftedHavingNodes.contains(node); + } + + /** + * Records that the given decorrelated join-condition parse node originated from the subquery with + * the given temp alias. + */ + public void putDecorrelatedSubqueryAlias(ParseNode joinConditionNode, String subqueryAlias) { + if (joinConditionNode != null && subqueryAlias != null) { + decorrelatedSubqueryAlias.put(joinConditionNode, subqueryAlias); + } + } + + /** + * Returns the subquery temp alias the given join-condition parse node was decorrelated from, or + * {@code null} if the node is not a decorrelated predicate. + */ + public String getDecorrelatedSubqueryAlias(ParseNode joinConditionNode) { + return decorrelatedSubqueryAlias.get(joinConditionNode); + } + + /** + * Records that the planner intentionally ignored {@code hint} for the given {@code reason}. + * Surfaced under {@code EXPLAIN (VERBOSE)} as a {@code /*- HINT(args) -- reason *}{@code /} + * comment. The first reason recorded for a hint wins. Diagnostic only. + */ + public void recordIgnoredHint(Hint hint, String reason) { + if (hint == null || reason == null) { + return; + } + ignoredHints.putIfAbsent(hint, reason); + } + + /** Returns the hints the planner intentionally ignored, mapped to the reason. */ + public Map<Hint, String> getIgnoredHints() { + return ignoredHints; + } + /** Returns true if this is the top-level (root) statement context, i.e. it has no parent. */ public boolean isRoot() { return parentContext == null; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java index 19f2ddcc76..b5fd3bcde4 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/SubqueryRewriter.java @@ -233,7 +233,7 @@ public class SubqueryRewriter extends ParseNodeRewriter { String subqueryTableTempAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor joinConditionExtractor = new JoinConditionExtractor( - subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias); + subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias, context); List<AliasedNode> newSubquerySelectAliasedNodes = null; ParseNode extractedJoinConditionParseNode = null; @@ -380,7 +380,7 @@ public class SubqueryRewriter extends ParseNodeRewriter { fixSubqueryStatement(subqueryParseNode.getSelectNode()); String subqueryTableTempAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor joinConditionExtractor = new JoinConditionExtractor( - subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias); + subquerySelectStatementToUse, columnResolver, connection, subqueryTableTempAlias, context); ParseNode whereParseNodeAfterExtract = subquerySelectStatementToUse.getWhere() == null ? null : subquerySelectStatementToUse.getWhere().accept(joinConditionExtractor); @@ -453,7 +453,7 @@ public class SubqueryRewriter extends ParseNodeRewriter { SelectStatement subquery = fixSubqueryStatement(subqueryNode.getSelectNode()); String rhsTableAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor conditionExtractor = - new JoinConditionExtractor(subquery, columnResolver, connection, rhsTableAlias); + new JoinConditionExtractor(subquery, columnResolver, connection, rhsTableAlias, context); ParseNode where = subquery.getWhere() == null ? null : subquery.getWhere().accept(conditionExtractor); if (where == subquery.getWhere()) { // non-correlated comparison subquery, add LIMIT 2, @@ -540,7 +540,7 @@ public class SubqueryRewriter extends ParseNodeRewriter { SelectStatement subquery = fixSubqueryStatement(subqueryNode.getSelectNode()); String rhsTableAlias = ParseNodeFactory.createTempAlias(); JoinConditionExtractor conditionExtractor = - new JoinConditionExtractor(subquery, columnResolver, connection, rhsTableAlias); + new JoinConditionExtractor(subquery, columnResolver, connection, rhsTableAlias, context); ParseNode where = subquery.getWhere() == null ? null : subquery.getWhere().accept(conditionExtractor); if (where == subquery.getWhere()) { // non-correlated any/all comparison subquery @@ -737,14 +737,19 @@ public class SubqueryRewriter extends ParseNodeRewriter { private static class JoinConditionExtractor extends AndRewriterBooleanParseNodeVisitor { private final TableName tableName; + private final String tableAlias; + private final StatementContext context; private ColumnResolveVisitor columnResolveVisitor; private List<AliasedNode> additionalSubselectSelectAliasedNodes; private List<ParseNode> joinConditionParseNodes; public JoinConditionExtractor(SelectStatement subquery, ColumnResolver outerResolver, - PhoenixConnection connection, String tableAlias) throws SQLException { + PhoenixConnection connection, String tableAlias, StatementContext context) + throws SQLException { super(NODE_FACTORY); this.tableName = NODE_FACTORY.table(null, tableAlias); + this.tableAlias = tableAlias; + this.context = context; ColumnResolver localResolver = FromCompiler.getResolverForQuery(subquery, connection); this.columnResolveVisitor = new ColumnResolveVisitor(localResolver, outerResolver); this.additionalSubselectSelectAliasedNodes = Lists.<AliasedNode> newArrayList(); @@ -807,7 +812,11 @@ public class SubqueryRewriter extends ParseNodeRewriter { this.additionalSubselectSelectAliasedNodes .add(NODE_FACTORY.aliasedNode(alias, node.getLHS())); ParseNode lhsNode = NODE_FACTORY.column(tableName, alias, null); - this.joinConditionParseNodes.add(NODE_FACTORY.equal(lhsNode, node.getRHS())); + ParseNode joinCondition = NODE_FACTORY.equal(lhsNode, node.getRHS()); + if (context != null) { + context.putDecorrelatedSubqueryAlias(joinCondition, tableAlias); + } + this.joinConditionParseNodes.add(joinCondition); return null; } if ( @@ -818,7 +827,11 @@ public class SubqueryRewriter extends ParseNodeRewriter { this.additionalSubselectSelectAliasedNodes .add(NODE_FACTORY.aliasedNode(alias, node.getRHS())); ParseNode rhsNode = NODE_FACTORY.column(tableName, alias, null); - this.joinConditionParseNodes.add(NODE_FACTORY.equal(node.getLHS(), rhsNode)); + ParseNode joinCondition = NODE_FACTORY.equal(node.getLHS(), rhsNode); + if (context != null) { + context.putDecorrelatedSubqueryAlias(joinCondition, tableAlias); + } + this.joinConditionParseNodes.add(joinCondition); return null; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java index 6dee0c2f51..4d4ac7717f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/WhereCompiler.java @@ -254,11 +254,32 @@ public class WhereCompiler { scan.withStopRow(whereCompiler.getScanEndKey()); } } + // Tag the residual predicate(s) that become the server-side filter with their "WHERE" origin + // so EXPLAIN can attribute each SERVER FILTER BY line. + tagWhereOrigins(context, expression); + setScanFilter(context, statement, expression, whereCompiler.disambiguateWithFamily); return expression; } + /** + * Tag the top-level conjuncts of the residual WHERE expression with their origin for VERBOSE + * predicate source attribution. + */ + private static void tagWhereOrigins(StatementContext context, Expression expression) { + if (expression == null) { + return; + } + if (expression instanceof AndExpression) { + for (Expression child : expression.getChildren()) { + context.tagPredicate(child, "WHERE"); + } + } else { + context.tagPredicate(expression, "WHERE"); + } + } + public static class WhereExpressionCompiler extends ExpressionCompiler { protected boolean disambiguateWithFamily; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java index 2c5d62b2a5..8a8ad87f8a 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/AggregatePlan.java @@ -330,7 +330,7 @@ public class AggregatePlan extends BaseQueryPlan { } if (having != null) { - aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having); + aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having, context); } if (statement.isDistinct() && statement.isAggregate()) { // Dedup on client if select distinct diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java index 4ecdb69117..b6b3e69219 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientAggregatePlan.java @@ -22,6 +22,7 @@ import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY; import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.Cell; @@ -29,6 +30,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; @@ -136,7 +138,7 @@ public class ClientAggregatePlan extends ClientProcessingPlan { public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { ResultIterator iterator = delegate.iterator(scanGrouper, scan); if (where != null) { - iterator = new FilterResultIterator(iterator, where); + iterator = new FilterResultIterator(iterator, where, context); } AggregatingResultIterator aggResultIterator; @@ -186,7 +188,7 @@ public class ClientAggregatePlan extends ClientProcessingPlan { } if (having != null) { - aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having); + aggResultIterator = new FilterAggregatingResultIterator(aggResultIterator, having, context); } if (statement.isDistinct() && statement.isAggregate()) { // Dedup on client if select distinct @@ -227,10 +229,21 @@ public class ClientAggregatePlan extends ClientProcessingPlan { ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); if (where != null) { - String step = "CLIENT FILTER BY " + where.toString(); - planSteps.add(step); - newBuilder.setClientFilterBy(where.toString()); - newBuilder.addClientStep(step); + if (context.isVerbose()) { + List<String> filterLines = new ArrayList<>(); + List<ExplainFilter> clientFilters = ExplainTable.renderVerboseFilters(context, where, + where.toString(), "CLIENT FILTER BY", filterLines); + for (String filterLine : filterLines) { + planSteps.add(filterLine); + newBuilder.addClientStep(filterLine); + } + newBuilder.setClientFilters(clientFilters); + } else { + String step = "CLIENT FILTER BY " + where.toString(); + planSteps.add(step); + newBuilder.setClientFilterBy(where.toString()); + newBuilder.addClientStep(step); + } } if (groupBy.isEmpty()) { String step = "CLIENT AGGREGATE INTO SINGLE ROW"; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java index ec9f15d801..895853ee6a 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/ClientScanPlan.java @@ -18,11 +18,13 @@ package org.apache.phoenix.execute; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.hadoop.hbase.client.Scan; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.QueryPlan; @@ -87,7 +89,7 @@ public class ClientScanPlan extends ClientProcessingPlan { public ResultIterator iterator(ParallelScanGrouper scanGrouper, Scan scan) throws SQLException { ResultIterator iterator = delegate.iterator(scanGrouper, scan); if (where != null) { - iterator = new FilterResultIterator(iterator, where); + iterator = new FilterResultIterator(iterator, where, context); } if (!orderBy.getOrderByExpressions().isEmpty()) { // TopN @@ -124,10 +126,21 @@ public class ClientScanPlan extends ClientProcessingPlan { ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); if (where != null) { - String step = "CLIENT FILTER BY " + where.toString(); - planSteps.add(step); - newBuilder.setClientFilterBy(where.toString()); - newBuilder.addClientStep(step); + if (context.isVerbose()) { + List<String> filterLines = new ArrayList<>(); + List<ExplainFilter> clientFilters = ExplainTable.renderVerboseFilters(context, where, + where.toString(), "CLIENT FILTER BY", filterLines); + for (String filterLine : filterLines) { + planSteps.add(filterLine); + newBuilder.addClientStep(filterLine); + } + newBuilder.setClientFilters(clientFilters); + } else { + String step = "CLIENT FILTER BY " + where.toString(); + planSteps.add(step); + newBuilder.setClientFilterBy(where.toString()); + newBuilder.addClientStep(step); + } } if (!orderBy.getOrderByExpressions().isEmpty()) { if (offset != null) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java index 9995c2f06c..7732d4d6e7 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java @@ -273,7 +273,7 @@ public class HashJoinPlan extends DelegateQueryPlan { ? delegate.iterator(scanGrouper, scan) : ((BaseQueryPlan) delegate).iterator(dependencies, scanGrouper, scan); if (statement.getInnerSelectStatement() != null && postFilter != null) { - iterator = new FilterResultIterator(iterator, postFilter); + iterator = new FilterResultIterator(iterator, postFilter, delegate.getContext()); } if (hasSubPlansWithPersistentCache) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java index 413f046d22..bc2e1b200f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/TupleProjectionPlan.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; @@ -40,6 +41,7 @@ import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.OrderByExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.iterate.DelegateResultIterator; +import org.apache.phoenix.iterate.ExplainTable; import org.apache.phoenix.iterate.FilterResultIterator; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; @@ -53,6 +55,7 @@ public class TupleProjectionPlan extends DelegateQueryPlan { private final TupleProjector tupleProjector; private final Expression postFilter; private final ColumnResolver columnResolver; + private final StatementContext statementContext; private final List<OrderBy> actualOutputOrderBys; public TupleProjectionPlan(QueryPlan plan, TupleProjector tupleProjector, @@ -63,6 +66,7 @@ public class TupleProjectionPlan extends DelegateQueryPlan { } this.tupleProjector = tupleProjector; this.postFilter = postFilter; + this.statementContext = statementContext; if (statementContext != null) { this.columnResolver = statementContext.getResolver(); this.actualOutputOrderBys = this.convertInputOrderBys(plan); @@ -147,12 +151,22 @@ public class TupleProjectionPlan extends DelegateQueryPlan { List<String> planSteps = Lists.newArrayList(explainPlan.getPlanSteps()); ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes(); if (postFilter != null) { - String step = "CLIENT FILTER BY " + postFilter.toString(); - planSteps.add(step); ExplainPlanAttributesBuilder newBuilder = new ExplainPlanAttributesBuilder(explainPlanAttributes); - newBuilder.setClientFilterBy(postFilter.toString()); - newBuilder.addClientStep(step); + if (statementContext != null && statementContext.isVerbose()) { + int from = planSteps.size(); + List<ExplainFilter> filters = ExplainTable.renderVerboseFilters(statementContext, + postFilter, postFilter.toString(), "CLIENT FILTER BY", planSteps); + for (int i = from; i < planSteps.size(); i++) { + newBuilder.addClientStep(planSteps.get(i)); + } + newBuilder.setClientFilters(filters); + } else { + String step = "CLIENT FILTER BY " + postFilter.toString(); + planSteps.add(step); + newBuilder.setClientFilterBy(postFilter.toString()); + newBuilder.addClientStep(step); + } explainPlanAttributes = newBuilder.build(); } @@ -178,7 +192,7 @@ public class TupleProjectionPlan extends DelegateQueryPlan { }; if (postFilter != null) { - iterator = new FilterResultIterator(iterator, postFilter); + iterator = new FilterResultIterator(iterator, postFilter, statementContext); } return iterator; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 87956b1141..64ce3cb73e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -651,6 +651,11 @@ public abstract class BaseResultIterators extends ExplainTable implements Result return plan.getOptimizerDecision(); } + @Override + protected org.apache.phoenix.compile.RowProjector getProjector() { + return plan.getProjector(); + } + @Override public List<List<Scan>> getScans() { if (scans == null) return Collections.emptyList(); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index a096fd574c..dff466b3c8 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -37,15 +37,21 @@ import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.ColumnProjector; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; import org.apache.phoenix.compile.GroupByCompiler.GroupBy; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; +import org.apache.phoenix.compile.RowProjector; import org.apache.phoenix.compile.ScanRanges; import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.compile.StatementPlan; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.expression.AndExpression; import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.function.BsonConditionExpressionFunction; +import org.apache.phoenix.expression.function.JsonExistsFunction; import org.apache.phoenix.filter.BooleanExpressionFilter; import org.apache.phoenix.filter.DistinctPrefixFilter; import org.apache.phoenix.filter.EmptyColumnOnlyFilter; @@ -153,6 +159,14 @@ public abstract class ExplainTable { return null; } + /** + * The post-compile row projector for the plan this scan belongs to. Returns {@code null} when the + * plan has no projector. + */ + protected RowProjector getProjector() { + return null; + } + /** * Whether {@code rule} is a default rule whose {@code INDEX} comment is suppressed. The default * rules are {@link OptimizerReasons#RULE_DATA_TABLE} (no candidate indexes considered) and @@ -297,6 +311,7 @@ public abstract class ExplainTable { StringBuilder buf = new StringBuilder(prefix); ScanRanges scanRanges = context.getScanRanges(); Scan scan = context.getScan(); + boolean verbose = context.isVerbose(); if (scan.getConsistency() != Consistency.STRONG) { buf.append("TIMELINE-CONSISTENCY "); @@ -340,6 +355,8 @@ public abstract class ExplainTable { } } + emitProject(planSteps, explainPlanAttributesBuilder, verbose); + PTable.IndexType indexType = tableRef.getTable().getIndexType(); String explainIndexName = getExplainIndexName(tableRef.getTable()); String indexKind = null; @@ -367,7 +384,7 @@ public abstract class ExplainTable { indexLine.append(" /* ").append(decision.getRule()).append(" */"); } planSteps.add(indexLine.toString()); - if (decision != null) { + if (verbose && decision != null) { for (RejectedIndexEntry rejected : decision.getRejectedIndexes()) { planSteps .add(" /* !INDEX " + rejected.getName() + " -- " + rejected.getReason() + " */"); @@ -455,11 +472,17 @@ public abstract class ExplainTable { explainPlanAttributesBuilder.setServerEmptyColumnOnlyProjection(true); } } + emitIgnoredHints(planSteps, explainPlanAttributesBuilder, verbose); if (whereFilterStr != null) { - String serverWhereFilter = "SERVER FILTER BY " + whereFilterStr; - planSteps.add(" " + serverWhereFilter); - if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder.setServerWhereFilter(serverWhereFilter); + if (verbose) { + emitServerFilters(planSteps, explainPlanAttributesBuilder, + whereFilter == null ? null : whereFilter.getExpression(), whereFilterStr); + } else { + String serverWhereFilter = "SERVER FILTER BY " + whereFilterStr; + planSteps.add(" " + serverWhereFilter); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setServerWhereFilter(serverWhereFilter); + } } } if (distinctFilter != null) { @@ -562,6 +585,188 @@ public abstract class ExplainTable { } } + /** Emit the VERBOSE-only {@code PROJECT <cols>} line and populate {@code serverProject}. */ + private void emitProject(List<String> planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, boolean verbose) { + if (!verbose) { + return; + } + RowProjector projector = getProjector(); + if (projector == null) { + return; + } + List<? extends ColumnProjector> columnProjectors = projector.getColumnProjectors(); + if (columnProjectors == null || columnProjectors.isEmpty()) { + return; + } + List<String> columns = new ArrayList<>(columnProjectors.size()); + for (ColumnProjector columnProjector : columnProjectors) { + String name = columnProjector.getName(); + if (name == null || name.isEmpty()) { + name = String.valueOf(columnProjector.getExpression()); + } + columns.add(name); + } + planSteps.add(" PROJECT " + String.join(", ", columns)); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setServerProject(columns); + } + } + + /** + * Emit the VERBOSE-only ignored-hint comments, one {@code /*- HINT(args) -- reason *}{@code /} + * line per hint the planner intentionally ignored. + */ + private void emitIgnoredHints(List<String> planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, boolean verbose) { + if (!verbose) { + return; + } + Map<Hint, String> ignoredHints = context.getIgnoredHints(); + if (ignoredHints == null || ignoredHints.isEmpty()) { + return; + } + for (Map.Entry<Hint, String> entry : ignoredHints.entrySet()) { + Hint ignoredHint = entry.getKey(); + String args = hint == null ? null : hint.getHint(ignoredHint); + String rendered = ignoredHint.name() + (args == null ? "" : args); + planSteps.add(" /*- " + rendered + " -- " + entry.getValue() + " */"); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.addIgnoredHint(ignoredHint.name(), entry.getValue()); + } + } + } + + /** + * Emit the VERBOSE-only per-predicate {@code SERVER FILTER BY <expr> -- <origin>} lines and + * populate {@code serverFilters}. When the top-level filter is an {@link AndExpression} whose + * children all carry origin tags, one line is emitted per child. Otherwise a single line carries + * the comma-separated union of origins. + */ + private void emitServerFilters(List<String> planSteps, + ExplainPlanAttributesBuilder explainPlanAttributesBuilder, Expression whereExpression, + String whereFilterStr) { + List<ExplainFilter> filters = renderVerboseFilters(context, whereExpression, whereFilterStr, + " SERVER FILTER BY", planSteps); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setServerFilters(filters); + } + } + + /** + * Render the VERBOSE per-predicate filter breakdown. Each emitted line is appended to + * {@code planSteps}. + * @param context the statement context carrying the predicate-origin tags. + * @param filterExpression the residual filter expression (may be {@code null} when only the + * rendered string is available, e.g. an index-serialized filter). + * @param combinedFilterStr the combined filter string used for the single-line fallback. + * @param linePrefix the full leading token including any indentation, e.g. + * {@code " SERVER FILTER BY"} or {@code "CLIENT FILTER BY"}. + * @param planSteps the plan-steps list to append rendered lines to. + * @return the structured filters in render order (never {@code null}; never empty). + */ + public static List<ExplainFilter> renderVerboseFilters(StatementContext context, + Expression filterExpression, String combinedFilterStr, String linePrefix, + List<String> planSteps) { + List<ExplainFilter> filters = new ArrayList<>(); + if (filterExpression instanceof AndExpression) { + List<Expression> children = filterExpression.getChildren(); + boolean allTagged = !children.isEmpty(); + for (Expression child : children) { + if (context.getPredicateOrigins(child).isEmpty()) { + allTagged = false; + break; + } + } + if (allTagged) { + for (Expression child : children) { + filters.add(buildFilter(context, "(" + child + ")", child)); + } + } + } + if (filters.isEmpty()) { + filters.add(buildCombinedFilter(context, combinedFilterStr, filterExpression)); + } + for (ExplainFilter filter : filters) { + StringBuilder line = new StringBuilder(linePrefix).append(" ").append(filter.getExpr()); + String comment = renderOriginComment(filter); + if (comment != null) { + line.append(" -- ").append(comment); + } + planSteps.add(line.toString()); + } + return filters; + } + + private static ExplainFilter buildFilter(StatementContext context, String exprStr, + Expression expression) { + List<String> origins = + expression == null ? null : new ArrayList<>(context.getPredicateOrigins(expression)); + String pathTestSubtag = detectPathTestSubtag(expression); + return new ExplainFilter(exprStr, origins, pathTestSubtag); + } + + /** + * Build the {@link ExplainFilter} for a single combined line. Origin tags can live on the parent + * expression or its conjunct children, so when we collapse to one line union both sets to avoid + * dropping the attribution. + */ + private static ExplainFilter buildCombinedFilter(StatementContext context, String exprStr, + Expression expression) { + List<String> origins = expression == null ? null : combinedOrigins(context, expression); + String pathTestSubtag = detectPathTestSubtag(expression); + return new ExplainFilter(exprStr, origins, pathTestSubtag); + } + + /** Union of origin tags on {@code expression} and, when it is an AND, its conjunct children. */ + private static List<String> combinedOrigins(StatementContext context, Expression expression) { + Set<String> origins = new LinkedHashSet<>(context.getPredicateOrigins(expression)); + if (expression instanceof AndExpression) { + for (Expression child : expression.getChildren()) { + origins.addAll(context.getPredicateOrigins(child)); + } + } + return new ArrayList<>(origins); + } + + /** Render the trailing origin comment for a server filter. */ + private static String renderOriginComment(ExplainFilter filter) { + StringBuilder comment = new StringBuilder(); + List<String> origins = filter.getOrigin(); + if (origins != null && !origins.isEmpty()) { + comment.append(String.join(", ", origins)); + } + if (filter.getPathTestSubtag() != null) { + if (comment.length() > 0) { + comment.append(" "); + } + comment.append("(").append(filter.getPathTestSubtag()).append(")"); + } + return comment.length() == 0 ? null : comment.toString(); + } + + /** Detect a path test function anywhere in the expression tree and return its sub tag. */ + private static String detectPathTestSubtag(Expression expression) { + if (expression == null) { + return null; + } + if (expression instanceof JsonExistsFunction) { + return "JSON EXISTS"; + } + if (expression instanceof BsonConditionExpressionFunction) { + return "BSON CONDITION"; + } + if (expression.getChildren() != null) { + for (Expression child : expression.getChildren()) { + String subtag = detectPathTestSubtag(child); + if (subtag != null) { + return subtag; + } + } + } + return null; + } + /** * Retrieve region locations and set the values in the explain plan output. * @param planSteps list of plan steps to add explain plan output to. diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterAggregatingResultIterator.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterAggregatingResultIterator.java index 82d6ba6824..436e7dc435 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterAggregatingResultIterator.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterAggregatingResultIterator.java @@ -20,7 +20,9 @@ package org.apache.phoenix.iterate; import java.sql.SQLException; import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; +import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.aggregator.Aggregator; import org.apache.phoenix.schema.tuple.Tuple; @@ -37,12 +39,14 @@ import org.apache.phoenix.schema.types.PBoolean; public class FilterAggregatingResultIterator implements AggregatingResultIterator { private final AggregatingResultIterator delegate; private final Expression expression; + private final StatementContext context; private final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - public FilterAggregatingResultIterator(AggregatingResultIterator delegate, - Expression expression) { + public FilterAggregatingResultIterator(AggregatingResultIterator delegate, Expression expression, + StatementContext context) { this.delegate = delegate; this.expression = expression; + this.context = context; if (expression.getDataType() != PBoolean.INSTANCE) { throw new IllegalArgumentException( "FilterResultIterator requires a boolean expression, but got " + expression); @@ -81,10 +85,22 @@ public class FilterAggregatingResultIterator implements AggregatingResultIterato public void explain(List<String> planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { delegate.explain(planSteps, explainPlanAttributesBuilder); - explainPlanAttributesBuilder.setClientFilterBy(expression.toString()); - String step = "CLIENT FILTER BY " + expression.toString(); - planSteps.add(step); - explainPlanAttributesBuilder.addClientStep(step); + if (context != null && context.isVerbose() && explainPlanAttributesBuilder != null) { + int from = planSteps.size(); + List<ExplainFilter> filters = ExplainTable.renderVerboseFilters(context, expression, + expression.toString(), "CLIENT FILTER BY", planSteps); + for (int i = from; i < planSteps.size(); i++) { + explainPlanAttributesBuilder.addClientStep(planSteps.get(i)); + } + explainPlanAttributesBuilder.setClientFilters(filters); + } else { + String step = "CLIENT FILTER BY " + expression.toString(); + planSteps.add(step); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setClientFilterBy(expression.toString()); + explainPlanAttributesBuilder.addClientStep(step); + } + } } @Override diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterResultIterator.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterResultIterator.java index 9435f20464..680fdad71e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterResultIterator.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/FilterResultIterator.java @@ -20,7 +20,9 @@ package org.apache.phoenix.iterate; import java.sql.SQLException; import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; +import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PBoolean; @@ -36,15 +38,18 @@ import org.apache.phoenix.schema.types.PBoolean; public class FilterResultIterator extends LookAheadResultIterator { private final ResultIterator delegate; private final Expression expression; + private final StatementContext context; private final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - public FilterResultIterator(ResultIterator delegate, Expression expression) { + public FilterResultIterator(ResultIterator delegate, Expression expression, + StatementContext context) { if (delegate instanceof AggregatingResultIterator) { throw new IllegalArgumentException( "FilterResultScanner may not be used with an aggregate delegate. Use phoenix.iterate.FilterAggregateResultScanner instead"); } this.delegate = delegate; this.expression = expression; + this.context = context; if (expression.getDataType() != PBoolean.INSTANCE) { throw new IllegalArgumentException( "FilterResultIterator requires a boolean expression, but got " + expression); @@ -79,10 +84,22 @@ public class FilterResultIterator extends LookAheadResultIterator { public void explain(List<String> planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder) { delegate.explain(planSteps, explainPlanAttributesBuilder); - explainPlanAttributesBuilder.setClientFilterBy(expression.toString()); - String step = "CLIENT FILTER BY " + expression.toString(); - planSteps.add(step); - explainPlanAttributesBuilder.addClientStep(step); + if (context != null && context.isVerbose() && explainPlanAttributesBuilder != null) { + int from = planSteps.size(); + List<ExplainFilter> filters = ExplainTable.renderVerboseFilters(context, expression, + expression.toString(), "CLIENT FILTER BY", planSteps); + for (int i = from; i < planSteps.size(); i++) { + explainPlanAttributesBuilder.addClientStep(planSteps.get(i)); + } + explainPlanAttributesBuilder.setClientFilters(filters); + } else { + String step = "CLIENT FILTER BY " + expression.toString(); + planSteps.add(step); + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder.setClientFilterBy(expression.toString()); + explainPlanAttributesBuilder.addClientStep(step); + } + } } @Override diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java index b16a0ed22e..63fe8e5a8c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java @@ -250,6 +250,9 @@ public class QueryOptimizer { indexHint == null && dataPlan.getContext().getScanRanges().isPointLookup() && stopAtBestPlan && dataPlan.isApplicable() ) { + if (select.getHint().hasHint(Hint.NO_INDEX)) { + dataPlan.getContext().recordIgnoredHint(Hint.NO_INDEX, "point lookup short-circuit"); + } return Collections.<QueryPlan> singletonList( recordDecision(dataPlan, OptimizerReasons.RULE_POINT_LOOKUP, state)); } @@ -283,6 +286,9 @@ public class QueryOptimizer { table = cdcBuilder.build(); dataPlan.getTableRef().setTable(table); forCDC = true; + if (select.getHint().hasHint(Hint.NO_INDEX)) { + dataPlan.getContext().recordIgnoredHint(Hint.NO_INDEX, "CDC table"); + } } List<PTable> indexes = Lists.newArrayList(dataPlan.getTableRef().getTable().getIndexes()); @@ -292,6 +298,10 @@ public class QueryOptimizer { ) { if (select.getHint().hasHint(Hint.NO_INDEX)) { state.rejectAll(indexes, OptimizerReasons.REASON_EXCLUDED_BY_NO_INDEX_HINT); + if (indexes.isEmpty()) { + // NO_INDEX had no effect: the table has no indexes to exclude. + dataPlan.getContext().recordIgnoredHint(Hint.NO_INDEX, "no indexes on table"); + } } return Collections.< QueryPlan> singletonList(recordDecision(dataPlan, OptimizerReasons.RULE_DATA_TABLE, state)); @@ -334,6 +344,9 @@ public class QueryOptimizer { } plans.add(0, hintedPlan); } + } else if (indexHint != null) { + // An INDEX(...) hint was supplied but no hinted index could be built into a usable plan. + dataPlan.getContext().recordIgnoredHint(Hint.INDEX, "no matching index applicable"); } } @@ -395,7 +408,8 @@ public class QueryOptimizer { */ private static void carryForwardRewrites(StatementContext from, List<QueryPlan> plans) { List<String> rewrites = from.getAppliedRewrites(); - if (rewrites.isEmpty()) { + Map<Hint, String> ignoredHints = from.getIgnoredHints(); + if (rewrites.isEmpty() && (ignoredHints == null || ignoredHints.isEmpty())) { return; } for (QueryPlan plan : plans) { @@ -408,6 +422,11 @@ public class QueryOptimizer { to.addAppliedRewrite(rewrite); } } + if (ignoredHints != null) { + for (Map.Entry<Hint, String> entry : ignoredHints.entrySet()) { + to.recordIgnoredHint(entry.getKey(), entry.getValue()); + } + } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java index ca8f4bc4c3..776f4a7059 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ExplainOptions.java @@ -33,6 +33,7 @@ public final class ExplainOptions { public static final ExplainOptions DEFAULT = new ExplainOptions(false, false, Format.TEXT); public static final ExplainOptions WITH_REGIONS = new ExplainOptions(true, false, Format.TEXT); + public static final ExplainOptions VERBOSE = new ExplainOptions(false, true, Format.TEXT); private final boolean regions; private final boolean verbose; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java index dee96ceafc..89f6fc7ad7 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -2537,6 +2537,8 @@ public class PTableImpl implements PTable { ParseNode where = plan.getStatement().getWhere(); plan.getContext().setResolver(FromCompiler.getResolver(plan.getTableRef())); indexWhereExpression = transformDNF(where, plan.getContext()); + // Tag the partial-index WHERE predicate with its origin for VERBOSE attribution. + plan.getContext().tagPredicate(indexWhereExpression, "INDEX WHERE"); indexWhereColumns = Sets.newHashSetWithExpectedSize(plan.getContext().getWhereConditionColumns().size()); for (Pair<byte[], byte[]> column : plan.getContext().getWhereConditionColumns()) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java index e07eb68261..6739e09469 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainJsonNormalizer.java @@ -125,6 +125,34 @@ public final class ExplainJsonNormalizer { } } + // The VERBOSE serverFilters and clientFilters breakdowns carry a rendered predicate string per + // element. Rewrite any temp aliases inside each element's "expr" so the comparison is + // invariant under environment differences. + rewriteFilterExprs(obj.get("serverFilters"), aliases); + rewriteFilterExprs(obj.get("clientFilters"), aliases); + return obj; } + + /** + * Rewrite temp aliases inside the {@code expr} string of each element of a filter-breakdown array + * (e.g. {@code serverFilters} or {@code clientFilters}). + */ + private static void rewriteFilterExprs(JsonNode node, TempAliasRenumberer aliases) { + if (node == null || !node.isArray()) { + return; + } + for (JsonNode element : (ArrayNode) node) { + if (element != null && element.isObject()) { + JsonNode expr = element.get("expr"); + if (expr != null && expr.isTextual()) { + String original = expr.asText(); + String rewritten = aliases.rewrite(original); + if (!rewritten.equals(original)) { + ((ObjectNode) element).put("expr", rewritten); + } + } + } + } + } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java index 9e510fd9e9..03aa689c7b 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -46,9 +47,17 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.phoenix.compile.ExplainPlan; import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainFilter; import org.apache.phoenix.compile.ExplainPlanAttributes.ExplainPlanAttributesBuilder; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.expression.AndExpression; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.LiteralExpression; import org.apache.phoenix.iterate.ExplainTable; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; import org.apache.phoenix.optimize.OptimizerReasons; +import org.apache.phoenix.parse.ExplainOptions; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PColumn; @@ -973,11 +982,205 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { stmt.execute("CREATE LOCAL INDEX " + idx + " ON " + base + " (c1)"); String query = "SELECT c1, max(rowkey), max(c2) FROM " + base + " WHERE rowkey <= 'z' GROUP BY c1"; + // The structured indexRejected attribute is populated regardless of EXPLAIN mode. ExplainPlanTestUtil.assertPlan(conn, query).indexName(base) .indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS).indexRejectedCount(1) .indexRejected(0, idx, OptimizerReasons.REASON_NO_PK_PREFIX_BOUND); assertPlanContainsLine(conn, query, " INDEX " + base + " /* more bound PK columns */"); - assertPlanContainsLine(conn, query, " /* !INDEX " + idx + " -- no PK prefix bound */"); + // The !INDEX rejection comment text is VERBOSE-only. + assertNoPlanLineContains(conn, query, "!INDEX"); + List<String> verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue( + "expected VERBOSE plan to contain the !INDEX rejection comment but was " + verboseSteps, + verboseSteps.contains(" /* !INDEX " + idx + " -- no PK prefix bound */")); + } + } + + @Test + public void testVerboseProjectLine() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt + .execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR, c VARCHAR)"); + String query = "SELECT a, b FROM " + t; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverProject("A", "B"); + List<String> verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue("expected VERBOSE plan to contain the PROJECT line but was " + verboseSteps, + verboseSteps.contains(" PROJECT A, B")); + // Plain EXPLAIN carries no PROJECT line and no serverProject attribute. + ExplainPlanTestUtil.assertPlan(conn, query).serverProjectNone(); + assertNoPlanLineContains(conn, query, "PROJECT "); + } + } + + @Test + public void testVerboseServerFilterWhereFanout() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt + .execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR, c VARCHAR)"); + String query = "SELECT a FROM " + t + " WHERE b = 'x' AND c = 'y'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(2) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, null).serverFilterOrigin(1, "WHERE") + .serverFilterPathTest(1, null); + // Plain EXPLAIN keeps the combined single serverWhereFilter line, no per-predicate breakdown. + ExplainPlanTestUtil.assertPlan(conn, query).serverFiltersNone(); + } + } + + @Test + public void testVerboseServerFilterSinglePredicate() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR)"); + String query = "SELECT a FROM " + t + " WHERE b = 'x'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(1) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, null); + } + } + + @Test + public void testVerboseServerFilterJsonExistsSubtag() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (pk VARCHAR PRIMARY KEY, jsoncol JSON)"); + String query = "SELECT pk FROM " + t + " WHERE JSON_EXISTS(jsoncol, '$.info.address.town')"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(1) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, "JSON EXISTS"); + } + } + + @Test + public void testVerboseServerFilterBsonConditionSubtag() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (pk VARCHAR PRIMARY KEY, payload BSON)"); + String query = "SELECT pk FROM " + t + + " WHERE BSON_CONDITION_EXPRESSION(payload, '{\"$EXPR\": \"field_exists(Id)\"}')"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).serverFilterCount(1) + .serverFilterOrigin(0, "WHERE").serverFilterPathTest(0, "BSON CONDITION"); + } + } + + @Test + public void testVerboseClientFilterFanout() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + String query = "SELECT a_string FROM (SELECT a_string, a_integer FROM atable LIMIT 5)" + + " WHERE a_integer > 2"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).clientFilterCount(1) + .clientFilter(0, "A_INTEGER > 2").clientFilterOrigin(0, "WHERE") + .clientFilterPathTest(0, null); + List<String> verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue("expected VERBOSE plan to contain a CLIENT FILTER BY line but was " + verboseSteps, + verboseSteps.stream().anyMatch(s -> s.startsWith("CLIENT FILTER BY A_INTEGER > 2"))); + // Plain EXPLAIN keeps the combined clientFilterBy string and no structured breakdown. + ExplainPlanTestUtil.assertPlan(conn, query).clientFiltersNone() + .clientFilterBy("A_INTEGER > 2"); + } + } + + @Test + public void testVerboseClientFilterHavingFanout() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + String query = + "SELECT count(1) FROM atable GROUP BY a_string, b_string HAVING max(a_string) = 'a'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).clientFilterCount(1) + .clientFilter(0, "MAX(A_STRING) = 'a'").clientFilterOrigin(0, "HAVING") + .clientFilterPathTest(0, null); + // Plain EXPLAIN keeps the combined string and no structured breakdown. + ExplainPlanTestUtil.assertPlan(conn, query).clientFiltersNone() + .clientFilterBy("MAX(A_STRING) = 'a'"); + } + } + + /** + * When a top-level AND is only partially tagged, renderVerboseFilters collapses to a single + * combined line. That line must still union the origins recorded on the tagged conjunct(s) rather + * than reading only the parent expression. Normal WHERE/HAVING compilation always tags every + * conjunct, so this partial-tag state is reachable only when identity is lost during expression + * rewriting. + */ + @Test + public void testVerboseCombinedFilterUnionsConjunctOrigins() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps())) { + QueryPlan plan = conn.prepareStatement("SELECT * FROM atable") + .unwrap(PhoenixPreparedStatement.class).optimizeQuery(); + StatementContext context = plan.getContext(); + Expression tagged = LiteralExpression.newConstant(true); + Expression untagged = LiteralExpression.newConstant(false); + Expression and = new AndExpression(Arrays.asList(tagged, untagged)); + context.tagPredicate(tagged, "WHERE"); + List<String> planSteps = new ArrayList<>(); + List<ExplainFilter> filters = ExplainTable.renderVerboseFilters(context, and, and.toString(), + " SERVER FILTER BY", planSteps); + assertEquals("partial tagging must collapse to one combined line", 1, filters.size()); + assertEquals("combined line must union origins from tagged conjuncts", + Collections.singletonList("WHERE"), filters.get(0).getOrigin()); + assertEquals(1, planSteps.size()); + assertTrue("combined line should carry the unioned origin comment but was " + planSteps, + planSteps.get(0).endsWith("-- WHERE")); + } + } + + @Test + public void testVerboseIgnoredHintNoIndexNoIndexes() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR, b VARCHAR)"); + String query = "SELECT /*+ NO_INDEX */ a FROM " + t + " WHERE b = 'x'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).ignoredHint("NO_INDEX", + "no indexes on table"); + List<String> verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue( + "expected VERBOSE plan to disclose the ignored NO_INDEX hint but was " + verboseSteps, + verboseSteps.contains(" /*- NO_INDEX -- no indexes on table */")); + // Plain EXPLAIN does not disclose ignored hints. + ExplainPlanTestUtil.assertPlan(conn, query).ignoredHintsNone(); + assertNoPlanLineContains(conn, query, "/*- NO_INDEX"); + } + } + + @Test + public void testVerboseIgnoredHintIndexNoMatch() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + String idx = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + stmt.execute("CREATE INDEX " + idx + " ON " + t + " (v1) INCLUDE (v2)"); + // Hint references an index name that does not exist on the table. + String query = + "SELECT /*+ INDEX(" + t + " NONEXISTENT_IDX) */ k, v2 FROM " + t + " WHERE v1 = 'x'"; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).ignoredHint("INDEX", + "no matching index applicable"); + } + } + + @Test + public void testVerboseIgnoredHintSortMergeNoJoin() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl(), defaultProps()); + java.sql.Statement stmt = conn.createStatement()) { + String t = generateUniqueName(); + stmt.execute("CREATE TABLE " + t + " (k VARCHAR PRIMARY KEY, a VARCHAR)"); + String query = "SELECT /*+ USE_SORT_MERGE_JOIN */ a FROM " + t; + ExplainPlanTestUtil.assertPlanWithVerbose(conn, query).ignoredHint("USE_SORT_MERGE_JOIN", + "no join in query"); + List<String> verboseSteps = + ExplainPlanTestUtil.getPlanSteps(conn, query, ExplainOptions.VERBOSE); + assertTrue( + "expected VERBOSE plan to disclose the ignored USE_SORT_MERGE_JOIN hint but was " + + verboseSteps, + verboseSteps.contains(" /*- USE_SORT_MERGE_JOIN -- no join in query */")); } } @@ -1545,10 +1748,14 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { n.putNull("serverOffset"); n.putNull("serverRowLimit"); n.putNull("serverParsedProjections"); + n.putNull("serverProject"); + n.putNull("serverFilters"); + n.putNull("ignoredHints"); n.put("serverFirstKeyOnlyProjection", false); n.put("serverEmptyColumnOnlyProjection", false); n.putNull("serverAggregate"); n.putNull("clientFilterBy"); + n.putNull("clientFilters"); n.putNull("clientAggregate"); n.putNull("clientSortedBy"); n.putNull("clientAfterAggregate"); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java index 16f1f6c360..b239b9af50 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java @@ -81,6 +81,20 @@ public final class ExplainPlanTestUtil { return getExplainPlan(conn, query).getPlanSteps(); } + /** + * Optimize {@code query} with the given {@link ExplainOptions} applied to the plan's + * {@code StatementContext} and return its plan-steps text. + */ + public static List<String> getPlanSteps(Connection conn, String query, ExplainOptions options) + throws SQLException { + try (PhoenixPreparedStatement statement = + conn.prepareStatement(query).unwrap(PhoenixPreparedStatement.class)) { + QueryPlan plan = statement.optimizeQuery(); + plan.getContext().setExplainOptions(options); + return plan.getExplainPlan().getPlanSteps(); + } + } + /** Compile a mutation (UPSERT/DELETE) and return its {@link ExplainPlan}. */ public static ExplainPlan getMutationExplainPlan(Connection conn, String query) throws SQLException { @@ -118,6 +132,16 @@ public final class ExplainPlanTestUtil { return assertPlan(getExplainAttributes(conn, query, ExplainOptions.WITH_REGIONS)); } + /** + * Optimize {@code query} on {@code conn} with the {@code VERBOSE} option enabled and begin + * assertions on its plan attributes. Use this when asserting on VERBOSE-only attributes such as + * {@code serverProject}, {@code serverFilters}, and {@code ignoredHints}. + */ + public static ExplainPlanAssert assertPlanWithVerbose(Connection conn, String query) + throws SQLException { + return assertPlan(getExplainAttributes(conn, query, ExplainOptions.VERBOSE)); + } + /** * Optimize an already-prepared and, if needed, parameter-bound {@link PhoenixPreparedStatement} * and begin assertions on its plan attributes. @@ -507,6 +531,156 @@ public final class ExplainPlanTestUtil { return this; } + /** Assert the entire VERBOSE {@code PROJECT} column list matches {@code expected}, in order. */ + public ExplainPlanAssert serverProject(String... expected) { + List<String> actual = attributes.getServerProject(); + List<String> actualOrEmpty = actual == null ? Collections.<String> emptyList() : actual; + assertEquals(at("serverProject"), Arrays.asList(expected), actualOrEmpty); + return this; + } + + /** Assert the number of VERBOSE {@code PROJECT} columns. */ + public ExplainPlanAssert serverProjectCount(int expected) { + List<String> actual = attributes.getServerProject(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("serverProject.size"), expected, actualCount); + return this; + } + + /** Assert that no VERBOSE {@code PROJECT} disclosure was emitted (null or empty). */ + public ExplainPlanAssert serverProjectNone() { + List<String> actual = attributes.getServerProject(); + assertTrue(at("serverProject") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the number of VERBOSE server filter predicates. */ + public ExplainPlanAssert serverFilterCount(int expected) { + List<ExplainPlanAttributes.ExplainFilter> actual = attributes.getServerFilters(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("serverFilters.size"), expected, actualCount); + return this; + } + + /** Assert that no VERBOSE server filter breakdown was emitted (null or empty). */ + public ExplainPlanAssert serverFiltersNone() { + List<ExplainPlanAttributes.ExplainFilter> actual = attributes.getServerFilters(); + assertTrue(at("serverFilters") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the i-th VERBOSE server filter's rendered expression. */ + public ExplainPlanAssert serverFilter(int i, String expectedExpr) { + ExplainPlanAttributes.ExplainFilter filter = serverFilterAt(i); + assertEquals(at("serverFilters[" + i + "].expr"), expectedExpr, filter.getExpr()); + return this; + } + + /** Assert the i-th VERBOSE server filter's origin attribution, in order. */ + public ExplainPlanAssert serverFilterOrigin(int i, String... expectedOrigin) { + ExplainPlanAttributes.ExplainFilter filter = serverFilterAt(i); + List<String> actual = + filter.getOrigin() == null ? Collections.<String> emptyList() : filter.getOrigin(); + assertEquals(at("serverFilters[" + i + "].origin"), Arrays.asList(expectedOrigin), actual); + return this; + } + + /** Assert the i-th VERBOSE server filter's path-test sub-tag (or {@code null}). */ + public ExplainPlanAssert serverFilterPathTest(int i, String expectedSubtag) { + ExplainPlanAttributes.ExplainFilter filter = serverFilterAt(i); + assertEquals(at("serverFilters[" + i + "].pathTestSubtag"), expectedSubtag, + filter.getPathTestSubtag()); + return this; + } + + private ExplainPlanAttributes.ExplainFilter serverFilterAt(int i) { + List<ExplainPlanAttributes.ExplainFilter> filters = attributes.getServerFilters(); + assertNotNull(at("serverFilters") + " must not be null", filters); + assertTrue(at("serverFilters") + " has no index " + i + " (size=" + filters.size() + ")", + i >= 0 && i < filters.size()); + return filters.get(i); + } + + /** Assert the number of VERBOSE client filter predicates. */ + public ExplainPlanAssert clientFilterCount(int expected) { + List<ExplainPlanAttributes.ExplainFilter> actual = attributes.getClientFilters(); + int actualCount = actual == null ? 0 : actual.size(); + assertEquals(at("clientFilters.size"), expected, actualCount); + return this; + } + + /** Assert that no VERBOSE client filter breakdown was emitted (null or empty). */ + public ExplainPlanAssert clientFiltersNone() { + List<ExplainPlanAttributes.ExplainFilter> actual = attributes.getClientFilters(); + assertTrue(at("clientFilters") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + + /** Assert the i-th VERBOSE client filter's rendered expression. */ + public ExplainPlanAssert clientFilter(int i, String expectedExpr) { + ExplainPlanAttributes.ExplainFilter filter = clientFilterAt(i); + assertEquals(at("clientFilters[" + i + "].expr"), expectedExpr, filter.getExpr()); + return this; + } + + /** Assert the i-th VERBOSE client filter's origin attribution, in order. */ + public ExplainPlanAssert clientFilterOrigin(int i, String... expectedOrigin) { + ExplainPlanAttributes.ExplainFilter filter = clientFilterAt(i); + List<String> actual = + filter.getOrigin() == null ? Collections.<String> emptyList() : filter.getOrigin(); + assertEquals(at("clientFilters[" + i + "].origin"), Arrays.asList(expectedOrigin), actual); + return this; + } + + /** Assert the i-th VERBOSE client filter's path-test sub-tag (or {@code null}). */ + public ExplainPlanAssert clientFilterPathTest(int i, String expectedSubtag) { + ExplainPlanAttributes.ExplainFilter filter = clientFilterAt(i); + assertEquals(at("clientFilters[" + i + "].pathTestSubtag"), expectedSubtag, + filter.getPathTestSubtag()); + return this; + } + + private ExplainPlanAttributes.ExplainFilter clientFilterAt(int i) { + List<ExplainPlanAttributes.ExplainFilter> filters = attributes.getClientFilters(); + assertNotNull(at("clientFilters") + " must not be null", filters); + assertTrue(at("clientFilters") + " has no index " + i + " (size=" + filters.size() + ")", + i >= 0 && i < filters.size()); + return filters.get(i); + } + + /** Assert the entire VERBOSE ignored-hint map matches {@code expected}. */ + public ExplainPlanAssert ignoredHints(Map<String, String> expected) { + assertEquals(at("ignoredHints"), expected, attributes.getIgnoredHints()); + return this; + } + + /** Assert the ignored-hint map carries {@code hint} mapped to {@code reason}. */ + public ExplainPlanAssert ignoredHint(String hint, String reason) { + Map<String, String> actual = attributes.getIgnoredHints(); + assertNotNull(at("ignoredHints") + " must not be null", actual); + assertEquals(at("ignoredHints[" + hint + "]"), reason, actual.get(hint)); + return this; + } + + /** Assert that the ignored-hint map contains an entry for {@code hint}. */ + public ExplainPlanAssert hasIgnoredHint(String hint) { + Map<String, String> actual = attributes.getIgnoredHints(); + assertTrue(at("ignoredHints") + " expected to contain '" + hint + "' but was " + actual, + actual != null && actual.containsKey(hint)); + return this; + } + + /** Assert that no ignored-hint disclosure was emitted (null or empty). */ + public ExplainPlanAssert ignoredHintsNone() { + Map<String, String> actual = attributes.getIgnoredHints(); + assertTrue(at("ignoredHints") + " expected none but was " + actual, + actual == null || actual.isEmpty()); + return this; + } + public ExplainPlanAssert serverFirstKeyOnlyProjection(boolean expected) { assertEquals(at("serverFirstKeyOnlyProjection"), expected, attributes.isServerFirstKeyOnlyProjection());
