This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7876-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit a4f7665d2fa9b488ed5c119a99b5900a1329af33 Author: Andrew Purtell <[email protected]> AuthorDate: Sat Jun 6 20:30:41 2026 -0700 [WIP] Add OptimizerDecision data model and plumbing for chosen-index EXPLAIN Adds the decision data model and plumbing for chosen-index disclosure. A new org.apache.phoenix.optimize package introduces OptimizerDecision (chosenIndex, rule, rejectedIndexes), RejectedIndexEntry (name, reason), and OptimizerReasons, for the functional index rule. The plumbing wires this through the plan hierarchy: QueryPlan gains a default getOptimizerDecision() returning null and a default no-op setOptimizerDecision(...) so wrapping plans forward the call, BaseQueryPlan holds the optimizerDecision field with a public getter and setter, and DelegateQueryPlan delegates both accessors so wrapping plans inherit the decision. ExplainPlanAttributes gains indexRule and indexRejected fields. --- .../phoenix/compile/ExplainPlanAttributes.java | 42 ++++++++++++- .../java/org/apache/phoenix/compile/QueryPlan.java | 12 ++++ .../org/apache/phoenix/execute/BaseQueryPlan.java | 12 ++++ .../apache/phoenix/execute/DelegateQueryPlan.java | 11 ++++ .../apache/phoenix/optimize/OptimizerDecision.java | 63 +++++++++++++++++++ .../apache/phoenix/optimize/OptimizerReasons.java | 71 ++++++++++++++++++++++ .../phoenix/optimize/RejectedIndexEntry.java | 70 +++++++++++++++++++++ .../phoenix/end2end/SortMergeJoinMoreIT.java | 4 +- 8 files changed, 280 insertions(+), 5 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index 5bc1fe53b7..00aaa1534c 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.compile; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.ArrayList; @@ -25,6 +26,7 @@ import java.util.List; import java.util.Set; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.client.Consistency; +import org.apache.phoenix.optimize.RejectedIndexEntry; import org.apache.phoenix.parse.HintNode; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.schema.PColumn; @@ -65,6 +67,9 @@ public class ExplainPlanAttributes { private final String indexKind; private final Integer saltBuckets; private final Integer regionsPlanned; + // Optimizer index selection disclosure. + private final String indexRule; + private final List<RejectedIndexEntry> indexRejected; // Server-side operations private final boolean serverFirstKeyOnlyProjection; @@ -128,6 +133,8 @@ public class ExplainPlanAttributes { this.indexKind = null; this.saltBuckets = null; this.regionsPlanned = null; + this.indexRule = null; + this.indexRejected = null; this.serverFirstKeyOnlyProjection = false; this.serverEmptyColumnOnlyProjection = false; this.serverWhereFilter = null; @@ -176,8 +183,8 @@ public class ExplainPlanAttributes { List<String> clientSteps, ExplainPlanAttributes lhsJoinQueryExplainPlan, ExplainPlanAttributes rhsJoinQueryExplainPlan, List<ExplainPlanAttributes> subPlans, String dynamicServerFilter, String afterJoinFilter, Long joinScannerLimit, - boolean sortMergeSkipMerge, List<HRegionLocation> regionLocations, - int numRegionLocationLookups) { + boolean sortMergeSkipMerge, List<HRegionLocation> regionLocations, int numRegionLocationLookups, + String indexRule, List<RejectedIndexEntry> indexRejected) { this.abstractExplainPlan = abstractExplainPlan; this.hint = hint; this.explainScanType = explainScanType; @@ -197,6 +204,10 @@ public class ExplainPlanAttributes { this.indexKind = indexKind; this.saltBuckets = saltBuckets; this.regionsPlanned = regionsPlanned; + this.indexRule = indexRule; + this.indexRejected = (indexRejected == null || indexRejected.isEmpty()) + ? null + : Collections.unmodifiableList(new ArrayList<>(indexRejected)); this.serverFirstKeyOnlyProjection = serverFirstKeyOnlyProjection; this.serverEmptyColumnOnlyProjection = serverEmptyColumnOnlyProjection; this.serverWhereFilter = serverWhereFilter; @@ -308,6 +319,16 @@ public class ExplainPlanAttributes { return regionsPlanned; } + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getIndexRule() { + return indexRule; + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public List<RejectedIndexEntry> getIndexRejected() { + return indexRejected; + } + public boolean isServerFirstKeyOnlyProjection() { return serverFirstKeyOnlyProjection; } @@ -458,6 +479,8 @@ public class ExplainPlanAttributes { private String indexKind; private Integer saltBuckets; private Integer regionsPlanned; + private String indexRule; + private List<RejectedIndexEntry> indexRejected; private boolean serverFirstKeyOnlyProjection; private boolean serverEmptyColumnOnlyProjection; private String serverWhereFilter; @@ -514,6 +537,9 @@ public class ExplainPlanAttributes { this.indexKind = explainPlanAttributes.getIndexKind(); this.saltBuckets = explainPlanAttributes.getSaltBuckets(); this.regionsPlanned = explainPlanAttributes.getRegionsPlanned(); + this.indexRule = explainPlanAttributes.getIndexRule(); + List<RejectedIndexEntry> srcIndexRejected = explainPlanAttributes.getIndexRejected(); + this.indexRejected = srcIndexRejected == null ? null : new ArrayList<>(srcIndexRejected); this.serverFirstKeyOnlyProjection = explainPlanAttributes.isServerFirstKeyOnlyProjection(); this.serverEmptyColumnOnlyProjection = explainPlanAttributes.isServerEmptyColumnOnlyProjection(); @@ -644,6 +670,16 @@ public class ExplainPlanAttributes { return this; } + public ExplainPlanAttributesBuilder setIndexRule(String indexRule) { + this.indexRule = indexRule; + return this; + } + + public ExplainPlanAttributesBuilder setIndexRejected(List<RejectedIndexEntry> indexRejected) { + this.indexRejected = indexRejected == null ? null : new ArrayList<>(indexRejected); + return this; + } + public ExplainPlanAttributesBuilder setServerFirstKeyOnlyProjection(boolean serverFirstKeyOnlyProjection) { this.serverFirstKeyOnlyProjection = serverFirstKeyOnlyProjection; @@ -824,7 +860,7 @@ public class ExplainPlanAttributes { clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, clientSteps, lhsJoinQueryExplainPlan, rhsJoinQueryExplainPlan, subPlans, dynamicServerFilter, afterJoinFilter, joinScannerLimit, sortMergeSkipMerge, regionLocations, - numRegionLocationLookups); + numRegionLocationLookups, indexRule, indexRejected); } } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java index a56a262b80..3eabb3dd3f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java @@ -28,6 +28,7 @@ import org.apache.phoenix.execute.visitor.QueryPlanVisitor; import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.optimize.Cost; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.query.KeyRange; @@ -103,4 +104,15 @@ public interface QueryPlan extends StatementPlan { * </pre> */ public List<OrderBy> getOutputOrderBys(); + + /** + * The optimizer's index selection decision for this plan, or {@code null}. + */ + default OptimizerDecision getOptimizerDecision() { + return null; + } + + /** Record the optimizer's index selection decision on this plan. */ + default void setOptimizerDecision(OptimizerDecision optimizerDecision) { + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index e0dbc5afad..424fddda78 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -57,6 +57,7 @@ import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.parse.ParseNodeFactory; @@ -110,6 +111,7 @@ public abstract class BaseQueryPlan implements QueryPlan { protected Long estimateInfoTimestamp; private boolean getEstimatesCalled; protected boolean isApplicable = true; + private OptimizerDecision optimizerDecision; protected BaseQueryPlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projection, ParameterMetaData paramMetaData, Integer limit, Integer offset, @@ -585,6 +587,16 @@ public abstract class BaseQueryPlan implements QueryPlan { this.isApplicable = isApplicable; } + @Override + public OptimizerDecision getOptimizerDecision() { + return optimizerDecision; + } + + @Override + public void setOptimizerDecision(OptimizerDecision optimizerDecision) { + this.optimizerDecision = optimizerDecision; + } + private void getEstimates() throws SQLException { getEstimatesCalled = true; // Initialize a dummy iterator to get the estimates based on stats. diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java index b78a4ab614..860053e547 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java @@ -32,6 +32,7 @@ import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; import org.apache.phoenix.optimize.Cost; +import org.apache.phoenix.optimize.OptimizerDecision; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.TableRef; @@ -171,4 +172,14 @@ public abstract class DelegateQueryPlan implements QueryPlan { public boolean isApplicable() { return delegate.isApplicable(); } + + @Override + public OptimizerDecision getOptimizerDecision() { + return delegate.getOptimizerDecision(); + } + + @Override + public void setOptimizerDecision(OptimizerDecision optimizerDecision) { + delegate.setOptimizerDecision(optimizerDecision); + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java new file mode 100644 index 0000000000..250abe4d13 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.optimize; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Captures the {@link org.apache.phoenix.optimize.QueryOptimizer}'s index selection decision for a + * single query plan. + * @see OptimizerReasons + */ +public final class OptimizerDecision { + + private final String chosenIndex; + private final String rule; + private final List<RejectedIndexEntry> rejectedIndexes; + + /** + * @param chosenIndex the chosen index (or table) name + * @param rule a closed-set rule from {@link OptimizerReasons} ({@code RULE_*}), or a + * {@code matches <expr>} label built via + * {@link OptimizerReasons#matches(String)} + * @param rejectedIndexes the considered and rejected candidates + */ + public OptimizerDecision(String chosenIndex, String rule, + List<RejectedIndexEntry> rejectedIndexes) { + this.chosenIndex = chosenIndex; + this.rule = rule; + this.rejectedIndexes = (rejectedIndexes == null || rejectedIndexes.isEmpty()) + ? Collections.<RejectedIndexEntry> emptyList() + : Collections.unmodifiableList(new ArrayList<>(rejectedIndexes)); + } + + public String getChosenIndex() { + return chosenIndex; + } + + public String getRule() { + return rule; + } + + /** Returns the rejected candidates, never {@code null}. */ + public List<RejectedIndexEntry> getRejectedIndexes() { + return rejectedIndexes; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerReasons.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerReasons.java new file mode 100644 index 0000000000..8a68a77aa4 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerReasons.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.optimize; + +/** + * Closed-set vocabulary for {@link OptimizerDecision}. The {@code RULE_*} constants label why an + * index or the data table was chosen. The {@code REASON_*} constants label why a candidate index + * was rejected. + */ +public final class OptimizerReasons { + + private OptimizerReasons() { + } + + // Rules (index_rule in the EXPLAIN grammar): why the chosen target won. + public static final String RULE_POINT_LOOKUP = "point lookup"; + public static final String RULE_ORDER_PRESERVING = "order-preserving"; + public static final String RULE_MORE_BOUND_PK_COLUMNS = "more bound PK columns"; + public static final String RULE_PARTIAL_INDEX_APPLICABLE = "partial index applicable"; + public static final String RULE_NON_LOCAL_PREFERRED = "non-local preferred"; + public static final String RULE_COST_BASED = "cost-based"; + public static final String RULE_HINT = "hint"; + public static final String RULE_DATA_TABLE = "data table"; + public static final String RULE_ONLY_CANDIDATE = "only candidate"; + public static final String RULE_CDC_INDEX = "CDC index"; + + /** + * Prefix for the index match rule. The full label is built by appending the matched source + * expression, e.g. {@code "matches BSON_VALUE(PAYLOAD, 'user.id', 'VARCHAR')"}. + */ + public static final String RULE_MATCHES_PREFIX = "matches "; + + // Reasons why a candidate index was rejected. + public static final String REASON_NO_PK_PREFIX_BOUND = "no PK prefix bound"; + public static final String REASON_DOES_NOT_COVER_PROJECTION = "does not cover projection"; + public static final String REASON_PARTIAL_INDEX_PREDICATE_NOT_SATISFIED = + "partial index predicate not satisfied"; + public static final String REASON_EXCLUDED_BY_NO_INDEX_HINT = "excluded by NO_INDEX hint"; + public static final String REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE = + "local index loses to global by rule"; + public static final String REASON_DEGENERATE_RANGE = "degenerate range"; + public static final String REASON_COST_BASED_LOSS = "cost-based loss"; + public static final String REASON_FULL_SCAN_WOULD_BE_REQUIRED = "full scan would be required"; + public static final String REASON_NOT_APPLICABLE_TO_JOIN = "not applicable to join"; + public static final String REASON_PATH_EXPRESSION_DOES_NOT_MATCH = + "path expression does not match"; + + /** + * Build the functional-index match rule label for the given matched source expression. + * @param expression the source expression rendering (e.g. the result of {@code toString()}) + * @return {@code "matches " + expression} + */ + public static String matches(String expression) { + return RULE_MATCHES_PREFIX + expression; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/RejectedIndexEntry.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/RejectedIndexEntry.java new file mode 100644 index 0000000000..3ba3b29cd2 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/RejectedIndexEntry.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.optimize; + +import java.util.Objects; + +/** + * A single index candidate that the {@link org.apache.phoenix.optimize.QueryOptimizer} considered + * but did not choose, paired with the reason it was rejected. + * @see OptimizerReasons + */ +public final class RejectedIndexEntry { + + private final String name; + private final String reason; + + /** + * @param name the rejected index (or table) name + * @param reason a rejection reason from {@link OptimizerReasons} ({@code REASON_*}) + */ + public RejectedIndexEntry(String name, String reason) { + this.name = name; + this.reason = reason; + } + + public String getName() { + return name; + } + + public String getReason() { + return reason; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RejectedIndexEntry that = (RejectedIndexEntry) o; + return Objects.equals(name, that.name) && Objects.equals(reason, that.reason); + } + + @Override + public int hashCode() { + return Objects.hash(name, reason); + } + + @Override + public String toString() { + return "!INDEX " + name + " -- " + reason; + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java index 1d0980b60f..f26473b482 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinMoreIT.java @@ -433,8 +433,8 @@ public class SortMergeJoinMoreIT extends ParallelStatsDisabledIT { .serverDistinctFilter("SERVER DISTINCT PREFIX FILTER OVER [BUCKET, TIMESTAMP, LOCATION]") .serverAggregate( "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [BUCKET, TIMESTAMP, LOCATION]") - .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[BUCKET, TIMESTAMP]") - .end().rhs().iteratorType("PARALLEL").scanType("SKIP SCAN ON 2 RANGES").table(t[i]) + .clientSortAlgo("CLIENT MERGE SORT").clientSortedBy("[BUCKET, TIMESTAMP]").end().rhs() + .iteratorType("PARALLEL").scanType("SKIP SCAN ON 2 RANGES").table(t[i]) .keyRanges(rhsKeyRanges).serverFirstKeyOnlyProjection(true) .serverWhereFilter("SRC_LOCATION = DST_LOCATION") .serverDistinctFilter(
