This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7876-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit d821301c6a7adadb6e2f4cd14900353b5ebd4076 Author: Andrew Purtell <[email protected]> AuthorDate: Fri Jun 12 11:21:40 2026 -0700 PHOENIX-7903 Functional index match rule EXPLAIN disclosure (#2523) Co-authored-by: Claude Opus 4.8[1m] <[email protected]> --- .../apache/phoenix/compile/StatementContext.java | 48 +++++++++++ .../apache/phoenix/optimize/QueryOptimizer.java | 96 ++++++++++++++++++---- .../parse/IndexExpressionParseNodeRewriter.java | 52 +++++++++++- .../apache/phoenix/compile/QueryOptimizerTest.java | 42 ++++++++++ 4 files changed, 221 insertions(+), 17 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java index d5eac400e1..c2ad404027 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java @@ -21,6 +21,7 @@ import java.sql.SQLException; import java.text.Format; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Deque; import java.util.Iterator; @@ -106,6 +107,8 @@ public class StatementContext { private List<String> appliedRewrites; private int derivedTableFlattenCount; private List<Pair<ParseNode, String>> indexExpressionSubstitutions; + private Map<String, List<String>> appliedIndexExpressionMatches; + private Set<String> functionalIndexNames; private Set<Pair<String, String>> partialIndexCheckedSet; private Map<String, List<Expression>> serverParsedProjections; private StatementContext parentContext; @@ -148,6 +151,8 @@ public class StatementContext { this.appliedRewrites = context.appliedRewrites; this.derivedTableFlattenCount = context.derivedTableFlattenCount; this.indexExpressionSubstitutions = context.indexExpressionSubstitutions; + this.appliedIndexExpressionMatches = context.appliedIndexExpressionMatches; + this.functionalIndexNames = context.functionalIndexNames; this.partialIndexCheckedSet = context.partialIndexCheckedSet; this.serverParsedProjections = context.serverParsedProjections; this.parentContext = context.parentContext; @@ -216,6 +221,8 @@ public class StatementContext { this.appliedRewrites = new ArrayList<>(); this.derivedTableFlattenCount = 0; this.indexExpressionSubstitutions = new ArrayList<>(); + this.appliedIndexExpressionMatches = Maps.newLinkedHashMap(); + this.functionalIndexNames = Sets.newHashSet(); this.partialIndexCheckedSet = Sets.newHashSet(); this.serverParsedProjections = null; this.parentContext = null; @@ -509,6 +516,8 @@ public class StatementContext { this.appliedRewrites = source.appliedRewrites; this.derivedTableFlattenCount = source.derivedTableFlattenCount; this.indexExpressionSubstitutions = source.indexExpressionSubstitutions; + this.appliedIndexExpressionMatches = source.appliedIndexExpressionMatches; + this.functionalIndexNames = source.functionalIndexNames; this.partialIndexCheckedSet = source.partialIndexCheckedSet; this.serverParsedProjections = source.serverParsedProjections; } @@ -530,6 +539,45 @@ public class StatementContext { indexExpressionSubstitutions.add(new Pair<>(source, indexColumnName)); } + /** + * Records the path expressions that actually substituted against this query for the given + * functional index. Used by the optimizer to label the chosen index's rule as + * {@code matches <expr>} and to distinguish functional index rejections that matched no query + * expression. Deduplicated per index, preserving first seen order. + */ + public void recordAppliedIndexExpressionMatches(String indexName, + Collection<String> expressions) { + if (expressions == null || expressions.isEmpty()) { + return; + } + List<String> matches = + appliedIndexExpressionMatches.computeIfAbsent(indexName, k -> new ArrayList<>()); + for (String expression : expressions) { + if (!matches.contains(expression)) { + matches.add(expression); + } + } + } + + /** + * Returns the path expressions that actually substituted against this query for the given + * functional index, or an empty list if none were recorded. + */ + public List<String> getAppliedIndexExpressionMatches(String indexName) { + List<String> matches = appliedIndexExpressionMatches.get(indexName); + return matches == null ? Collections.emptyList() : matches; + } + + /** Marks the given index as a functional index. */ + public void recordFunctionalIndex(String indexName) { + functionalIndexNames.add(indexName); + } + + /** Returns whether the given index was recorded as a functional index during rewriting. */ + public boolean isFunctionalIndex(String indexName) { + return functionalIndexNames.contains(indexName); + } + /** * Dedup set used by the optimizer so a partial index applicability breadcrumb is recorded only * once per (table, index) pair even when the same index is scored across multiple paths. diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java index b2cea9c761..b16a0ed22e 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java @@ -495,6 +495,7 @@ public class QueryOptimizer { isHinted, indexSelect, resolver); } + @SuppressWarnings("rawtypes") private AddPlanResult addPlan(PhoenixStatement statement, SelectStatement select, PTable index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, QueryPlan dataPlan, boolean isHinted, SelectStatement indexSelect, ColumnResolver resolver) @@ -531,12 +532,22 @@ public class QueryOptimizer { } // translate nodes that match expressions that are indexed to the // associated column parse node + IndexExpressionParseNodeRewriter indexExpressionRewriter = + new IndexExpressionParseNodeRewriter(index, null, statement.getConnection(), + indexSelect.getUdfParseNodes()); SelectStatement rewrittenIndexSelect = - ParseNodeRewriter.rewrite(indexSelect, new IndexExpressionParseNodeRewriter(index, null, - statement.getConnection(), indexSelect.getUdfParseNodes())); + ParseNodeRewriter.rewrite(indexSelect, indexExpressionRewriter); + // Record which functional index expressions actually matched this query, so the + // optimizer can label the chosen index's rule and distinguish functional index rejections + // that matched nothing. + if (indexExpressionRewriter.hasFunctionalColumns()) { + dataPlan.getContext().recordFunctionalIndex(index.getTableName().getString()); + } + dataPlan.getContext().recordAppliedIndexExpressionMatches(index.getTableName().getString(), + indexExpressionRewriter.getAppliedFunctionalSubstitutions().values()); QueryCompiler compiler = new QueryCompiler(statement, rewrittenIndexSelect, resolver, targetColumns, parallelIteratorFactory, dataPlan.getContext().getSequenceManager(), - isProjected, true, dataPlans); + isProjected, true, dataPlans).withRewriteContext(dataPlan.getContext()); QueryPlan plan = compiler.compile(); if (indexTable.getIndexType() == IndexType.UNCOVERED_GLOBAL) { @@ -579,10 +590,8 @@ public class QueryOptimizer { maintainer.getIndexedColumnInfo(); for (org.apache.hadoop.hbase.util.Pair<String, String> pair : indexedColumns) { // The first member of the pair is the column family. For the data table PK columns, the - // column - // family is set to null. The data PK columns should not be added to the set of data - // columns - // to join back to index rows + // column family is set to null. The data PK columns should not be added to the set of + // data columns to join back to index rows if (pair.getFirst() != null) { PColumn pColumn = dataTable.getColumnForColumnName(pair.getSecond()); // The following adds the column to the set @@ -598,8 +607,7 @@ public class QueryOptimizer { indexTable = indexTableRef.getTable(); indexState = indexTable.getIndexState(); // Checking number of columns handles the wildcard cases correctly, as in that case the - // index - // must contain all columns from the data table to be able to be used. + // index must contain all columns from the data table to be able to be used. if ( indexState == PIndexState.ACTIVE || indexState == PIndexState.PENDING_ACTIVE || (indexState == PIndexState.PENDING_DISABLE && isUnderPendingDisableThreshold( @@ -687,7 +695,8 @@ public class QueryOptimizer { return AddPlanResult.rejected(index, OptimizerReasons.REASON_DEGENERATE_RANGE); } } - return AddPlanResult.rejected(index, OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); + return AddPlanResult.rejected(index, adjustReasonForFunctionalIndex(index, + OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION, dataPlan.getContext())); } // returns true if we can still use the index @@ -935,12 +944,59 @@ public class QueryOptimizer { * can use this inline at a {@code return} site. */ private static QueryPlan recordDecision(QueryPlan winner, String rule, DecisionState state) { + String functionalRule = functionalIndexRule(winner); + if (functionalRule != null) { + rule = functionalRule; + } winner.setOptimizerDecision( new OptimizerDecision(winner.getTableRef().getTable().getTableName().getString(), rule, state == null ? null : state.getRejections())); return winner; } + /** + * Returns {@link OptimizerReasons#matches(String)} for {@code winner} when it is a functional + * index whose indexed expression actually substituted a path expression in the user's query, or + * {@code null} when the functional index rule override does not apply. The applied matches are + * only ever recorded for genuine expression substitutions, so a non-empty match list implies the + * chosen plan is a functional index that matched. + */ + private static String functionalIndexRule(QueryPlan winner) { + PTable table = winner.getTableRef().getTable(); + List<String> matches = + winner.getContext().getAppliedIndexExpressionMatches(table.getTableName().getString()); + if (matches.isEmpty()) { + return null; + } + return OptimizerReasons.matches(matches.get(0)); + } + + /** + * Swaps a generic coverage driven rejection reason for + * {@link OptimizerReasons#REASON_PATH_EXPRESSION_DOES_NOT_MATCH} when {@code index} is a + * functional index for which no query expression matched its indexed expression. Other reasons, + * and functional indexes that did match an expression, keep their original reason. + */ + private static String adjustReasonForFunctionalIndex(PTable index, String reason, + StatementContext rewriteContext) { + if (!rewriteContext.isFunctionalIndex(index.getTableName().getString())) { + return reason; + } + if ( + !OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION.equals(reason) + && !OptimizerReasons.REASON_NO_PK_PREFIX_BOUND.equals(reason) + && !OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE.equals(reason) + ) { + return reason; + } + if ( + !rewriteContext.getAppliedIndexExpressionMatches(index.getTableName().getString()).isEmpty() + ) { + return reason; + } + return OptimizerReasons.REASON_PATH_EXPRESSION_DOES_NOT_MATCH; + } + /** First plan in {@code plans} that is not {@code exclude}, or {@code null} if there is none. */ private static QueryPlan firstOther(List<QueryPlan> plans, QueryPlan exclude) { for (QueryPlan plan : plans) { @@ -998,9 +1054,11 @@ public class QueryOptimizer { continue; } if (adjustedBoundCount(plan, boundRanges) < winnerBound) { - state.reject(table, OptimizerReasons.REASON_NO_PK_PREFIX_BOUND); + state.reject(table, adjustReasonForFunctionalIndex(table, + OptimizerReasons.REASON_NO_PK_PREFIX_BOUND, winner.getContext())); } else if (table.getIndexType() == IndexType.LOCAL && winnerNonLocal) { - state.reject(table, OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE); + state.reject(table, adjustReasonForFunctionalIndex(table, + OptimizerReasons.REASON_LOCAL_INDEX_LOSES_TO_GLOBAL_BY_RULE, winner.getContext())); } } } @@ -1055,10 +1113,20 @@ public class QueryOptimizer { IndexStatementRewriter.translate(FACTORY.select(select, newFrom), resolver, replacement); for (TableRef indexTableRef : replacement.values()) { // replace expressions with corresponding matching columns for functional indexes - indexSelect = ParseNodeRewriter.rewrite(indexSelect, + IndexExpressionParseNodeRewriter indexExpressionRewriter = new IndexExpressionParseNodeRewriter(indexTableRef.getTable(), indexTableRef.getTableAlias(), connection, indexSelect.getUdfParseNodes(), - breadcrumbContext)); + breadcrumbContext); + indexSelect = ParseNodeRewriter.rewrite(indexSelect, indexExpressionRewriter); + // Surface which functional index expressions actually matched so the optimizer can label + // the chosen index's rule and distinguish functional index rejections that matched nothing. + if (indexExpressionRewriter.hasFunctionalColumns()) { + breadcrumbContext + .recordFunctionalIndex(indexTableRef.getTable().getTableName().getString()); + } + breadcrumbContext.recordAppliedIndexExpressionMatches( + indexTableRef.getTable().getTableName().getString(), + indexExpressionRewriter.getAppliedFunctionalSubstitutions().values()); } return indexSelect; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java index 8d93f6415f..de3faef662 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/IndexExpressionParseNodeRewriter.java @@ -19,6 +19,7 @@ package org.apache.phoenix.parse; import java.sql.SQLException; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.apache.phoenix.compile.ColumnResolver; @@ -44,6 +45,19 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter { private final Map<ParseNode, ParseNode> indexedParseNodeToColumnParseNodeMap; + /** + * Maps the rewritten parse node of a functional index column to a [colName, expressionStr] pair. + * Plain PK columns are excluded. + */ + private final Map<ParseNode, String[]> indexedParseNodeToFunctionalColumn; + + /** + * Records the functional index substitutions that actually fired against the query, mapping the + * index column name to the expression string. Insertion order is preserved so callers can report + * matches in first-seen order. + */ + private final Map<String, String> appliedFunctionalSubstitutions = new LinkedHashMap<>(); + public IndexExpressionParseNodeRewriter(PTable index, String alias, PhoenixConnection connection, Map<String, UDFParseNode> udfParseNodes) throws SQLException { this(index, alias, connection, udfParseNodes, null); @@ -54,6 +68,7 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter { throws SQLException { indexedParseNodeToColumnParseNodeMap = Maps.newHashMapWithExpectedSize(index.getColumns().size()); + indexedParseNodeToFunctionalColumn = Maps.newHashMapWithExpectedSize(index.getColumns().size()); NamedTableNode tableNode = NamedTableNode.create(alias, TableName.create(index.getParentSchemaName().getString(), index.getParentTableName().getString()), Collections.<ColumnDef> emptyList()); @@ -80,6 +95,14 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter { columnParseNode = NODE_FACTORY.cast(columnParseNode, expressionDataType, null, null); } indexedParseNodeToColumnParseNodeMap.put(indexedParseNode, columnParseNode); + // Only true functional columns defined over an expression get an applied match entry. A + // plain indexed or PK column's expression string parses to a bare column reference. An + // expression column (e.g. UPPER(NAME)) parses to a compound node. + if (!(expressionParseNode instanceof ColumnParseNode)) { + // Trim leading/trailing whitespace + indexedParseNodeToFunctionalColumn.put(indexedParseNode, + new String[] { colName, expressionStr.trim() }); + } if (breadcrumbContext != null) { breadcrumbContext.addAppliedRewrite("INDEX EXPRESSION " + expressionStr + " AS " + colName); breadcrumbContext.addIndexExpressionSubstitution(indexedParseNode, colName); @@ -90,9 +113,32 @@ public class IndexExpressionParseNodeRewriter extends ParseNodeRewriter { @Override protected ParseNode leaveCompoundNode(CompoundParseNode node, List<ParseNode> children, CompoundNodeFactory factory) { - return indexedParseNodeToColumnParseNodeMap.containsKey(node) - ? indexedParseNodeToColumnParseNodeMap.get(node) - : super.leaveCompoundNode(node, children, factory); + ParseNode replacement = indexedParseNodeToColumnParseNodeMap.get(node); + if (replacement == null) { + return super.leaveCompoundNode(node, children, factory); + } + // A functional index substitution actually fired for this query node. + String[] functionalColumn = indexedParseNodeToFunctionalColumn.get(node); + if (functionalColumn != null) { + appliedFunctionalSubstitutions.put(functionalColumn[0], functionalColumn[1]); + } + return replacement; + } + + /** + * Returns the functional index substitutions that actually fired against the query, keyed by + * index column name with the expression string as the value, in first-seen order. + */ + public Map<String, String> getAppliedFunctionalSubstitutions() { + return Collections.unmodifiableMap(appliedFunctionalSubstitutions); + } + + /** + * Returns {@code true} if the index this rewriter was built for has at least one functional + * column, regardless of whether any such column matched the query. + */ + public boolean hasFunctionalColumns() { + return !indexedParseNodeToFunctionalColumn.isEmpty(); } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java index 9ee22f54f7..abd027ca1a 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java @@ -511,6 +511,48 @@ public class QueryOptimizerTest extends BaseConnectionlessQueryTest { .indexRejected(0, "INDEX_TEST_TABLE_INDEX_D", OptimizerReasons.REASON_NO_PK_PREFIX_BOUND); } + @Test + public void testFunctionalIndexChosenRuleMatchesExpression() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute( + "create table fe_match (id varchar not null primary key, name varchar, val varchar)"); + conn.createStatement().execute("create index fe_match_upper_idx on fe_match (UPPER(name))"); + // The functional index's indexed expression matches the query's UPPER(NAME) path expression, + // so the chosen index's rule is overridden to the "matches <expr>" form. Assert on the rule + // prefix. + assertPlan(conn, "select id from fe_match where UPPER(name) = 'ABC'") + .table("FE_MATCH_UPPER_IDX").indexRule(OptimizerReasons.matches("UPPER(NAME)")); + } + + @Test + public void testFunctionalIndexRejectedPathExpressionDoesNotMatch() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute( + "create table fe_nomatch (id varchar not null primary key, name varchar, val varchar, other varchar)"); + conn.createStatement().execute("create index fe_nomatch_upper_idx on fe_nomatch (UPPER(name))"); + conn.createStatement() + .execute("create index fe_nomatch_val_idx on fe_nomatch (val) include (other)"); + // The query never references UPPER(NAME), so the functional index matched no path expression + // and its generic "does not cover projection" rejection is swapped for the more specific + // "path expression does not match". + assertPlan(conn, "select val, other from fe_nomatch where val = 'x'") + .table("FE_NOMATCH_VAL_IDX").indexRejectedContains("FE_NOMATCH_UPPER_IDX", + OptimizerReasons.REASON_PATH_EXPRESSION_DOES_NOT_MATCH); + } + + @Test + public void testFunctionalIndexRejectedDoesNotCoverProjection() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute( + "create table fe_cover (id varchar not null primary key, name varchar, val varchar)"); + conn.createStatement().execute("create index fe_cover_upper_idx on fe_cover (UPPER(name))"); + // The functional index's expression DOES match UPPER(NAME) in the query, but VAL is not in the + // index, so the rejection stays "does not cover projection" rather than being swapped. + assertPlan(conn, "select UPPER(name), val from fe_cover where UPPER(name) = 'ABC'") + .table("FE_COVER").indexRejectedContains("FE_COVER_UPPER_IDX", + OptimizerReasons.REASON_DOES_NOT_COVER_PROJECTION); + } + @Test public void testCharArrayLength() throws Exception { Connection conn = DriverManager.getConnection(getUrl());
