This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7876-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit d8b35a5868cf07637b4f9ec79fca9b5fcc2f800b Author: Andrew Purtell <[email protected]> AuthorDate: Thu Jun 18 18:15:48 2026 -0700 PHOENIX-7932 Sweep tests for remaining EXPLAIN improvement issues (#2538) Co-authored-by: Claude Opus 4.8[1m] <[email protected]> --- .../phoenix/compile/ExplainPlanAttributes.java | 38 ++++--- .../org/apache/phoenix/execute/BaseQueryPlan.java | 1 + .../org/apache/phoenix/iterate/ExplainTable.java | 117 +++++++++++++++------ .../apache/phoenix/optimize/OptimizerDecision.java | 25 +++-- .../apache/phoenix/optimize/QueryOptimizer.java | 10 +- .../GlobalIndexCheckerEventualGenerateIT.java | 5 + .../index/GlobalIndexCheckerEventualIT.java | 5 + .../end2end/index/GlobalIndexCheckerIT.java | 57 ++++++++-- .../apache/phoenix/end2end/index/IndexUsageIT.java | 22 ++-- .../UncoveredGlobalIndexRegionScanner2IT.java | 56 ++++++---- .../index/UncoveredGlobalIndexRegionScannerIT.java | 56 ++++++---- .../phoenix/end2end/json/JsonFunctionsIT.java | 2 +- .../apache/phoenix/compile/QueryOptimizerTest.java | 7 +- .../phoenix/query/explain/ExplainPlanTest.java | 15 +-- .../phoenix/query/explain/ExplainPlanTestUtil.java | 17 ++- 15 files changed, 301 insertions(+), 132 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java index a068ec5fb4..e233931f71 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java @@ -42,18 +42,19 @@ import org.apache.phoenix.schema.PColumn; @JsonPropertyOrder({ "tenantId", "viewName", "viewBaseName", "cdcScopes", "txnProvider", "rewrites", "estimatedRows", "estimatedSizeInBytes", "estimateInfoTs", "abstractExplainPlan", "onDuplicateKeyAction", "serverUpdateSet", "returningRow", "hint", "explainScanType", - "consistency", "tableName", "keyRanges", "indexName", "indexKind", "indexRule", "indexRejected", - "saltBuckets", "regionsPlanned", "scanTimeRangeMin", "scanTimeRangeMax", "splitsChunk", - "useRoundRobinIterator", "samplingRate", "hexStringRVCOffset", "iteratorTypeAndScanSize", - "scanEstimatedRows", "scanEstimatedSizeInBytes", "serverWhereFilter", "serverDistinctFilter", - "serverMergeColumns", "serverParsedProjections", "serverProject", "serverFilters", "ignoredHints", - "serverFirstKeyOnlyProjection", "serverEmptyColumnOnlyProjection", "serverAggregate", - "serverGroupByLimit", "serverSortedBy", "serverOffset", "serverRowLimit", "clientFilterBy", - "clientFilters", "clientAggregate", "clientDistinctFilter", "clientAfterAggregate", - "clientSortAlgo", "clientSortedBy", "clientOffset", "clientRowLimit", "clientSequenceCount", - "clientCursorName", "clientSteps", "lhsJoinQueryExplainPlan", "rhsJoinQueryExplainPlan", - "subPlans", "dynamicServerFilter", "afterJoinFilter", "joinScannerLimit", "sortMergeSkipMerge", - "regionLocations", "regionLocationsTotalSize", "numRegionLocationLookups" }) + "consistency", "tableName", "keyRanges", "indexName", "indexKind", "indexRule", "functionalMatch", + "indexRejected", "saltBuckets", "regionsPlanned", "scanTimeRangeMin", "scanTimeRangeMax", + "splitsChunk", "useRoundRobinIterator", "samplingRate", "hexStringRVCOffset", + "iteratorTypeAndScanSize", "scanEstimatedRows", "scanEstimatedSizeInBytes", "serverWhereFilter", + "serverDistinctFilter", "serverMergeColumns", "serverParsedProjections", "serverProject", + "serverFilters", "ignoredHints", "serverFirstKeyOnlyProjection", + "serverEmptyColumnOnlyProjection", "serverAggregate", "serverGroupByLimit", "serverSortedBy", + "serverOffset", "serverRowLimit", "clientFilterBy", "clientFilters", "clientAggregate", + "clientDistinctFilter", "clientAfterAggregate", "clientSortAlgo", "clientSortedBy", + "clientOffset", "clientRowLimit", "clientSequenceCount", "clientCursorName", "clientSteps", + "lhsJoinQueryExplainPlan", "rhsJoinQueryExplainPlan", "subPlans", "dynamicServerFilter", + "afterJoinFilter", "joinScannerLimit", "sortMergeSkipMerge", "regionLocations", + "regionLocationsTotalSize", "numRegionLocationLookups" }) public class ExplainPlanAttributes { // Top-of-plan disclosures (populated only on the root plan) @@ -82,6 +83,7 @@ public class ExplainPlanAttributes { private final String indexName; private final String indexKind; private final String indexRule; + private final String functionalMatch; private final List<RejectedIndexEntry> indexRejected; private final Integer saltBuckets; private final Integer regionsPlanned; @@ -171,6 +173,7 @@ public class ExplainPlanAttributes { this.indexName = b.indexName; this.indexKind = b.indexKind; this.indexRule = b.indexRule; + this.functionalMatch = b.functionalMatch; this.indexRejected = (b.indexRejected == null || b.indexRejected.isEmpty()) ? null : Collections.unmodifiableList(new ArrayList<>(b.indexRejected)); @@ -311,6 +314,10 @@ public class ExplainPlanAttributes { return indexRule; } + public String getFunctionalMatch() { + return functionalMatch; + } + public List<RejectedIndexEntry> getIndexRejected() { return indexRejected; } @@ -608,6 +615,7 @@ public class ExplainPlanAttributes { private String indexName; private String indexKind; private String indexRule; + private String functionalMatch; private List<RejectedIndexEntry> indexRejected; private Integer saltBuckets; private Integer regionsPlanned; @@ -686,6 +694,7 @@ public class ExplainPlanAttributes { this.indexName = explainPlanAttributes.getIndexName(); this.indexKind = explainPlanAttributes.getIndexKind(); this.indexRule = explainPlanAttributes.getIndexRule(); + this.functionalMatch = explainPlanAttributes.getFunctionalMatch(); List<RejectedIndexEntry> srcIndexRejected = explainPlanAttributes.getIndexRejected(); this.indexRejected = srcIndexRejected == null ? null : new ArrayList<>(srcIndexRejected); this.saltBuckets = explainPlanAttributes.getSaltBuckets(); @@ -855,6 +864,11 @@ public class ExplainPlanAttributes { return this; } + public ExplainPlanAttributesBuilder setFunctionalMatch(String functionalMatch) { + this.functionalMatch = functionalMatch; + return this; + } + public ExplainPlanAttributesBuilder setIndexRule(String indexRule) { this.indexRule = indexRule; return this; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index b955437b83..08076542c4 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -550,6 +550,7 @@ public abstract class BaseQueryPlan implements QueryPlan { OptimizerDecision decision = getOptimizerDecision(); if (decision != null) { builder.setIndexRule(decision.getRule()); + builder.setFunctionalMatch(decision.getFunctionalMatch()); builder.setIndexRejected(decision.getRejectedIndexes()); } if (context.isRoot()) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java index 115e88d329..00e2564ab0 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java @@ -376,8 +376,25 @@ public abstract class ExplainTable { if (indexKind != null) { indexLine.append(" ").append(indexKind); } - if (decision != null && !isDefaultRule(decision.getRule())) { - indexLine.append(" /* ").append(decision.getRule()).append(" */"); + if (decision != null) { + // Disclose the selection rule (unless it is a suppressed default) and, when the chosen plan + // is a functional index that matched a query expression, the separate "matches <expr>" + // disclosure. Both may appear together as "/* <rule>, matches <expr> */". + boolean showRule = !isDefaultRule(decision.getRule()); + String functionalMatch = decision.getFunctionalMatch(); + if (showRule || functionalMatch != null) { + indexLine.append(" /* "); + if (showRule) { + indexLine.append(decision.getRule()); + } + if (functionalMatch != null) { + if (showRule) { + indexLine.append(", "); + } + indexLine.append(functionalMatch); + } + indexLine.append(" */"); + } } planSteps.add(indexLine.toString()); if (verbose && decision != null) { @@ -772,33 +789,38 @@ public abstract class ExplainTable { private void getRegionLocations(List<String> planSteps, ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List<HRegionLocation> regionLocations) { - // Region locations are emitted as text and in the structured attributes only when the - // EXPLAIN statement requested them via the REGIONS option (or the legacy WITH REGIONS alias). - if (!context.getExplainOptions().isRegions()) { + // Region locations are computed during scan planning, so always record them in the structured + // attributes. Consumers that read the attributes directly (e.g. the connection activity logger, + // and JSON output) rely on them being present. Only build and append the (potentially large) + // text representation to the plan steps when the EXPLAIN statement requested them via the + // REGIONS option (or the legacy WITH REGIONS alias). + RegionLocationsExplainInfo regionLocationsInfo = + populateRegionLocationAttributes(explainPlanAttributesBuilder, regionLocations); + if (regionLocationsInfo == null || !context.getExplainOptions().isRegions()) { return; } - String regionLocationPlan = - getRegionLocationsForExplainPlan(explainPlanAttributesBuilder, regionLocations); + String regionLocationPlan = renderRegionLocationsForExplainPlan(regionLocationsInfo); if (regionLocationPlan.length() > 0) { planSteps.add(regionLocationPlan); } } /** - * Retrieve region locations from hbase client and set the values for the explain plan output. If - * the list of region locations exceed max limit, print only list with the max limit and print num - * of total list size. + * Deduplicate the region locations by region boundary, trim to the configured max size, and + * record the result (and the total size) in the structured explain plan attributes. This is + * always done so that consumers reading the attributes directly have the values available, + * regardless of whether the text representation is requested. * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance. * @param regionLocationsFromResultIterator region locations. - * @return region locations to be added to the explain plan output. + * @return the deduplicated (and possibly trimmed) region locations along with the total + * deduplicated size, or {@code null} when no region locations were available. */ - private String getRegionLocationsForExplainPlan( + private RegionLocationsExplainInfo populateRegionLocationAttributes( ExplainPlanAttributesBuilder explainPlanAttributesBuilder, List<HRegionLocation> regionLocationsFromResultIterator) { if (regionLocationsFromResultIterator == null) { - return ""; + return null; } - StringBuilder buf = new StringBuilder().append(REGION_LOCATIONS); Set<String> regionBoundaries = new LinkedHashSet<>(); List<HRegionLocation> regionLocations = new ArrayList<>(); for (HRegionLocation regionLocation : regionLocationsFromResultIterator) { @@ -811,29 +833,62 @@ public abstract class ExplainTable { int maxLimitRegionLoc = context.getConnection().getQueryServices().getConfiguration().getInt( QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN); - if (regionLocations.size() > maxLimitRegionLoc) { - int originalSize = regionLocations.size(); - List<HRegionLocation> trimmedRegionLocations = regionLocations.subList(0, maxLimitRegionLoc); - if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder - .setRegionLocations(Collections.unmodifiableList(trimmedRegionLocations)) - .setRegionLocationsTotalSize(originalSize); - } - buf.append(trimmedRegionLocations); + int totalSize = regionLocations.size(); + List<HRegionLocation> trimmedRegionLocations = totalSize > maxLimitRegionLoc + ? regionLocations.subList(0, maxLimitRegionLoc) + : regionLocations; + if (explainPlanAttributesBuilder != null) { + explainPlanAttributesBuilder + .setRegionLocations(Collections.unmodifiableList(trimmedRegionLocations)) + .setRegionLocationsTotalSize(totalSize); + } + return new RegionLocationsExplainInfo(trimmedRegionLocations, totalSize); + } + + /** + * Render the region locations text for the explain plan output. If the region locations were + * trimmed (i.e. the deduplicated total exceeded the configured max limit), the trimmed list is + * printed followed by the total deduplicated size. + * @param regionLocationsInfo the deduplicated (and possibly trimmed) region locations along with + * the total deduplicated size. + * @return region locations to be added to the explain plan output. + */ + private String + renderRegionLocationsForExplainPlan(RegionLocationsExplainInfo regionLocationsInfo) { + List<HRegionLocation> trimmedRegionLocations = regionLocationsInfo.getTrimmedRegionLocations(); + StringBuilder buf = new StringBuilder().append(REGION_LOCATIONS); + buf.append(trimmedRegionLocations); + if (trimmedRegionLocations.size() < regionLocationsInfo.getTotalSize()) { buf.append("...total size = "); - buf.append(originalSize); - } else { - buf.append(regionLocations); - if (explainPlanAttributesBuilder != null) { - explainPlanAttributesBuilder - .setRegionLocations(Collections.unmodifiableList(regionLocations)) - .setRegionLocationsTotalSize(regionLocations.size()); - } + buf.append(regionLocationsInfo.getTotalSize()); } buf.append(") "); return buf.toString(); } + /** + * Holder for the deduplicated (and possibly trimmed) region locations plus the total deduplicated + * size, used to keep attribute population separate from text rendering. + */ + private static final class RegionLocationsExplainInfo { + private final List<HRegionLocation> trimmedRegionLocations; + private final int totalSize; + + private RegionLocationsExplainInfo(List<HRegionLocation> trimmedRegionLocations, + int totalSize) { + this.trimmedRegionLocations = trimmedRegionLocations; + this.totalSize = totalSize; + } + + private List<HRegionLocation> getTrimmedRegionLocations() { + return trimmedRegionLocations; + } + + private int getTotalSize() { + return totalSize; + } + } + @SuppressWarnings("rawtypes") private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex, boolean changeViewIndexId) { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java index be137a0bca..ef9af32eb7 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/OptimizerDecision.java @@ -22,21 +22,18 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -/** - * Records the optimizer's index selection rationale: the chosen index (or data table), the rule - * that selected it, and the indexes that were considered but rejected. The {@code rule} is one of - * the closed-set {@code RULE_*} labels in {@link OptimizerReasons}. The entry for each rejected - * index carries a {@code REASON_*} label. - */ +/** Records the optimizer's index selection rationale. */ public final class OptimizerDecision { private final String chosenIndex; private final String rule; + private final String functionalMatch; private final List<RejectedIndexEntry> rejectedIndexes; - public OptimizerDecision(String chosenIndex, String rule, + public OptimizerDecision(String chosenIndex, String rule, String functionalMatch, List<RejectedIndexEntry> rejectedIndexes) { this.chosenIndex = chosenIndex; this.rule = rule; + this.functionalMatch = functionalMatch; this.rejectedIndexes = rejectedIndexes == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(rejectedIndexes)); @@ -50,6 +47,13 @@ public final class OptimizerDecision { return rule; } + /** + * The functional-index match disclosure of the form {@code "matches <expr>"}, or {@code null}. + */ + public String getFunctionalMatch() { + return functionalMatch; + } + /** Never null; unmodifiable; possibly empty. */ public List<RejectedIndexEntry> getRejectedIndexes() { return rejectedIndexes; @@ -65,17 +69,18 @@ public final class OptimizerDecision { } OptimizerDecision that = (OptimizerDecision) o; return Objects.equals(chosenIndex, that.chosenIndex) && Objects.equals(rule, that.rule) + && Objects.equals(functionalMatch, that.functionalMatch) && rejectedIndexes.equals(that.rejectedIndexes); } @Override public int hashCode() { - return Objects.hash(chosenIndex, rule, rejectedIndexes); + return Objects.hash(chosenIndex, rule, functionalMatch, rejectedIndexes); } @Override public String toString() { - return "OptimizerDecision{chosenIndex=" + chosenIndex + ", rule=" + rule + ", rejectedIndexes=" - + rejectedIndexes + "}"; + return "OptimizerDecision{chosenIndex=" + chosenIndex + ", rule=" + rule + ", functionalMatch=" + + functionalMatch + ", rejectedIndexes=" + rejectedIndexes + "}"; } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java index 20711d5412..04d25dfd13 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java @@ -964,13 +964,13 @@ public class QueryOptimizer { * can use this inline at a {@code return} site. */ private static QueryPlan recordDecision(QueryPlan winner, String rule, DecisionState state) { - String functionalRule = functionalIndexRule(winner); - if (functionalRule != null) { - rule = functionalRule; - } + // The rule names the selection reason (e.g. hint, more bound PK columns). When the winner is a + // functional index that matched a query expression, the "matches <expr>" disclosure is + // recorded separately so both the selection reason and the functional match are surfaced. + String functionalMatch = functionalIndexRule(winner); winner.setOptimizerDecision( new OptimizerDecision(winner.getTableRef().getTable().getTableName().getString(), rule, - state == null ? null : state.getRejections())); + functionalMatch, state == null ? null : state.getRejections())); recordFunctionalIndexExpressionBreadcrumbs(winner); return winner; } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java index a862df07ad..2d5de2dbed 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualGenerateIT.java @@ -74,6 +74,11 @@ public class GlobalIndexCheckerEventualGenerateIT extends GlobalIndexCheckerIT { Thread.sleep(18000); } + @Override + protected boolean isEventualConsistency() { + return true; + } + @Parameterized.Parameters(name = "async={0},encoded={1}") public static synchronized Collection<Object[]> data() { List<Object[]> list = Lists.newArrayListWithExpectedSize(4); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java index d7b5a35e1c..a54d5903be 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerEventualIT.java @@ -74,6 +74,11 @@ public class GlobalIndexCheckerEventualIT extends GlobalIndexCheckerIT { Thread.sleep(15000); } + @Override + protected boolean isEventualConsistency() { + return true; + } + @Parameterized.Parameters(name = "async={0},encoded={1}") public static synchronized Collection<Object[]> data() { List<Object[]> list = Lists.newArrayListWithExpectedSize(4); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java index f0ffe1fd9f..1c6384f22c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java @@ -153,6 +153,24 @@ public class GlobalIndexCheckerIT extends BaseTest { // Verify the query is served by a RANGE SCAN over the index table not the data table. ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); assertPlan(attributes).scanType("RANGE SCAN").indexRule(expectedRule); + assertScannedTableIsIndex(attributes, indexTableFullName); + } + + /** + * Asserts the selection {@code expectedRule} and the separate + * {@code "matches <expectedFunctionalMatchExpr>"} functional index match. + */ + public static void assertExplainPlan(Connection conn, String selectSql, String dataTableFullName, + String indexTableFullName, String expectedRule, String expectedFunctionalMatchExpr) + throws SQLException { + ExplainPlanAttributes attributes = getExplainAttributes(conn, selectSql); + assertPlan(attributes).scanType("RANGE SCAN").indexRule(expectedRule) + .functionalMatch(expectedFunctionalMatchExpr); + assertScannedTableIsIndex(attributes, indexTableFullName); + } + + private static void assertScannedTableIsIndex(ExplainPlanAttributes attributes, + String indexTableFullName) { String actualTable = attributes.getTableName() == null ? null : attributes.getTableName().replaceAll(":", "."); String expectedTable = SchemaUtil.normalizeIdentifier(indexTableFullName); @@ -249,7 +267,8 @@ public class GlobalIndexCheckerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -268,7 +287,8 @@ public class GlobalIndexCheckerIT extends BaseTest { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -281,7 +301,8 @@ public class GlobalIndexCheckerIT extends BaseTest { + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -293,10 +314,15 @@ public class GlobalIndexCheckerIT extends BaseTest { String noIndexQuery = "SELECT /*+ NO_INDEX */ val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; - // Verify that we will read from the data table + // Verify that we will read from the data table. The NO_INDEX hint rejects every + // secondary index candidate. Under STRONG consistency only the user index exists. + // Under EVENTUAL consistency the user index is paired with an auto-created CDC index, + // so two candidates are rejected. Match the user index by name rather than position + // since the rejection order is not guaranteed. + int expectedRejected = isEventualConsistency() ? 2 : 1; assertPlan(conn, noIndexQuery).scanType("FULL SCAN").table(dataTableName) - .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedCount(1) - .indexRejected(0, indexTableName, OptimizerReasons.REASON_EXCLUDED_BY_NO_INDEX_HINT); + .indexRule(OptimizerReasons.RULE_DATA_TABLE).indexRejectedCount(expectedRejected) + .indexRejectedContains(indexTableName, OptimizerReasons.REASON_EXCLUDED_BY_NO_INDEX_HINT); rs = conn.createStatement().executeQuery(noIndexQuery); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -316,7 +342,8 @@ public class GlobalIndexCheckerIT extends BaseTest { query = "SELECT val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -346,7 +373,8 @@ public class GlobalIndexCheckerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertExplainPlan(conn, query, dataTableName, indexTableName, + OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("ab", rs.getString(1)); @@ -383,7 +411,10 @@ public class GlobalIndexCheckerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; - assertExplainPlan(conn, query, dataTableName, indexTableName, OptimizerReasons.RULE_HINT); + // This query carries an explicit INDEX hint, so the selection rule is "hint". The functional + // index match over PHOENIX_ROW_TIMESTAMP() is disclosed separately. + assertExplainPlan(conn, query, dataTableName, indexTableName, OptimizerReasons.RULE_HINT, + "PHOENIX_ROW_TIMESTAMP()"); waitForEventualConsistency(); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -1538,7 +1569,7 @@ public class GlobalIndexCheckerIT extends BaseTest { assertPlan(attributes).table(indexName); assertNotNull("expected a server distinct prefix filter, plan=" + attributes, attributes.getServerDistinctFilter()); - List actualValues = Lists.newArrayList(); + List<String> actualValues = Lists.newArrayList(); while (rs.next()) { actualValues.add(rs.getString(1)); } @@ -1877,7 +1908,7 @@ public class GlobalIndexCheckerIT extends BaseTest { assertPlan(attributes).table(indexTableName); assertNotNull("expected a server distinct prefix filter, plan=" + attributes, attributes.getServerDistinctFilter()); - List actualValues = Lists.newArrayList(); + List<String> actualValues = Lists.newArrayList(); while (rs.next()) { actualValues.add(rs.getString(1)); } @@ -1901,6 +1932,10 @@ public class GlobalIndexCheckerIT extends BaseTest { protected void waitForEventualConsistency() throws Exception { } + protected boolean isEventualConsistency() { + return false; + } + protected void verifyTableHealth(Connection conn, String dataTableName, String indexTableName) throws Exception { // Add two rows and check everything is still okay diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java index 2ca74c462f..9b0d4c20f5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexUsageIT.java @@ -135,7 +135,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { .iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true) .serverAggregate("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY " + "[TO_BIGINT(\"(A.INT_COL1 + B.INT_COL2)\")]") - .indexRuleMatches("(A.INT_COL1 + B.INT_COL2)").indexRejectedNone(); + .functionalMatch("(A.INT_COL1 + B.INT_COL2)").indexRejectedNone(); if (localIndex) { basePlan.scanType("RANGE SCAN") .table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")") @@ -192,7 +192,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { "SERVER DISTINCT PREFIX FILTER OVER " + "[TO_BIGINT(\"(A.INT_COL1 + 1)\")]") .serverAggregate( "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY " + "[TO_BIGINT(\"(A.INT_COL1 + 1)\")]") - .indexRuleMatches("(A.INT_COL1 + 1)").indexRejectedNone(); + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); if (localIndex) { basePlan.table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")") .keyRanges(" [1,0] - [1,*]").clientSortAlgo("CLIENT MERGE SORT"); @@ -247,7 +247,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, sql) .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").serverFirstKeyOnlyProjection(true) - .indexRuleMatches("(A.INT_COL1 + 1)").indexRejectedNone(); + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); if (localIndex) { basePlan.table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")") .keyRanges(" [1,2]").clientSortAlgo("CLIENT MERGE SORT"); @@ -301,7 +301,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, sql).iteratorType("PARALLEL 1-WAY").serverFirstKeyOnlyProjection(true) - .indexRuleMatches("(A.INT_COL1 + 1)").indexRejectedNone(); + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); if (localIndex) { basePlan.scanType("RANGE SCAN") .table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")").keyRanges(" [1]") @@ -376,7 +376,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { + " WHERE (\"V1\" || '_' || \"v2\") = 'x_1'"; ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") - .indexRuleMatches("(\"cf1\".\"V1\" || '_' || \"CF2\".\"v2\")").indexRejectedNone(); + .functionalMatch("(\"cf1\".\"V1\" || '_' || \"CF2\".\"v2\")").indexRejectedNone(); if (localIndex) { basePlan.table(indexName + "(" + dataTableName + ")").keyRanges(" [1,'x_1']") .clientSortAlgo("CLIENT MERGE SORT"); @@ -401,7 +401,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { "SELECT \"V1\", \"V1\" as foo1, (\"V1\" || '_' || \"v2\") as foo, (\"V1\" || '_' || \"v2\") as \"Foo1\", (\"V1\" || '_' || \"v2\") FROM " + dataTableName + " ORDER BY foo"; basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY") - .indexRuleMatches("(\"cf1\".\"V1\" || '_' || \"CF2\".\"v2\")").indexRejectedNone(); + .functionalMatch("(\"cf1\".\"V1\" || '_' || \"CF2\".\"v2\")").indexRejectedNone(); if (localIndex) { basePlan.scanType("RANGE SCAN").table(indexName + "(" + dataTableName + ")") .keyRanges(" [1]").clientSortAlgo("CLIENT MERGE SORT"); @@ -482,7 +482,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { basePlan.scanType("RANGE SCAN") .table("INDEX_TEST." + indexName + "(" + fullDataTableName + ")").keyRanges(" [1,2]") .clientSortAlgo("CLIENT MERGE SORT").serverFirstKeyOnlyProjection(true) - .indexRuleMatches("(A.INT_COL1 + 1)").indexRejectedNone(); + .functionalMatch("(A.INT_COL1 + 1)").indexRejectedNone(); } else { basePlan.scanType("FULL SCAN").table(fullDataTableName).clientSortAlgo(null) .serverWhereFilter("SERVER FILTER BY (A.INT_COL1 + 1) = 2") @@ -537,7 +537,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { String query = "SELECT k1, k2, k3, s1, s2 FROM " + viewName + " WHERE k1+k2+k3 = 173.0"; ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") - .indexRuleMatches("(K1 + K2 + K3)").indexRejectedNone(); + .functionalMatch("(K1 + K2 + K3)").indexRejectedNone(); if (local) { basePlan.table(indexName1 + "(" + dataTableName + ")").keyRanges(" [1,173]") .clientSortAlgo("CLIENT MERGE SORT"); @@ -560,7 +560,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { query = "SELECT k1, k2, s1||'_'||s2 FROM " + viewName + " WHERE (s1||'_'||s2)='foo2_bar2'"; basePlan = assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") - .serverFirstKeyOnlyProjection(true).indexRuleMatches("(S1 || '_' || S2)") + .serverFirstKeyOnlyProjection(true).functionalMatch("(S1 || '_' || S2)") .indexRejectedCount(1); if (local) { basePlan.table(indexName2 + "(" + dataTableName + ")") @@ -625,7 +625,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { assertPlan(conn, query).iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN") .table(indexName2).keyRanges(" [1,'abc_cab','foo']").serverFirstKeyOnlyProjection(true) - .indexRuleMatches("(S2 || '_' || S3)").indexRejectedCount(1); + .functionalMatch("(S2 || '_' || S3)").indexRejectedCount(1); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -724,7 +724,7 @@ public class IndexUsageIT extends ParallelStatsDisabledIT { ExplainPlanTestUtil.ExplainPlanAssert basePlan = assertPlan(conn, query) .iteratorType("PARALLEL 1-WAY").scanType("RANGE SCAN").serverFirstKeyOnlyProjection(true) - .indexRuleMatches("REGEXP_SUBSTR(V,'id:\\\\w+')").indexRejectedNone(); + .functionalMatch("REGEXP_SUBSTR(V,'id:\\\\w+')").indexRejectedNone(); if (localIndex) { basePlan.table(indexName + "(" + dataTableName + ")").keyRanges(" [1,'id:id1']") .clientSortAlgo("CLIENT MERGE SORT"); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java index 5be82ccdd9..d6c716e438 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScanner2IT.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; @@ -63,6 +64,7 @@ import org.apache.phoenix.exception.PhoenixParserException; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; @@ -325,7 +327,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -346,7 +348,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -363,7 +365,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -395,7 +397,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -424,7 +426,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -537,7 +539,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -558,7 +560,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -573,7 +575,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -604,7 +606,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -631,7 +633,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -709,6 +711,22 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { } } + /** + * Asserts the query is served by the index, disclosing the optimizer's selection rule (and, for a + * functional index, the separate {@code matches <expr>}). + */ + private void assertIndexPlan(Connection conn, String sql, String dataTableName, + String indexTableName, String functionalExpr) throws SQLException { + String rule = sql.contains("/*+ INDEX(") + ? OptimizerReasons.RULE_HINT + : OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS; + if (functionalExpr == null) { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule); + } else { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule, functionalExpr); + } + } + @Test public void testUncoveredQuery() throws Exception { String dataTableName = generateUniqueName(); @@ -764,7 +782,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); @@ -784,7 +802,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0' GROUP BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -802,7 +820,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0'"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(3, rs.getInt(1)); @@ -817,7 +835,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { selectSql = "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -862,11 +880,11 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { // is not included by the index table selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } else { // Verify that an index hint is not necessary for an uncovered index selectSql = "SELECT val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } ResultSet rs = conn.createStatement().executeQuery(selectSql); @@ -958,7 +976,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "Count(v3) from " + dataTableName + " where v1 = 5"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(count, rs.getInt(1)); @@ -996,7 +1014,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "val2, val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the first index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); moveRegionsOfTable(dataTableName); @@ -1021,7 +1039,7 @@ public class UncoveredGlobalIndexRegionScanner2IT extends BaseTest { String selectSql = "SELECT id from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java index 79772ff1f0..642b25c6c2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/UncoveredGlobalIndexRegionScannerIT.java @@ -28,6 +28,7 @@ import static org.junit.Assert.fail; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; import java.util.Arrays; @@ -45,6 +46,7 @@ import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.exception.PhoenixParserException; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.hbase.index.IndexRegionObserver; +import org.apache.phoenix.optimize.OptimizerReasons; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; @@ -192,7 +194,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -212,7 +214,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -228,7 +230,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + " WHERE val1 = 'bc' AND " + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -259,7 +261,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -286,7 +288,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -373,7 +375,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() < TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -392,7 +394,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { conn.createStatement() .execute("upsert into " + dataTableName + " values ('c', 'bc', 'ccc', 'cccc')"); conn.commit(); - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -407,7 +409,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + after + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("bc", rs.getString(1)); @@ -438,7 +440,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + " val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE val1 = 'de'"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("de", rs.getString(1)); @@ -465,7 +467,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')"; // Verify that we will read from the index table - assertExplainPlan(conn, query, dataTableName, indexTableName); + assertIndexPlan(conn, query, dataTableName, indexTableName, "PHOENIX_ROW_TIMESTAMP()"); rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); @@ -529,6 +531,22 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { } } + /** + * Asserts the query is served by the index, disclosing the optimizer's selection rule (and, for a + * functional index, the separate {@code matches <expr>}). + */ + private void assertIndexPlan(Connection conn, String sql, String dataTableName, + String indexTableName, String functionalExpr) throws SQLException { + String rule = sql.contains("/*+ INDEX(") + ? OptimizerReasons.RULE_HINT + : OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS; + if (functionalExpr == null) { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule); + } else { + assertExplainPlan(conn, sql, dataTableName, indexTableName, rule, functionalExpr); + } + } + @Test public void testUncoveredQuery() throws Exception { String dataTableName = generateUniqueName(); @@ -582,7 +600,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("b", rs.getString(1)); @@ -602,7 +620,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0' GROUP BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); @@ -616,7 +634,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { selectSql = "SELECT count(val3) from " + dataTableName + " where val1 > '0'"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(3, rs.getInt(1)); @@ -631,7 +649,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { selectSql = "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1"; } // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("abcd", rs.getString(1)); @@ -670,11 +688,11 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { // is not included by the index table selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } else { // Verify that an index hint is not necessary for an uncovered index selectSql = "SELECT val4 from " + dataTableName + " WHERE val1 = 'bc' AND val2 = 'bcdd'"; - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); } ResultSet rs = conn.createStatement().executeQuery(selectSql); @@ -756,7 +774,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "Count(v3) from " + dataTableName + " where v1 = 5"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals(count, rs.getInt(1)); @@ -792,7 +810,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { "SELECT" + (uncovered ? " " : "/*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ ") + "val2, val3 from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the first index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("abc", rs.getString(1)); @@ -813,7 +831,7 @@ public class UncoveredGlobalIndexRegionScannerIT extends BaseTest { String selectSql = "SELECT id from " + dataTableName + " WHERE val1 = 'ab'"; // Verify that we will read from the index table - assertExplainPlan(conn, selectSql, dataTableName, indexTableName); + assertIndexPlan(conn, selectSql, dataTableName, indexTableName, null); ResultSet rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); assertEquals("a", rs.getString(1)); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java index d6eddcda1e..ce5bdaf950 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/json/JsonFunctionsIT.java @@ -369,7 +369,7 @@ public class JsonFunctionsIT extends ParallelStatsDisabledIT { "SELECT JSON_VALUE(JSONCOL,'$.type'), " + "JSON_VALUE(JSONCOL,'$.info.address.town') FROM " + tableName + " WHERE JSON_VALUE(JSONCOL,'$.type') = 'Basic'"; assertPlan(conn, selectSql).scanType("RANGE SCAN").table(indexName) - .indexRuleMatches("JSON_VALUE(JSONCOL.JSONCOL,'$.type')").indexRejectedNone(); + .functionalMatch("JSON_VALUE(JSONCOL.JSONCOL,'$.type')").indexRejectedNone(); // Validate the total count of rows String countSql = "SELECT COUNT(1) FROM " + tableName; ResultSet rs = conn.createStatement().executeQuery(countSql); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java index 29d609ba1c..e178d4eef6 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java @@ -521,10 +521,11 @@ public class QueryOptimizerTest extends BaseConnectionlessQueryTest { "create table fe_match (id varchar not null primary key, name varchar, val varchar)"); conn.createStatement().execute("create index fe_match_upper_idx on fe_match (UPPER(name))"); // The functional index's indexed expression matches the query's UPPER(NAME) path expression, - // so the chosen index's rule is overridden to the "matches <expr>" form. Assert on the rule - // prefix. + // so the chosen index carries a separate "matches <expr>" functional match disclosure with + // its selection rule. assertPlan(conn, "select id from fe_match where UPPER(name) = 'ABC'") - .table("FE_MATCH_UPPER_IDX").indexRule(OptimizerReasons.matches("UPPER(NAME)")); + .table("FE_MATCH_UPPER_IDX").indexRule(OptimizerReasons.RULE_MORE_BOUND_PK_COLUMNS) + .functionalMatch("UPPER(NAME)"); } @Test diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java index 72fa018b75..50764d2793 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTest.java @@ -1001,10 +1001,11 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { /** * A query whose path expression matches a covered functional index's indexed expression must - * choose that index, label the disclosed rule {@code "matches <expr>"}, and emit exactly one - * {@code INDEX EXPRESSION <expr> AS <col>} rewrite breadcrumb on the chosen plan's context. The - * breadcrumb is rendered as a {@code REWRITE INDEX EXPRESSION ...} top-of-plan line in plain - * EXPLAIN text and as a single entry in the structured {@code rewrites} attribute. + * choose that index, disclose the separate {@code "matches <expr>"} functional match label, and + * emit exactly one {@code INDEX EXPRESSION <expr> AS <col>} rewrite breadcrumb on the chosen + * plan's context. The breadcrumb is rendered as a {@code REWRITE INDEX EXPRESSION ...} + * top-of-plan line in plain EXPLAIN text and as a single entry in the structured {@code rewrites} + * attribute. */ @Test public void testIndexExpressionRewriteEmittedForChosenFunctionalIndex() throws Exception { @@ -1018,9 +1019,10 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { String query = "SELECT BSON_VALUE(payload, 'k', 'VARCHAR') FROM " + base + " WHERE BSON_VALUE(payload, 'k', 'VARCHAR') = 'x'"; - // The functional index is chosen and the rule comment names the matched expression. + // The functional index is chosen and the separate functional match disclosure names the + // matched expression. ExplainPlanTestUtil.assertPlan(conn, query).indexName(idx) - .indexRuleMatches("BSON_VALUE(PAYLOAD,'k','VARCHAR')") + .functionalMatch("BSON_VALUE(PAYLOAD,'k','VARCHAR')") // Exactly one breadcrumb (one applied substitution; no eager per-PK-column emissions). .rewriteCount(1).rewrite(0, "INDEX EXPRESSION BSON_VALUE(PAYLOAD,'k','VARCHAR') AS \":BSON_VALUE(PAYLOAD,'k','VARCHAR')\""); @@ -1837,6 +1839,7 @@ public class ExplainPlanTest extends BaseConnectionlessQueryTest { n.putNull("indexName"); n.putNull("indexKind"); n.putNull("indexRule"); + n.putNull("functionalMatch"); n.putNull("indexRejected"); n.putNull("saltBuckets"); n.putNull("regionsPlanned"); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java index 711f5dcd53..135180db83 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/explain/ExplainPlanTestUtil.java @@ -256,11 +256,20 @@ public final class ExplainPlanTestUtil { } /** - * Assert the optimizer chose a functional index whose rule label is exactly - * {@code "matches <expression>"} (see {@link OptimizerReasons#matches(String)}). + * Assert the optimizer's separate functional-index match disclosure equals + * {@code "matches <expression>"} (see {@link OptimizerReasons#matches(String)}). The selection + * {@code indexRule} is disclosed independently and is asserted with {@link #indexRule(String)}. */ - public ExplainPlanAssert indexRuleMatches(String expression) { - return indexRule(OptimizerReasons.matches(expression)); + public ExplainPlanAssert functionalMatch(String expression) { + assertEquals(at("functionalMatch"), OptimizerReasons.matches(expression), + attributes.getFunctionalMatch()); + return this; + } + + /** Assert no functional-index match disclosure was recorded for this plan. */ + public ExplainPlanAssert functionalMatchNone() { + assertEquals(at("functionalMatch"), null, attributes.getFunctionalMatch()); + return this; } /** Assert the number of rejected index candidates recorded for this plan. */
