From 9dfce9e88e9a2648e1f569f3913abdc12751e3f4 Mon Sep 17 00:00:00 2001 From: Alexandre Felipe Date: Thu, 6 Aug 2026 21:50:44 +0100 Subject: [PATCH-v1 1/2] SKIP-MERGE Implementation This commit introduces skip-merge plan, it is not itself a plan node but a strategy that combines index scans and merge append nodes to take advantage of multi-column ordered indices to satisfy both filtering and (partially) query order. If an index have a prefix constrained to a discrete set of values and a suffix that matches the query order. the output can be produced by merging ordered disjoing subsets of the data produced by separate index scans, and combined by a Merge Append node. This also adds parameters enable_indexskip_merge and max_index_merge_scans. max_index_merge_scans limits the number of concurrent index scan nodes that can be produced. It is added to QUERY_TUNING_COST group, but maybe there is a better group for it. --- src/backend/optimizer/path/costsize.c | 2 + src/backend/optimizer/path/indxpath.c | 335 +++++++++++++++++++++- src/backend/utils/misc/guc_parameters.dat | 15 + src/include/optimizer/cost.h | 3 + 4 files changed, 353 insertions(+), 2 deletions(-) diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index ac523ecf9a8..65d468c0579 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -145,6 +145,7 @@ int max_parallel_workers_per_gather = 2; bool enable_seqscan = true; bool enable_indexscan = true; +bool enable_index_skip_merge = true; bool enable_indexonlyscan = true; bool enable_bitmapscan = true; bool enable_tidscan = true; @@ -166,6 +167,7 @@ bool enable_partition_pruning = true; bool enable_presorted_aggregate = true; bool enable_async_append = true; +int max_index_merge_scans = 32; typedef struct { PlannerInfo *root; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 3f5d4fa3182..c44ff59f614 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -15,6 +15,7 @@ */ #include "postgres.h" +#include "access/cmptype.h" #include "access/stratnum.h" #include "access/sysattr.h" #include "access/transam.h" @@ -33,10 +34,10 @@ #include "optimizer/placeholder.h" #include "optimizer/prep.h" #include "optimizer/restrictinfo.h" +#include "utils/array.h" #include "utils/lsyscache.h" #include "utils/selfuncs.h" - /* XXX see PartCollMatchesExprColl */ #define IndexCollMatchesExprColl(idxcollation, exprcollation) \ ((idxcollation) == InvalidOid || (idxcollation) == (exprcollation)) @@ -103,6 +104,23 @@ static bool eclass_already_used(EquivalenceClass *parent_ec, Relids oldrelids, static void get_index_paths(PlannerInfo *root, RelOptInfo *rel, IndexOptInfo *index, IndexClauseSet *clauses, List **bitindexpaths); +static void consider_index_skip_merge_path(PlannerInfo *root, RelOptInfo *rel, + IndexOptInfo *index, + IndexClauseSet *clauses); +static IndexClause *make_eq_indexclause_from_saop(PlannerInfo *root, + IndexClause *saop_iclause, + Datum prefix_value, + bool prefix_isnull, + IndexOptInfo *index); +static List *expand_saop_to_eq_clauses(PlannerInfo *root, + IndexClause *iclause, + IndexOptInfo *index); +static bool build_prefix_col_constraints(PlannerInfo *root, + IndexOptInfo *index, + IndexClauseSet *clauses, + int suffix_indexcol, + List ***col_eq_clauses_out, + int *num_prefixes_out); static List *build_index_paths(PlannerInfo *root, RelOptInfo *rel, IndexOptInfo *index, IndexClauseSet *clauses, bool useful_predicate, @@ -769,6 +787,13 @@ get_index_paths(PlannerInfo *root, RelOptInfo *rel, NULL); *bitindexpaths = list_concat(*bitindexpaths, indexpaths); } + + + /* + * Consider index suffix scan: expand IN-list on a prefix column into + * per-value index scans merged with MergeAppend. + */ + consider_index_skip_merge_path(root, rel, index, clauses); } /* @@ -826,7 +851,9 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, bool index_only_scan; int indexcol; - Assert(skip_nonnative_saop != NULL || scantype == ST_BITMAPSCAN); + Assert(skip_nonnative_saop != NULL || + scantype == ST_BITMAPSCAN || + scantype == ST_INDEXSCAN); /* * Check that index supports the desired scan type(s) @@ -4459,3 +4486,307 @@ is_pseudo_constant_for_index(PlannerInfo *root, Node *expr, IndexOptInfo *index) return false; /* no good, volatile comparison value */ return true; } + +/* + * make_eq_indexclause_from_saop + * Build an IndexClause for "leftop = value" from a ScalarArrayOpExpr. + */ +static IndexClause * +make_eq_indexclause_from_saop(PlannerInfo *root, + IndexClause *saop_iclause, + Datum prefix_value, + bool prefix_isnull, + IndexOptInfo *index) +{ + ScalarArrayOpExpr *saop = (ScalarArrayOpExpr *) saop_iclause->rinfo->clause; + Node *leftop = (Node *) linitial(saop->args); + Oid typid = exprType(leftop); + Expr *opexpr; + RestrictInfo *rinfo; + IndexClause *iclause; + + opexpr = make_opclause(saop->opno, BOOLOID, false, + (Expr *) copyObject(leftop), + (Expr *) makeConst(typid, -1, saop->inputcollid, + get_typlen(typid), + prefix_value, prefix_isnull, + get_typbyval(typid)), + InvalidOid, saop->inputcollid); + rinfo = make_restrictinfo(root, opexpr, + true, false, false, true, 0, + index->rel->relids, NULL, NULL); + + iclause = makeNode(IndexClause); + iclause->rinfo = rinfo; + iclause->indexquals = list_make1(rinfo); + iclause->lossy = false; + iclause->indexcol = saop_iclause->indexcol; + iclause->indexcols = NIL; + + return iclause; +} + +/* + * expand_saop_to_eq_clauses + * Expand a useOr ScalarArrayOpExpr into one equality IndexClause per + * array element. Returns NIL unless the array is a non-null Const. + */ +static List * +expand_saop_to_eq_clauses(PlannerInfo *root, IndexClause *iclause, + IndexOptInfo *index) +{ + ScalarArrayOpExpr *saop = (ScalarArrayOpExpr *) iclause->rinfo->clause; + Node *arrayarg = (Node *) lsecond(saop->args); + Const *aconst; + ArrayType *arr; + Datum *values; + bool *nulls; + int nvalues; + int i; + Oid elemtype; + int16 elemlen; + bool elembyval; + char elemalign; + List *result = NIL; + + Assert(saop->useOr); + + if (!IsA(arrayarg, Const)) + return NIL; + + aconst = (Const *) arrayarg; + if (aconst->constisnull) + return NIL; + + arr = DatumGetArrayTypeP(aconst->constvalue); + elemtype = ARR_ELEMTYPE(arr); + get_typlenbyvalalign(elemtype, &elemlen, &elembyval, &elemalign); + deconstruct_array(arr, elemtype, elemlen, elembyval, elemalign, + &values, &nulls, &nvalues); + + for (i = 0; i < nvalues; i++) + result = lappend(result, + make_eq_indexclause_from_saop(root, iclause, + values[i], nulls[i], + index)); + + pfree(values); + pfree(nulls); + return result; +} + +/* + * build_prefix_col_constraints + * For each index column before suffix_indexcol, collect equality + * IndexClauses (expanding IN-lists). Returns false if any prefix column + * lacks a plan-time equality, or if the cartesian product size is outside + * [2, index_suffix_scan_max_prefixes]. + */ +static bool +build_prefix_col_constraints(PlannerInfo *root, + IndexOptInfo *index, + IndexClauseSet *clauses, + int suffix_indexcol, + List ***col_eq_clauses_out, + int *num_prefixes_out) +{ + List **col_clauses; + int col; + int num_prefixes = 1; + + col_clauses = palloc0(suffix_indexcol * sizeof(List *)); + + for (col = 0; col < suffix_indexcol; col++) + { + ListCell *lc; + List *eq_clauses = NIL; + + foreach(lc, clauses->indexclauses[col]) + { + IndexClause *iclause = (IndexClause *) lfirst(lc); + RestrictInfo *rinfo = iclause->rinfo; + + if (IsA(rinfo->clause, ScalarArrayOpExpr)) + { + ScalarArrayOpExpr *saop = (ScalarArrayOpExpr *) rinfo->clause; + + if (!saop->useOr) + continue; /* try another clause on this column */ + + eq_clauses = expand_saop_to_eq_clauses(root, iclause, index); + if (eq_clauses == NIL) + goto fail; + break; + } + if (IsA(rinfo->clause, OpExpr)) + { + /* + * IndexClause is no_copy_equal; share the pointer. We do not + * mutate it, and planner memory context reclaims on failure. + */ + eq_clauses = list_make1(iclause); + break; + } + } + + if (eq_clauses == NIL) + goto fail; + + /* Reject early (and avoid int overflow) once over the GUC limit. */ + if (list_length(eq_clauses) > max_index_merge_scans / num_prefixes) + goto fail; + + col_clauses[col] = eq_clauses; + num_prefixes *= list_length(eq_clauses); + } + + if (num_prefixes < 2) + goto fail; + + *col_eq_clauses_out = col_clauses; + *num_prefixes_out = num_prefixes; + return true; + +fail: + /* Shallow free only: lists may share IndexClauses with the input set. */ + for (col = 0; col < suffix_indexcol; col++) + list_free(col_clauses[col]); + pfree(col_clauses); + return false; +} + +/* + * consider_index_skip_merge_path + * Build a MergeAppend of per-prefix IndexPaths when the query has + * equality/IN on leading index columns and ORDER BY on a later column. + */ +static void +consider_index_skip_merge_path(PlannerInfo *root, RelOptInfo *rel, + IndexOptInfo *index, IndexClauseSet *clauses) +{ + List *subpaths = NIL; + List **col_eq_clauses; + PathKey *query_first_pk; + int suffix_indexcol; + int num_prefixes; + int indexcol; + int i; + bool forward; + ScanDirection scandirection; + MergeAppendPath *mapath; + double total_matching_rows = 0; + + if (!enable_index_skip_merge || !enable_indexscan) + return; + + if (index->nkeycolumns < 2 || + index->sortopfamily == NULL || + !index->amhasgettuple || + root->query_pathkeys == NIL || + !bms_is_empty(rel->lateral_relids)) + return; + + /* First non-leading index key matching the leading ORDER BY pathkey. */ + query_first_pk = (PathKey *) linitial(root->query_pathkeys); + suffix_indexcol = -1; + for (indexcol = 1; indexcol < index->nkeycolumns; indexcol++) + { + TargetEntry *indextle = (TargetEntry *) list_nth(index->indextlist, + indexcol); + + if (find_ec_member_matching_expr(query_first_pk->pk_eclass, + indextle->expr, + index->rel->relids) != NULL) + { + suffix_indexcol = indexcol; + break; + } + } + + if (suffix_indexcol < 1) + return; + + /* Match ASC/DESC and nulls ordering (as build_index_pathkeys does). */ + forward = ((query_first_pk->pk_cmptype == COMPARE_GT) == + index->reverse_sort[suffix_indexcol]); + scandirection = forward ? ForwardScanDirection : BackwardScanDirection; + if (query_first_pk->pk_nulls_first != + (forward ? index->nulls_first[suffix_indexcol] + : !index->nulls_first[suffix_indexcol])) + return; + + if (!build_prefix_col_constraints(root, index, clauses, suffix_indexcol, + &col_eq_clauses, &num_prefixes)) + return; + + for (i = 0; i < num_prefixes; i++) + { + IndexClauseSet child_clauses = *clauses; + List *child_indexpaths; + IndexPath *ipath; + int idx = i; + int col; + + /* Fix each prefix column to one value of the cartesian product. */ + for (col = suffix_indexcol - 1; col >= 0; col--) + { + int n = list_length(col_eq_clauses[col]); + + child_clauses.indexclauses[col] = + list_make1(list_nth(col_eq_clauses[col], idx % n)); + idx /= n; + } + + child_indexpaths = build_index_paths(root, rel, index, &child_clauses, + index->predOK, ST_INDEXSCAN, + NULL); + if (child_indexpaths == NIL) + { + subpaths = NIL; + break; + } + + ipath = (IndexPath *) linitial(child_indexpaths); + + /* + * Prefix columns are fixed, so each child is ordered by the query's + * suffix-first pathkeys even though the index is prefix-first. + */ + ipath->path.pathkeys = root->query_pathkeys; + ipath->indexscandir = scandirection; + + subpaths = lappend(subpaths, ipath); + total_matching_rows += ipath->path.rows; + } + + if (list_length(subpaths) < 2) + { + list_free_deep(subpaths); + return; + } + + mapath = create_merge_append_path(root, rel, subpaths, NIL, + root->query_pathkeys, NULL); + + /* + * With a LIMIT, a suffix scan reads at most (limit + K - 1) index tuples + * across all branches, not every matching row. + */ + if (root->limit_tuples > 0) + { + double merge_rows = root->limit_tuples + list_length(subpaths) - 1; + + if (merge_rows < mapath->path.rows) + { + double ratio = merge_rows / mapath->path.rows; + + mapath->path.total_cost = mapath->path.startup_cost + + (mapath->path.total_cost - mapath->path.startup_cost) * ratio; + mapath->path.rows = merge_rows; + } + } + else + mapath->path.rows = Min(mapath->path.rows, total_matching_rows); + + add_path(rel, (Path *) mapath); +} diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index e2b48ea69e9..95700e8a21f 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -946,6 +946,12 @@ variable => 'enable_indexscan', boot_val => 'true', }, +{ name => 'enable_indexskipmerge', type => 'bool', context => 'PGC_USERSET', group => 'QUERY_TUNING_METHOD', + short_desc => 'Enables the planer\'s use of index skip merge plans.', + flags => 'GUC_EXPLAIN', + variable => 'enable_index_skip_merge', + boot_val => 'true', +}, { name => 'enable_material', type => 'bool', context => 'PGC_USERSET', group => 'QUERY_TUNING_METHOD', short_desc => 'Enables the planner\'s use of materialization.', @@ -2026,6 +2032,15 @@ max => 'INDEX_MAX_KEYS', }, +{ name => 'max_index_merge_scans', type => 'int', context => 'PGC_USERSET', group => 'QUERY_TUNING_COST', + short_desc => 'Set a limit for the number of scans created when considering a skip-merge plan.', + flags => 'GUC_EXPLAIN', + variable => 'max_index_merge_scans', + boot_val => '32', + min => '0', + max => 'INT_MAX', +}, + # See also CheckRequiredParameterValues() if this parameter changes { name => 'max_locks_per_transaction', type => 'int', context => 'PGC_POSTMASTER', group => 'LOCK_MANAGEMENT', short_desc => 'Sets the maximum number of locks per transaction.', diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index bda3f1690c0..5a5d6bb23c9 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -51,6 +51,7 @@ extern PGDLLIMPORT Cost disable_cost; extern PGDLLIMPORT int max_parallel_workers_per_gather; extern PGDLLIMPORT bool enable_seqscan; extern PGDLLIMPORT bool enable_indexscan; +extern PGDLLIMPORT bool enable_index_skip_merge; extern PGDLLIMPORT bool enable_indexonlyscan; extern PGDLLIMPORT bool enable_bitmapscan; extern PGDLLIMPORT bool enable_tidscan; @@ -71,7 +72,9 @@ extern PGDLLIMPORT bool enable_parallel_hash; extern PGDLLIMPORT bool enable_partition_pruning; extern PGDLLIMPORT bool enable_presorted_aggregate; extern PGDLLIMPORT bool enable_async_append; + extern PGDLLIMPORT int constraint_exclusion; +extern PGDLLIMPORT int max_index_merge_scans; extern double index_pages_fetched(double tuples_fetched, BlockNumber pages, double index_pages, PlannerInfo *root); -- 2.53.0