From be0f4c4dc3a97e4075ab80f4b4b685ee5a151d5e Mon Sep 17 00:00:00 2001 From: William Bernbaum Date: Tue, 25 Aug 2026 19:22:36 -0700 Subject: [PATCH 2/4] semijoin-v1-patch-b Co-authored-by: Cursor --- doc/src/sgml/config.sgml | 22 + src/backend/optimizer/README | 48 +++ src/backend/optimizer/plan/analyzejoins.c | 387 ++++++++++++++++++ src/backend/optimizer/plan/planmain.c | 7 + src/backend/utils/misc/guc_parameters.dat | 8 + src/backend/utils/misc/postgresql.conf.sample | 1 + src/include/optimizer/planmain.h | 2 + .../regress/expected/semijoin_conversion.out | 346 ++++++++++++++++ src/test/regress/expected/sysviews.out | 3 +- src/test/regress/parallel_schedule | 5 + src/test/regress/sql/semijoin_conversion.sql | 186 +++++++++ 11 files changed, 1014 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/semijoin_conversion.out create mode 100644 src/test/regress/sql/semijoin_conversion.sql diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 0165eb9ec02..bc707985a5f 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6234,6 +6234,28 @@ ANY num_sync ( + enable_semijoin_conversion (boolean) + + enable_semijoin_conversion configuration parameter + + + + + Enables or disables the optimization that plans an inner join as a + semijoin when the joined relation only determines which rows survive. + The relation must not contribute any values beyond the join + conditions, and the query must not depend on how many times a row is + duplicated. SELECT DISTINCT qualifies, as does + GROUP BY without aggregates. Matching rows + then do not multiply the other inputs. Only queries whose joins are + all inner joins are considered, and one relation at a time, so a + chain of such joins is left alone. The default is + off. + + + + enable_seqscan (boolean) diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README index 78a307cc523..ae6e7966db3 100644 --- a/src/backend/optimizer/README +++ b/src/backend/optimizer/README @@ -273,6 +273,54 @@ such cases don't arise often so it's not clear that it's worth developing a more complicated system. +Converting Filtering Joins to Semijoins +--------------------------------------- + +An inner join sometimes serves only to restrict which rows of the other inputs +survive. The joined relation appears only in the join clauses, while the +target list, the grouping and ordering clauses and the aggregate arguments all +draw on the other inputs. When the query discards duplicate rows, emitting one +output row per match is wasted work. For example, in + + SELECT DISTINCT a.x FROM a, b WHERE b.a_id = a.id + +each A row is emitted once however many B rows match, so B needs a probe for +existence. convert_joins_to_semijoins() adds a JOIN_SEMI SpecialJoinInfo +for B. + +Two conditions must hold. First, the query must discard duplicates. Second, +nothing outside the joins may reference the relation, which is the attr_needed +test join_is_removable() uses. + +DISTINCT qualifies, and so does GROUP BY without aggregates. An aggregate may +count its input rows, so a query containing one is left alone. DISTINCT ON +keeps the duplicates, since the sort order picks the surviving row. So do window +functions, set-returning functions in the target list, row locking, grouping +sets and volatile output expressions. A set operation keeps them too, since +each arm is planned as a separate query level and a UNION above stays invisible +from within an arm. + +Adding the SpecialJoinInfo does not dictate a plan shape. join_is_legal() +still allows the righthand side to be unique-ified and then inner-joined, so the +planner keeps choosing between that shape and a semijoin on cost. What the +transformation removes is the alternative that fans the other inputs out. A +righthand side must be unique-ifiable, and a righthand side already unique for +its join clauses is left alone. + +One relation at a time is lifted. Several joined rels could be lifted together +as one righthand side, but with a selective lefthand side and no index, in a +handful of adversarial cases, the transformation could cost more than +preventing the fanout would save. + +enable_semijoin_conversion controls the transformation. Only queries whose +joins are all plain inner joins are considered, and lateral references and +PlaceHolderVars rule a query out, since either may need a column from a +relation about to stop emitting columns. The joinlist must also be a single +flat list of base relations: beyond join_collapse_limit relations the jointree +stays nested and make_rel_from_joinlist() plans each sub-list on its own, +while a synthesized semijoin needs both of its sides in one sub-list. + + Pulling Up Subqueries --------------------- diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index cbd36f85417..3b09db63af5 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -24,6 +24,7 @@ #include "catalog/pg_class.h" #include "nodes/nodeFuncs.h" +#include "optimizer/clauses.h" #include "optimizer/joininfo.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" @@ -52,6 +53,7 @@ typedef struct } SelfJoinCandidate; bool enable_self_join_elimination; +bool enable_semijoin_conversion; /* local functions */ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); @@ -83,6 +85,13 @@ static bool is_innerrel_unique_for(PlannerInfo *root, static int self_join_candidates_cmp(const void *a, const void *b); static bool replace_relid_callback(Node *node, ChangeVarNodes_context *context); +static bool query_discards_duplicates(PlannerInfo *root); +static bool rel_is_output_irrelevant(PlannerInfo *root, RelOptInfo *rel, + Relids inputrelids); +static Relids semijoin_rhs_component(PlannerInfo *root, Relids pool, int seed); +static List *semijoin_join_clauses(PlannerInfo *root, Relids lhs, Relids rhs); +static bool convert_one_join_to_semijoin(PlannerInfo *root, Relids lhs, + Relids rhs); /* @@ -1102,6 +1111,384 @@ reduce_unique_semijoins(PlannerInfo *root) } } +/* + * convert_joins_to_semijoins + * Represent inner joins that only filter as semijoins instead. + * + * See "Converting Filtering Joins to Semijoins" in src/backend/optimizer/README. + */ +void +convert_joins_to_semijoins(PlannerInfo *root, List *joinlist) +{ + Relids candidates = NULL; + Relids lhs; + int relid; + ListCell *lc; + + if (!enable_semijoin_conversion) + return; + + /* + * Beyond join_collapse_limit relations the jointree stays nested and + * make_rel_from_joinlist() plans each sub-list on its own, while a + * synthesized semijoin needs both sides in one sub-list. + */ + if (list_length(joinlist) != bms_num_members(root->all_baserels)) + return; + foreach(lc, joinlist) + { + if (!IsA(lfirst(lc), RangeTblRef)) + return; + } + + /* Only plain inner joins: an outer join constrains the join order. */ + if (root->join_info_list != NIL) + return; + + /* + * A lifted relation stops emitting columns, which a lateral reference or + * a PlaceHolderVar may need. + */ + if (root->hasLateralRTEs || root->placeholder_list != NIL) + return; + + if (!query_discards_duplicates(root)) + return; + + relid = -1; + while ((relid = bms_next_member(root->all_baserels, relid)) > 0) + { + RelOptInfo *rel = root->simple_rel_array[relid]; + RangeTblEntry *rte = root->simple_rte_array[relid]; + + if (rel == NULL || rel->reloptkind != RELOPT_BASEREL) + continue; + if (relid == root->parse->resultRelation) + continue; + + switch (rte->rtekind) + { + case RTE_RELATION: + case RTE_SUBQUERY: + break; + + default: + continue; + } + if (!bms_is_empty(rel->lateral_relids)) + continue; + + /* A column read above the join disqualifies the candidate. */ + if (!rel_is_output_irrelevant(root, rel, root->all_baserels)) + continue; + + candidates = bms_add_member(candidates, relid); + } + + if (bms_is_empty(candidates)) + return; + + /* Something has to be left to return rows from. */ + lhs = bms_difference(root->all_baserels, candidates); + if (bms_is_empty(lhs)) + return; + + while (!bms_is_empty(candidates)) + { + int seed = bms_next_member(candidates, -1); + Relids rhs = semijoin_rhs_component(root, candidates, seed); + + candidates = bms_del_members(candidates, rhs); + + /* + * Lift one relation only. A semijoin joins its righthand side as a + * unit, and against a selective lefthand side that can cost more than + * the fanout it removes. + */ + if (bms_membership(rhs) != BMS_SINGLETON) + continue; + + convert_one_join_to_semijoin(root, lhs, rhs); + } +} + +/* + * semijoin_rhs_component + * Collect the relations in "pool" that are joined, directly or + * transitively, to the one identified by "seed". + * + * Unconnected relations carry independent existence tests, so one shared + * righthand side would make a cartesian product of them. + */ +static Relids +semijoin_rhs_component(PlannerInfo *root, Relids pool, int seed) +{ + Relids component = bms_make_singleton(seed); + bool grew = true; + + while (grew) + { + int relid = -1; + + grew = false; + while ((relid = bms_next_member(pool, relid)) > 0) + { + RelOptInfo *rel; + bool joined = false; + ListCell *lc; + + if (bms_is_member(relid, component)) + continue; + + rel = root->simple_rel_array[relid]; + + foreach(lc, rel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + if (bms_overlap(rinfo->required_relids, component)) + { + joined = true; + break; + } + } + + /* + * An ordinary "a.x = b.y" clause is absorbed into an equivalence + * class and never reaches joininfo, so sharing a class counts + * too. + */ + if (!joined) + { + foreach(lc, root->eq_classes) + { + EquivalenceClass *ec = (EquivalenceClass *) lfirst(lc); + + if (bms_is_member(relid, ec->ec_relids) && + bms_overlap(ec->ec_relids, component)) + { + joined = true; + break; + } + } + } + + if (joined) + { + component = bms_add_member(component, relid); + grew = true; + } + } + } + + return component; +} + +/* + * semijoin_join_clauses + * Find the clauses that would become the join conditions of a semijoin + * between "lhs" and "rhs". + */ +static List * +semijoin_join_clauses(PlannerInfo *root, Relids lhs, Relids rhs) +{ + List *result = NIL; + Relids joinrelids = bms_union(lhs, rhs); + int relid = -1; + + while ((relid = bms_next_member(rhs, relid)) > 0) + { + RelOptInfo *rel = root->simple_rel_array[relid]; + List *derived; + ListCell *lc; + + derived = generate_join_implied_equalities(root, joinrelids, lhs, + rel, NULL); + foreach(lc, derived) + { + if (!list_member_ptr(result, lfirst(lc))) + result = lappend(result, lfirst(lc)); + } + + foreach(lc, rel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + if (!bms_overlap(rinfo->required_relids, lhs)) + continue; + if (!list_member_ptr(result, rinfo)) + result = lappend(result, rinfo); + } + } + + return result; +} + +/* + * convert_one_join_to_semijoin + * Add the SpecialJoinInfo describing "lhs SEMI rhs", if worthwhile. + * + * Returns true if the semijoin was installed. + */ +static bool +convert_one_join_to_semijoin(PlannerInfo *root, Relids lhs, Relids rhs) +{ + SpecialJoinInfo *sjinfo; + List *rinfos; + List *clauses = NIL; + Relids clause_relids = NULL; + Relids strict_relids = NULL; + Relids min_lefthand; + ListCell *lc; + + rinfos = semijoin_join_clauses(root, lhs, rhs); + + /* + * With no clause tying the sides together the join is a cartesian + * product. + */ + if (rinfos == NIL) + return false; + + foreach(lc, rinfos) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + + clauses = lappend(clauses, rinfo->clause); + clause_relids = bms_add_members(clause_relids, rinfo->clause_relids); + strict_relids = bms_add_members(strict_relids, + find_nonnullable_rels((Node *) rinfo->clause)); + } + + min_lefthand = bms_intersect(clause_relids, lhs); + /* An empty minimum is not allowed. */ + if (bms_is_empty(min_lefthand)) + min_lefthand = bms_copy(lhs); + + sjinfo = makeNode(SpecialJoinInfo); + sjinfo->jointype = JOIN_SEMI; + sjinfo->syn_lefthand = bms_copy(lhs); + sjinfo->syn_righthand = bms_copy(rhs); + sjinfo->min_lefthand = min_lefthand; + sjinfo->min_righthand = bms_copy(rhs); + sjinfo->ojrelid = 0; /* semijoins have no RT index */ + sjinfo->commute_above_l = NULL; + sjinfo->commute_above_r = NULL; + sjinfo->commute_below_l = NULL; + sjinfo->commute_below_r = NULL; + sjinfo->lhs_strict = bms_overlap(strict_relids, lhs); + + compute_semijoin_info(root, sjinfo, clauses); + + /* Without unique-ification the semijoin would be the only shape left. */ + if (!sjinfo->semi_can_btree && !sjinfo->semi_can_hash) + return false; + + /* + * An already-unique righthand side has no duplicates to remove. + * reduce_unique_semijoins() has already run and never revisits a semijoin + * added later, so check here. + */ + if (is_innerrel_unique_for(root, bms_union(lhs, rhs), lhs, + find_base_rel(root, bms_singleton_member(rhs)), + JOIN_SEMI, rinfos, NULL)) + return false; + + root->join_info_list = lappend(root->join_info_list, sjinfo); + + return true; +} + +/* + * query_discards_duplicates + * Does this query level discard duplicate rows? + */ +static bool +query_discards_duplicates(PlannerInfo *root) +{ + Query *parse = root->parse; + + if (parse->commandType != CMD_SELECT) + return false; + + /* A UNION above is planned separately and stays invisible from here. */ + if (parse->setOperations != NULL) + return false; + + /* A window function can count its partition's rows. */ + if (parse->hasWindowFuncs) + return false; + + /* An SRF in the targetlist expands rows after any deduplication. */ + if (parse->hasTargetSRFs) + return false; + + if (parse->hasModifyingCTE) + return false; + + /* Row locking makes the number of scanned rows externally visible. */ + if (parse->rowMarks != NIL) + return false; + + if (parse->groupingSets != NIL) + return false; + + /* A volatile output expression can differ between two copies of a row. */ + if (contain_volatile_functions((Node *) root->processed_tlist)) + return false; + + /* An aggregate can count its input rows. */ + if (parse->hasAggs) + return false; + + /* Under DISTINCT ON, arrival order picks the surviving row of a tie. */ + if (parse->distinctClause != NIL) + return !parse->hasDistinctOn; + + if (parse->groupClause != NIL) + return true; + + return false; +} + +/* + * rel_is_output_irrelevant + * Does this relation do anything besides restrict which rows of the + * other relations survive? + * + * "inputrelids" holds the contemplated join's inputs, and attr_needed must stay + * inside them, as in join_is_removable(). + */ +static bool +rel_is_output_irrelevant(PlannerInfo *root, RelOptInfo *rel, + Relids inputrelids) +{ + int attroff; + ListCell *l; + + if (rel->attr_needed == NULL) + return false; + + for (attroff = rel->max_attr - rel->min_attr; attroff >= 0; attroff--) + { + if (!bms_is_subset(rel->attr_needed[attroff], inputrelids)) + return false; + } + + foreach(l, root->placeholder_list) + { + PlaceHolderInfo *phinfo = (PlaceHolderInfo *) lfirst(l); + + if (bms_overlap(phinfo->ph_lateral, rel->relids)) + return false; + if (!bms_overlap(phinfo->ph_eval_at, rel->relids)) + continue; + if (!bms_is_subset(phinfo->ph_needed, inputrelids)) + return false; + } + + return true; +} /* * rel_supports_distinctness diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 02495e22e24..fe5d84a7cbc 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -241,6 +241,13 @@ query_planner(PlannerInfo *root, */ joinlist = remove_useless_self_joins(root, joinlist); + /* + * Represent inner joins that only filter as semijoins, so that matching + * rows don't fan out the other inputs. This has to follow the join + * removals above, since they can change which relations are referenced. + */ + convert_joins_to_semijoins(root, joinlist); + /* * Now distribute "placeholders" to base rels as needed. This has to be * done after join removal because removal could change whether a diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 3c5e16ad1e7..d4d645e1526 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -1026,6 +1026,14 @@ boot_val => 'true', }, +{ name => 'enable_semijoin_conversion', type => 'bool', context => 'PGC_USERSET', group => 'QUERY_TUNING_METHOD', + short_desc => 'Enables representing filtering inner joins as semijoins.', + long_desc => 'When a joined relation only restricts which rows survive and nothing can observe how often rows are duplicated, the join is planned as a semijoin so that matching rows do not fan out the other inputs.', + flags => 'GUC_EXPLAIN', + variable => 'enable_semijoin_conversion', + boot_val => 'false', +}, + { name => 'enable_seqscan', type => 'bool', context => 'PGC_USERSET', group => 'QUERY_TUNING_METHOD', short_desc => 'Enables the planner\'s use of sequential-scan plans.', flags => 'GUC_EXPLAIN', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e759f06b50f..516361f3f7f 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -451,6 +451,7 @@ #enable_group_by_reordering = on #enable_distinct_reordering = on #enable_self_join_elimination = on +#enable_semijoin_conversion = off #enable_eager_aggregate = on # - Planner Cost Constants - diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index 850277f2f2f..ff78284fbf4 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -21,6 +21,7 @@ #define DEFAULT_CURSOR_TUPLE_FRACTION 0.1 extern PGDLLIMPORT double cursor_tuple_fraction; extern PGDLLIMPORT bool enable_self_join_elimination; +extern PGDLLIMPORT bool enable_semijoin_conversion; /* query_planner callback to compute query_pathkeys */ typedef void (*query_pathkeys_callback) (PlannerInfo *root, void *extra); @@ -112,6 +113,7 @@ extern void compute_semijoin_info(PlannerInfo *root, SpecialJoinInfo *sjinfo, */ extern List *remove_useless_joins(PlannerInfo *root, List *joinlist); extern void reduce_unique_semijoins(PlannerInfo *root); +extern void convert_joins_to_semijoins(PlannerInfo *root, List *joinlist); extern bool query_supports_distinctness(Query *query); extern bool query_is_distinct_for(Query *query, List *distinct_cols); extern bool innerrel_is_unique(PlannerInfo *root, diff --git a/src/test/regress/expected/semijoin_conversion.out b/src/test/regress/expected/semijoin_conversion.out new file mode 100644 index 00000000000..fbde99261bc --- /dev/null +++ b/src/test/regress/expected/semijoin_conversion.out @@ -0,0 +1,346 @@ +-- +-- SEMIJOIN CONVERSION +-- Inner joins that only filter can be planned as semijoins, so that matching +-- rows do not fan out the other inputs. The tests below check two things: the +-- transformation applies where the query discards the duplicates, and declines +-- where the query keeps them. +-- +CREATE TABLE sjc_driver (id int PRIMARY KEY, grp int, payload text); +CREATE TABLE sjc_filter (id int PRIMARY KEY, driver_id int, flag bool); +CREATE TABLE sjc_deep (id int PRIMARY KEY, filter_id int); +CREATE TABLE sjc_unique (id int PRIMARY KEY, driver_id int UNIQUE); +CREATE TABLE sjc_uniq2 (id int PRIMARY KEY, unique_id int UNIQUE); +CREATE TABLE sjc_bygrp (id int PRIMARY KEY, grp int); +INSERT INTO sjc_driver + SELECT g, g % 5, 'p' || g FROM generate_series(1, 40) g; +INSERT INTO sjc_filter + SELECT g, (g % 40) + 1, g % 4 <> 0 FROM generate_series(1, 200) g; +INSERT INTO sjc_deep + SELECT g, (g % 200) + 1 FROM generate_series(1, 400) g; +INSERT INTO sjc_unique + SELECT g, g FROM generate_series(1, 40) g; +INSERT INTO sjc_uniq2 + SELECT g, g FROM generate_series(1, 40) g; +INSERT INTO sjc_bygrp + SELECT g, g % 5 FROM generate_series(1, 400) g; +ANALYZE sjc_driver; +ANALYZE sjc_filter; +ANALYZE sjc_deep; +ANALYZE sjc_unique; +ANALYZE sjc_uniq2; +ANALYZE sjc_bygrp; +SET enable_semijoin_conversion = on; +-- +-- Cases where the duplicates cannot be observed, so the join is converted +-- +-- SELECT DISTINCT discards them +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + QUERY PLAN +-------------------------------------------- + HashAggregate + Group Key: d.id, d.payload + -> Hash Right Semi Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(8 rows) + +-- GROUP BY without aggregates discards them +EXPLAIN (COSTS OFF) +SELECT d.grp + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + GROUP BY d.grp; + QUERY PLAN +-------------------------------------------- + HashAggregate + Group Key: d.grp + -> Hash Right Semi Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(8 rows) + +-- a chain of filtering joins forms one group of two relations, and a group of +-- two is declined +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f.id + WHERE f.flag; + QUERY PLAN +-------------------------------------------------- + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f.driver_id = d.id) + -> Hash Join + Hash Cond: (e.filter_id = f.id) + -> Seq Scan on sjc_deep e + -> Hash + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(12 rows) + +-- two filters joined to each other through an equivalence class also form one +-- group, and are declined for the same reason +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_unique u ON u.driver_id = d.id + WHERE f.flag AND u.id < 30; + QUERY PLAN +------------------------------------------------------ + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (f.driver_id = d.id) + -> Hash Join + Hash Cond: (f.driver_id = u.driver_id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_unique u + Filter: (id < 30) + -> Hash + -> Seq Scan on sjc_driver d +(13 rows) + +-- filters with no clause connecting them form separate groups of one, so each +-- filter is lifted on its own +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_bygrp b ON b.grp = d.grp + WHERE f.flag AND b.id < 300; + QUERY PLAN +-------------------------------------------------- + HashAggregate + Group Key: d.id + -> Hash Join + Hash Cond: (d.grp = b.grp) + -> Hash Right Semi Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d + -> Hash + -> HashAggregate + Group Key: b.grp + -> Seq Scan on sjc_bygrp b + Filter: (id < 300) +(15 rows) + +-- +-- Cases where the duplicates are observable, so the join is left alone +-- +-- nothing deduplicates +EXPLAIN (COSTS OFF) +SELECT d.id + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + QUERY PLAN +-------------------------------------- + Hash Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(6 rows) + +-- the inner relation is projected, and so does more than filter +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, f.id + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + QUERY PLAN +-------------------------------------------- + HashAggregate + Group Key: d.id, f.id + -> Hash Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(8 rows) + +-- a window function can see the partition's row count +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, count(*) OVER () AS n + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + QUERY PLAN +-------------------------------------------------- + HashAggregate + Group Key: d.id, count(*) OVER w1 + -> WindowAgg + Window: w1 AS () + -> Hash Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(10 rows) + +-- DISTINCT ON keeps one row per group, but which one depends on the sort +EXPLAIN (COSTS OFF) +SELECT DISTINCT ON (d.grp) d.grp, d.id + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + ORDER BY d.grp, d.id; + QUERY PLAN +-------------------------------------------------- + Unique + -> Sort + Sort Key: d.grp, d.id + -> Hash Join + Hash Cond: (f.driver_id = d.id) + -> Seq Scan on sjc_filter f + Filter: flag + -> Hash + -> Seq Scan on sjc_driver d +(9 rows) + +-- the inner relation is already unique on its join key, so no fanout remains +-- to remove and the transformation declines +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_unique u ON u.driver_id = d.id; + QUERY PLAN +-------------------------------------------- + HashAggregate + Group Key: d.id, d.payload + -> Hash Join + Hash Cond: (d.id = u.driver_id) + -> Seq Scan on sjc_driver d + -> Hash + -> Seq Scan on sjc_unique u +(7 rows) + +-- a chain of unique relations is declined as a group of two, before uniqueness +-- comes into question +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, d.payload + FROM sjc_driver d + JOIN sjc_unique u ON u.driver_id = d.id + JOIN sjc_uniq2 u2 ON u2.unique_id = u.id + WHERE u2.id < 30; + QUERY PLAN +-------------------------------------------------- + HashAggregate + Group Key: d.id, d.payload + -> Hash Join + Hash Cond: (u.driver_id = d.id) + -> Hash Join + Hash Cond: (u.id = u2.unique_id) + -> Seq Scan on sjc_unique u + -> Hash + -> Seq Scan on sjc_uniq2 u2 + Filter: (id < 30) + -> Hash + -> Seq Scan on sjc_driver d +(12 rows) + +-- Beyond join_collapse_limit the jointree stays nested and is planned one +-- sub-list at a time, and a semijoin must not span two sub-lists. Only the +-- result is checked, a nine-way join plan being too unstable to compare. +SET join_collapse_limit = 8; +SELECT count(*) FROM ( + SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_unique u ON u.driver_id = d.id + JOIN sjc_uniq2 u2 ON u2.unique_id = u.id + JOIN sjc_filter f1 ON f1.driver_id = d.id + JOIN sjc_filter f2 ON f2.driver_id = d.id + JOIN sjc_filter f3 ON f3.driver_id = d.id + JOIN sjc_filter f4 ON f4.driver_id = d.id + JOIN sjc_filter f5 ON f5.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f1.id + WHERE d.id = 7 AND f1.flag) s; + count +------- + 1 +(1 row) + +RESET join_collapse_limit; +-- +-- Results must be identical either way +-- +SELECT count(*), sum(id), sum(length(payload)) FROM ( + SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag) s; + count | sum | sum +-------+-----+----- + 30 | 630 | 84 +(1 row) + +SELECT count(*), sum(grp) FROM ( + SELECT d.grp + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + GROUP BY d.grp) s; + count | sum +-------+----- + 5 | 10 +(1 row) + +SELECT count(*), sum(id) FROM ( + SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f.id + WHERE f.flag) s; + count | sum +-------+----- + 30 | 630 +(1 row) + +SET enable_semijoin_conversion = off; +SELECT count(*), sum(id), sum(length(payload)) FROM ( + SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag) s; + count | sum | sum +-------+-----+----- + 30 | 630 | 84 +(1 row) + +SELECT count(*), sum(grp) FROM ( + SELECT d.grp + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + GROUP BY d.grp) s; + count | sum +-------+----- + 5 | 10 +(1 row) + +SELECT count(*), sum(id) FROM ( + SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f.id + WHERE f.flag) s; + count | sum +-------+----- + 30 | 630 +(1 row) + +DROP TABLE sjc_driver, sjc_filter, sjc_deep, sjc_unique, sjc_uniq2, sjc_bygrp; diff --git a/src/test/regress/expected/sysviews.out b/src/test/regress/expected/sysviews.out index 1e327c2afa4..16f50803435 100644 --- a/src/test/regress/expected/sysviews.out +++ b/src/test/regress/expected/sysviews.out @@ -178,10 +178,11 @@ select name, setting from pg_settings where name like 'enable%'; enable_partitionwise_join | off enable_presorted_aggregate | on enable_self_join_elimination | on + enable_semijoin_conversion | off enable_seqscan | on enable_sort | on enable_tidscan | on -(26 rows) +(27 rows) -- There are always wait event descriptions for various types. InjectionPoint -- may be present or absent, depending on history since last postmaster start. diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 8fa0a6c47fb..7d19f6050d4 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -68,6 +68,11 @@ test: select_into select_distinct select_distinct_on select_implicit select_havi # ---------- test: brin gin gist spgist privileges init_privs security_label collate matview lock replica_identity rowsecurity object_address tablesample groupingsets drop_operator password identity generated_stored join_hash +# ---------- +# Semijoin conversion, which needs its own tables and plan stability +# ---------- +test: semijoin_conversion + # ---------- # Additional BRIN tests # ---------- diff --git a/src/test/regress/sql/semijoin_conversion.sql b/src/test/regress/sql/semijoin_conversion.sql new file mode 100644 index 00000000000..27999d6a48c --- /dev/null +++ b/src/test/regress/sql/semijoin_conversion.sql @@ -0,0 +1,186 @@ +-- +-- SEMIJOIN CONVERSION +-- Inner joins that only filter can be planned as semijoins, so that matching +-- rows do not fan out the other inputs. The tests below check two things: the +-- transformation applies where the query discards the duplicates, and declines +-- where the query keeps them. +-- + +CREATE TABLE sjc_driver (id int PRIMARY KEY, grp int, payload text); +CREATE TABLE sjc_filter (id int PRIMARY KEY, driver_id int, flag bool); +CREATE TABLE sjc_deep (id int PRIMARY KEY, filter_id int); +CREATE TABLE sjc_unique (id int PRIMARY KEY, driver_id int UNIQUE); +CREATE TABLE sjc_uniq2 (id int PRIMARY KEY, unique_id int UNIQUE); +CREATE TABLE sjc_bygrp (id int PRIMARY KEY, grp int); + +INSERT INTO sjc_driver + SELECT g, g % 5, 'p' || g FROM generate_series(1, 40) g; +INSERT INTO sjc_filter + SELECT g, (g % 40) + 1, g % 4 <> 0 FROM generate_series(1, 200) g; +INSERT INTO sjc_deep + SELECT g, (g % 200) + 1 FROM generate_series(1, 400) g; +INSERT INTO sjc_unique + SELECT g, g FROM generate_series(1, 40) g; +INSERT INTO sjc_uniq2 + SELECT g, g FROM generate_series(1, 40) g; +INSERT INTO sjc_bygrp + SELECT g, g % 5 FROM generate_series(1, 400) g; + +ANALYZE sjc_driver; +ANALYZE sjc_filter; +ANALYZE sjc_deep; +ANALYZE sjc_unique; +ANALYZE sjc_uniq2; +ANALYZE sjc_bygrp; + +SET enable_semijoin_conversion = on; + +-- +-- Cases where the duplicates cannot be observed, so the join is converted +-- + +-- SELECT DISTINCT discards them +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + +-- GROUP BY without aggregates discards them +EXPLAIN (COSTS OFF) +SELECT d.grp + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + GROUP BY d.grp; + +-- a chain of filtering joins forms one group of two relations, and a group of +-- two is declined +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f.id + WHERE f.flag; + +-- two filters joined to each other through an equivalence class also form one +-- group, and are declined for the same reason +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_unique u ON u.driver_id = d.id + WHERE f.flag AND u.id < 30; + +-- filters with no clause connecting them form separate groups of one, so each +-- filter is lifted on its own +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_bygrp b ON b.grp = d.grp + WHERE f.flag AND b.id < 300; + +-- +-- Cases where the duplicates are observable, so the join is left alone +-- + +-- nothing deduplicates +EXPLAIN (COSTS OFF) +SELECT d.id + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + +-- the inner relation is projected, and so does more than filter +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, f.id + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + +-- a window function can see the partition's row count +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, count(*) OVER () AS n + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag; + +-- DISTINCT ON keeps one row per group, but which one depends on the sort +EXPLAIN (COSTS OFF) +SELECT DISTINCT ON (d.grp) d.grp, d.id + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + ORDER BY d.grp, d.id; + +-- the inner relation is already unique on its join key, so no fanout remains +-- to remove and the transformation declines +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_unique u ON u.driver_id = d.id; + +-- a chain of unique relations is declined as a group of two, before uniqueness +-- comes into question +EXPLAIN (COSTS OFF) +SELECT DISTINCT d.id, d.payload + FROM sjc_driver d + JOIN sjc_unique u ON u.driver_id = d.id + JOIN sjc_uniq2 u2 ON u2.unique_id = u.id + WHERE u2.id < 30; + +-- Beyond join_collapse_limit the jointree stays nested and is planned one +-- sub-list at a time, and a semijoin must not span two sub-lists. Only the +-- result is checked, a nine-way join plan being too unstable to compare. +SET join_collapse_limit = 8; +SELECT count(*) FROM ( + SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_unique u ON u.driver_id = d.id + JOIN sjc_uniq2 u2 ON u2.unique_id = u.id + JOIN sjc_filter f1 ON f1.driver_id = d.id + JOIN sjc_filter f2 ON f2.driver_id = d.id + JOIN sjc_filter f3 ON f3.driver_id = d.id + JOIN sjc_filter f4 ON f4.driver_id = d.id + JOIN sjc_filter f5 ON f5.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f1.id + WHERE d.id = 7 AND f1.flag) s; +RESET join_collapse_limit; + +-- +-- Results must be identical either way +-- + +SELECT count(*), sum(id), sum(length(payload)) FROM ( + SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag) s; + +SELECT count(*), sum(grp) FROM ( + SELECT d.grp + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + GROUP BY d.grp) s; + +SELECT count(*), sum(id) FROM ( + SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f.id + WHERE f.flag) s; + +SET enable_semijoin_conversion = off; + +SELECT count(*), sum(id), sum(length(payload)) FROM ( + SELECT DISTINCT d.id, d.payload + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag) s; + +SELECT count(*), sum(grp) FROM ( + SELECT d.grp + FROM sjc_driver d JOIN sjc_filter f ON f.driver_id = d.id + WHERE f.flag + GROUP BY d.grp) s; + +SELECT count(*), sum(id) FROM ( + SELECT DISTINCT d.id + FROM sjc_driver d + JOIN sjc_filter f ON f.driver_id = d.id + JOIN sjc_deep e ON e.filter_id = f.id + WHERE f.flag) s; + +DROP TABLE sjc_driver, sjc_filter, sjc_deep, sjc_unique, sjc_uniq2, sjc_bygrp;