From 548a753d7af2eb1186bc745172decb2db3359266 Mon Sep 17 00:00:00 2001 From: "dgrowley@gmail.com" Date: Mon, 29 Oct 2018 20:25:53 +1300 Subject: [PATCH v2] Allow Pathkeys to derive their order from a parent key This parent PathKey concept has been given the name "Super Keys". The term "super" has been borrowed out of class hierarchy from OOP. A Path containing a PathKey which has a pk_superkey set must also be ordered by that super key. The reverse is not true, meaning the super key describes a more strict ordering than any subkey which uses it. A new column has been added to pg_proc to allow an optional setting to mention which of the function arguments are the order key. Giving a function a valid order key argument is only valid for precision loss functions or functions which return an exactly equivalent value (e.g casts to a wider type). Some examples are: date_trunc() truncates precision from the 2nd argument, so the 2nd argument can become a super PathKey allowing a btree index on that column to provide sorted input to a GROUP BY clause containing that function, which would allow a GroupAggregate to be used instead of a HashAggregate, which can improve performance significantly, especially so when only a small subset of all groups are required. The functionality is likely also useful for casts between types, however when modifying existing casts functions to give them an order key care must be taken to ensure the sort order of the input and output types match exactly. --- doc/src/sgml/catalogs.sgml | 8 ++ src/backend/catalog/pg_proc.c | 1 + src/backend/nodes/copyfuncs.c | 2 +- src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfuncs.c | 1 + src/backend/optimizer/README | 11 ++ src/backend/optimizer/path/pathkeys.c | 196 ++++++++++++++++++++++++++++++-- src/backend/optimizer/plan/createplan.c | 18 ++- src/backend/utils/cache/lsyscache.c | 18 +++ src/include/catalog/pg_class.dat | 2 +- src/include/catalog/pg_proc.dat | 10 +- src/include/catalog/pg_proc.h | 6 + src/include/nodes/relation.h | 6 + src/include/utils/lsyscache.h | 1 + src/test/regress/expected/indexing.out | 34 ++++++ src/test/regress/sql/indexing.sql | 15 +++ 16 files changed, 313 insertions(+), 17 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 4256516c08..62c9db235b 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -5304,6 +5304,14 @@ SCRAM-SHA-256$<iteration count>:&l Number of input arguments + + proorderkeyarg + int2 + + 0-based number defining the input argument which the return value + of the function is ordered by. + + pronargdefaults int2 diff --git a/src/backend/catalog/pg_proc.c b/src/backend/catalog/pg_proc.c index 0c817047cd..2e15898c1e 100644 --- a/src/backend/catalog/pg_proc.c +++ b/src/backend/catalog/pg_proc.c @@ -326,6 +326,7 @@ ProcedureCreate(const char *procedureName, values[Anum_pg_proc_provolatile - 1] = CharGetDatum(volatility); values[Anum_pg_proc_proparallel - 1] = CharGetDatum(parallel); values[Anum_pg_proc_pronargs - 1] = UInt16GetDatum(parameterCount); + values[Anum_pg_proc_proorderkeyarg - 1] = Int16GetDatum(-1); values[Anum_pg_proc_pronargdefaults - 1] = UInt16GetDatum(list_length(parameterDefaults)); values[Anum_pg_proc_prorettype - 1] = ObjectIdGetDatum(returnType); values[Anum_pg_proc_proargtypes - 1] = PointerGetDatum(parameterTypes); diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index e8ea59e34a..13fe15f7cb 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2216,7 +2216,7 @@ _copyPathKey(const PathKey *from) COPY_SCALAR_FIELD(pk_opfamily); COPY_SCALAR_FIELD(pk_strategy); COPY_SCALAR_FIELD(pk_nulls_first); - + COPY_NODE_FIELD(pk_superkey); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 3bb91c9595..69ea95d1f6 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -824,6 +824,7 @@ _equalPathKey(const PathKey *a, const PathKey *b) COMPARE_SCALAR_FIELD(pk_opfamily); COMPARE_SCALAR_FIELD(pk_strategy); COMPARE_SCALAR_FIELD(pk_nulls_first); + COMPARE_NODE_FIELD(pk_superkey); return true; } diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 69731ccdea..54a69272f2 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2490,6 +2490,7 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_OID_FIELD(pk_opfamily); WRITE_INT_FIELD(pk_strategy); WRITE_BOOL_FIELD(pk_nulls_first); + WRITE_NODE_FIELD(pk_superkey); } static void diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README index 9c852a15ef..bfc8f3d523 100644 --- a/src/backend/optimizer/README +++ b/src/backend/optimizer/README @@ -553,6 +553,7 @@ the result. Each PathKey contains these fields: * a btree opfamily OID (must match one of those in the EC) * a sort direction (ascending or descending) * a nulls-first-or-last flag + * an optional link to a "super" pathkey. The EquivalenceClass represents the value being sorted on. Since the various members of an EquivalenceClass are known equal according to the @@ -667,6 +668,16 @@ if a path was sorted by {a.x} below an outer join, we'll re-sort if that sort ordering was important; and so using the same PathKey for both sort orderings doesn't create any real problem. +Pathkeys may also contain an optional "super key". The super key is a link +to another pathkey to which this pathkey's order is derived from. For +example a pathkey with the expression "date_trunc('hour', timestamp)" +could contain a super key containing the expression "timestamp". This is +because this particular function truncates away precision, meaning any +Path which is ordered by "timestamp" is also ordered by +date_trunc('hour', timestamp). When checking if a set of path keys is +contained in another set we make use of any super keys to increase the +likelihood of a match. For this to hold true the first parameter of the +function must be constant. Order of processing for EquivalenceClasses and PathKeys ------------------------------------------------------- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index ec66cb9c3c..d260c5422b 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -25,10 +25,18 @@ #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/tlist.h" +#include "parser/parse_oper.h" /* XXX shouldn't be included here */ #include "utils/lsyscache.h" static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); +static PathKey *make_pathkey_from_sortop(PlannerInfo *root, + Expr *expr, + Relids nullable_relids, + Oid ordering_op, + bool nulls_first, + Index sortref, + bool create_it); static bool right_merge_direction(PlannerInfo *root, PathKey *pathkey); @@ -146,6 +154,54 @@ pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) return false; } +/* + * extract_orderkey_expr + * Analyze 'expr' and attempt to determine if the expr has a suitable + * order key. If it has return it, else return NULL. + * + * Note: We require that all non-orderkey parameters are Consts. + */ +static Expr * +extract_orderkey_expr(const Expr *expr) +{ + if (IsA(expr, FuncExpr)) + { + FuncExpr *func = (FuncExpr *)expr; + int16 orderkeyarg = get_func_orderkeyarg(func->funcid); + + if (orderkeyarg >= 0) + { + ListCell *lc; + int16 argnum; + Expr *orderkeyexpr; + + Assert(orderkeyarg < list_length(func->args)); + + /* + * Process each argument checking to ensure all arguments apart + * from the order key arg are Const. Along the way we can + * collect the actual order key arg to save from having to do a + * list_nth on the arg list afterwards. + */ + argnum = 0; + foreach(lc, func->args) + { + Expr *argexpr = lfirst(lc); + + if (argnum == orderkeyarg) + orderkeyexpr = argexpr; + else if (!IsA(argexpr, Const)) + return NULL; + argnum++; + } + + return orderkeyexpr; + } + } + + return NULL; +} + /* * make_pathkey_from_sortinfo * Given an expression and sort-order information, create a PathKey. @@ -179,10 +235,12 @@ make_pathkey_from_sortinfo(PlannerInfo *root, Relids rel, bool create_it) { + PathKey *pathkey; int16 strategy; Oid equality_op; List *opfamilies; EquivalenceClass *eclass; + Expr *orderkeyexpr; strategy = reverse_sort ? BTGreaterStrategyNumber : BTLessStrategyNumber; @@ -214,8 +272,39 @@ make_pathkey_from_sortinfo(PlannerInfo *root, return NULL; /* And finally we can find or create a PathKey node */ - return make_canonical_pathkey(root, eclass, opfamily, - strategy, nulls_first); + pathkey = make_canonical_pathkey(root, eclass, opfamily, + strategy, nulls_first); + + /* Determine if 'expr' contains any orderkeys */ + orderkeyexpr = extract_orderkey_expr(expr); + + if (orderkeyexpr != NULL) + { + Oid sortop; + Oid eqop; + bool hashable; + + if (pathkey->pk_strategy == BTLessStrategyNumber) + get_sort_group_operators(exprType((Node *) orderkeyexpr), /* XXX this is not the right function to call */ + true, true, false, + &sortop, &eqop, NULL, + &hashable); + else + get_sort_group_operators(exprType((Node *) orderkeyexpr), + false, true, true, + NULL, &eqop, &sortop, + &hashable); + + pathkey->pk_superkey = make_pathkey_from_sortop(root, + orderkeyexpr, + nullable_relids, + sortop, + nulls_first, + sortref, + create_it); + } + + return pathkey; } /* @@ -311,20 +400,107 @@ compare_pathkeys(List *keys1, List *keys2) /* * pathkeys_contained_in * Common special case of compare_pathkeys: we just want to know - * if keys2 are at least as well sorted as keys1. + * if keys2 are at least as well sorted as keys1. keys1 can exploit any + * super keys to determine if the order matches. */ bool pathkeys_contained_in(List *keys1, List *keys2) { - switch (compare_pathkeys(keys1, keys2)) + ListCell *key1, + *key2; + + /* + * Fall out quickly if we are passed two identical lists. This mostly + * catches the case where both are NIL, but that's common enough to + * warrant the test. + */ + if (keys1 == keys2) + return true; + + key1 = list_head(keys1); + + foreach(key2, keys2) { - case PATHKEYS_EQUAL: - case PATHKEYS_BETTER2: - return true; - default: - break; + PathKey *pathkey1; + PathKey *pathkey2 = (PathKey *) lfirst(key2); + bool first = true; + + for (;;) + { + if (key1 == NULL) + goto out; + + pathkey1 = (PathKey *) lfirst(key1); + + if (pathkey1 != pathkey2) + { + bool found = false; + + /* + * No match on the main key... see if any super keys exist + * which do match. + */ + + pathkey1 = pathkey1->pk_superkey; + while (pathkey1 != NULL) + { + if (pathkey1 == pathkey2) + { + found = true; + break; + } + pathkey1 = pathkey1->pk_superkey; + } + + if (found) + { + /* + * When we find a matching super key we must try to match + * the next pathkey1 to the same pathkey2. There may be + * multiple pathkey1s relying on this pathkey2. If we + * don't find a subsequent match then we can just try to + * match to the next pathkey2. + */ + key1 = lnext(key1); + first = false; + continue; + } + else if (first) + { + /* + * Lack of match is only a problem when its the first + * attempt to match fails + */ + return false; + } + else + { + /* + * Not found, but we've already matched to this key1, so + * skip to next key2. + */ + break; + } + } + else + { + key1 = lnext(key1); + /* main key matched, just skip to next key2. */ + break; + } + } } - return false; + +out: + /* + * If we reached the end of only one list, the other is longer and + * therefore not a subset. + */ + if (key1 != NULL) + return false; /* key1 is longer */ + if (key2 != NULL) + return true; /* key2 is longer */ + return true; } /* diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index ae46b0140e..015ecc2244 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -5734,6 +5734,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * WindowFunc in a sort expression, treat it as a variable. */ Expr *sortexpr = NULL; + Oid em_datatype = InvalidOid; foreach(j, ec->ec_members) { @@ -5758,6 +5759,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, continue; sortexpr = em->em_expr; + em_datatype = em->em_datatype; exprvars = pull_var_clause((Node *) sortexpr, PVC_INCLUDE_AGGREGATES | PVC_INCLUDE_WINDOWFUNCS | @@ -5775,7 +5777,21 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, } } if (!j) - elog(ERROR, "could not find pathkey item to sort"); + { + /* + * Hard error if we were unable to find an EquivalenceMember + * in order to determine the data type of the sort key + */ + if (sortexpr == NULL) + elog(ERROR, "could not find pathkey item to sort"); + + /* + * Otherwise just use the datatype from the EquivalenceMember + * and we'll add a new target list item for the sortexpr + * below. + */ + pk_datatype = em_datatype; + } /* * Do we need to insert a Result node? diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 892ddc0d48..1ce373fdd4 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -1440,6 +1440,24 @@ get_func_nargs(Oid funcid) ReleaseSysCache(tp); return result; } +/* + * get_func_orderkeyarg + * Given procedure id, return the 0-based order key arg number. + */ +int16 +get_func_orderkeyarg(Oid funcid) +{ + HeapTuple tp; + int16 result; + + tp = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for function %u", funcid); + + result = ((Form_pg_proc) GETSTRUCT(tp))->proorderkeyarg; + ReleaseSysCache(tp); + return result; +} /* * get_func_signature diff --git a/src/include/catalog/pg_class.dat b/src/include/catalog/pg_class.dat index 9fffdef379..48ab7b2ec8 100644 --- a/src/include/catalog/pg_class.dat +++ b/src/include/catalog/pg_class.dat @@ -47,7 +47,7 @@ reloftype => '0', relowner => 'PGUID', relam => '0', relfilenode => '0', reltablespace => '0', relpages => '0', reltuples => '0', relallvisible => '0', reltoastrelid => '0', relhasindex => 'f', relisshared => 'f', - relpersistence => 'p', relkind => 'r', relnatts => '28', relchecks => '0', + relpersistence => 'p', relkind => 'r', relnatts => '29', relchecks => '0', relhasoids => 't', relhasrules => 'f', relhastriggers => 'f', relhassubclass => 'f', relrowsecurity => 'f', relforcerowsecurity => 'f', relispopulated => 't', relreplident => 'n', relispartition => 'f', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4026018ba9..8c8dbf9aad 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -2198,7 +2198,7 @@ proargtypes => 'text interval', prosrc => 'interval_part' }, { oid => '1174', descr => 'convert date to timestamp with time zone', proname => 'timestamptz', provolatile => 's', prorettype => 'timestamptz', - proargtypes => 'date', prosrc => 'date_timestamptz' }, + proargtypes => 'date', prosrc => 'date_timestamptz', proorderkeyarg => 0 }, { oid => '2711', descr => 'promote groups of 24 hours to numbers of days and promote groups of 30 days to numbers of months', proname => 'justify_interval', prorettype => 'interval', @@ -2279,7 +2279,8 @@ { oid => '1217', descr => 'truncate timestamp with time zone to specified units', proname => 'date_trunc', provolatile => 's', prorettype => 'timestamptz', - proargtypes => 'text timestamptz', prosrc => 'timestamptz_trunc' }, + proargtypes => 'text timestamptz', prosrc => 'timestamptz_trunc', + proorderkeyarg => 1 }, { oid => '1218', descr => 'truncate interval to specified units', proname => 'date_trunc', prorettype => 'interval', proargtypes => 'text interval', prosrc => 'interval_trunc' }, @@ -5470,13 +5471,14 @@ proargtypes => 'timestamptz', prosrc => 'timestamptz_time' }, { oid => '2020', descr => 'truncate timestamp to specified units', proname => 'date_trunc', prorettype => 'timestamp', - proargtypes => 'text timestamp', prosrc => 'timestamp_trunc' }, + proargtypes => 'text timestamp', prosrc => 'timestamp_trunc', + proorderkeyarg => 1 }, { oid => '2021', descr => 'extract field from timestamp', proname => 'date_part', prorettype => 'float8', proargtypes => 'text timestamp', prosrc => 'timestamp_part' }, { oid => '2024', descr => 'convert date to timestamp', proname => 'timestamp', prorettype => 'timestamp', proargtypes => 'date', - prosrc => 'date_timestamp' }, + prosrc => 'date_timestamp', proorderkeyarg => 0 }, { oid => '2025', descr => 'convert date and time to timestamp', proname => 'timestamp', prorettype => 'timestamp', proargtypes => 'date time', prosrc => 'datetime_timestamp' }, diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index a34b2596fa..718a21a003 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -79,6 +79,12 @@ CATALOG(pg_proc,1255,ProcedureRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(81,Proce /* Note: need not be given in pg_proc.dat; genbki.pl will compute it */ int16 pronargs; + /* + * 0-based argument number of the argument that defines the sort order + * of the return value, or -1 when the function does not support this. + */ + int16 proorderkeyarg BKI_DEFAULT(-1); + /* number of arguments with defaults */ int16 pronargdefaults BKI_DEFAULT(0); diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 88d37236f7..c5e27784e1 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -977,6 +977,10 @@ typedef struct EquivalenceMember * equivalent and closely-related orderings. (See optimizer/README for more * information.) * + * PathKeys may also have a "super key". If present describes that the order + * described by the key can be satisfied by a path which is ordered by its + * 'pk_superkey'. A super key may in turn have its own super key defined. + * * Note: pk_strategy is either BTLessStrategyNumber (for ASC) or * BTGreaterStrategyNumber (for DESC). We assume that all ordering-capable * index types will use btree-compatible strategy numbers. @@ -989,6 +993,8 @@ typedef struct PathKey Oid pk_opfamily; /* btree opfamily defining the ordering */ int pk_strategy; /* sort direction (ASC or DESC) */ bool pk_nulls_first; /* do NULLs come before normal values? */ + struct PathKey *pk_superkey; /* Link to path key which induces this + * pathkey. */ } PathKey; diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index ff1705ad2b..9096442b41 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -111,6 +111,7 @@ extern char *get_func_name(Oid funcid); extern Oid get_func_namespace(Oid funcid); extern Oid get_func_rettype(Oid funcid); extern int get_func_nargs(Oid funcid); +extern int16 get_func_orderkeyarg(Oid funcid); extern Oid get_func_signature(Oid funcid, Oid **argtypes, int *nargs); extern Oid get_func_variadictype(Oid funcid); extern bool get_func_retset(Oid funcid); diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index ca27346f18..8150678658 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -1404,3 +1404,37 @@ insert into covidxpart values (4, 1); insert into covidxpart values (4, 1); ERROR: duplicate key value violates unique constraint "covidxpart4_a_b_idx" DETAIL: Key (a)=(4) already exists. +-- Test super path keys +create table tstbl (ts timestamp, a int); +create index on tstbl (ts, a); +set enable_sort = 0; +explain (costs off) select date_trunc('year', ts), a, count(*) from tstbl group by 1,2 order by 1,2; + QUERY PLAN +----------------------------------------------------- + GroupAggregate + Group Key: date_trunc('year'::text, ts), a + -> Index Only Scan using tstbl_ts_a_idx on tstbl +(3 rows) + +-- Ensure the we don't use the index to provide pre-sorted input for a +-- GroupAggregate when the order key expr function has non-constant +-- non-orderkey arguments. +explain (costs off) select date_trunc(case random() > 0.5 when true then 'year' else 'month' end, ts), a, count(*) from tstbl group by 1,2 order by 1,2; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------- + Sort + Sort Key: (date_trunc(CASE (random() > '0.5'::double precision) WHEN CASE_TEST_EXPR THEN 'year'::text ELSE 'month'::text END, ts)), a + -> HashAggregate + Group Key: date_trunc(CASE (random() > '0.5'::double precision) WHEN CASE_TEST_EXPR THEN 'year'::text ELSE 'month'::text END, ts), a + -> Index Only Scan using tstbl_ts_a_idx on tstbl +(5 rows) + +-- Test a more complex case where the superkey can be matched to multiple pathkeys +explain (costs off) select date_trunc('year', ts), date_trunc('month', ts), a, count(*) from tstbl group by 1,2,3 order by 1,2,3; + QUERY PLAN +----------------------------------------------------------------------------- + GroupAggregate + Group Key: date_trunc('year'::text, ts), date_trunc('month'::text, ts), a + -> Index Only Scan using tstbl_ts_a_idx on tstbl +(3 rows) + diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 400b7eb7ba..9d24e74a7d 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -753,3 +753,18 @@ create unique index on covidxpart4 (a); alter table covidxpart attach partition covidxpart4 for values in (4); insert into covidxpart values (4, 1); insert into covidxpart values (4, 1); + +-- Test super path keys +create table tstbl (ts timestamp, a int); +create index on tstbl (ts, a); +set enable_sort = 0; + +explain (costs off) select date_trunc('year', ts), a, count(*) from tstbl group by 1,2 order by 1,2; + +-- Ensure the we don't use the index to provide pre-sorted input for a +-- GroupAggregate when the order key expr function has non-constant +-- non-orderkey arguments. +explain (costs off) select date_trunc(case random() > 0.5 when true then 'year' else 'month' end, ts), a, count(*) from tstbl group by 1,2 order by 1,2; + +-- Test a more complex case where the superkey can be matched to multiple pathkeys +explain (costs off) select date_trunc('year', ts), date_trunc('month', ts), a, count(*) from tstbl group by 1,2,3 order by 1,2,3; -- 2.16.2.windows.1