From 69855fffacf69575471beb69da761babadc9f75c Mon Sep 17 00:00:00 2001 From: amitlan Date: Wed, 22 Dec 2021 16:55:17 +0900 Subject: [PATCH v29 2/2] In GetCachedPlan(), only lock unpruned partitions This does two things mainly: * The planner now removes the RT indexes of "initially prunable" partitions from PlannedStmt.minLockRelids such that the set only contains the relations not subject to initial partition pruning. So, AcquireExecutorLocks only locks a subset of the relations contained in a plan, deferring the locking of prunable relations to the caller. * GetCachedPlans(), if there are prunable relations in the plan, performs the initial partition pruning using available EXTERN params and locks the partitions remaining after that, so the the CachedPlan that's returned is valid in a race-free manner including for any partitions that will be scanned during execution. To make the pruning possible before entering ExecutorStart(), this also adds a ExecPartitionDoInitialPruning(), which can be called by GetCachedPlan() for a given PlannedStmt. The result of performing initial partition pruning this way is made available to the actual execution via PartitionPruneResult, of which there is one for every ParttionPruneInfo contained in the PlannedStmt. List of PartitionPruneResult for a given PlannedStmt are returned to to the callers of GetCachedPlan() via its new output parameter of type CachedPlanExtra, whose members currently only include said List. --- src/backend/commands/copyto.c | 2 +- src/backend/commands/createas.c | 2 +- src/backend/commands/explain.c | 7 +- src/backend/commands/extension.c | 2 +- src/backend/commands/matview.c | 2 +- src/backend/commands/prepare.c | 28 ++- src/backend/executor/README | 31 ++- src/backend/executor/execMain.c | 2 + src/backend/executor/execParallel.c | 25 ++- src/backend/executor/execPartition.c | 215 +++++++++++++++++---- src/backend/executor/execUtils.c | 1 + src/backend/executor/functions.c | 2 +- src/backend/executor/nodeAppend.c | 11 +- src/backend/executor/nodeMergeAppend.c | 5 +- src/backend/executor/spi.c | 31 ++- src/backend/optimizer/plan/setrefs.c | 36 ++++ src/backend/tcop/postgres.c | 9 +- src/backend/tcop/pquery.c | 28 ++- src/backend/utils/cache/plancache.c | 257 +++++++++++++++++++++++-- src/backend/utils/mmgr/portalmem.c | 16 ++ src/include/commands/explain.h | 4 +- src/include/executor/execPartition.h | 7 +- src/include/executor/execdesc.h | 3 + src/include/nodes/execnodes.h | 1 + src/include/nodes/pathnodes.h | 4 +- src/include/nodes/plannodes.h | 31 ++- src/include/utils/plancache.h | 11 +- src/include/utils/portal.h | 3 + 28 files changed, 694 insertions(+), 82 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index f26cc0d162..401a2280a3 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -558,7 +558,7 @@ BeginCopyTo(ParseState *pstate, ((DR_copy *) dest)->cstate = cstate; /* Create a QueryDesc requesting no output */ - cstate->queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext, + cstate->queryDesc = CreateQueryDesc(plan, NIL, pstate->p_sourcetext, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, NULL, 0); diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 152c29b551..942449544c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -325,7 +325,7 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, UpdateActiveSnapshotCommandId(); /* Create a QueryDesc, redirecting output to our tuple receiver */ - queryDesc = CreateQueryDesc(plan, pstate->p_sourcetext, + queryDesc = CreateQueryDesc(plan, NIL, pstate->p_sourcetext, GetActiveSnapshot(), InvalidSnapshot, dest, params, queryEnv, 0); diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index f86983c660..2f2b558608 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -407,7 +407,7 @@ ExplainOneQuery(Query *query, int cursorOptions, } /* run it (if needed) and produce output */ - ExplainOnePlan(plan, into, es, queryString, params, queryEnv, + ExplainOnePlan(plan, NIL, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL)); } } @@ -515,7 +515,8 @@ ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, * to call it. */ void -ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, +ExplainOnePlan(PlannedStmt *plannedstmt, List *part_prune_results, + IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage) @@ -563,7 +564,7 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, dest = None_Receiver; /* Create a QueryDesc for the query */ - queryDesc = CreateQueryDesc(plannedstmt, queryString, + queryDesc = CreateQueryDesc(plannedstmt, part_prune_results, queryString, GetActiveSnapshot(), InvalidSnapshot, dest, params, queryEnv, instrument_option); diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c index cf1b1ca571..904cbcba4a 100644 --- a/src/backend/commands/extension.c +++ b/src/backend/commands/extension.c @@ -779,7 +779,7 @@ execute_sql_string(const char *sql) { QueryDesc *qdesc; - qdesc = CreateQueryDesc(stmt, + qdesc = CreateQueryDesc(stmt, NIL, sql, GetActiveSnapshot(), NULL, dest, NULL, NULL, 0); diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 9ac0383459..65c8d0aa59 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -408,7 +408,7 @@ refresh_matview_datafill(DestReceiver *dest, Query *query, UpdateActiveSnapshotCommandId(); /* Create a QueryDesc, redirecting output to our tuple receiver */ - queryDesc = CreateQueryDesc(plan, queryString, + queryDesc = CreateQueryDesc(plan, NIL, queryString, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, NULL, 0); diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index 9e29584d93..729384a9a6 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -154,6 +154,7 @@ ExecuteQuery(ParseState *pstate, { PreparedStatement *entry; CachedPlan *cplan; + CachedPlanExtra *cplan_extra = NULL; List *plan_list; ParamListInfo paramLI = NULL; EState *estate = NULL; @@ -193,7 +194,11 @@ ExecuteQuery(ParseState *pstate, entry->plansource->query_string); /* Replan if needed, and increment plan refcount for portal */ - cplan = GetCachedPlan(entry->plansource, paramLI, NULL, NULL); + cplan = GetCachedPlan(entry->plansource, paramLI, NULL, NULL, + &cplan_extra); + Assert(cplan_extra == NULL || + (list_length(cplan->stmt_list) == + list_length(cplan_extra->part_prune_results_list))); plan_list = cplan->stmt_list; /* @@ -207,6 +212,9 @@ ExecuteQuery(ParseState *pstate, plan_list, cplan); + if (cplan_extra) + PortalSaveCachedPlanExtra(portal, cplan_extra); + /* * For CREATE TABLE ... AS EXECUTE, we must verify that the prepared * statement is one that produces tuples. Currently we insist that it be @@ -575,6 +583,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, PreparedStatement *entry; const char *query_string; CachedPlan *cplan; + CachedPlanExtra *cplan_extra = NULL; List *plan_list; ListCell *p; ParamListInfo paramLI = NULL; @@ -619,7 +628,11 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, /* Replan if needed, and acquire a transient refcount */ cplan = GetCachedPlan(entry->plansource, paramLI, - CurrentResourceOwner, queryEnv); + CurrentResourceOwner, queryEnv, + &cplan_extra); + Assert(cplan_extra == NULL || + (list_length(cplan->stmt_list) == + list_length(cplan_extra->part_prune_results_list))); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); @@ -637,10 +650,17 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, foreach(p, plan_list) { PlannedStmt *pstmt = lfirst_node(PlannedStmt, p); + List *part_prune_results = NIL; + + if (cplan_extra) + part_prune_results = list_nth_node(List, + cplan_extra->part_prune_results_list, + foreach_current_index(p)); if (pstmt->commandType != CMD_UTILITY) - ExplainOnePlan(pstmt, into, es, query_string, paramLI, queryEnv, - &planduration, (es->buffers ? &bufusage : NULL)); + ExplainOnePlan(pstmt, part_prune_results, into, es, query_string, + paramLI, queryEnv, &planduration, + (es->buffers ? &bufusage : NULL)); else ExplainOneUtility(pstmt->utilityStmt, into, es, query_string, paramLI, queryEnv); diff --git a/src/backend/executor/README b/src/backend/executor/README index 17775a49e2..2222b3ed6f 100644 --- a/src/backend/executor/README +++ b/src/backend/executor/README @@ -63,7 +63,36 @@ if the executor determines that an entire subplan is not required due to execution time partition pruning determining that no matching records will be found there. This currently only occurs for Append and MergeAppend nodes. In this case the non-required subplans are ignored and the executor state's -subnode array will become out of sequence to the plan's subplan list. +subnode array will become out of sequence to the plan's subplan list. Note +that this is referred to as "initial" pruning, because it needs to occur only +once during the execution startup, and uses a set of pruning steps called +initial pruning steps (see PartitionedRelPruneInfo.initial_pruning_steps). + +Actually, "initial" pruning may occur even before the execution startup in +in some cases. For example, when a cached generic plan is validated for +execution, which works by locking all the relations that will be scanned by +that plan during execution. If the generic plan contains plan nodes that have +prunable child subnodes, then this validation locking is performed after +pruning child subnodes that need not be scanned during execution, that is, +using initial pruning steps. When such a generic plan is forwarded for +execution, it must be accompanied by the set of PartitionPruneResult nodes that +contain the result of that pruning, which basically consists of a bitmapset of +child subnode indexes that survived the pruning and thus whose relations would +have been locked for execution. This is important, because, unlike the +plan-time pruning and actual executor-startup pruning, this does not actually +remove the pruned subnodes from the plan tree, but only marks them as being +pruned. So, the executor code (core or third party), especially one that runs +before ExecutorStart() and thus looks at bare Plan trees (not PlanState trees) +must beware of plan nodes that may actually have been pruned and thus subject +to being invalidated by concurrent schema changes. For plan nodes that can +have prunable child subnodes and thus contain a PartitionPruneInfo, such code +must always check if the corresponding PartitionPruneResult exists +in EState.es_part_prune_results at given part_prune_index and use that to +decide which subplans are valid for execution instead of redoing the pruning. +Note that that is not just a performance optimization but also necessary to +avoid possibly ending up considering a different set of child subnodes as valid +than the set CachedPlanLockPartitions() would have locked the relations of, if +the pruning steps produce a different result when executed multiple times. Each Plan node may have expression trees associated with it, to represent its target list, qualification conditions, etc. These trees are also diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 2c2b3a8874..229f61f72e 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -798,6 +798,7 @@ InitPlan(QueryDesc *queryDesc, int eflags) { CmdType operation = queryDesc->operation; PlannedStmt *plannedstmt = queryDesc->plannedstmt; + List *part_prune_results = queryDesc->part_prune_results; Plan *plan = plannedstmt->planTree; List *rangeTable = plannedstmt->rtable; EState *estate = queryDesc->estate; @@ -819,6 +820,7 @@ InitPlan(QueryDesc *queryDesc, int eflags) estate->es_plannedstmt = plannedstmt; estate->es_part_prune_infos = plannedstmt->partPruneInfos; + estate->es_part_prune_results = part_prune_results; /* * Next, build the ExecRowMark array from the PlanRowMark(s), if any. diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c index 65c4b63bbd..9745eba0af 100644 --- a/src/backend/executor/execParallel.c +++ b/src/backend/executor/execParallel.c @@ -66,6 +66,7 @@ #define PARALLEL_KEY_QUERY_TEXT UINT64CONST(0xE000000000000008) #define PARALLEL_KEY_JIT_INSTRUMENTATION UINT64CONST(0xE000000000000009) #define PARALLEL_KEY_WAL_USAGE UINT64CONST(0xE00000000000000A) +#define PARALLEL_KEY_PARTITION_PRUNE_RESULTS UINT64CONST(0xE00000000000000B) #define PARALLEL_TUPLE_QUEUE_SIZE 65536 @@ -599,12 +600,15 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate, FixedParallelExecutorState *fpes; char *pstmt_data; char *pstmt_space; + char *part_prune_results_data; + char *part_prune_results_space; char *paramlistinfo_space; BufferUsage *bufusage_space; WalUsage *walusage_space; SharedExecutorInstrumentation *instrumentation = NULL; SharedJitInstrumentation *jit_instrumentation = NULL; int pstmt_len; + int part_prune_results_len; int paramlistinfo_len; int instrumentation_len = 0; int jit_instrumentation_len = 0; @@ -633,6 +637,7 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate, /* Fix up and serialize plan to be sent to workers. */ pstmt_data = ExecSerializePlan(planstate->plan, estate); + part_prune_results_data = nodeToString(estate->es_part_prune_results); /* Create a parallel context. */ pcxt = CreateParallelContext("postgres", "ParallelQueryMain", nworkers); @@ -659,6 +664,11 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate, shm_toc_estimate_chunk(&pcxt->estimator, pstmt_len); shm_toc_estimate_keys(&pcxt->estimator, 1); + /* Estimate space for serialized List of PartitionPruneResult. */ + part_prune_results_len = strlen(part_prune_results_data) + 1; + shm_toc_estimate_chunk(&pcxt->estimator, part_prune_results_len); + shm_toc_estimate_keys(&pcxt->estimator, 1); + /* Estimate space for serialized ParamListInfo. */ paramlistinfo_len = EstimateParamListSpace(estate->es_param_list_info); shm_toc_estimate_chunk(&pcxt->estimator, paramlistinfo_len); @@ -753,6 +763,12 @@ ExecInitParallelPlan(PlanState *planstate, EState *estate, memcpy(pstmt_space, pstmt_data, pstmt_len); shm_toc_insert(pcxt->toc, PARALLEL_KEY_PLANNEDSTMT, pstmt_space); + /* Store serialized List of PartitionPruneResult */ + part_prune_results_space = shm_toc_allocate(pcxt->toc, part_prune_results_len); + memcpy(part_prune_results_space, part_prune_results_data, part_prune_results_len); + shm_toc_insert(pcxt->toc, PARALLEL_KEY_PARTITION_PRUNE_RESULTS, + part_prune_results_space); + /* Store serialized ParamListInfo. */ paramlistinfo_space = shm_toc_allocate(pcxt->toc, paramlistinfo_len); shm_toc_insert(pcxt->toc, PARALLEL_KEY_PARAMLISTINFO, paramlistinfo_space); @@ -1234,8 +1250,10 @@ ExecParallelGetQueryDesc(shm_toc *toc, DestReceiver *receiver, int instrument_options) { char *pstmtspace; + char *part_prune_results_space; char *paramspace; PlannedStmt *pstmt; + List *part_prune_results; ParamListInfo paramLI; char *queryString; @@ -1246,12 +1264,17 @@ ExecParallelGetQueryDesc(shm_toc *toc, DestReceiver *receiver, pstmtspace = shm_toc_lookup(toc, PARALLEL_KEY_PLANNEDSTMT, false); pstmt = (PlannedStmt *) stringToNode(pstmtspace); + /* Reconstruct leader-supplied PartitionPruneResult. */ + part_prune_results_space = + shm_toc_lookup(toc, PARALLEL_KEY_PARTITION_PRUNE_RESULTS, false); + part_prune_results = (List *) stringToNode(part_prune_results_space); + /* Reconstruct ParamListInfo. */ paramspace = shm_toc_lookup(toc, PARALLEL_KEY_PARAMLISTINFO, false); paramLI = RestoreParamList(¶mspace); /* Create a QueryDesc for the query. */ - return CreateQueryDesc(pstmt, + return CreateQueryDesc(pstmt, part_prune_results, queryString, GetActiveSnapshot(), InvalidSnapshot, receiver, paramLI, NULL, instrument_options); diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c index 5b62157712..dcd2bb0f90 100644 --- a/src/backend/executor/execPartition.c +++ b/src/backend/executor/execPartition.c @@ -25,6 +25,7 @@ #include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/makefuncs.h" +#include "parser/parsetree.h" #include "partitioning/partbounds.h" #include "partitioning/partdesc.h" #include "partitioning/partprune.h" @@ -185,7 +186,11 @@ static char *ExecBuildSlotPartitionKeyDescription(Relation rel, static List *adjust_partition_colnos(List *colnos, ResultRelInfo *leaf_part_rri); static List *adjust_partition_colnos_using_map(List *colnos, AttrMap *attrMap); static PartitionPruneState *CreatePartitionPruneState(PlanState *planstate, - PartitionPruneInfo *pruneinfo); + PartitionPruneInfo *pruneinfo, + bool consider_initial_steps, + bool consider_exec_steps, + List *rtable, ExprContext *econtext, + PartitionDirectory partdir); static void InitPartitionPruneContext(PartitionPruneContext *context, List *pruning_steps, PartitionDesc partdesc, @@ -198,7 +203,8 @@ static void PartitionPruneFixSubPlanMap(PartitionPruneState *prunestate, static void find_matching_subplans_recurse(PartitionPruningData *prunedata, PartitionedRelPruningData *pprune, bool initial_prune, - Bitmapset **validsubplans); + Bitmapset **validsubplans, + Bitmapset **scan_leafpart_rtis); /* @@ -1742,7 +1748,8 @@ adjust_partition_colnos_using_map(List *colnos, AttrMap *attrMap) * considered to be a stable expression, it can change value from one plan * node scan to the next during query execution. Stable comparison * expressions that don't involve such Params allow partition pruning to be - * done once during executor startup. Expressions that do involve such Params + * done once during executor startup or even before that, such as when called + * from CachedPlanLockPartitions(). Expressions that do involve such Params * require us to prune separately for each scan of the parent plan node. * * Note that pruning away unneeded subplans during executor startup has the @@ -1760,6 +1767,12 @@ adjust_partition_colnos_using_map(List *colnos, AttrMap *attrMap) * account for initial pruning possibly having eliminated some of the * subplans. * + * ExecPartitionDoInitialPruning: + * Do initial pruning with the information contained in a given + * PartitionPruneInfo to determine the set of the parent plan node's + * child subnodes that are valid for execution and also the set of the RT + * indexes of leaf partitions scanned by those subnodes. + * * ExecFindMatchingSubPlans: * Returns indexes of matching subplans after evaluating the expressions * that are safe to evaluate at a given point. This function is first @@ -1780,8 +1793,10 @@ adjust_partition_colnos_using_map(List *colnos, AttrMap *attrMap) * * On return, *initially_valid_subplans is assigned the set of indexes of * child subplans that must be initialized along with the parent plan node. - * Initial pruning is performed here if needed and in that case only the - * surviving subplans' indexes are added. + * That set is computed by either performing the "initial pruning" here or + * reusing the one present in EState.es_part_prune_results[part_prune_index] + * if it has been set, which it would be if CachedPlanLockPartitions() would + * have done the initial pruning. * * If subplans are indeed pruned, subplan_map arrays contained in the returned * PartitionPruneState are re-sequenced to not count those, though only if the @@ -1794,9 +1809,10 @@ ExecInitPartitionPruning(PlanState *planstate, Bitmapset *root_parent_relids, Bitmapset **initially_valid_subplans) { - PartitionPruneState *prunestate; + PartitionPruneState *prunestate = NULL; EState *estate = planstate->state; PartitionPruneInfo *pruneinfo; + PartitionPruneResult *pruneresult = NULL; /* Obtain the pruneinfo we need, and make sure it's the right one */ pruneinfo = list_nth(estate->es_part_prune_infos, part_prune_index); @@ -1812,20 +1828,62 @@ ExecInitPartitionPruning(PlanState *planstate, /* We may need an expression context to evaluate partition exprs */ ExecAssignExprContext(estate, planstate); - /* Create the working data structure for pruning */ - prunestate = CreatePartitionPruneState(planstate, pruneinfo); + /* Initial pruning already done if es_part_prune_results has been set. */ + if (estate->es_part_prune_results) + { + pruneresult = list_nth_node(PartitionPruneResult, + estate->es_part_prune_results, + part_prune_index); + if (!bms_equal(root_parent_relids, pruneinfo->root_parent_relids)) + ereport(ERROR, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg_internal("mismatching PartitionPruneInfo and PartitionPruneResult at part_prune_index %d", + part_prune_index), + errdetail_internal("prunresult relids %s, pruneinfo relids %s", + bmsToString(pruneresult->root_parent_relids), + bmsToString(pruneinfo->root_parent_relids))); + } + + if (pruneresult == NULL || pruneinfo->needs_exec_pruning) + { + /* We may need an expression context to evaluate partition exprs */ + ExecAssignExprContext(estate, planstate); + + /* For data reading, executor always omits detached partitions */ + if (estate->es_partition_directory == NULL) + estate->es_partition_directory = + CreatePartitionDirectory(estate->es_query_cxt, false); + + /* + * Create the working data structure for pruning. No need to consider + * initial pruning steps if we have a PartitionPruneResult. + */ + prunestate = CreatePartitionPruneState(planstate, pruneinfo, + pruneresult == NULL, + pruneinfo->needs_exec_pruning, + NIL, planstate->ps_ExprContext, + estate->es_partition_directory); + } /* * Perform an initial partition prune pass, if required. */ - if (prunestate->do_initial_prune) - *initially_valid_subplans = ExecFindMatchingSubPlans(prunestate, true); + if (pruneresult) + { + *initially_valid_subplans = bms_copy(pruneresult->valid_subplan_offs); + } + else if (prunestate && prunestate->do_initial_prune) + { + *initially_valid_subplans = ExecFindMatchingSubPlans(prunestate, true, + NULL); + } else { - /* No pruning, so we'll need to initialize all subplans */ + /* No initial pruning, so we'll need to initialize all subplans */ Assert(n_total_subplans > 0); *initially_valid_subplans = bms_add_range(NULL, 0, n_total_subplans - 1); + return prunestate; } /* @@ -1833,7 +1891,8 @@ ExecInitPartitionPruning(PlanState *planstate, * that were removed above due to initial pruning. No need to do this if * no steps were removed. */ - if (bms_num_members(*initially_valid_subplans) < n_total_subplans) + if (prunestate && + bms_num_members(*initially_valid_subplans) < n_total_subplans) { /* * We can safely skip this when !do_exec_prune, even though that @@ -1849,11 +1908,58 @@ ExecInitPartitionPruning(PlanState *planstate, return prunestate; } +/* + * ExecPartitionDoInitialPruning + * Perform initial pruning using given PartitionPruneInfo to determine + * the set of the parent plan node's child subnodes that are valid for + * execution + * + * On return, *scan_leafpart_rtis will contain the RT indexes of leaf + * partitions scanned by those valid subnodes. + * + * Note that this does not share state with the actual execution, so must do + * with the information present in the PlannedStmt. For example, there isn't + * a PlanState for the parent plan node yet, so we must create a standalone + * ExprContext to evaluate pruning expressions, equipped with the information + * about the EXTERN parameters that we do have. Note that that's okay because + * the initial pruning steps do not contain anything that would require the + * execution to have started. Likewise, we create our own PartitionDirectory + * to look up the PartitionDescs to use. + */ +Bitmapset * +ExecPartitionDoInitialPruning(PlannedStmt *plannedstmt, ParamListInfo params, + PartitionPruneInfo *pruneinfo, + Bitmapset **scan_leafpart_rtis) +{ + List *rtable = plannedstmt->rtable; + ExprContext *econtext; + PartitionDirectory pdir; + PartitionPruneState *prunestate; + Bitmapset *valid_subplan_offs; + + /* Don't omit detached partitions, just like during execution proper. */ + pdir = CreatePartitionDirectory(CurrentMemoryContext, false); + econtext = CreateStandaloneExprContext(); + econtext->ecxt_param_list_info = params; + prunestate = CreatePartitionPruneState(NULL, pruneinfo, true, false, + rtable, econtext, pdir); + valid_subplan_offs = ExecFindMatchingSubPlans(prunestate, true, + scan_leafpart_rtis); + + FreeExprContext(econtext, true); + DestroyPartitionDirectory(pdir); + + return valid_subplan_offs; +} + /* * CreatePartitionPruneState * Build the data structure required for calling ExecFindMatchingSubPlans * - * 'planstate' is the parent plan node's execution state. + * 'planstate', if not NULL, is the parent plan node's execution state. It + * can be NULL if being called before ExecutorStart(), in which case, + * 'rtable' (range table), 'econtext', and 'partdir' must be explicitly + * provided. * * 'pruneinfo' is a PartitionPruneInfo as generated by * make_partition_pruneinfo. Here we build a PartitionPruneState containing a @@ -1867,19 +1973,21 @@ ExecInitPartitionPruning(PlanState *planstate, * PartitionedRelPruneInfo. */ static PartitionPruneState * -CreatePartitionPruneState(PlanState *planstate, PartitionPruneInfo *pruneinfo) +CreatePartitionPruneState(PlanState *planstate, + PartitionPruneInfo *pruneinfo, + bool consider_initial_steps, + bool consider_exec_steps, + List *rtable, ExprContext *econtext, + PartitionDirectory partdir) { - EState *estate = planstate->state; + EState *estate = planstate ? planstate->state : NULL; PartitionPruneState *prunestate; int n_part_hierarchies; ListCell *lc; int i; - ExprContext *econtext = planstate->ps_ExprContext; - /* For data reading, executor always omits detached partitions */ - if (estate->es_partition_directory == NULL) - estate->es_partition_directory = - CreatePartitionDirectory(estate->es_query_cxt, false); + Assert((estate != NULL) || + (partdir != NULL && econtext != NULL && rtable != NIL)); n_part_hierarchies = list_length(pruneinfo->prune_infos); Assert(n_part_hierarchies > 0); @@ -1934,15 +2042,39 @@ CreatePartitionPruneState(PlanState *planstate, PartitionPruneInfo *pruneinfo) PartitionKey partkey; /* - * We can rely on the copies of the partitioned table's partition - * key and partition descriptor appearing in its relcache entry, - * because that entry will be held open and locked for the - * duration of this executor run. + * Must open the relation by ourselves when called before the + * execution has started, such as, when called from + * CachedPlanLockPartitions(). In that case, sub-partitions must + * be locked, because AcquirePlannerLocks() would have locked only + * the root parent. + */ + if (estate == NULL) + { + RangeTblEntry *rte = rt_fetch(pinfo->rtindex, rtable); + int lockmode = (j == 0) ? NoLock : rte->rellockmode; + + partrel = table_open(rte->relid, lockmode); + } + else + partrel = ExecGetRangeTableRelation(estate, pinfo->rtindex); + + /* + * We can rely on the copy of the partitioned table's partition + * key from in its relcache entry, because it can't change (or + * get destroyed) as long as the relation is locked. Partition + * descriptor is taken from the PartitionDirectory associated with + * the table that is held open long enough for the descriptor to + * remain valid while it's used to perform the pruning steps. */ - partrel = ExecGetRangeTableRelation(estate, pinfo->rtindex); partkey = RelationGetPartitionKey(partrel); - partdesc = PartitionDirectoryLookup(estate->es_partition_directory, - partrel); + partdesc = PartitionDirectoryLookup(partdir, partrel); + + /* + * Must close partrel, keeping the lock taken, if we're not using + * EState's entry. + */ + if (estate == NULL) + table_close(partrel, NoLock); /* * Initialize the subplan_map and subpart_map. @@ -2050,7 +2182,7 @@ CreatePartitionPruneState(PlanState *planstate, PartitionPruneInfo *pruneinfo) * Initialize pruning contexts as needed. */ pprune->initial_pruning_steps = pinfo->initial_pruning_steps; - if (pinfo->initial_pruning_steps) + if (consider_initial_steps && pinfo->initial_pruning_steps) { InitPartitionPruneContext(&pprune->initial_context, pinfo->initial_pruning_steps, @@ -2060,7 +2192,7 @@ CreatePartitionPruneState(PlanState *planstate, PartitionPruneInfo *pruneinfo) prunestate->do_initial_prune = true; } pprune->exec_pruning_steps = pinfo->exec_pruning_steps; - if (pinfo->exec_pruning_steps) + if (consider_exec_steps && pinfo->exec_pruning_steps) { InitPartitionPruneContext(&pprune->exec_context, pinfo->exec_pruning_steps, @@ -2288,10 +2420,14 @@ PartitionPruneFixSubPlanMap(PartitionPruneState *prunestate, * Pass initial_prune if PARAM_EXEC Params cannot yet be evaluated. This * differentiates the initial executor-time pruning step from later * runtime pruning. + * + * RT indexes of leaf partitions scanned by the chosen subplans are added to + * *scan_leafpart_rtis if the pointer is non-NULL. */ Bitmapset * ExecFindMatchingSubPlans(PartitionPruneState *prunestate, - bool initial_prune) + bool initial_prune, + Bitmapset **scan_leafpart_rtis) { Bitmapset *result = NULL; MemoryContext oldcontext; @@ -2326,7 +2462,7 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate, */ pprune = &prunedata->partrelprunedata[0]; find_matching_subplans_recurse(prunedata, pprune, initial_prune, - &result); + &result, scan_leafpart_rtis); /* Expression eval may have used space in ExprContext too */ if (pprune->exec_pruning_steps) @@ -2340,6 +2476,8 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate, /* Copy result out of the temp context before we reset it */ result = bms_copy(result); + if (scan_leafpart_rtis) + *scan_leafpart_rtis = bms_copy(*scan_leafpart_rtis); MemoryContextReset(prunestate->prune_context); @@ -2350,13 +2488,15 @@ ExecFindMatchingSubPlans(PartitionPruneState *prunestate, * find_matching_subplans_recurse * Recursive worker function for ExecFindMatchingSubPlans * - * Adds valid (non-prunable) subplan IDs to *validsubplans + * Adds valid (non-prunable) subplan IDs to *validsubplans and RT indexes of + * of the corresponding leaf partitions to *scan_leafpart_rtis (if asked for). */ static void find_matching_subplans_recurse(PartitionPruningData *prunedata, PartitionedRelPruningData *pprune, bool initial_prune, - Bitmapset **validsubplans) + Bitmapset **validsubplans, + Bitmapset **scan_leafpart_rtis) { Bitmapset *partset; int i; @@ -2383,8 +2523,14 @@ find_matching_subplans_recurse(PartitionPruningData *prunedata, while ((i = bms_next_member(partset, i)) >= 0) { if (pprune->subplan_map[i] >= 0) + { *validsubplans = bms_add_member(*validsubplans, pprune->subplan_map[i]); + Assert(pprune->rti_map[i] > 0); + if (scan_leafpart_rtis) + *scan_leafpart_rtis = bms_add_member(*scan_leafpart_rtis, + pprune->rti_map[i]); + } else { int partidx = pprune->subpart_map[i]; @@ -2392,7 +2538,8 @@ find_matching_subplans_recurse(PartitionPruningData *prunedata, if (partidx >= 0) find_matching_subplans_recurse(prunedata, &prunedata->partrelprunedata[partidx], - initial_prune, validsubplans); + initial_prune, validsubplans, + scan_leafpart_rtis); else { /* diff --git a/src/backend/executor/execUtils.c b/src/backend/executor/execUtils.c index 87f4d53ca7..7d36c972d3 100644 --- a/src/backend/executor/execUtils.c +++ b/src/backend/executor/execUtils.c @@ -139,6 +139,7 @@ CreateExecutorState(void) estate->es_param_exec_vals = NULL; estate->es_queryEnv = NULL; + estate->es_part_prune_results = NIL; estate->es_query_cxt = qcontext; diff --git a/src/backend/executor/functions.c b/src/backend/executor/functions.c index dc13625171..bffb42ce71 100644 --- a/src/backend/executor/functions.c +++ b/src/backend/executor/functions.c @@ -842,7 +842,7 @@ postquel_start(execution_state *es, SQLFunctionCachePtr fcache) else dest = None_Receiver; - es->qd = CreateQueryDesc(es->stmt, + es->qd = CreateQueryDesc(es->stmt, NIL, fcache->src, GetActiveSnapshot(), InvalidSnapshot, diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c index 99830198bd..3b917584de 100644 --- a/src/backend/executor/nodeAppend.c +++ b/src/backend/executor/nodeAppend.c @@ -156,7 +156,8 @@ ExecInitAppend(Append *node, EState *estate, int eflags) * subplan, we can fill as_valid_subplans immediately, preventing * later calls to ExecFindMatchingSubPlans. */ - if (!prunestate->do_exec_prune && nplans > 0) + if (appendstate->as_prune_state == NULL || + (!appendstate->as_prune_state->do_exec_prune && nplans > 0)) appendstate->as_valid_subplans = bms_add_range(NULL, 0, nplans - 1); } else @@ -578,7 +579,7 @@ choose_next_subplan_locally(AppendState *node) } else if (node->as_valid_subplans == NULL) node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); whichplan = -1; } @@ -643,7 +644,7 @@ choose_next_subplan_for_leader(AppendState *node) if (node->as_valid_subplans == NULL) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); /* * Mark each invalid plan as finished to allow the loop below to @@ -718,7 +719,7 @@ choose_next_subplan_for_worker(AppendState *node) else if (node->as_valid_subplans == NULL) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); mark_invalid_subplans_as_finished(node); } @@ -869,7 +870,7 @@ ExecAppendAsyncBegin(AppendState *node) if (node->as_valid_subplans == NULL) { node->as_valid_subplans = - ExecFindMatchingSubPlans(node->as_prune_state, false); + ExecFindMatchingSubPlans(node->as_prune_state, false, NULL); classify_matching_subplans(node); } diff --git a/src/backend/executor/nodeMergeAppend.c b/src/backend/executor/nodeMergeAppend.c index f370f9f287..ccfa083945 100644 --- a/src/backend/executor/nodeMergeAppend.c +++ b/src/backend/executor/nodeMergeAppend.c @@ -104,7 +104,8 @@ ExecInitMergeAppend(MergeAppend *node, EState *estate, int eflags) * subplan, we can fill ms_valid_subplans immediately, preventing * later calls to ExecFindMatchingSubPlans. */ - if (!prunestate->do_exec_prune && nplans > 0) + if (mergestate->ms_prune_state == NULL || + (!mergestate->ms_prune_state->do_exec_prune && nplans > 0)) mergestate->ms_valid_subplans = bms_add_range(NULL, 0, nplans - 1); } else @@ -219,7 +220,7 @@ ExecMergeAppend(PlanState *pstate) */ if (node->ms_valid_subplans == NULL) node->ms_valid_subplans = - ExecFindMatchingSubPlans(node->ms_prune_state, false); + ExecFindMatchingSubPlans(node->ms_prune_state, false, NULL); /* * First time through: pull the first tuple from each valid subplan, diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index fd5796f1b9..2ecb9193aa 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -1577,6 +1577,7 @@ SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, { CachedPlanSource *plansource; CachedPlan *cplan; + CachedPlanExtra *cplan_extra; List *stmt_list; char *query_string; Snapshot snapshot; @@ -1657,7 +1658,11 @@ SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, */ /* Replan if needed, and increment plan refcount for portal */ - cplan = GetCachedPlan(plansource, paramLI, NULL, _SPI_current->queryEnv); + cplan = GetCachedPlan(plansource, paramLI, NULL, _SPI_current->queryEnv, + &cplan_extra); + Assert(cplan_extra == NULL || + (list_length(cplan->stmt_list) == + list_length(cplan_extra->part_prune_results_list))); stmt_list = cplan->stmt_list; if (!plan->saved) @@ -1685,6 +1690,9 @@ SPI_cursor_open_internal(const char *name, SPIPlanPtr plan, stmt_list, cplan); + if (cplan_extra) + PortalSaveCachedPlanExtra(portal, cplan_extra); + /* * Set up options for portal. Default SCROLL type is chosen the same way * as PerformCursorOpen does it. @@ -2067,6 +2075,7 @@ SPI_plan_get_cached_plan(SPIPlanPtr plan) { CachedPlanSource *plansource; CachedPlan *cplan; + CachedPlanExtra *cplan_extra = NULL; SPICallbackArg spicallbackarg; ErrorContextCallback spierrcontext; @@ -2092,8 +2101,12 @@ SPI_plan_get_cached_plan(SPIPlanPtr plan) /* Get the generic plan for the query */ cplan = GetCachedPlan(plansource, NULL, plan->saved ? CurrentResourceOwner : NULL, - _SPI_current->queryEnv); + _SPI_current->queryEnv, + &cplan_extra); Assert(cplan == plansource->gplan); + Assert(cplan_extra == NULL || + (list_length(cplan->stmt_list) == + list_length(cplan_extra->part_prune_results_list))); /* Pop the error context stack */ error_context_stack = spierrcontext.previous; @@ -2399,6 +2412,7 @@ _SPI_execute_plan(SPIPlanPtr plan, const SPIExecuteOptions *options, SPICallbackArg spicallbackarg; ErrorContextCallback spierrcontext; CachedPlan *cplan = NULL; + CachedPlanExtra *cplan_extra = NULL; ListCell *lc1; /* @@ -2549,8 +2563,12 @@ _SPI_execute_plan(SPIPlanPtr plan, const SPIExecuteOptions *options, * plan, the refcount must be backed by the plan_owner. */ cplan = GetCachedPlan(plansource, options->params, - plan_owner, _SPI_current->queryEnv); + plan_owner, _SPI_current->queryEnv, + &cplan_extra); + Assert(cplan_extra == NULL || + (list_length(cplan->stmt_list) == + list_length(cplan_extra->part_prune_results_list))); stmt_list = cplan->stmt_list; /* @@ -2592,9 +2610,14 @@ _SPI_execute_plan(SPIPlanPtr plan, const SPIExecuteOptions *options, foreach(lc2, stmt_list) { PlannedStmt *stmt = lfirst_node(PlannedStmt, lc2); + List *part_prune_results = NIL; bool canSetTag = stmt->canSetTag; DestReceiver *dest; + if (cplan_extra) + part_prune_results = list_nth_node(List, + cplan_extra->part_prune_results_list, + foreach_current_index(lc2)); /* * Reset output state. (Note that if a non-SPI receiver is used, * _SPI_current->processed will stay zero, and that's what we'll @@ -2663,7 +2686,7 @@ _SPI_execute_plan(SPIPlanPtr plan, const SPIExecuteOptions *options, else snap = InvalidSnapshot; - qdesc = CreateQueryDesc(stmt, + qdesc = CreateQueryDesc(stmt, part_prune_results, plansource->query_string, snap, crosscheck_snapshot, dest, diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index ed43d5936d..db27cae297 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -372,6 +372,7 @@ set_plan_references(PlannerInfo *root, Plan *plan) { PartitionPruneInfo *pruneinfo = lfirst(lc); ListCell *l; + Bitmapset *leafpart_rtis = NULL; pruneinfo->root_parent_relids = offset_relid_set(pruneinfo->root_parent_relids, rtoffset); @@ -383,17 +384,52 @@ set_plan_references(PlannerInfo *root, Plan *plan) foreach(l2, prune_infos) { PartitionedRelPruneInfo *pinfo = lfirst(l2); + int i; /* RT index of the table to which the pinfo belongs. */ pinfo->rtindex += rtoffset; + + /* Also of the leaf partitions that might be scanned. */ + for (i = 0; i < pinfo->nparts; i++) + { + if (pinfo->rti_map[i] > 0 && pinfo->subplan_map[i] >= 0) + { + pinfo->rti_map[i] += rtoffset; + leafpart_rtis = bms_add_member(leafpart_rtis, + pinfo->rti_map[i]); + } + } } } + if (pruneinfo->needs_init_pruning) + { + glob->containsInitialPruning = true; + + /* + * Delete the leaf partition RTIs from the set of relations to be + * locked by AcquireExecutorLocks(). The actual set of leaf + * partitions to be locked is computed by + * CachedPlanLockPartitions(). + */ + glob->minLockRelids = bms_del_members(glob->minLockRelids, + leafpart_rtis); + } + glob->partPruneInfos = lappend(glob->partPruneInfos, pruneinfo); glob->containsInitialPruning |= pruneinfo->needs_init_pruning; } + /* + * It seems worth doing a bms_copy() on glob->minLockRelids if we deleted + * bits from it above to get rid of any empty tail bits. It seems better + * for the loop over this set in AcquireExecutorLocks() to not have to go + * through those useless bit words. + */ + if (glob->containsInitialPruning) + glob->minLockRelids = bms_copy(glob->minLockRelids); + return result; } diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index f8808d2191..9c1c7bfa9e 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -1598,6 +1598,7 @@ exec_bind_message(StringInfo input_message) int16 *rformats = NULL; CachedPlanSource *psrc; CachedPlan *cplan; + CachedPlanExtra *cplan_extra = NULL; Portal portal; char *query_string; char *saved_stmt_name; @@ -1972,7 +1973,10 @@ exec_bind_message(StringInfo input_message) * will be generated in MessageContext. The plan refcount will be * assigned to the Portal, so it will be released at portal destruction. */ - cplan = GetCachedPlan(psrc, params, NULL, NULL); + cplan = GetCachedPlan(psrc, params, NULL, NULL, &cplan_extra); + Assert(cplan_extra == NULL || + (list_length(cplan->stmt_list) == + list_length(cplan_extra->part_prune_results_list))); /* * Now we can define the portal. @@ -1987,6 +1991,9 @@ exec_bind_message(StringInfo input_message) cplan->stmt_list, cplan); + if (cplan_extra) + PortalSaveCachedPlanExtra(portal, cplan_extra); + /* Done with the snapshot used for parameter I/O and parsing/planning */ if (snapshot_set) PopActiveSnapshot(); diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 52e2db6452..32e6b7b767 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -35,7 +35,7 @@ Portal ActivePortal = NULL; -static void ProcessQuery(PlannedStmt *plan, +static void ProcessQuery(PlannedStmt *plan, List *part_prune_results, const char *sourceText, ParamListInfo params, QueryEnvironment *queryEnv, @@ -65,6 +65,7 @@ static void DoPortalRewind(Portal portal); */ QueryDesc * CreateQueryDesc(PlannedStmt *plannedstmt, + List *part_prune_results, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, @@ -77,6 +78,7 @@ CreateQueryDesc(PlannedStmt *plannedstmt, qd->operation = plannedstmt->commandType; /* operation */ qd->plannedstmt = plannedstmt; /* plan */ + qd->part_prune_results = part_prune_results; qd->sourceText = sourceText; /* query text */ qd->snapshot = RegisterSnapshot(snapshot); /* snapshot */ /* RI check snapshot */ @@ -122,6 +124,7 @@ FreeQueryDesc(QueryDesc *qdesc) * PORTAL_ONE_RETURNING, or PORTAL_ONE_MOD_WITH portal * * plan: the plan tree for the query + * part_prune_results: pruning results returned by CachedPlanLockPartitions() * sourceText: the source text of the query * params: any parameters needed * dest: where to send results @@ -134,6 +137,7 @@ FreeQueryDesc(QueryDesc *qdesc) */ static void ProcessQuery(PlannedStmt *plan, + List *part_prune_results, const char *sourceText, ParamListInfo params, QueryEnvironment *queryEnv, @@ -145,7 +149,7 @@ ProcessQuery(PlannedStmt *plan, /* * Create the QueryDesc object */ - queryDesc = CreateQueryDesc(plan, sourceText, + queryDesc = CreateQueryDesc(plan, part_prune_results, sourceText, GetActiveSnapshot(), InvalidSnapshot, dest, params, queryEnv, 0); @@ -491,8 +495,13 @@ PortalStart(Portal portal, ParamListInfo params, /* * Create QueryDesc in portal's context; for the moment, set * the destination to DestNone. + * + * There is no PartitionPruneResult unless the PlannedStmt is + * from a CachedPlan. */ queryDesc = CreateQueryDesc(linitial_node(PlannedStmt, portal->stmts), + portal->cplan_extra == NULL ? NIL : + linitial(portal->cplan_extra->part_prune_results_list), portal->sourceText, GetActiveSnapshot(), InvalidSnapshot, @@ -1225,6 +1234,8 @@ PortalRunMulti(Portal portal, if (pstmt->utilityStmt == NULL) { + List *part_prune_results = NIL; + /* * process a plannable query. */ @@ -1271,10 +1282,19 @@ PortalRunMulti(Portal portal, else UpdateActiveSnapshotCommandId(); + /* + * Determine if there's a corresponding List of PartitionPruneResult + * for this PlannedStmt. + */ + if (portal->cplan_extra) + part_prune_results = list_nth_node(List, + portal->cplan_extra->part_prune_results_list, + foreach_current_index(stmtlist_item)); + if (pstmt->canSetTag) { /* statement can set tag string */ - ProcessQuery(pstmt, + ProcessQuery(pstmt, part_prune_results, portal->sourceText, portal->portalParams, portal->queryEnv, @@ -1283,7 +1303,7 @@ PortalRunMulti(Portal portal, else { /* stmt added by rewrite cannot set tag */ - ProcessQuery(pstmt, + ProcessQuery(pstmt, part_prune_results, portal->sourceText, portal->portalParams, portal->queryEnv, diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index 339bb603f7..7bd94e7632 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -59,6 +59,7 @@ #include "access/transam.h" #include "catalog/namespace.h" #include "executor/executor.h" +#include "executor/execPartition.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" @@ -96,17 +97,20 @@ static dlist_head saved_plan_list = DLIST_STATIC_INIT(saved_plan_list); */ static dlist_head cached_expression_list = DLIST_STATIC_INIT(cached_expression_list); +static CachedPlan *GetCachedPlanInternal(CachedPlanSource *plansource, + ParamListInfo boundParams, ResourceOwner owner, + QueryEnvironment *queryEnv, bool *hasUnlockedParts); static void ReleaseGenericPlan(CachedPlanSource *plansource); static List *RevalidateCachedQuery(CachedPlanSource *plansource, QueryEnvironment *queryEnv); -static bool CheckCachedPlan(CachedPlanSource *plansource); +static bool CheckCachedPlan(CachedPlanSource *plansource, bool *hasUnlockedParts); static CachedPlan *BuildCachedPlan(CachedPlanSource *plansource, List *qlist, ParamListInfo boundParams, QueryEnvironment *queryEnv); static bool choose_custom_plan(CachedPlanSource *plansource, ParamListInfo boundParams); static double cached_plan_cost(CachedPlan *plan, bool include_planner); static Query *QueryListGetPrimaryStmt(List *stmts); -static void AcquireExecutorLocks(List *stmt_list, bool acquire); +static bool AcquireExecutorLocks(List *stmt_list, bool acquire); static void AcquirePlannerLocks(List *stmt_list, bool acquire); static void ScanQueryForLocks(Query *parsetree, bool acquire); static bool ScanQueryWalker(Node *node, bool *acquire); @@ -783,16 +787,23 @@ RevalidateCachedQuery(CachedPlanSource *plansource, } /* - * CheckCachedPlan: see if the CachedPlanSource's generic plan is valid. + * CheckCachedPlan: see if the CachedPlanSource's generic plan is valid and + * set *hasUnlockedParts if any PlannedStmt contains "initially" prunable + * subnodes; partitions are not locked till initial pruning is done. * * Caller must have already called RevalidateCachedQuery to verify that the * querytree is up to date. * - * On a "true" return, we have acquired the locks needed to run the plan. + * On a "true" return, we have acquired the minimal set of locks needed to run + * the plan, that is, excluding partitions that are subject to being pruned + * before execution. The caller must lock partitions after pruning those and + * locking the ones that remain before actually telling the world that the + * plan is "valid". + * * (We must do this for the "true" result to be race-condition-free.) */ static bool -CheckCachedPlan(CachedPlanSource *plansource) +CheckCachedPlan(CachedPlanSource *plansource, bool *hasUnlockedParts) { CachedPlan *plan = plansource->gplan; @@ -826,7 +837,7 @@ CheckCachedPlan(CachedPlanSource *plansource) */ Assert(plan->refcount > 0); - AcquireExecutorLocks(plan->stmt_list, true); + *hasUnlockedParts = AcquireExecutorLocks(plan->stmt_list, true); /* * If plan was transient, check to see if TransactionXmin has @@ -848,7 +859,7 @@ CheckCachedPlan(CachedPlanSource *plansource) } /* Oops, the race case happened. Release useless locks. */ - AcquireExecutorLocks(plan->stmt_list, false); + (void) AcquireExecutorLocks(plan->stmt_list, false); } /* @@ -1120,7 +1131,125 @@ cached_plan_cost(CachedPlan *plan, bool include_planner) } /* - * GetCachedPlan: get a cached plan from a CachedPlanSource. + * For each PlannedStmt in plan->stmt_list, do initial partition pruning if + * needed and lock partitions that survive. + * + * The returned list of the same length as plan->stmt_list will contains either + * a NIL if the PlannedStmt did not contain any PartitionPruneInfos requiring + * initial pruning or a List of PartitionPruneResult that in turn contains + * an element for each PartitionPruneInfo found in stmt->partPruneInfos. + * + * Also, on return, *lockedRelids_per_stmt, that will be made of the same + * length as plan->stmt_list, will contain either a NULL if no additional + * relations needed to be locked for the PlannedStmt, or a bitmapset of RT + * indexes of partitions locked. + */ +static bool +CachedPlanLockPartitions(CachedPlan *plan, + ParamListInfo boundParams, + ResourceOwner owner, + List **part_prune_results_list, + List **lockedRelids_per_stmt) +{ + List *my_part_prune_results_list = NIL; + List *my_lockedRelids_per_stmt = NIL; + ListCell *lc1; + MemoryContext oldcontext, + tmpcontext; + + *part_prune_results_list = NIL; + *lockedRelids_per_stmt = NIL; + + /* + * Create a temporary context for memory allocations required while + * executing partition pruning steps. + */ + tmpcontext = AllocSetContextCreate(CurrentMemoryContext, + "CachedPlanLockPartitions() working data", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(tmpcontext); + foreach(lc1, plan->stmt_list) + { + PlannedStmt *plannedstmt = lfirst_node(PlannedStmt, lc1); + Bitmapset *lockPartRelids = NULL; + int rti; + List *part_prune_results = NIL; + Bitmapset *lockedRelids = NULL; + + if (plannedstmt->commandType == CMD_UTILITY) + { + /* + * Ignore utility statements, because AcquireExecutorLocks on the + * parent CachedPlan would have dealt with these. Though, do let + * the caller know that no pruning is applicable to this statement. + */ + my_part_prune_results_list = lappend(my_part_prune_results_list, + NIL); + *lockedRelids_per_stmt = lappend(*lockedRelids_per_stmt, NULL); + continue; + } + + /* Figure out the partitions that would need to be locked. */ + if (plannedstmt->containsInitialPruning) + { + ListCell *lc2; + + foreach(lc2, plannedstmt->partPruneInfos) + { + PartitionPruneInfo *pruneinfo = lfirst_node(PartitionPruneInfo, lc2); + PartitionPruneResult *pruneresult = makeNode(PartitionPruneResult); + + pruneresult->root_parent_relids = + bms_copy(pruneinfo->root_parent_relids); + pruneresult->valid_subplan_offs = + ExecPartitionDoInitialPruning(plannedstmt, boundParams, + pruneinfo, + &lockPartRelids); + part_prune_results = lappend(part_prune_results, pruneresult); + } + } + + rti = -1; + while ((rti = bms_next_member(lockPartRelids, rti)) > 0) + { + RangeTblEntry *rte = rt_fetch(rti, plannedstmt->rtable); + + Assert(rte->rtekind == RTE_RELATION); + + /* + * Acquire the appropriate type of lock on each relation OID. Note + * that we don't actually try to open the rel, and hence will not + * fail if it's been dropped entirely --- we'll just transiently + * acquire a non-conflicting lock. + */ + LockRelationOid(rte->relid, rte->rellockmode); + lockedRelids = bms_add_member(lockedRelids, rti); + } + + my_part_prune_results_list = lappend(my_part_prune_results_list, + part_prune_results); + my_lockedRelids_per_stmt = lappend(my_lockedRelids_per_stmt, + lockedRelids); + } + + /* + * If the plan is still valid, copy the prune results and lockRelids + * bitmapsets into the caller's context. + */ + MemoryContextSwitchTo(oldcontext); + if (plan->is_valid) + { + *part_prune_results_list = copyObject(my_part_prune_results_list); + *lockedRelids_per_stmt = copyObject(my_lockedRelids_per_stmt); + } + + /* Clear up the temporary context. */ + MemoryContextDelete(tmpcontext); + return plan->is_valid; +} + +/* + * GetCachedPlan: get a cached plan from a CachedPlanSource * * This function hides the logic that decides whether to use a generic * plan or a custom plan for the given parameters: the caller does not know @@ -1139,7 +1268,97 @@ cached_plan_cost(CachedPlan *plan, bool include_planner) */ CachedPlan * GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, - ResourceOwner owner, QueryEnvironment *queryEnv) + ResourceOwner owner, QueryEnvironment *queryEnv, + CachedPlanExtra **extra) +{ + CachedPlan *plan; + + Assert(extra != NULL); + *extra = NULL; + for (;;) + { + bool hasUnlockedParts = false; + + /* Actually get the plan. */ + plan = GetCachedPlanInternal(plansource, boundParams, owner, queryEnv, + &hasUnlockedParts); + Assert(plan->is_valid); + + /* Nothing to do if all relations already locked. */ + if (!hasUnlockedParts) + return plan; + else + { + /* + * Do initial pruning to filter out partitions that need not be + * locked for execution. + */ + ListCell *lc1, + *lc2; + List *part_prune_results_list; + List *lockedRelids_per_stmt; + + /* Only a generic plan can ever have unlocked partitions in it. */ + Assert(plan == plansource->gplan); + + /* + * This does: + * + * 1) the pruning, returning in part_prune_results_list the + * PartitionPruneResult Lists for all statements + * + * 2) lock partitions that survive in each statement, returning + * in lockedRelids_per_stmt the RT indexes of those locked. + * + * True is returned if the plan is still valid after locking all + * partitions; false otherwise, in which case we must get a new + * plan. + */ + if (CachedPlanLockPartitions(plan, boundParams, owner, + &part_prune_results_list, + &lockedRelids_per_stmt)) + { + Assert(plan->is_valid); + *extra = (CachedPlanExtra *) palloc(sizeof(CachedPlanExtra)); + (*extra)->part_prune_results_list = part_prune_results_list; + return plan; + } + + /* + * Release the locks and start over. This is the same as what + * CheckCachedPlan does when doing AcquireExecutorLocks() causes + * the plan to be invalidated. + */ + forboth(lc1, plan->stmt_list, lc2, lockedRelids_per_stmt) + { + PlannedStmt *plannedstmt = lfirst(lc1); + Bitmapset *lockedRelids = lfirst(lc2); + int rti; + + if (plannedstmt->commandType == CMD_UTILITY) + continue; + rti = -1; + while ((rti = bms_next_member(lockedRelids, rti)) > 0) + { + RangeTblEntry *rte = rt_fetch(rti, plannedstmt->rtable); + + Assert(rte->rtekind == RTE_RELATION); + + UnlockRelationOid(rte->relid, rte->rellockmode); + } + } + } + } + + Assert(false); + return NULL; +} + +/* Internal workhorse of GetCachedPlan() */ +static CachedPlan * +GetCachedPlanInternal(CachedPlanSource *plansource, ParamListInfo boundParams, + ResourceOwner owner, QueryEnvironment *queryEnv, + bool *hasUnlockedParts) { CachedPlan *plan = NULL; List *qlist; @@ -1160,7 +1379,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, if (!customplan) { - if (CheckCachedPlan(plansource)) + if (CheckCachedPlan(plansource, hasUnlockedParts)) { /* We want a generic plan, and we already have a valid one */ plan = plansource->gplan; @@ -1738,11 +1957,16 @@ QueryListGetPrimaryStmt(List *stmts) /* * AcquireExecutorLocks: acquire locks needed for execution of a cached plan; * or release them if acquire is false. + * + * If some PlannedStmt(s) contain "initially prunable" partitions, they are not + * locked here. Instead, the caller is informed of their existence so that it + * can lock them after doing the initial pruning. */ -static void +static bool AcquireExecutorLocks(List *stmt_list, bool acquire) { ListCell *lc1; + bool hasUnlockedParts = false; foreach(lc1, stmt_list) { @@ -1763,10 +1987,17 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) Assert(plannedstmt->minLockRelids == NULL); if (query) - ScanQueryForLocks(query, acquire); + ScanQueryForLocks(query, true); continue; } + /* + * If partitions can be pruned before execution, defer their locking to + * the caller. + */ + if (plannedstmt->containsInitialPruning) + hasUnlockedParts = true; + allLockRelids = plannedstmt->minLockRelids; rti = -1; while ((rti = bms_next_member(allLockRelids, rti)) > 0) @@ -1788,6 +2019,8 @@ AcquireExecutorLocks(List *stmt_list, bool acquire) UnlockRelationOid(rte->relid, rte->rellockmode); } } + + return hasUnlockedParts; } /* diff --git a/src/backend/utils/mmgr/portalmem.c b/src/backend/utils/mmgr/portalmem.c index 7b1ae6fdcf..94a9db84e3 100644 --- a/src/backend/utils/mmgr/portalmem.c +++ b/src/backend/utils/mmgr/portalmem.c @@ -303,6 +303,22 @@ PortalDefineQuery(Portal portal, portal->status = PORTAL_DEFINED; } +/* + * Copies the given CachedPlanExtra struct into the portal. + */ +void +PortalSaveCachedPlanExtra(Portal portal, CachedPlanExtra *extra) +{ + MemoryContext oldcxt = MemoryContextSwitchTo(portal->portalContext); + + Assert(portal->cplan_extra == NULL && extra != NULL); + portal->cplan_extra = (CachedPlanExtra *) + palloc(sizeof(CachedPlanExtra)); + portal->cplan_extra->part_prune_results_list = + copyObject(extra->part_prune_results_list); + MemoryContextSwitchTo(oldcxt); +} + /* * PortalReleaseCachedPlan * Release a portal's reference to its cached plan, if any. diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 9ebde089ae..269cc4d562 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -87,7 +87,9 @@ extern void ExplainOneUtility(Node *utilityStmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv); -extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, +extern void ExplainOnePlan(PlannedStmt *plannedstmt, + List *part_prune_results, + IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, diff --git a/src/include/executor/execPartition.h b/src/include/executor/execPartition.h index aeeaeb7884..4b98d0d2ef 100644 --- a/src/include/executor/execPartition.h +++ b/src/include/executor/execPartition.h @@ -129,5 +129,10 @@ extern PartitionPruneState *ExecInitPartitionPruning(PlanState *planstate, Bitmapset *root_parent_relids, Bitmapset **initially_valid_subplans); extern Bitmapset *ExecFindMatchingSubPlans(PartitionPruneState *prunestate, - bool initial_prune); + bool initial_prune, + Bitmapset **scan_leafpart_rtis); +extern Bitmapset *ExecPartitionDoInitialPruning(PlannedStmt *plannedstmt, + ParamListInfo params, + PartitionPruneInfo *pruneinfo, + Bitmapset **scan_leafpart_rtis); #endif /* EXECPARTITION_H */ diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index e79e2c001f..5a7d075750 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -35,6 +35,8 @@ typedef struct QueryDesc /* These fields are provided by CreateQueryDesc */ CmdType operation; /* CMD_SELECT, CMD_UPDATE, etc. */ PlannedStmt *plannedstmt; /* planner's output (could be utility, too) */ + List *part_prune_results; /* PartitionPruneResults returned by + * CachedPlanLockPartitions() */ const char *sourceText; /* source text of the query */ Snapshot snapshot; /* snapshot to use for query */ Snapshot crosscheck_snapshot; /* crosscheck for RI update/delete */ @@ -57,6 +59,7 @@ typedef struct QueryDesc /* in pquery.c */ extern QueryDesc *CreateQueryDesc(PlannedStmt *plannedstmt, + List *part_prune_results, const char *sourceText, Snapshot snapshot, Snapshot crosscheck_snapshot, diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 9a64a830a2..f1374057e5 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -617,6 +617,7 @@ typedef struct EState List *es_rteperminfos; /* List of RTEPermissionInfo */ PlannedStmt *es_plannedstmt; /* link to top of plan tree */ List *es_part_prune_infos; /* PlannedStmt.partPruneInfos */ + List *es_part_prune_results; /* QueryDesc.part_prune_results */ const char *es_sourceText; /* Source text from QueryDesc */ JunkFilter *es_junkFilter; /* top-level junk filter, if any */ diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 4337e7aa34..10f12e780e 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -134,8 +134,8 @@ typedef struct PlannerGlobal bool containsInitialPruning; /* - * Indexes of all range table entries; for AcquireExecutorLocks()'s - * perusal. + * Indexes of all range table entries except those of leaf partitions + * scanned by prunable subplans; for AcquireExecutorLocks() perusal. */ Bitmapset *minLockRelids; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index eb0a007946..ab8bc74e4a 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -82,7 +82,9 @@ typedef struct PlannedStmt List *permInfos; /* list of RTEPermissionInfo nodes for rtable * entries needing one */ - Bitmapset *minLockRelids; /* Indexes of all range table entries; for + Bitmapset *minLockRelids; /* Indexes of all range table entries except + * those of leaf partitions scanned by + * prunable subplans; for * AcquireExecutorLocks()'s perusal */ /* rtable indexes of target relations for INSERT/UPDATE/DELETE/MERGE */ @@ -1575,6 +1577,33 @@ typedef struct PartitionPruneStepCombine List *source_stepids; } PartitionPruneStepCombine; +/*---------------- + * PartitionPruneResult + * + * The result of performing ExecPartitionDoInitialPruning() on a given + * PartitionPruneInfo. + * + * root_parent_relids is same as PartitionPruneInfo.root_parent_relids. It's + * there for cross-checking in ExecInitPartitionPruning() that the + * PartitionPruneResult and the PartitionPruneInfo at a given index in + * EState.es_part_prune_results and EState.es_part_prune_infos, respectively, + * belong to the same parent plan node. + * + * valid_subplans_offs contains the indexes of subplans remaining after + * performing initial pruning by calling ExecFindMatchingSubPlans() on the + * PartitionPruneInfo. + * + * This is used to store the result of initial partition pruning that is + * peformed before the execution has started, such as in + * CachedPlanLockPartitions(). + */ +typedef struct PartitionPruneResult +{ + NodeTag type; + + Bitmapset *root_parent_relids; + Bitmapset *valid_subplan_offs; +} PartitionPruneResult; /* * Plan invalidation info diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h index 0499635f59..4ac66d2761 100644 --- a/src/include/utils/plancache.h +++ b/src/include/utils/plancache.h @@ -160,6 +160,14 @@ typedef struct CachedPlan MemoryContext context; /* context containing this CachedPlan */ } CachedPlan; +/* + * Additional information to pass the executor when executing a CachedPlan. + */ +typedef struct CachedPlanExtra +{ + List *part_prune_results_list; +} CachedPlanExtra; + /* * CachedExpression is a low-overhead mechanism for caching the planned form * of standalone scalar expressions. While such expressions are not usually @@ -220,7 +228,8 @@ extern List *CachedPlanGetTargetList(CachedPlanSource *plansource, extern CachedPlan *GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, ResourceOwner owner, - QueryEnvironment *queryEnv); + QueryEnvironment *queryEnv, + CachedPlanExtra **extra); extern void ReleaseCachedPlan(CachedPlan *plan, ResourceOwner owner); extern bool CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, diff --git a/src/include/utils/portal.h b/src/include/utils/portal.h index aeddbdafe5..49bb00cda5 100644 --- a/src/include/utils/portal.h +++ b/src/include/utils/portal.h @@ -138,6 +138,8 @@ typedef struct PortalData QueryCompletion qc; /* command completion data for executed query */ List *stmts; /* list of PlannedStmts */ CachedPlan *cplan; /* CachedPlan, if stmts are from one */ + CachedPlanExtra *cplan_extra; /* CachedPlanExtra for cplan in Portal's + * memory */ ParamListInfo portalParams; /* params to pass to query */ QueryEnvironment *queryEnv; /* environment for query */ @@ -242,6 +244,7 @@ extern void PortalDefineQuery(Portal portal, CommandTag commandTag, List *stmts, CachedPlan *cplan); +extern void PortalSaveCachedPlanExtra(Portal portal, CachedPlanExtra *extra); extern PlannedStmt *PortalGetPrimaryStmt(Portal portal); extern void PortalCreateHoldStore(Portal portal); extern void PortalHashTableDeleteAll(void); -- 2.35.3