doc/src/sgml/custom-scan.sgml | 41 +++++++++++----- doc/src/sgml/fdwhandler.sgml | 22 ++++++++- src/backend/commands/explain.c | 2 +- src/backend/executor/nodeCustom.c | 4 +- src/backend/nodes/copyfuncs.c | 21 ++++++-- src/backend/nodes/nodes.c | 74 ++++++++++++++++++++++++++++ src/backend/nodes/outfuncs.c | 46 ++++++++++++++++-- src/backend/nodes/readfuncs.c | 51 ++++++++++++++----- src/include/nodes/execnodes.h | 2 +- src/include/nodes/nodes.h | 100 ++++++++++++++++++++++++++++++++++++++ src/include/nodes/plannodes.h | 5 +- src/include/nodes/relation.h | 3 +- 12 files changed, 328 insertions(+), 43 deletions(-) diff --git a/doc/src/sgml/custom-scan.sgml b/doc/src/sgml/custom-scan.sgml index 1ca9247..1a83b6a 100644 --- a/doc/src/sgml/custom-scan.sgml +++ b/doc/src/sgml/custom-scan.sgml @@ -80,12 +80,17 @@ typedef struct CustomPath custom_private can be used to store the custom path's private data. Private data should be stored in a form that can be handled by nodeToString, so that debugging routines that attempt to - print the custom path will work as designed. methods must - point to a (usually statically allocated) object implementing the required - custom path methods, of which there is currently only one. The - LibraryName and SymbolName fields must also - be initialized so that the dynamic loader can resolve them to locate the - method table. + print the custom path will work as designed. methods must + point to the statically allocated function pointers table of + CustomPathMethods that implements the required custom + path methods. + The first field of CustomPathMethods is + ExtensibleNodeMethods. If custom scan provider defines own + structure larger than CustomPathMethods to keep private + information on the extra area, it must support callbacks of the + ExtensibleNodeMethods to treat these private fields. + See the source code comments in src/include/nodes/nodes.h + for more details. @@ -185,10 +190,17 @@ typedef struct CustomScan Plan trees must be able to be duplicated using copyObject, so all the data stored within the custom fields must consist of - nodes that that function can handle. Furthermore, custom scan providers - cannot substitute a larger structure that embeds - a CustomScan for the structure itself, as would be possible - for a CustomPath or CustomScanState. + nodes that function can handle. + On the other hands, custom scan provider can substitute a larger structure + that embeds CustomScan for the CustomScan + itself, as would be possible for CustomPath or + CustomScanState. + If custom scan provider keeps its private fields in the extra area of the + above own defined structure, it also have to implement each callbacks of + the ExtensibleNodeMethods to support serialization and + deserialization. + See the source code comments in src/include/nodes/nodes.h + for more details. @@ -235,9 +247,12 @@ typedef struct CustomScanState CustomPath and CustomScan. methods must point to a (usually statically allocated) object implementing the required custom scan state methods, which are - further detailed below. Typically, a CustomScanState, which - need not support copyObject, will actually be a larger - structure embedding the above as its first member. + further detailed below. Right now, CustomScanState node + is not supported by copyObject or relevant node functions, + so we don't need to implement callbacks of the + ExtensibleNodeMethods, even if CustomScanState + is embedded in a larger private structure. However, it is a good idea to + care about private fields for code compatibility to the future enhancement. diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index c6b60fa..b56597a 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -155,6 +155,14 @@ GetForeignPaths (PlannerInfo *root, estimates, and can contain any FDW-private information that is needed to identify the specific scan method intended. + + The ForeignPath node can be embedded in a self-define + larger structure, to keep private fields in the extra area, instead of + the fdw_private, if extension prefers. + In this case, extnodename has to be a valid name of the + registered ExtensibleNodeMethods, of NULL + elsewhere. + See for additional information. @@ -186,7 +194,19 @@ GetForeignPlan (PlannerInfo *root, This function must create and return a ForeignScan plan node; it's recommended to use make_foreignscan to build the - ForeignScan node. + ForeignScan node, if it is not embedded in a self-defined + larger structure. + + + + On the other hands, ForeignScan node can be embedded in + a self-defined larger structure, to keep private fields in the extra + area, if extension prefers this way. + In this case, extnodename has to be a valid name of the + registered ExtensibleNodeMethods, of NULL + elsewhere. + See the source code comments in src/include/nodes/nodes.h + for more details. diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 25d8ca0..2fc16de 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -892,7 +892,7 @@ ExplainNode(PlanState *planstate, List *ancestors, break; case T_CustomScan: sname = "Custom Scan"; - custom_name = ((CustomScan *) plan)->methods->CustomName; + custom_name = ((CustomScan *) plan)->methods->xnode.extnodename; if (custom_name) pname = psprintf("Custom Scan (%s)", custom_name); else diff --git a/src/backend/executor/nodeCustom.c b/src/backend/executor/nodeCustom.c index 322abca..953c46a 100644 --- a/src/backend/executor/nodeCustom.c +++ b/src/backend/executor/nodeCustom.c @@ -146,7 +146,7 @@ ExecCustomMarkPos(CustomScanState *node) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("custom scan \"%s\" does not support MarkPos", - node->methods->CustomName))); + node->methods->xnode.extnodename))); node->methods->MarkPosCustomScan(node); } @@ -157,7 +157,7 @@ ExecCustomRestrPos(CustomScanState *node) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("custom scan \"%s\" does not support MarkPos", - node->methods->CustomName))); + node->methods->xnode.extnodename))); node->methods->RestrPosCustomScan(node); } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index a8b79fa..59b4b66 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -636,8 +636,16 @@ _copyWorkTableScan(const WorkTableScan *from) static ForeignScan * _copyForeignScan(const ForeignScan *from) { - ForeignScan *newnode = makeNode(ForeignScan); + const ExtensibleNodeMethods *methods = NULL; + ForeignScan *newnode; + if (from->extnodename) + methods = GetExtensibleNodeMethods(from->extnodename); + + newnode = (ForeignScan *) newNode(!methods + ? sizeof(ForeignScan) + : methods->node_size, + T_ForeignScan); /* * copy node superclass fields */ @@ -646,6 +654,7 @@ _copyForeignScan(const ForeignScan *from) /* * copy remainder of node */ + COPY_STRING_FIELD(extnodename); COPY_SCALAR_FIELD(fs_server); COPY_NODE_FIELD(fdw_exprs); COPY_NODE_FIELD(fdw_private); @@ -653,6 +662,8 @@ _copyForeignScan(const ForeignScan *from) COPY_NODE_FIELD(fdw_recheck_quals); COPY_BITMAPSET_FIELD(fs_relids); COPY_SCALAR_FIELD(fsSystemCol); + if (methods) + methods->nodeCopy((Node *) newnode, (const Node *) from); return newnode; } @@ -663,8 +674,9 @@ _copyForeignScan(const ForeignScan *from) static CustomScan * _copyCustomScan(const CustomScan *from) { - CustomScan *newnode = makeNode(CustomScan); - + const ExtensibleNodeMethods *methods = &from->methods->xnode; + CustomScan *newnode = (CustomScan *) newNode(methods->node_size, + T_CustomScan); /* * copy node superclass fields */ @@ -687,6 +699,9 @@ _copyCustomScan(const CustomScan *from) */ COPY_SCALAR_FIELD(methods); + if (methods->nodeCopy) + methods->nodeCopy((Node *) newnode, (const Node *) from); + return newnode; } diff --git a/src/backend/nodes/nodes.c b/src/backend/nodes/nodes.c index aea3880..463104b 100644 --- a/src/backend/nodes/nodes.c +++ b/src/backend/nodes/nodes.c @@ -19,6 +19,9 @@ #include "postgres.h" #include "nodes/nodes.h" +#include "nodes/pg_list.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" /* * Support for newNode() macro @@ -29,3 +32,74 @@ */ Node *newNodeMacroHolder; + +/* + * Extensible node support - We allow extension to have own structure that + * contains well known structure (like ForeignScan or CustomScan), to keep + * their private data fields. These structure also have to support + * serialization / deserialization using node functions. A collection of + * callback enables to handle private fields also, not only well known + * portion. + */ +static HTAB *extensible_node_methods = NULL; + +typedef struct +{ + char extnodename[NAMEDATALEN]; + const ExtensibleNodeMethods *methods; +} ExtensibleNodeEntry; + +void +RegisterExtensibleNodeMethods(const ExtensibleNodeMethods *methods) +{ + ExtensibleNodeEntry *entry; + bool found; + + if (!extensible_node_methods) + { + HASHCTL ctl; + + memset(&ctl, 0, sizeof(HASHCTL)); + ctl.keysize = NAMEDATALEN; + ctl.entrysize = sizeof(const ExtensibleNodeEntry); + extensible_node_methods = hash_create("Extensible Node Methods", + 100, &ctl, HASH_ELEM); + } + + if (strlen(methods->extnodename) >= NAMEDATALEN) + ereport(ERROR, + (errcode(ERRCODE_NAME_TOO_LONG), + errmsg("ExtensibleNodeMethods \"%s\" name too long", + methods->extnodename))); + + entry = (ExtensibleNodeEntry *) hash_search(extensible_node_methods, + methods->extnodename, + HASH_ENTER, &found); + if (found) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("ExtensibleNodeMethods \"%s\" already exists", + methods->extnodename))); + /* setup entry */ + entry->methods = methods; +} + +const ExtensibleNodeMethods * +GetExtensibleNodeMethods(const char *extnodename) +{ + ExtensibleNodeEntry *entry = NULL; + + if (extensible_node_methods) + { + entry = (ExtensibleNodeEntry *) hash_search(extensible_node_methods, + extnodename, + HASH_FIND, NULL); + } + + if (!entry) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("ExtensibleNodeMethods \"%s\" was not registered", + extnodename))); + return entry->methods; +} diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d59b954..2cf0178 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -140,6 +140,13 @@ _outToken(StringInfo str, const char *s) } } +/* outToken - entrypoint for extensions */ +void +outToken(StringInfo str, const char *s) +{ + _outToken(str, s); +} + static void _outList(StringInfo str, const List *node) { @@ -196,6 +203,13 @@ _outBitmapset(StringInfo str, const Bitmapset *bms) appendStringInfoChar(str, ')'); } +/* outBitmapset - entrypoint for extensions */ +void +outBitmapset(StringInfo str, const Bitmapset *bms) +{ + _outBitmapset(str, bms); +} + /* * Print the value of a Datum given its type. */ @@ -588,6 +602,11 @@ _outWorkTableScan(StringInfo str, const WorkTableScan *node) static void _outForeignScan(StringInfo str, const ForeignScan *node) { + const ExtensibleNodeMethods *methods = NULL; + + if (node->extnodename) + methods = GetExtensibleNodeMethods(node->extnodename); + WRITE_NODE_TYPE("FOREIGNSCAN"); _outScanInfo(str, (const Scan *) node); @@ -599,11 +618,15 @@ _outForeignScan(StringInfo str, const ForeignScan *node) WRITE_NODE_FIELD(fdw_recheck_quals); WRITE_BITMAPSET_FIELD(fs_relids); WRITE_BOOL_FIELD(fsSystemCol); + if (methods) + methods->nodeOut(str, (const Node *) node); } static void _outCustomScan(StringInfo str, const CustomScan *node) { + const ExtensibleNodeMethods *methods = &node->methods->xnode; + WRITE_NODE_TYPE("CUSTOMSCAN"); _outScanInfo(str, (const Scan *) node); @@ -614,11 +637,12 @@ _outCustomScan(StringInfo str, const CustomScan *node) WRITE_NODE_FIELD(custom_private); WRITE_NODE_FIELD(custom_scan_tlist); WRITE_BITMAPSET_FIELD(custom_relids); - /* Dump library and symbol name instead of raw pointer */ + /* Dump CustomName; alias of extnodename */ appendStringInfoString(str, " :methods "); - _outToken(str, node->methods->LibraryName); - appendStringInfoChar(str, ' '); - _outToken(str, node->methods->SymbolName); + _outToken(str, methods->extnodename); + /* Dump private fields if any */ + if (methods && methods->nodeOut) + methods->nodeOut(str, (const Node *) node); } static void @@ -1685,17 +1709,27 @@ _outTidPath(StringInfo str, const TidPath *node) static void _outForeignPath(StringInfo str, const ForeignPath *node) { + const ExtensibleNodeMethods *methods = NULL; + + if (node->extnodename) + methods = GetExtensibleNodeMethods(node->extnodename); + WRITE_NODE_TYPE("FOREIGNPATH"); _outPathInfo(str, (const Path *) node); + WRITE_STRING_FIELD(extnodename); WRITE_NODE_FIELD(fdw_outerpath); WRITE_NODE_FIELD(fdw_private); + if (methods) + methods->nodeOut(str, (Node *) node); } static void _outCustomPath(StringInfo str, const CustomPath *node) { + const ExtensibleNodeMethods *methods = &node->methods->xnode; + WRITE_NODE_TYPE("CUSTOMPATH"); _outPathInfo(str, (const Path *) node); @@ -1704,7 +1738,9 @@ _outCustomPath(StringInfo str, const CustomPath *node) WRITE_NODE_FIELD(custom_paths); WRITE_NODE_FIELD(custom_private); appendStringInfoString(str, " :methods "); - _outToken(str, node->methods->CustomName); + _outToken(str, methods->extnodename); + if (methods && methods->nodeOut) + methods->nodeOut(str, (Node *)node); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 6c46151..3e9e1ae 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -218,6 +218,12 @@ _readBitmapset(void) return result; } +/* readBitmapset - entrypoint for extensions */ +Bitmapset * +readBitmapset(void) +{ + return _readBitmapset(); +} /* * _readQuery @@ -1797,6 +1803,7 @@ _readForeignScan(void) ReadCommonScan(&local_node->scan); + READ_STRING_FIELD(extnodename); READ_OID_FIELD(fs_server); READ_NODE_FIELD(fdw_exprs); READ_NODE_FIELD(fdw_private); @@ -1805,6 +1812,19 @@ _readForeignScan(void) READ_BITMAPSET_FIELD(fs_relids); READ_BOOL_FIELD(fsSystemCol); + if (local_node->extnodename) + { + const ExtensibleNodeMethods *methods + = GetExtensibleNodeMethods(local_node->extnodename); + ForeignScan *local_temp = palloc0(methods->node_size); + + Assert(methods->node_size >= sizeof(ForeignScan)); + memcpy(local_temp, local_node, sizeof(ForeignScan)); + methods->nodeRead((Node *) local_temp); + + pfree(local_node); + local_node = local_temp; + } READ_DONE(); } @@ -1814,10 +1834,9 @@ _readForeignScan(void) static CustomScan * _readCustomScan(void) { + const ExtensibleNodeMethods *methods; + const char *extnodename; READ_LOCALS(CustomScan); - char *library_name; - char *symbol_name; - const CustomScanMethods *methods; ReadCommonScan(&local_node->scan); @@ -1832,18 +1851,24 @@ _readCustomScan(void) * Reconstruction of methods using library and symbol name */ token = pg_strtok(&length); /* skip methods: */ - token = pg_strtok(&length); /* LibraryName */ - library_name = nullable_string(token, length); - token = pg_strtok(&length); /* SymbolName */ - symbol_name = nullable_string(token, length); + token = pg_strtok(&length); /* CustomName (alias of extnodename) */ + extnodename = nullable_string(token, length); + methods = GetExtensibleNodeMethods(extnodename); - methods = (const CustomScanMethods *) - load_external_function(library_name, symbol_name, true, NULL); - Assert(strcmp(methods->LibraryName, library_name) == 0 && - strcmp(methods->SymbolName, symbol_name) == 0); - local_node->methods = methods; + if (methods->nodeRead) + { + CustomScan *local_temp = palloc0(methods->node_size); - READ_DONE(); + Assert(methods->node_size >= sizeof(CustomScan)); + memcpy(local_temp, local_node, sizeof(CustomScan)); + methods->nodeRead((Node *) local_temp); + + pfree(local_node); + local_node = local_temp; + } + local_node->methods = (const CustomScanMethods *) methods; + + READ_DONE(); } /* diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 064a050..881ce6b 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1611,7 +1611,7 @@ struct CustomScanState; typedef struct CustomExecMethods { - const char *CustomName; + ExtensibleNodeMethods xnode; /* collection of node operations */ /* Executor methods: mark/restore are optional, the rest are required */ void (*BeginCustomScan) (struct CustomScanState *node, diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index cf09db4..54423db 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -527,10 +527,17 @@ extern PGDLLIMPORT Node *newNodeMacroHolder; */ extern char *nodeToString(const void *obj); +struct Bitmapset; /* not to include bitmapset.h here */ +struct StringInfoData; /* not to include stringinfo.h here */ +extern void outToken(struct StringInfoData *str, const char *s); +extern void outBitmapset(struct StringInfoData *str, + const struct Bitmapset *bms); + /* * nodes/{readfuncs.c,read.c} */ extern void *stringToNode(char *str); +extern struct Bitmapset *readBitmapset(void); /* * nodes/copyfuncs.c @@ -542,6 +549,99 @@ extern void *copyObject(const void *obj); */ extern bool equal(const void *a, const void *b); +/* + * ExtensibleNodeMethods - A collection of node operation callbacks + * + * Some of node types allows extension to define a larger structure that + * embeds itself, typically, to keep private data without own form and + * deforming. + * Below is an example of a larger structure that emdeds the well known + * CustomScan node, in addition to the private variables. + * + * typedef struct myFooScan { + * CustomScan cscan; + * char *foo_name; + * int var_index; + * int baz_value; + * } myFooScan; + * + * In this case, extension has to pay attention how core backend processes + * the node object. Of course, only extension shall touch its private fields. + * It also means the core backend ignores the private fields when it copies, + * serializes and de-serialized the node or compares with others. + * + * ExtensibleNodeMethods provides an infrastructure to coordinate between + * the core backend and extension to handle these private fields properly. + * + * Extensions who want to use larger structures than well known form, has to + * register a statically allocated ExtensibleNodeMethods, likely at the time + * of _PG_init(), using RegisterExtensibleNodeMethods(), with a unique name. + * The registered ExtensibleNodeMethods contains the following fields and + * callbacks. It is role of extension to implement them properly. + * + * const char *extnodename + * ------------------------ + * This field is a unique name of the ExtensibleNodeMethods. It shouldn't + * be duplicated to others. + * + * Size node_size + * ------------------- + * This field introduces size of the larger structure including private + * fileds, to be allocated on its creation time. We don't support variable- + * length fields in this schema. If needed, allocate a separated array + * individually. + * + * void (*nodeCopy)(Node *newnode, const Node *oldnode) + * --------------------------------------------------------- + * This callback supplies a newly allocated node and the original; both of + * them are relevant to ths ExtensibleNodeMethods. Extension has to make + * copy of the private fields + * The portion of the well known node embedded in the private larger structure + * is already processed by the copyfunc.c, so all extension has to copy is + * its private fields. + * + * bool (*nodeEqual)(const Node *a, const Node *b) + * ---------------------------------------------------- + * This callback supplies two node objects that is relevant to same extensible + * node methods. The portion of the well known node embedded in the private + * larger structure is already checked, so all all extension has to compare is + * its private fields. + * + * NOTE: At this moment, we have no node types that support both of equal() + * and extensible node. + * + * void (*nodeOut)(StringInfo str, const Node *node) + * ------------------------------------------------------ + * This callback serializes the supplied node object onto the StringInfo + * buffer. The portion of the well known node embedded in the private larger + * structure is already serialized (written out), so all extension has to + * write is its private fields. + * The written text form has to follow the manner in outfuncs.c, and also + * has to be reproducible using the 'nodeRead' callback below. + * + * void (*nodeRead)(Node *node) + * --------------------------------- + * This callback de-serializes the supplied node object according to the + * extra tokens that reflect the private fields written by the above 'nodeOut' + * callback. The portion of the well known node embedded in the private larger + * structure is already de-serialized (read in), so all extension has to + * reconstruct from the upcoming tokens is its private fields. + * Extension can fetch next token using pg_strtok() from the current input + * stream, then reconstruct the private fields according to the manner in + * readfuncs.c. + */ +typedef struct ExtensibleNodeMethods +{ + const char *extnodename; + Size node_size; + void (*nodeCopy)(Node *newnode, const Node *oldnode); + bool (*nodeEqual)(const Node *a, const Node *b); + void (*nodeOut)(struct StringInfoData *str, const Node *node); + void (*nodeRead)(Node *node); +} ExtensibleNodeMethods; + +extern void RegisterExtensibleNodeMethods(const ExtensibleNodeMethods *method); +extern const ExtensibleNodeMethods *GetExtensibleNodeMethods(const char *name); /* * Typedefs for identifying qualifier selectivities and plan costs as such. diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 55d6bbe..5c0d645 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -531,6 +531,7 @@ typedef struct WorkTableScan typedef struct ForeignScan { Scan scan; + const char *extnodename; /* collection of node operation callbacks */ Oid fs_server; /* OID of foreign server */ List *fdw_exprs; /* expressions that FDW may evaluate */ List *fdw_private; /* private data for FDW */ @@ -557,9 +558,7 @@ struct CustomScan; typedef struct CustomScanMethods { - const char *CustomName; - const char *LibraryName; - const char *SymbolName; + ExtensibleNodeMethods xnode; /* collection of node operations */ /* Create execution state (CustomScanState) from a CustomScan plan node */ Node *(*CreateCustomScanState) (struct CustomScan *cscan); diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 595438c..eec4403 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -920,6 +920,7 @@ typedef struct TidPath typedef struct ForeignPath { Path path; + const char *extnodename; Path *fdw_outerpath; List *fdw_private; } ForeignPath; @@ -949,7 +950,7 @@ struct CustomPath; typedef struct CustomPathMethods { - const char *CustomName; + ExtensibleNodeMethods xnode; /* collection of node operations */ /* Convert Path to a Plan */ struct Plan *(*PlanCustomPath) (PlannerInfo *root,