From 26ac277594a0fd6853c9b09afa10bf56e9f2818b Mon Sep 17 00:00:00 2001 From: Alexander Korotkov Date: Sun, 14 Jan 2024 02:09:32 +0200 Subject: [PATCH] Add new COPY option SAVE_ERROR_TO Currently, when source data contains unexpected data regarding data type or range, the entire COPY fails. However, in some cases, such data can be ignored and just copying normal data is preferable. This commit adds a new option SAVE_ERROR_TO, which specifies where to save the error information. When this option is specified, COPY skips soft errors and continues copying. Currently, SAVE_ERROR_TO only supports "none". This indicates error information is not saved and COPY just skips the unexpected data and continues running. Later works are expected to add more choices, such as 'log' and 'table'. Author: Damir Belyalov, Atsushi Torikoshi, Alex Shulgin, Jian He Discussion: https://postgr.es/m/87k31ftoe0.fsf_-_%40commandprompt.com Reviewed-by: Pavel Stehule, Andres Freund, Tom Lane, Daniel Gustafsson, Reviewed-by: Alena Rybakina, Andy Fan, Andrei Lepikhov, Masahiko Sawada Reviewed-by: Vignesh C --- doc/src/sgml/ref/copy.sgml | 21 ++++++++++- src/backend/commands/copy.c | 25 +++++++++++++ src/backend/commands/copyfrom.c | 46 ++++++++++++++++++++++++ src/backend/commands/copyfromparse.c | 17 ++++++--- src/bin/psql/tab-complete.c | 7 +++- src/include/commands/copy.h | 11 ++++++ src/include/commands/copyfrom_internal.h | 5 +++ src/test/regress/expected/copy2.out | 40 +++++++++++++++++++++ src/test/regress/sql/copy2.sql | 34 ++++++++++++++++++ src/tools/pgindent/typedefs.list | 1 + 10 files changed, 200 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index e2ffbbdf84e..e15d5a621b8 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -43,6 +43,7 @@ COPY { table_name [ ( column_name [, ...] ) | * } FORCE_NOT_NULL { ( column_name [, ...] ) | * } FORCE_NULL { ( column_name [, ...] ) | * } + SAVE_ERROR_TO 'location' ENCODING 'encoding_name' @@ -373,6 +374,23 @@ COPY { table_name [ ( + + SAVE_ERROR_TO + + + Specifies to save error information to + location when there is malformed data in the input. + If this option is specified, COPY skips malformed data + and continues copying data. + Currently, only the none value is supported. + If this option is omitted, COPY stops operation at the + first error. + This option is allowed only in COPY FROM, and only when + not using binary format. + + + + ENCODING @@ -556,7 +574,8 @@ COPY count - COPY stops operation at the first error. This + COPY stops operation at the first error when + SAVE_ERROR_TO is not specified. This should not lead to problems in the event of a COPY TO, but the target table will already have received earlier rows in a COPY FROM. These rows will not diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index fe4cf957d77..8fc54e028a3 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -571,6 +571,26 @@ ProcessCopyOptions(ParseState *pstate, defel->defname), parser_errposition(pstate, defel->location))); } + else if (strcmp(defel->defname, "save_error_to") == 0) + { + char *location = defGetString(defel); + + if (opts_out->save_error_to != COPY_SAVE_ERROR_TO_UNSPECIFIED) + errorConflictingDefElem(defel, pstate); + else if (strcmp(location, "none") == 0) + opts_out->save_error_to = COPY_SAVE_ERROR_TO_NONE; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY save_error_to \"%s\" not recognized", location), + parser_errposition(pstate, defel->location))); + + if (!is_from) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY SAVE_ERROR_TO cannot be used with COPY TO"), + parser_errposition(pstate, defel->location))); + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -598,6 +618,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot specify DEFAULT in BINARY mode"))); + if (opts_out->binary && opts_out->save_error_to != COPY_SAVE_ERROR_TO_UNSPECIFIED) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify SAVE_ERROR_TO in BINARY mode"))); + /* Set defaults for omitted options */ if (!opts_out->delim) opts_out->delim = opts_out->csv_mode ? "," : "\t"; diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 37836a769c7..d86c24e3140 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -42,6 +42,7 @@ #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "miscadmin.h" +#include "nodes/miscnodes.h" #include "optimizer/optimizer.h" #include "pgstat.h" #include "rewrite/rewriteHandler.h" @@ -656,6 +657,9 @@ CopyFrom(CopyFromState cstate) Assert(cstate->rel); Assert(list_length(cstate->range_table) == 1); + if (cstate->opts.save_error_to != COPY_SAVE_ERROR_TO_UNSPECIFIED) + Assert(cstate->escontext); + /* * The target must be a plain, foreign, or partitioned relation, or have * an INSTEAD OF INSERT row trigger. (Currently, such triggers are only @@ -992,6 +996,25 @@ CopyFrom(CopyFromState cstate) if (!NextCopyFrom(cstate, econtext, myslot->tts_values, myslot->tts_isnull)) break; + if (cstate->opts.save_error_to != COPY_SAVE_ERROR_TO_UNSPECIFIED && + cstate->escontext->error_occurred) + { + /* + * Soft error occured, skip this tuple and save error information + * according to SAVE_ERROR_TO. + */ + if (cstate->opts.save_error_to == COPY_SAVE_ERROR_TO_NONE) + + /* + * Just make ErrorSaveContext ready for the next NextCopyFrom. + * Since we don't set details_wanted and error_data is not to + * be filled, just resetting error_occurred is enough. + */ + cstate->escontext->error_occurred = false; + + continue; + } + ExecStoreVirtualTuple(myslot); /* @@ -1284,6 +1307,12 @@ CopyFrom(CopyFromState cstate) /* Done, clean up */ error_context_stack = errcallback.previous; + if (cstate->opts.save_error_to != COPY_SAVE_ERROR_TO_UNSPECIFIED && + cstate->num_errors > 0) + ereport(NOTICE, + errmsg("%zd rows were skipped due to data type incompatibility", + cstate->num_errors)); + if (bistate != NULL) FreeBulkInsertState(bistate); @@ -1419,6 +1448,23 @@ BeginCopyFrom(ParseState *pstate, } } + /* Set up soft error handler for SAVE_ERROR_TO */ + if (cstate->opts.save_error_to != COPY_SAVE_ERROR_TO_UNSPECIFIED) + { + cstate->escontext = makeNode(ErrorSaveContext); + cstate->escontext->type = T_ErrorSaveContext; + cstate->escontext->error_occurred = false; + + /* + * Currently we only support COPY_SAVE_ERROR_TO_NONE. We'll add other + * options later + */ + if (cstate->opts.save_error_to == COPY_SAVE_ERROR_TO_NONE) + cstate->escontext->details_wanted = false; + } + else + cstate->escontext = NULL; + /* Convert FORCE_NULL name list to per-column flags, check validity */ cstate->opts.force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); if (cstate->opts.force_null_all) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index af4c36f6450..7207eb26983 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -70,6 +70,7 @@ #include "libpq/pqformat.h" #include "mb/pg_wchar.h" #include "miscadmin.h" +#include "nodes/miscnodes.h" #include "pgstat.h" #include "port/pg_bswap.h" #include "utils/builtins.h" @@ -955,11 +956,17 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, values[m] = ExecEvalExpr(defexprs[m], econtext, &nulls[m]); } - else - values[m] = InputFunctionCall(&in_functions[m], - string, - typioparams[m], - att->atttypmod); + /* If SAVE_ERROR_TO is specified, skip rows with soft errors */ + else if (!InputFunctionCallSafe(&in_functions[m], + string, + typioparams[m], + att->atttypmod, + (Node *) cstate->escontext, + &values[m])) + { + cstate->num_errors++; + return true; + } cstate->cur_attname = NULL; cstate->cur_attval = NULL; diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 09914165e42..efe2b7cc101 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -2898,12 +2898,17 @@ psql_completion(const char *text, int start, int end) else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(")) COMPLETE_WITH("FORMAT", "FREEZE", "DELIMITER", "NULL", "HEADER", "QUOTE", "ESCAPE", "FORCE_QUOTE", - "FORCE_NOT_NULL", "FORCE_NULL", "ENCODING", "DEFAULT"); + "FORCE_NOT_NULL", "FORCE_NULL", "ENCODING", "DEFAULT", + "SAVE_ERROR_TO"); /* Complete COPY FROM|TO filename WITH (FORMAT */ else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "FORMAT")) COMPLETE_WITH("binary", "csv", "text"); + /* Complete COPY FROM filename WITH (SAVE_ERROR_TO */ + else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "SAVE_ERROR_TO")) + COMPLETE_WITH("none"); + /* Complete COPY FROM WITH () */ else if (Matches("COPY|\\copy", MatchAny, "FROM", MatchAny, "WITH", MatchAny)) COMPLETE_WITH("WHERE"); diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index e6c1867a2fc..7d1a6286a6f 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -30,6 +30,16 @@ typedef enum CopyHeaderChoice COPY_HEADER_MATCH, } CopyHeaderChoice; +/* + * Represents where to save input processing errors. More values to be added + * in the future. + */ +typedef enum CopySaveErrorToChoice +{ + COPY_SAVE_ERROR_TO_UNSPECIFIED = 0, /* immediately throw errors */ + COPY_SAVE_ERROR_TO_NONE, /* ignore errors */ +} CopySaveErrorToChoice; + /* * A struct to hold COPY options, in a parsed form. All of these are related * to formatting, except for 'freeze', which doesn't really belong here, but @@ -62,6 +72,7 @@ typedef struct CopyFormatOptions bool force_null_all; /* FORCE_NULL *? */ bool *force_null_flags; /* per-column CSV FN flags */ bool convert_selectively; /* do selective binary conversion? */ + CopySaveErrorToChoice save_error_to; /* where to save error information */ List *convert_select; /* list of column names (can be NIL) */ } CopyFormatOptions; diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 715939a9071..cad52fcc783 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -16,6 +16,7 @@ #include "commands/copy.h" #include "commands/trigger.h" +#include "nodes/miscnodes.h" /* * Represents the different source cases we need to worry about at @@ -94,6 +95,10 @@ typedef struct CopyFromStateData * default value */ FmgrInfo *in_functions; /* array of input functions for each attrs */ Oid *typioparams; /* array of element types for in_functions */ + ErrorSaveContext *escontext; /* soft error trapper during in_functions + * execution */ + uint64 num_errors; /* total number of rows which contained soft + * errors */ int *defmap; /* array of default att numbers related to * missing att */ ExprState **defexprs; /* array of default att expressions for all diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index c4178b9c07c..100fbf1dd1a 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -77,11 +77,21 @@ COPY x from stdin (encoding 'sql_ascii', encoding 'sql_ascii'); ERROR: conflicting or redundant options LINE 1: COPY x from stdin (encoding 'sql_ascii', encoding 'sql_ascii... ^ +COPY x from stdin (save_error_to none,save_error_to none); +ERROR: conflicting or redundant options +LINE 1: COPY x from stdin (save_error_to none,save_error_to none); + ^ -- incorrect options COPY x to stdin (format BINARY, delimiter ','); ERROR: cannot specify DELIMITER in BINARY mode COPY x to stdin (format BINARY, null 'x'); ERROR: cannot specify NULL in BINARY mode +COPY x from stdin (format BINARY, save_error_to none); +ERROR: cannot specify SAVE_ERROR_TO in BINARY mode +COPY x to stdin (save_error_to none); +ERROR: COPY SAVE_ERROR_TO cannot be used with COPY TO +LINE 1: COPY x to stdin (save_error_to none); + ^ COPY x to stdin (format TEXT, force_quote(a)); ERROR: COPY FORCE_QUOTE requires CSV mode COPY x from stdin (format CSV, force_quote(a)); @@ -94,6 +104,10 @@ COPY x to stdout (format TEXT, force_null(a)); ERROR: COPY FORCE_NULL requires CSV mode COPY x to stdin (format CSV, force_null(a)); ERROR: COPY FORCE_NULL cannot be used with COPY TO +COPY x to stdin (format BINARY, save_error_to unsupported); +ERROR: COPY save_error_to "unsupported" not recognized +LINE 1: COPY x to stdin (format BINARY, save_error_to unsupported); + ^ -- too many columns in column list: should fail COPY x (a, b, c, d, e, d, c) from stdin; ERROR: column "d" specified more than once @@ -710,6 +724,30 @@ SELECT * FROM instead_of_insert_tbl; (2 rows) COMMIT; +-- tests for SAVE_ERROR_TO option +CREATE TABLE check_ign_err (n int, m int[], k int); +COPY check_ign_err FROM STDIN WITH (save_error_to none); +NOTICE: 4 rows were skipped due to data type incompatibility +SELECT * FROM check_ign_err; + n | m | k +---+-----+--- + 1 | {1} | 1 + 5 | {5} | 5 +(2 rows) + +-- test datatype error that can't be handled as soft: should fail +CREATE TABLE hard_err(foo widget); +COPY hard_err FROM STDIN WITH (save_error_to none); +ERROR: invalid input syntax for type widget: "1" +CONTEXT: COPY hard_err, line 1, column foo: "1" +-- test missing data: should fail +COPY check_ign_err FROM STDIN WITH (save_error_to none); +ERROR: missing data for column "k" +CONTEXT: COPY check_ign_err, line 1: "1 {1}" +-- test extra data: should fail +COPY check_ign_err FROM STDIN WITH (save_error_to none); +ERROR: extra data after last expected column +CONTEXT: COPY check_ign_err, line 1: "1 {1} 3 abc" -- clean up DROP TABLE forcetest; DROP TABLE vistest; @@ -724,6 +762,8 @@ DROP TABLE instead_of_insert_tbl; DROP VIEW instead_of_insert_tbl_view; DROP VIEW instead_of_insert_tbl_view_2; DROP FUNCTION fun_instead_of_insert_tbl(); +DROP TABLE check_ign_err; +DROP TABLE hard_err; -- -- COPY FROM ... DEFAULT -- diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index a5486f60867..f3c24647d4a 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -66,16 +66,20 @@ COPY x from stdin (force_not_null (a), force_not_null (b)); COPY x from stdin (force_null (a), force_null (b)); COPY x from stdin (convert_selectively (a), convert_selectively (b)); COPY x from stdin (encoding 'sql_ascii', encoding 'sql_ascii'); +COPY x from stdin (save_error_to none,save_error_to none); -- incorrect options COPY x to stdin (format BINARY, delimiter ','); COPY x to stdin (format BINARY, null 'x'); +COPY x from stdin (format BINARY, save_error_to none); +COPY x to stdin (save_error_to none); COPY x to stdin (format TEXT, force_quote(a)); COPY x from stdin (format CSV, force_quote(a)); COPY x to stdout (format TEXT, force_not_null(a)); COPY x to stdin (format CSV, force_not_null(a)); COPY x to stdout (format TEXT, force_null(a)); COPY x to stdin (format CSV, force_null(a)); +COPY x to stdin (format BINARY, save_error_to unsupported); -- too many columns in column list: should fail COPY x (a, b, c, d, e, d, c) from stdin; @@ -494,6 +498,34 @@ test1 SELECT * FROM instead_of_insert_tbl; COMMIT; +-- tests for SAVE_ERROR_TO option +CREATE TABLE check_ign_err (n int, m int[], k int); +COPY check_ign_err FROM STDIN WITH (save_error_to none); +1 {1} 1 +a {2} 2 +3 {3} 3333333333 +4 {a, 4} 4 + +5 {5} 5 +\. +SELECT * FROM check_ign_err; + +-- test datatype error that can't be handled as soft: should fail +CREATE TABLE hard_err(foo widget); +COPY hard_err FROM STDIN WITH (save_error_to none); +1 +\. + +-- test missing data: should fail +COPY check_ign_err FROM STDIN WITH (save_error_to none); +1 {1} +\. + +-- test extra data: should fail +COPY check_ign_err FROM STDIN WITH (save_error_to none); +1 {1} 3 abc +\. + -- clean up DROP TABLE forcetest; DROP TABLE vistest; @@ -508,6 +540,8 @@ DROP TABLE instead_of_insert_tbl; DROP VIEW instead_of_insert_tbl_view; DROP VIEW instead_of_insert_tbl_view_2; DROP FUNCTION fun_instead_of_insert_tbl(); +DROP TABLE check_ign_err; +DROP TABLE hard_err; -- -- COPY FROM ... DEFAULT diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f582eb59e7d..29fd1cae641 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4041,3 +4041,4 @@ manifest_writer rfile ws_options ws_file_info +CopySaveErrorToChoice -- 2.39.3 (Apple Git-145)