From a416e54e306129d818c9dd4806a7dcc34055d557 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Sun, 9 Jul 2023 11:19:15 +0800 Subject: [PATCH] COPY FROM enable FORCE_NULL/FORCE_NOT_NULL on all columns We already have FORCE_NULL/FORCE_NOT_NULL options to force null string to be converted to NULL or empty string. But users must set the columns one by one. This commit enables FORCE_NULL/FORCE_NOT_NULL options to select all columns of a table to be converted. Like FORCE_QUOTE * , the sql format are FORCE_NULL * and FORCE_NOT_NULL *. FORCE_NULL * and FORCE_NOT_NULL * can be used simultaneously as before. BEGIN; COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); 4,,"" \. COMMIT; SELECT b, c FROM forcetest WHERE a = 4; b | c ---+------ | NULL (1 row) --- doc/src/sgml/ref/copy.sgml | 6 ++-- src/backend/commands/copy.c | 12 +++++--- src/backend/commands/copyfrom.c | 8 ++++-- src/backend/parser/gram.y | 8 ++++++ src/include/commands/copy.h | 2 ++ src/test/regress/expected/copy2.out | 44 +++++++++++++++++++++++++++++ src/test/regress/sql/copy2.sql | 30 ++++++++++++++++++++ 7 files changed, 102 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 5e591ed2e6..4baf62295e 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -40,8 +40,8 @@ COPY { table_name [ ( quote_character' ESCAPE 'escape_character' FORCE_QUOTE { ( column_name [, ...] ) | * } - FORCE_NOT_NULL ( column_name [, ...] ) - FORCE_NULL ( column_name [, ...] ) + FORCE_NOT_NULL ( column_name [, ...] ) | * ) + FORCE_NULL ( column_name [, ...] ) | * ) ENCODING 'encoding_name' DEFAULT 'default_string' @@ -337,6 +337,7 @@ COPY { table_name [ ( * is specified, it will be applied in all columns. This option is allowed only in COPY FROM, and only when using CSV format. @@ -351,6 +352,7 @@ COPY { table_name [ ( NULL. In the default case where the null string is empty, this converts a quoted empty string into NULL. + If * is specified, it will be applied in all columns. This option is allowed only in COPY FROM, and only when using CSV format. diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index f14fae3308..c5d7d78645 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -512,9 +512,11 @@ ProcessCopyOptions(ParseState *pstate, } else if (strcmp(defel->defname, "force_not_null") == 0) { - if (opts_out->force_notnull) + if (opts_out->force_notnull || opts_out->force_notnull_all) errorConflictingDefElem(defel, pstate); - if (defel->arg && IsA(defel->arg, List)) + if (defel->arg && IsA(defel->arg, A_Star)) + opts_out->force_notnull_all = true; + else if (defel->arg && IsA(defel->arg, List)) opts_out->force_notnull = castNode(List, defel->arg); else ereport(ERROR, @@ -525,9 +527,11 @@ ProcessCopyOptions(ParseState *pstate, } else if (strcmp(defel->defname, "force_null") == 0) { - if (opts_out->force_null) + if (opts_out->force_null || opts_out->force_null_all) errorConflictingDefElem(defel, pstate); - if (defel->arg && IsA(defel->arg, List)) + if (defel->arg && IsA(defel->arg, A_Star)) + opts_out->force_null_all = true; + else if (defel->arg && IsA(defel->arg, List)) opts_out->force_null = castNode(List, defel->arg); else ereport(ERROR, diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 80bca79cd0..23a6e23b4b 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1393,7 +1393,9 @@ BeginCopyFrom(ParseState *pstate, /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */ cstate->opts.force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); - if (cstate->opts.force_notnull) + if (cstate->opts.force_notnull_all) + MemSet(cstate->opts.force_notnull_flags, true, num_phys_attrs * sizeof(bool)); + else if (cstate->opts.force_notnull) { List *attnums; ListCell *cur; @@ -1416,7 +1418,9 @@ BeginCopyFrom(ParseState *pstate, /* Convert FORCE_NULL name list to per-column flags, check validity */ cstate->opts.force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); - if (cstate->opts.force_null) + if (cstate->opts.force_null_all) + MemSet(cstate->opts.force_null_flags, true, num_phys_attrs * sizeof(bool)); + else if (cstate->opts.force_null) { List *attnums; ListCell *cur; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 39ab7eac0d..7faaa4bbe4 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3421,10 +3421,18 @@ copy_opt_item: { $$ = makeDefElem("force_not_null", (Node *) $4, @1); } + | FORCE NOT NULL_P '*' + { + $$ = makeDefElem("force_not_null", (Node *) makeNode(A_Star), @1); + } | FORCE NULL_P columnList { $$ = makeDefElem("force_null", (Node *) $3, @1); } + | FORCE NULL_P '*' + { + $$ = makeDefElem("force_null", (Node *) makeNode(A_Star), @1); + } | ENCODING Sconst { $$ = makeDefElem("encoding", (Node *) makeString($2), @1); diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 33175868f6..15bdafd27a 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -56,8 +56,10 @@ typedef struct CopyFormatOptions bool force_quote_all; /* FORCE_QUOTE *? */ bool *force_quote_flags; /* per-column CSV FQ flags */ List *force_notnull; /* list of column names */ + bool force_notnull_all; /* FORCE_NOT_NULL * */ bool *force_notnull_flags; /* per-column CSV FNN flags */ List *force_null; /* list of column names */ + bool force_null_all; /* FORCE_NULL * */ bool *force_null_flags; /* per-column CSV FN flags */ bool convert_selectively; /* do selective binary conversion? */ List *convert_select; /* list of column names (can be NIL) */ diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 8e33eee719..dc3929bfee 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -520,6 +520,50 @@ BEGIN; COPY forcetest (d, e) FROM STDIN WITH (FORMAT csv, FORCE_NULL(b)); ERROR: FORCE_NULL column "b" not referenced by COPY ROLLBACK; +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 4; + b | c +---+------ + | NULL +(1 row) + +-- should succeed with effect ("b" remains an empty string) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 5; + b | c +---+--- + | +(1 row) + +-- should succeed with effect ("c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 6; + b | c +---+------ + b | NULL +(1 row) + +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NOT_NULL(b)); +ERROR: conflicting or redundant options +LINE 1: ...c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NOT_... + ^ +ROLLBACK; +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *, FORCE_NULL(b)); +ERROR: conflicting or redundant options +LINE 1: ... b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *, FORCE_NULL... + ^ +ROLLBACK; \pset null '' -- test case with whole-row Var in a check constraint create table check_con_tbl (f1 int); diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index d759635068..a5486f6086 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -344,6 +344,36 @@ ROLLBACK; BEGIN; COPY forcetest (d, e) FROM STDIN WITH (FORMAT csv, FORCE_NULL(b)); ROLLBACK; +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); +4,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 4; +-- should succeed with effect ("b" remains an empty string) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *); +5,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 5; +-- should succeed with effect ("c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *); +6,"b","" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 6; +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NOT_NULL(b)); +ROLLBACK; +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *, FORCE_NULL(b)); +ROLLBACK; + \pset null '' -- test case with whole-row Var in a check constraint -- 2.34.1