See attached.
One issue is that the pg_dump tests are no longer exercising the COMPRESSION
clause. I don't know how to improve on that, since lz4 may not be available.
..unless we changed attcompression='\0' to mean (for varlena) "the default
compression". Rather than "resolving" to the default compression at the time
the table is created, columns without an explicit compression set would "defer"
to the GUC (of course, that only affects newly-inserted data).
Then, I think pg_dump would generate an COMPRESSION clause for columns with any
compression other than a null byte, and then the tests could "SET COMPRESSION
pglz" and check the output, since it's set to a specific compression, not just
inheriting the default.
I'm not sure if that'd be desirable, but I think that's similar to tablespaces,
where (if I recall) reltablespace=0 means "this database's default tblspc".
--
Justin
>From 58335318a9e72d0dbdf8c2009ba6d195a6cba862 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 19:12:53 -0500
Subject: [PATCH 1/9] Add docs for default_toast_compression..
bbe0a81db69bd10bd166907c3701492a29aca294
---
doc/src/sgml/config.sgml | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ee4925d6d9..5cb851a5eb 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -8151,6 +8151,29 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv;
</listitem>
</varlistentry>
+ <varlistentry id="guc-default-toast-compression" xreflabel="default_toast_compression">
+ <term><varname>default_toast_compression</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>default_toast_compression</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ This parameter specifies the default compression method to use
+ when compressing data in <acronym>TOAST</acronym> tables.
+ It applies only to variable-width data types.
+ It may be overriden by compression clauses in the
+ <command>CREATE</command> command, or changed after the relation is
+ created by <command>ALTER TABLE ... SET COMPRESSION</command>.
+
+ The supported compression methods are <literal>pglz</literal> and
+ (if configured at the time <productname>PostgreSQL</productname> was
+ built) <literal>lz4</literal>.
+ The default is <literal>pglz</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-temp-tablespaces" xreflabel="temp_tablespaces">
<term><varname>temp_tablespaces</varname> (<type>string</type>)
<indexterm>
--
2.17.0
>From ae52d1db7fa5f7731bc5c1170dfb2ec5c39b111a Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 21:52:28 -0500
Subject: [PATCH 2/9] doc: pg_dump --no-toast-compression
---
doc/src/sgml/ref/pg_dump.sgml | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index bcbb7a25fb..4a521186fb 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -931,6 +931,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-toast-compression</option></term>
+ <listitem>
+ <para>
+ Do not output commands to set <acronym>TOAST</acronym>compression
+ methods.
+ With this option, all objects will be created using whichever
+ compression method is the default during restore.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-unlogged-table-data</option></term>
<listitem>
--
2.17.0
>From c47fafc376f847f4463b146f467c6046e44e5afa Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 20:23:40 -0500
Subject: [PATCH 3/9] Compression method is an char not an OID
---
src/backend/commands/tablecmds.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 9b2800bf5e..8e756e59d5 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -7847,6 +7847,7 @@ SetIndexStorageProperties(Relation rel, Relation attrelation,
index_close(indrel, lockmode);
}
}
+
/*
* ALTER TABLE ALTER COLUMN SET STORAGE
*
@@ -15070,7 +15071,7 @@ ATExecSetCompression(AlteredTableInfo *tab,
AttrNumber attnum;
char *compression;
char typstorage;
- Oid cmoid;
+ char cmethod;
Datum values[Natts_pg_attribute];
bool nulls[Natts_pg_attribute];
bool replace[Natts_pg_attribute];
@@ -15111,9 +15112,9 @@ ATExecSetCompression(AlteredTableInfo *tab,
memset(replace, false, sizeof(replace));
/* get the attribute compression method. */
- cmoid = GetAttributeCompression(atttableform, compression);
+ cmethod = GetAttributeCompression(atttableform, compression);
- atttableform->attcompression = cmoid;
+ atttableform->attcompression = cmethod;
CatalogTupleUpdate(attrel, &tuple->t_self, tuple);
InvokeObjectPostAlterHook(RelationRelationId,
@@ -15123,7 +15124,7 @@ ATExecSetCompression(AlteredTableInfo *tab,
ReleaseSysCache(tuple);
/* apply changes to the index column as well */
- SetIndexStorageProperties(rel, attrel, attnum, cmoid, '\0', lockmode);
+ SetIndexStorageProperties(rel, attrel, attnum, cmethod, '\0', lockmode);
table_close(attrel, RowExclusiveLock);
/* make changes visible */
--
2.17.0
>From f577d25165815993b0289c75d5395d23409863cc Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 20:07:31 -0500
Subject: [PATCH 4/9] Remove duplicative macro
---
src/include/access/toast_compression.h | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 514df0bed1..09af6e7810 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -50,8 +50,6 @@ typedef enum ToastCompressionId
errdetail("This functionality requires the server to be built with lz4 support."), \
errhint("You need to rebuild PostgreSQL using --with-lz4.")))
-#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod)
-
#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \
(storage) != TYPSTORAGE_EXTERNAL)
--
2.17.0
>From 40aabd1aa36878974e84fc04ccff31d1e90f93c5 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 20:19:55 -0500
Subject: [PATCH 5/9] Error on invalid compression in CREATE and ALTER
---
src/backend/commands/tablecmds.c | 10 +++++++---
src/test/regress/expected/compression.out | 6 ++++++
src/test/regress/sql/compression.sql | 5 +++++
3 files changed, 18 insertions(+), 3 deletions(-)
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 8e756e59d5..7efe27ac6c 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -17865,9 +17865,13 @@ GetAttributeCompression(Form_pg_attribute att, char *compression)
/* fallback to default compression if it's not specified */
if (compression == NULL)
- cmethod = GetDefaultToastCompression();
- else
- cmethod = CompressionNameToMethod(compression);
+ return GetDefaultToastCompression();
+
+ cmethod = CompressionNameToMethod(compression);
+ if (!CompressionMethodIsValid(cmethod))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("invalid compression method \"%s\"", compression)));
return cmethod;
}
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 3de2886de0..501e0ead14 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -344,4 +344,10 @@ SELECT length(f1) FROM cmmove3;
10040
(2 rows)
+CREATE TABLE t (a text COMPRESSION I_Do_Not_Exist_Compression); -- fails
+ERROR: invalid compression method "i_do_not_exist_compression"
+CREATE TABLE t (a text);
+ALTER TABLE t ALTER a SET COMPRESSION I_Do_Not_Exist_Compression; -- fails
+ERROR: invalid compression method "i_do_not_exist_compression"
+DROP TABLE t;
\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index d97e26b6ee..afddd5a134 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -133,4 +133,9 @@ SELECT length(f1) FROM cmmove1;
SELECT length(f1) FROM cmmove2;
SELECT length(f1) FROM cmmove3;
+CREATE TABLE t (a text COMPRESSION I_Do_Not_Exist_Compression); -- fails
+CREATE TABLE t (a text);
+ALTER TABLE t ALTER a SET COMPRESSION I_Do_Not_Exist_Compression; -- fails
+DROP TABLE t;
+
\set HIDE_TOAST_COMPRESSION true
--
2.17.0
>From f9c8398bad190f85729c8bdece0fbd205e7b55db Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 19:39:10 -0500
Subject: [PATCH 6/9] WIP: Change default_toast_compression GUC to an enum..
..since the final version of this patch doesn't require catalog access anymore
---
src/backend/access/common/toast_compression.c | 49 +------------------
src/backend/utils/misc/guc.c | 30 +++++++-----
src/include/access/toast_compression.h | 10 ++--
src/test/regress/expected/compression.out | 4 +-
4 files changed, 28 insertions(+), 65 deletions(-)
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index a6f8b79a9e..edcd09f9e9 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -23,8 +23,8 @@
#include "fmgr.h"
#include "utils/builtins.h"
-/* Compile-time default */
-char *default_toast_compression = DEFAULT_TOAST_COMPRESSION;
+/* GUC */
+int default_toast_compression = TOAST_PGLZ_COMPRESSION_ID;
/*
* Compress a varlena using PGLZ.
@@ -266,48 +266,3 @@ toast_get_compression_id(struct varlena *attr)
return cmid;
}
-
-/*
- * Validate a new value for the default_toast_compression GUC.
- */
-bool
-check_default_toast_compression(char **newval, void **extra, GucSource source)
-{
- if (**newval == '\0')
- {
- GUC_check_errdetail("%s cannot be empty.",
- "default_toast_compression");
- return false;
- }
-
- if (strlen(*newval) >= NAMEDATALEN)
- {
- GUC_check_errdetail("%s is too long (maximum %d characters).",
- "default_toast_compression", NAMEDATALEN - 1);
- return false;
- }
-
- if (!CompressionMethodIsValid(CompressionNameToMethod(*newval)))
- {
- /*
- * When source == PGC_S_TEST, don't throw a hard error for a
- * nonexistent compression method, only a NOTICE. See comments in
- * guc.h.
- */
- if (source == PGC_S_TEST)
- {
- ereport(NOTICE,
- (errcode(ERRCODE_UNDEFINED_OBJECT),
- errmsg("compression method \"%s\" does not exist",
- *newval)));
- }
- else
- {
- GUC_check_errdetail("Compression method \"%s\" does not exist.",
- *newval);
- return false;
- }
- }
-
- return true;
-}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fb31c8b05c..0556f2e748 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -510,6 +510,14 @@ static struct config_enum_entry shared_memory_options[] = {
{NULL, 0, false}
};
+static struct config_enum_entry default_toast_compression_options[] = {
+ {"pglz", TOAST_PGLZ_COMPRESSION_ID, false},
+#ifdef USE_LZ4
+ {"lz4", TOAST_LZ4_COMPRESSION_ID, false},
+#endif
+ {NULL, 0, false}
+};
+
/*
* Options for enum values stored in other modules
*/
@@ -3953,17 +3961,6 @@ static struct config_string ConfigureNamesString[] =
check_default_table_access_method, NULL, NULL
},
- {
- {"default_toast_compression", PGC_USERSET, CLIENT_CONN_STATEMENT,
- gettext_noop("Sets the default compression for new columns."),
- NULL,
- GUC_IS_NAME
- },
- &default_toast_compression,
- DEFAULT_TOAST_COMPRESSION,
- check_default_toast_compression, NULL, NULL
- },
-
{
{"default_tablespace", PGC_USERSET, CLIENT_CONN_STATEMENT,
gettext_noop("Sets the default tablespace to create tables and indexes in."),
@@ -4605,6 +4602,17 @@ static struct config_enum ConfigureNamesEnum[] =
NULL, NULL, NULL
},
+ {
+ {"default_toast_compression", PGC_USERSET, CLIENT_CONN_STATEMENT,
+ gettext_noop("Sets the default compression for new columns."),
+ NULL,
+ GUC_IS_NAME
+ },
+ &default_toast_compression,
+ 0, /* pglz */
+ default_toast_compression_options, NULL, NULL
+ },
+
{
{"default_transaction_isolation", PGC_USERSET, CLIENT_CONN_STATEMENT,
gettext_noop("Sets the transaction isolation level of each new transaction."),
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 09af6e7810..88a46e264d 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -16,10 +16,7 @@
#include "utils/guc.h"
/* GUCs */
-extern char *default_toast_compression;
-
-/* default compression method if not specified. */
-#define DEFAULT_TOAST_COMPRESSION "pglz"
+extern int default_toast_compression;
/*
* Built-in compression method-id. The toast compression header will store
@@ -100,7 +97,10 @@ CompressionNameToMethod(char *compression)
static inline char
GetDefaultToastCompression(void)
{
- return CompressionNameToMethod(default_toast_compression);
+ /* See also guc.c */
+ char *const enumtoname[] = {"pglz", "lz4"};
+ Assert(default_toast_compression < lengthof(enumtoname));
+ return CompressionNameToMethod(enumtoname[default_toast_compression]);
}
/* pglz compression/decompression routines */
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 501e0ead14..5cc76cdcd6 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -232,10 +232,10 @@ DETAIL: pglz versus lz4
-- test default_toast_compression GUC
SET default_toast_compression = '';
ERROR: invalid value for parameter "default_toast_compression": ""
-DETAIL: default_toast_compression cannot be empty.
+HINT: Available values: pglz, lz4.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-DETAIL: Compression method "I do not exist compression" does not exist.
+HINT: Available values: pglz, lz4.
SET default_toast_compression = 'lz4';
DROP TABLE cmdata2;
CREATE TABLE cmdata2 (f1 text);
--
2.17.0
>From a8a477118e7df9316efa2e7f1cdf43d316b51a05 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sat, 20 Mar 2021 01:55:59 -0500
Subject: [PATCH 7/9] attcompression is a list of chars, and never null
Robert seems to have noticed half of this here: CA+TgmoYRsapWz2vDSQXFAEL_BAMq1aY37avcq9GAB=suuvf...@mail.gmail.com
---
src/bin/pg_dump/pg_dump.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f8bec3ffcc..aa7c004147 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -8759,7 +8759,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid));
tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *));
tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *));
- tbinfo->attcompression = (char *) pg_malloc(ntups * sizeof(char *));
+ tbinfo->attcompression = (char *) pg_malloc(ntups * sizeof(char));
tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *));
@@ -15908,8 +15908,7 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
/*
* Attribute compression
*/
- if (!dopt->no_toast_compression &&
- tbinfo->attcompression != NULL)
+ if (!dopt->no_toast_compression)
{
char *cmname;
--
2.17.0
>From e9e84a09c0e63a9bb487bc6a1800de90492e4ad6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 22:49:59 -0500
Subject: [PATCH 8/9] WIP: pg_dump: output default_toast_compression..
This is unlike default_tablespace in that tablespaces are associated with an
entire relation, whereas TOAST compression is associated with individual
columns, so cannot SET default_toast_compression between CREATE commands.
Rather, each column should have a COMPRESSION clause if its compression differs
from the dump-time default. This commit sets the default during restore to
match the dump-time default so that does the right thing.
XXX: need to bump archive version?
---
src/bin/pg_dump/pg_backup.h | 1 +
src/bin/pg_dump/pg_backup_archiver.c | 20 +++++++++++-
src/bin/pg_dump/pg_dump.c | 48 ++++++++++++++++++++++++++--
3 files changed, 66 insertions(+), 3 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 0296b9bb5e..dbe2689f23 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -209,6 +209,7 @@ typedef struct Archive
/* other important stuff */
char *searchpath; /* search_path to set during restore */
+ char *default_toast_compression; /* default TOAST compression to set during restore */
char *use_role; /* Issue SET ROLE to this */
/* error handling */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 35bf971220..d7fcbc5410 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -86,6 +86,7 @@ static void _selectTableAccessMethod(ArchiveHandle *AH, const char *tableam);
static void processEncodingEntry(ArchiveHandle *AH, TocEntry *te);
static void processStdStringsEntry(ArchiveHandle *AH, TocEntry *te);
static void processSearchPathEntry(ArchiveHandle *AH, TocEntry *te);
+static void processToastCompressionEntry(ArchiveHandle *AH, TocEntry *te);
static int _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH);
static RestorePass _tocEntryRestorePass(TocEntry *te);
static bool _tocEntryIsACL(TocEntry *te);
@@ -2696,6 +2697,8 @@ ReadToc(ArchiveHandle *AH)
processStdStringsEntry(AH, te);
else if (strcmp(te->desc, "SEARCHPATH") == 0)
processSearchPathEntry(AH, te);
+ else if (strcmp(te->desc, "TOAST_COMPRESSION") == 0)
+ processToastCompressionEntry(AH, te);
}
}
@@ -2753,6 +2756,16 @@ processSearchPathEntry(ArchiveHandle *AH, TocEntry *te)
AH->public.searchpath = pg_strdup(te->defn);
}
+static void
+processToastCompressionEntry(ArchiveHandle *AH, TocEntry *te)
+{
+ /*
+ * te->defn should contain a command to set default_toast_compression.
+ * We just copy it verbatim for use later.
+ */
+ AH->public.default_toast_compression = pg_strdup(te->defn);
+}
+
static void
StrictNamesCheck(RestoreOptions *ropt)
{
@@ -2812,7 +2825,8 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
/* These items are treated specially */
if (strcmp(te->desc, "ENCODING") == 0 ||
strcmp(te->desc, "STDSTRINGS") == 0 ||
- strcmp(te->desc, "SEARCHPATH") == 0)
+ strcmp(te->desc, "SEARCHPATH") == 0 ||
+ strcmp(te->desc, "TOAST_COMPRESSION") == 0)
return REQ_SPECIAL;
/*
@@ -3135,6 +3149,10 @@ _doSetFixedOutputState(ArchiveHandle *AH)
if (AH->public.searchpath)
ahprintf(AH, "%s", AH->public.searchpath);
+ /* Select the dump-time default_toast_compression */
+ if (AH->public.default_toast_compression)
+ ahprintf(AH, "%s", AH->public.default_toast_compression);
+
/* Make sure function checking is disabled */
ahprintf(AH, "SET check_function_bodies = false;\n");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index aa7c004147..12d1ebf466 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -270,6 +270,7 @@ static void dumpDatabaseConfig(Archive *AH, PQExpBuffer outbuf,
static void dumpEncoding(Archive *AH);
static void dumpStdStrings(Archive *AH);
static void dumpSearchPath(Archive *AH);
+static void dumpToastCompression(Archive *AH);
static void binary_upgrade_set_type_oids_by_type_oid(Archive *fout,
PQExpBuffer upgrade_buffer,
Oid pg_type_oid,
@@ -909,10 +910,14 @@ main(int argc, char **argv)
* order.
*/
- /* First the special ENCODING, STDSTRINGS, and SEARCHPATH entries. */
+ /*
+ * First the special entries for ENCODING, STDSTRINGS, SEARCHPATH and
+ * TOAST_COMPRESSION.
+ */
dumpEncoding(fout);
dumpStdStrings(fout);
dumpSearchPath(fout);
+ dumpToastCompression(fout);
/* The database items are always next, unless we don't want them at all */
if (dopt.outputCreateDB)
@@ -1048,7 +1053,7 @@ help(const char *progname)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
- printf(_(" --no-toast-compression do not dump toast compression methods\n"));
+ printf(_(" --no-toast-compression do not dump toast compression methods\n"));
printf(_(" --no-synchronized-snapshots do not use synchronized snapshots in parallel jobs\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
printf(_(" --no-unlogged-table-data do not dump unlogged table data\n"));
@@ -3260,6 +3265,45 @@ dumpStdStrings(Archive *AH)
destroyPQExpBuffer(qry);
}
+/*
+ * dumpToastCompression: save the dump-time default TOAST compression in the
+ * archive
+ */
+static void
+dumpToastCompression(Archive *AH)
+{
+ PQExpBuffer qry = createPQExpBuffer();
+ const char *toast_compression;
+ PGresult *res;
+
+ if (AH->dopt->no_toast_compression)
+ return;
+
+ res = ExecuteSqlQueryForSingleRow(AH, "SHOW default_toast_compression"); /* XXX: show ?? */
+ toast_compression = PQgetvalue(res, 0, 0);
+
+ pg_log_info("saving default_toast_compression = %s", toast_compression);
+
+ appendPQExpBufferStr(qry, "SET default_toast_compression = ");
+ appendStringLiteralAH(qry, toast_compression, AH);
+ appendPQExpBufferStr(qry, ";\n");
+
+ ArchiveEntry(AH, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = "TOAST_COMPRESSION",
+ .description = "TOAST_COMPRESSION",
+ .section = SECTION_PRE_DATA,
+ .createStmt = qry->data));
+
+ /*
+ * Also save it in AH->default_toast_compression, in case we're doing plain
+ * text dump
+ */
+ AH->default_toast_compression = pg_strdup(qry->data);
+
+ PQclear(res);
+ destroyPQExpBuffer(qry);
+}
+
/*
* dumpSearchPath: record the active search_path in the archive
*/
--
2.17.0
>From f23438aff8df88092c8c36cecec3f8980eb51db2 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 23:43:32 -0500
Subject: [PATCH 9/9] WIP: pg_dump: use ALTER SET COMPRESSION rather than
compression clauses within CREATE TABLE()..
This allows dumps with non-default compression to be restored to a cluster
before v14 compression support, or v14 --without-lz4. Note that allows
pg_restore |psql, but pg_restore -d will still fail, since the entire CREATE
TABLE..ALTER TABLE is execute within a simple query, and a single transaction.
Discussion:
XXX: the list archives are inaccessible...
20210308172915.gf29...@telsasoft.com
4193854.1616199...@sss.pgh.pa.us
20210320000625.gp11...@telsasoft.com
---
src/bin/pg_dump/pg_backup_archiver.c | 24 +++++++++---
src/bin/pg_dump/pg_dump.c | 58 +++++++++++++++-------------
src/bin/pg_dump/t/002_pg_dump.pl | 12 +++---
3 files changed, 56 insertions(+), 38 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index d7fcbc5410..b8a4f5933a 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -2759,11 +2759,22 @@ processSearchPathEntry(ArchiveHandle *AH, TocEntry *te)
static void
processToastCompressionEntry(ArchiveHandle *AH, TocEntry *te)
{
- /*
- * te->defn should contain a command to set default_toast_compression.
- * We just copy it verbatim for use later.
- */
- AH->public.default_toast_compression = pg_strdup(te->defn);
+ /* te->defn should have the form SET default_toast_compression = 'x'; */
+ char *defn = pg_strdup(te->defn);
+ char *ptr1,
+ *ptr2 = NULL;
+
+ ptr1 = strchr(defn, '\'');
+ if (ptr1)
+ ptr2 = strchr(++ptr1, '\'');
+ if (ptr2)
+ {
+ *ptr2 = '\0';
+ AH->public.default_toast_compression = pg_strdup(ptr1);
+ }
+ else
+ fatal("invalid TOAST_COMPRESSION item: %s",
+ te->defn);
}
static void
@@ -3151,7 +3162,8 @@ _doSetFixedOutputState(ArchiveHandle *AH)
/* Select the dump-time default_toast_compression */
if (AH->public.default_toast_compression)
- ahprintf(AH, "%s", AH->public.default_toast_compression);
+ ahprintf(AH, "SET default_toast_compression = '%s';\n",
+ AH->public.default_toast_compression);
/* Make sure function checking is disabled */
ahprintf(AH, "SET check_function_bodies = false;\n");
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 12d1ebf466..5e4baa6e5e 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -3296,9 +3296,11 @@ dumpToastCompression(Archive *AH)
/*
* Also save it in AH->default_toast_compression, in case we're doing plain
- * text dump
+ * text dump. The compression name is saved, not the create statement, to
+ * allow easily comparing it with each column's compression clause.
+ * XXX: maybe instead should just add a 2nd field to AH for the raw compression.
*/
- AH->default_toast_compression = pg_strdup(qry->data);
+ AH->default_toast_compression = pg_strdup(toast_compression);
PQclear(res);
destroyPQExpBuffer(qry);
@@ -15949,30 +15951,6 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
tbinfo->atttypnames[j]);
}
- /*
- * Attribute compression
- */
- if (!dopt->no_toast_compression)
- {
- char *cmname;
-
- switch (tbinfo->attcompression[j])
- {
- case 'p':
- cmname = "pglz";
- break;
- case 'l':
- cmname = "lz4";
- break;
- default:
- cmname = NULL;
- break;
- }
-
- if (cmname != NULL)
- appendPQExpBuffer(q, " COMPRESSION %s", cmname);
- }
-
if (print_default)
{
if (tbinfo->attgenerated[j] == ATTRIBUTE_GENERATED_STORED)
@@ -16391,6 +16369,34 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
qualrelname,
fmtId(tbinfo->attnames[j]),
tbinfo->attfdwoptions[j]);
+
+ /*
+ * Dump per-column compression
+ */
+ if (!dopt->no_toast_compression)
+ {
+ char *cmname;
+
+ switch (tbinfo->attcompression[j])
+ {
+ case 'p':
+ cmname = "pglz";
+ break;
+ case 'l':
+ cmname = "lz4";
+ break;
+ default:
+ cmname = NULL;
+ break;
+ }
+
+ if (cmname != NULL &&
+ strcmp(cmname, fout->default_toast_compression) != 0)
+ appendPQExpBuffer(q, "ALTER %sTABLE ONLY %s ALTER COLUMN %s SET COMPRESSION %s;\n",
+ foreign, qualrelname,
+ fmtId(tbinfo->attnames[j]),
+ cmname);
+ }
}
if (ftoptions)
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index bc91bb12ac..e7885fdd51 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -2284,9 +2284,9 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_table (\E\n
\s+\Qcol1 integer NOT NULL,\E\n
- \s+\Qcol2 text COMPRESSION\E\D*,\n
- \s+\Qcol3 text COMPRESSION\E\D*,\n
- \s+\Qcol4 text COMPRESSION\E\D*,\n
+ \s+\Qcol2 text,\E\n
+ \s+\Qcol3 text,\E\n
+ \s+\Qcol4 text,\E\n
\s+\QCONSTRAINT test_table_col1_check CHECK ((col1 <= 1000))\E\n
\Q)\E\n
\QWITH (autovacuum_enabled='false', fillfactor='80');\E\n/xm,
@@ -2326,7 +2326,7 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_second_table (\E
\n\s+\Qcol1 integer,\E
- \n\s+\Qcol2 text COMPRESSION\E\D*
+ \n\s+\Qcol2 text\E
\n\);
/xm,
like =>
@@ -2441,7 +2441,7 @@ my %tests = (
\n\s+\Qcol1 integer,\E
\n\s+\Qcol2 boolean,\E
\n\s+\Qcol3 boolean,\E
- \n\s+\Qcol4 bit(5) COMPRESSION\E\D*,
+ \n\s+\Qcol4 bit(5),\E*,
\n\s+\Qcol5 double precision\E
\n\);
/xm,
@@ -2459,7 +2459,7 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_table_identity (\E\n
\s+\Qcol1 integer NOT NULL,\E\n
- \s+\Qcol2 text COMPRESSION\E\D*\n
+ \s+\Qcol2 text\E\n
\);
.*
\QALTER TABLE dump_test.test_table_identity ALTER COLUMN col1 ADD GENERATED ALWAYS AS IDENTITY (\E\n
--
2.17.0
