Rebased on HEAD.
0005 forgot to update compression_1.out.
Included changes to ./configure.ac and some other patches, but not Tomas's,
since it'll make CFBOT get mad as soon as that's pushed.
--
Justin
>From bf1336d284792b29e30b42af2ec820bb0c256916 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 19:12:53 -0500
Subject: [PATCH v2 01/10] Add docs for default_toast_compression..
bbe0a81db69bd10bd166907c3701492a29aca294
---
doc/src/sgml/config.sgml | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ee4925d6d9..5cb851a5eb 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -8151,6 +8151,29 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv;
</listitem>
</varlistentry>
+ <varlistentry id="guc-default-toast-compression" xreflabel="default_toast_compression">
+ <term><varname>default_toast_compression</varname> (<type>string</type>)
+ <indexterm>
+ <primary><varname>default_toast_compression</varname> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ This parameter specifies the default compression method to use
+ when compressing data in <acronym>TOAST</acronym> tables.
+ It applies only to variable-width data types.
+ It may be overriden by compression clauses in the
+ <command>CREATE</command> command, or changed after the relation is
+ created by <command>ALTER TABLE ... SET COMPRESSION</command>.
+
+ The supported compression methods are <literal>pglz</literal> and
+ (if configured at the time <productname>PostgreSQL</productname> was
+ built) <literal>lz4</literal>.
+ The default is <literal>pglz</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-temp-tablespaces" xreflabel="temp_tablespaces">
<term><varname>temp_tablespaces</varname> (<type>string</type>)
<indexterm>
--
2.17.0
>From 3184c87f129ec935fa46773728869c7c6af88025 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 21:52:28 -0500
Subject: [PATCH v2 02/10] doc: pg_dump --no-toast-compression
---
doc/src/sgml/ref/pg_dump.sgml | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index bcbb7a25fb..4a521186fb 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -931,6 +931,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-toast-compression</option></term>
+ <listitem>
+ <para>
+ Do not output commands to set <acronym>TOAST</acronym>compression
+ methods.
+ With this option, all objects will be created using whichever
+ compression method is the default during restore.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-unlogged-table-data</option></term>
<listitem>
--
2.17.0
>From 2212020a50478c66a830f15b23c3410e4d7bb501 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 20:23:40 -0500
Subject: [PATCH v2 03/10] Compression method is an char not an OID
---
src/backend/commands/tablecmds.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 9b2800bf5e..8e756e59d5 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -7847,6 +7847,7 @@ SetIndexStorageProperties(Relation rel, Relation attrelation,
index_close(indrel, lockmode);
}
}
+
/*
* ALTER TABLE ALTER COLUMN SET STORAGE
*
@@ -15070,7 +15071,7 @@ ATExecSetCompression(AlteredTableInfo *tab,
AttrNumber attnum;
char *compression;
char typstorage;
- Oid cmoid;
+ char cmethod;
Datum values[Natts_pg_attribute];
bool nulls[Natts_pg_attribute];
bool replace[Natts_pg_attribute];
@@ -15111,9 +15112,9 @@ ATExecSetCompression(AlteredTableInfo *tab,
memset(replace, false, sizeof(replace));
/* get the attribute compression method. */
- cmoid = GetAttributeCompression(atttableform, compression);
+ cmethod = GetAttributeCompression(atttableform, compression);
- atttableform->attcompression = cmoid;
+ atttableform->attcompression = cmethod;
CatalogTupleUpdate(attrel, &tuple->t_self, tuple);
InvokeObjectPostAlterHook(RelationRelationId,
@@ -15123,7 +15124,7 @@ ATExecSetCompression(AlteredTableInfo *tab,
ReleaseSysCache(tuple);
/* apply changes to the index column as well */
- SetIndexStorageProperties(rel, attrel, attnum, cmoid, '\0', lockmode);
+ SetIndexStorageProperties(rel, attrel, attnum, cmethod, '\0', lockmode);
table_close(attrel, RowExclusiveLock);
/* make changes visible */
--
2.17.0
>From d73643d7569ef74b9008d232f8c0ea33cb96ff3c Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 20:07:31 -0500
Subject: [PATCH v2 04/10] Remove duplicative macro
---
src/include/access/toast_compression.h | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 514df0bed1..09af6e7810 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -50,8 +50,6 @@ typedef enum ToastCompressionId
errdetail("This functionality requires the server to be built with lz4 support."), \
errhint("You need to rebuild PostgreSQL using --with-lz4.")))
-#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod)
-
#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \
(storage) != TYPSTORAGE_EXTERNAL)
--
2.17.0
>From 79e6dd441a0624f611ee1f7ee037285feccda426 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 20:19:55 -0500
Subject: [PATCH v2 05/10] Error on invalid compression in CREATE and ALTER
---
src/backend/commands/tablecmds.c | 10 +++++++---
src/test/regress/expected/compression.out | 6 ++++++
src/test/regress/expected/compression_1.out | 6 ++++++
src/test/regress/sql/compression.sql | 5 +++++
4 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 8e756e59d5..7efe27ac6c 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -17865,9 +17865,13 @@ GetAttributeCompression(Form_pg_attribute att, char *compression)
/* fallback to default compression if it's not specified */
if (compression == NULL)
- cmethod = GetDefaultToastCompression();
- else
- cmethod = CompressionNameToMethod(compression);
+ return GetDefaultToastCompression();
+
+ cmethod = CompressionNameToMethod(compression);
+ if (!CompressionMethodIsValid(cmethod))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("invalid compression method \"%s\"", compression)));
return cmethod;
}
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 3de2886de0..501e0ead14 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -344,4 +344,10 @@ SELECT length(f1) FROM cmmove3;
10040
(2 rows)
+CREATE TABLE t (a text COMPRESSION I_Do_Not_Exist_Compression); -- fails
+ERROR: invalid compression method "i_do_not_exist_compression"
+CREATE TABLE t (a text);
+ALTER TABLE t ALTER a SET COMPRESSION I_Do_Not_Exist_Compression; -- fails
+ERROR: invalid compression method "i_do_not_exist_compression"
+DROP TABLE t;
\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index 40aad81fa1..2a704e9b14 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -337,4 +337,10 @@ SELECT length(f1) FROM cmmove3;
10000
(1 row)
+CREATE TABLE t (a text COMPRESSION I_Do_Not_Exist_Compression); -- fails
+ERROR: invalid compression method "i_do_not_exist_compression"
+CREATE TABLE t (a text);
+ALTER TABLE t ALTER a SET COMPRESSION I_Do_Not_Exist_Compression; -- fails
+ERROR: invalid compression method "i_do_not_exist_compression"
+DROP TABLE t;
\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index d97e26b6ee..afddd5a134 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -133,4 +133,9 @@ SELECT length(f1) FROM cmmove1;
SELECT length(f1) FROM cmmove2;
SELECT length(f1) FROM cmmove3;
+CREATE TABLE t (a text COMPRESSION I_Do_Not_Exist_Compression); -- fails
+CREATE TABLE t (a text);
+ALTER TABLE t ALTER a SET COMPRESSION I_Do_Not_Exist_Compression; -- fails
+DROP TABLE t;
+
\set HIDE_TOAST_COMPRESSION true
--
2.17.0
>From 2382b3fe3e609e99ee1742b872de5b4ae61b2ecc Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sat, 20 Mar 2021 17:09:36 -0500
Subject: [PATCH v2 06/10] ./configure: Avoid warnings on Mac --with-lz4
Reported by Andrey Borodin and Tom Lane.
>>> configure: WARNING: lz4.h: accepted by the compiler, rejected by the preprocessor!
>>> configure: WARNING: lz4.h: proceeding with the compiler's result
Possibly we could instead just *remove* the AC_CHECK_HEADERS?
---
configure | 6 ++++++
configure.ac | 6 ++++++
2 files changed, 12 insertions(+)
diff --git a/configure b/configure
index 6d34243dca..fa2b7692ca 100755
--- a/configure
+++ b/configure
@@ -13522,6 +13522,10 @@ fi
fi
if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
for ac_header in lz4/lz4.h
do :
ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
@@ -13549,6 +13553,8 @@ fi
done
+
+ CPPFLAGS=$ac_save_CPPFLAGS
fi
if test "$with_gssapi" = yes ; then
diff --git a/configure.ac b/configure.ac
index e54e2fb632..bd92643a3b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1426,8 +1426,14 @@ Use --without-zlib to disable zlib support.])])
fi
if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
AC_CHECK_HEADERS(lz4/lz4.h, [],
[AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
fi
if test "$with_gssapi" = yes ; then
--
2.17.0
>From 533a7281906e42db7329c6e44ca236094ab60ab9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sat, 20 Mar 2021 16:13:44 -0500
Subject: [PATCH v2 07/10] Commentary about slicing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
My language, per gripe from Álvaro Herrera:
| I updated the coverage script to use --with-lz4; results are updated.
| While eyeballing the results I noticed this bit in
| lz4_decompress_datum_slice():
|
| + /* slice decompression not supported prior to 1.8.3 */
| + if (LZ4_versionNumber() < 10803)
| + return lz4_decompress_datum(value);
|
| which I read as returning the complete decompressed datum if slice
| decompression is not supported. I thought that was a bug, but looking
| at the caller I realize that this isn't really a problem, since it's
| detoast_attr_slice's responsibility to slice the result further -- no
| bug, it's just wasteful. I suggest to add comments to this effect,
| perhaps as the attached (feel free to reword, I think mine is awkward.)
---
src/backend/access/common/detoast.c | 3 +++
src/backend/access/common/toast_compression.c | 3 +++
2 files changed, 6 insertions(+)
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 2fef40c2e9..316f5715e1 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -497,6 +497,9 @@ toast_decompress_datum(struct varlena *attr)
* Decompress the front of a compressed version of a varlena datum.
* offset handling happens in detoast_attr_slice.
* Here we just decompress a slice from the front.
+ *
+ * If slice decompression is not supported, the full datum is decompressed, and
+ * then sliced by detoast_attr_slice.
*/
static struct varlena *
toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index a6f8b79a9e..dd31a54b77 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -203,6 +203,9 @@ lz4_decompress_datum(const struct varlena *value)
/*
* Decompress part of a varlena that was compressed using LZ4.
+ *
+ * If slice decompression is not supported, the full datum is decompressed, and
+ * then sliced by detoast_attr_slice.
*/
struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
--
2.17.0
>From adc6eb10f79d1edf3df0d241557b7d23106b7cc6 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sat, 20 Mar 2021 17:25:46 -0500
Subject: [PATCH v2 08/10] specially handle SET default_toast_compression=lz4
---
src/backend/access/common/toast_compression.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index dd31a54b77..1617c382c2 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -304,6 +304,11 @@ check_default_toast_compression(char **newval, void **extra, GucSource source)
errmsg("compression method \"%s\" does not exist",
*newval)));
}
+ else if (strcmp(*newval, "lz4") == 0)
+ {
+ GUC_check_errdetail("This functionality requires the server to be built with lz4 support.");
+ GUC_check_errhint("You need to rebuild PostgreSQL using --with-lz4.");
+ }
else
{
GUC_check_errdetail("Compression method \"%s\" does not exist.",
--
2.17.0
>From 232987c6b79de7cf635664c28fa0062498485146 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 19 Mar 2021 19:39:10 -0500
Subject: [PATCH v2 09/10] Alternately: WIP: Change default_toast_compression
GUC to an enum..
..since the final version of the TOAST patch doesn't require catalog access
---
src/backend/access/common/toast_compression.c | 54 +------------------
src/backend/utils/misc/guc.c | 30 +++++++----
src/include/access/toast_compression.h | 10 ++--
src/test/regress/expected/compression.out | 4 +-
4 files changed, 28 insertions(+), 70 deletions(-)
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 1617c382c2..9d9ffc6f97 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -23,8 +23,8 @@
#include "fmgr.h"
#include "utils/builtins.h"
-/* Compile-time default */
-char *default_toast_compression = DEFAULT_TOAST_COMPRESSION;
+/* GUC */
+int default_toast_compression = TOAST_PGLZ_COMPRESSION_ID;
/*
* Compress a varlena using PGLZ.
@@ -269,53 +269,3 @@ toast_get_compression_id(struct varlena *attr)
return cmid;
}
-
-/*
- * Validate a new value for the default_toast_compression GUC.
- */
-bool
-check_default_toast_compression(char **newval, void **extra, GucSource source)
-{
- if (**newval == '\0')
- {
- GUC_check_errdetail("%s cannot be empty.",
- "default_toast_compression");
- return false;
- }
-
- if (strlen(*newval) >= NAMEDATALEN)
- {
- GUC_check_errdetail("%s is too long (maximum %d characters).",
- "default_toast_compression", NAMEDATALEN - 1);
- return false;
- }
-
- if (!CompressionMethodIsValid(CompressionNameToMethod(*newval)))
- {
- /*
- * When source == PGC_S_TEST, don't throw a hard error for a
- * nonexistent compression method, only a NOTICE. See comments in
- * guc.h.
- */
- if (source == PGC_S_TEST)
- {
- ereport(NOTICE,
- (errcode(ERRCODE_UNDEFINED_OBJECT),
- errmsg("compression method \"%s\" does not exist",
- *newval)));
- }
- else if (strcmp(*newval, "lz4") == 0)
- {
- GUC_check_errdetail("This functionality requires the server to be built with lz4 support.");
- GUC_check_errhint("You need to rebuild PostgreSQL using --with-lz4.");
- }
- else
- {
- GUC_check_errdetail("Compression method \"%s\" does not exist.",
- *newval);
- return false;
- }
- }
-
- return true;
-}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 4272cf4821..eb1b343c09 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -510,6 +510,14 @@ static struct config_enum_entry shared_memory_options[] = {
{NULL, 0, false}
};
+static struct config_enum_entry default_toast_compression_options[] = {
+ {"pglz", TOAST_PGLZ_COMPRESSION_ID, false},
+#ifdef USE_LZ4
+ {"lz4", TOAST_LZ4_COMPRESSION_ID, false},
+#endif
+ {NULL, 0, false}
+};
+
/*
* Options for enum values stored in other modules
*/
@@ -3953,17 +3961,6 @@ static struct config_string ConfigureNamesString[] =
check_default_table_access_method, NULL, NULL
},
- {
- {"default_toast_compression", PGC_USERSET, CLIENT_CONN_STATEMENT,
- gettext_noop("Sets the default compression for new columns."),
- NULL,
- GUC_IS_NAME
- },
- &default_toast_compression,
- DEFAULT_TOAST_COMPRESSION,
- check_default_toast_compression, NULL, NULL
- },
-
{
{"default_tablespace", PGC_USERSET, CLIENT_CONN_STATEMENT,
gettext_noop("Sets the default tablespace to create tables and indexes in."),
@@ -4605,6 +4602,17 @@ static struct config_enum ConfigureNamesEnum[] =
NULL, NULL, NULL
},
+ {
+ {"default_toast_compression", PGC_USERSET, CLIENT_CONN_STATEMENT,
+ gettext_noop("Sets the default compression for new columns."),
+ NULL,
+ GUC_IS_NAME
+ },
+ &default_toast_compression,
+ 0, /* pglz */
+ default_toast_compression_options, NULL, NULL
+ },
+
{
{"default_transaction_isolation", PGC_USERSET, CLIENT_CONN_STATEMENT,
gettext_noop("Sets the transaction isolation level of each new transaction."),
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 09af6e7810..88a46e264d 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -16,10 +16,7 @@
#include "utils/guc.h"
/* GUCs */
-extern char *default_toast_compression;
-
-/* default compression method if not specified. */
-#define DEFAULT_TOAST_COMPRESSION "pglz"
+extern int default_toast_compression;
/*
* Built-in compression method-id. The toast compression header will store
@@ -100,7 +97,10 @@ CompressionNameToMethod(char *compression)
static inline char
GetDefaultToastCompression(void)
{
- return CompressionNameToMethod(default_toast_compression);
+ /* See also guc.c */
+ char *const enumtoname[] = {"pglz", "lz4"};
+ Assert(default_toast_compression < lengthof(enumtoname));
+ return CompressionNameToMethod(enumtoname[default_toast_compression]);
}
/* pglz compression/decompression routines */
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 501e0ead14..5cc76cdcd6 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -232,10 +232,10 @@ DETAIL: pglz versus lz4
-- test default_toast_compression GUC
SET default_toast_compression = '';
ERROR: invalid value for parameter "default_toast_compression": ""
-DETAIL: default_toast_compression cannot be empty.
+HINT: Available values: pglz, lz4.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-DETAIL: Compression method "I do not exist compression" does not exist.
+HINT: Available values: pglz, lz4.
SET default_toast_compression = 'lz4';
DROP TABLE cmdata2;
CREATE TABLE cmdata2 (f1 text);
--
2.17.0
