From 4ffaae8b743c5288f7512097bb60da41f131e116 Mon Sep 17 00:00:00 2001 From: Aleksander Alekseev Date: Wed, 15 Jun 2022 16:34:50 +0300 Subject: [PATCH v7] Allow specifying STORAGE attribute for a new table. Also make the code and the documentation for STORAGE and COMPRESSION attributes consistent. Author: Teodor Sigaev Author: Aleksander Alekseev Reviewed-by: Peter Eisentraut Reviewed-by: wenjing zeng Reviewed-by: Matthias van de Meent Reviewed-by: Kyotaro Horiguchi Discussion: https://postgr.es/m/de83407a-ae3d-a8e1-a788-920eb334f25b@sigaev.ru --- doc/src/sgml/ref/alter_table.sgml | 40 +--------- doc/src/sgml/ref/create_table.sgml | 23 +++++- src/backend/commands/tablecmds.c | 93 +++++++++++++---------- src/backend/parser/gram.y | 24 ++++-- src/include/nodes/parsenodes.h | 1 + src/test/regress/expected/alter_table.out | 6 +- src/test/regress/sql/alter_table.sql | 5 +- 7 files changed, 103 insertions(+), 89 deletions(-) diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index a3c62bf056..e83837088a 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -367,7 +367,7 @@ WITH ( MODULUS numeric_literal, REM - SET STORAGE + SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } TOAST per-column storage settings @@ -375,22 +375,7 @@ WITH ( MODULUS numeric_literal, REM - This form sets the storage mode for a column. This controls whether this - column is held inline or in a secondary TOAST table, and - whether the data - should be compressed or not. PLAIN must be used - for fixed-length values such as integer and is - inline, uncompressed. MAIN is for inline, - compressible data. EXTERNAL is for external, - uncompressed data, and EXTENDED is for external, - compressed data. EXTENDED is the default for most - data types that support non-PLAIN storage. - Use of EXTERNAL will make substring operations on - very large text and bytea values run faster, - at the penalty of increased storage space. Note that - SET STORAGE doesn't itself change anything in the table, - it just sets the strategy to be pursued during future table updates. - See for more information. + See for details. @@ -401,26 +386,7 @@ WITH ( MODULUS numeric_literal, REM - This form sets the compression method for a column, determining how - values inserted in future will be compressed (if the storage mode - permits compression at all). - This does not cause the table to be rewritten, so existing data may still - be compressed with other compression methods. If the table is restored - with pg_restore, then all values are rewritten - with the configured compression method. - However, when data is inserted from another relation (for example, - by INSERT ... SELECT), values from the source table are - not necessarily detoasted, so any previously compressed data may retain - its existing compression method, rather than being recompressed with the - compression method of the target column. - The supported compression - methods are pglz and lz4. - (lz4 is available only if - was used when building PostgreSQL.) In - addition, compression_method - can be default, which selects the default behavior of - consulting the setting - at the time of data insertion to determine the method to use. + See for details. diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 6c9918b0a1..da90698046 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -22,7 +22,7 @@ PostgreSQL documentation CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] table_name ( [ - { column_name data_type [ COMPRESSION compression_method ] [ COLLATE collation ] [ column_constraint [ ... ] ] + { column_name data_type [ STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } ] [ COMPRESSION compression_method ] [ COLLATE collation ] [ column_constraint [ ... ] ] | table_constraint | LIKE source_table [ like_option ... ] } [, ... ] @@ -297,6 +297,27 @@ WITH ( MODULUS numeric_literal, REM + + STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + + + This form sets the storage mode for the column. This controls whether this + column is held inline or in a secondary TOAST table, + and whether the data should be compressed or not. PLAIN + must be used for fixed-length values such as integer and is + inline, uncompressed. MAIN is for inline, compressible + data. EXTERNAL is for external, uncompressed data, and + EXTENDED is for external, compressed data. + EXTENDED is the default for most data types that + support non-PLAIN storage. Use of + EXTERNAL will make substring operations on very large + text and bytea values run faster, at the penalty + of increased storage space. See for more + information. + + + + COMPRESSION compression_method diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index ef5b34a312..1df3a4c1d0 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -593,7 +593,7 @@ static void ATExecReplicaIdentity(Relation rel, ReplicaIdentityStmt *stmt, LOCKM static void ATExecGenericOptions(Relation rel, List *options); static void ATExecSetRowSecurity(Relation rel, bool rls); static void ATExecForceNoForceRowSecurity(Relation rel, bool force_rls); -static ObjectAddress ATExecSetCompression(AlteredTableInfo *tab, Relation rel, +static ObjectAddress ATExecSetCompression(Relation rel, const char *column, Node *newValue, LOCKMODE lockmode); static void index_copy_data(Relation rel, RelFileLocator newrlocator); @@ -633,6 +633,7 @@ static void refuseDupeIndexAttach(Relation parentIdx, Relation partIdx, static List *GetParentedForeignKeyRefs(Relation partition); static void ATDetachCheckNoForeignKeyRefs(Relation partition); static char GetAttributeCompression(Oid atttypid, char *compression); +static char GetAttributeStorage(Oid atttypid, const char *storagemode); /* ---------------------------------------------------------------- @@ -931,6 +932,10 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (colDef->compression) attr->attcompression = GetAttributeCompression(attr->atttypid, colDef->compression); + + if (colDef->storage_name) + attr->attstorage = GetAttributeStorage(attr->atttypid, colDef->storage_name); + } /* @@ -4963,8 +4968,8 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, case AT_SetStorage: /* ALTER COLUMN SET STORAGE */ address = ATExecSetStorage(rel, cmd->name, cmd->def, lockmode); break; - case AT_SetCompression: - address = ATExecSetCompression(tab, rel, cmd->name, cmd->def, + case AT_SetCompression: /* ALTER COLUMN SET COMPRESSION */ + address = ATExecSetCompression(rel, cmd->name, cmd->def, lockmode); break; case AT_DropColumn: /* DROP COLUMN */ @@ -6820,7 +6825,11 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, attribute.atttypmod = typmod; attribute.attbyval = tform->typbyval; attribute.attalign = tform->typalign; - attribute.attstorage = tform->typstorage; + if (colDef->storage_name) + attribute.attstorage = GetAttributeStorage(typeOid, colDef->storage_name); + else + attribute.attstorage = tform->typstorage; + attribute.attcompression = GetAttributeCompression(typeOid, colDef->compression); attribute.attnotnull = colDef->is_not_null; @@ -8263,34 +8272,12 @@ SetIndexStorageProperties(Relation rel, Relation attrelation, static ObjectAddress ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE lockmode) { - char *storagemode; - char newstorage; Relation attrelation; HeapTuple tuple; Form_pg_attribute attrtuple; AttrNumber attnum; ObjectAddress address; - Assert(IsA(newValue, String)); - storagemode = strVal(newValue); - - if (pg_strcasecmp(storagemode, "plain") == 0) - newstorage = TYPSTORAGE_PLAIN; - else if (pg_strcasecmp(storagemode, "external") == 0) - newstorage = TYPSTORAGE_EXTERNAL; - else if (pg_strcasecmp(storagemode, "extended") == 0) - newstorage = TYPSTORAGE_EXTENDED; - else if (pg_strcasecmp(storagemode, "main") == 0) - newstorage = TYPSTORAGE_MAIN; - else - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("invalid storage type \"%s\"", - storagemode))); - newstorage = 0; /* keep compiler quiet */ - } - attrelation = table_open(AttributeRelationId, RowExclusiveLock); tuple = SearchSysCacheCopyAttName(RelationGetRelid(rel), colName); @@ -8309,17 +8296,8 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc errmsg("cannot alter system column \"%s\"", colName))); - /* - * safety check: do not allow toasted storage modes unless column datatype - * is TOAST-aware. - */ - if (newstorage == TYPSTORAGE_PLAIN || TypeIsToastable(attrtuple->atttypid)) - attrtuple->attstorage = newstorage; - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("column data type %s can only have storage PLAIN", - format_type_be(attrtuple->atttypid)))); + Assert(IsA(newValue, String)); + attrtuple->attstorage = GetAttributeStorage(attrtuple->atttypid, strVal(newValue)); CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); @@ -8327,17 +8305,17 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc RelationGetRelid(rel), attrtuple->attnum); - heap_freetuple(tuple); - /* * Apply the change to indexes as well (only for simple index columns, * matching behavior of index.c ConstructTupleDescriptor()). */ SetIndexStorageProperties(rel, attrelation, attnum, - true, newstorage, + true, attrtuple->attstorage, false, 0, lockmode); + heap_freetuple(tuple); + table_close(attrelation, RowExclusiveLock); ObjectAddressSubSet(address, RelationRelationId, @@ -16157,8 +16135,7 @@ ATExecGenericOptions(Relation rel, List *options) * Return value is the address of the modified column */ static ObjectAddress -ATExecSetCompression(AlteredTableInfo *tab, - Relation rel, +ATExecSetCompression(Relation rel, const char *column, Node *newValue, LOCKMODE lockmode) @@ -19289,3 +19266,35 @@ GetAttributeCompression(Oid atttypid, char *compression) return cmethod; } + +static char +GetAttributeStorage(Oid atttypid, const char *storagemode) +{ + char cstorage; + + if (pg_strcasecmp(storagemode, "plain") == 0) + cstorage = TYPSTORAGE_PLAIN; + else if (pg_strcasecmp(storagemode, "external") == 0) + cstorage = TYPSTORAGE_EXTERNAL; + else if (pg_strcasecmp(storagemode, "extended") == 0) + cstorage = TYPSTORAGE_EXTENDED; + else if (pg_strcasecmp(storagemode, "main") == 0) + cstorage = TYPSTORAGE_MAIN; + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid storage type \"%s\"", + storagemode))); + + /* + * safety check: do not allow toasted storage modes unless column datatype + * is TOAST-aware. + */ + if (!(cstorage == TYPSTORAGE_PLAIN || TypeIsToastable(atttypid))) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column data type %s can only have storage PLAIN", + format_type_be(atttypid)))); + + return cstorage; +} diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 0523013f53..c018140afe 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -595,7 +595,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type TableConstraint TableLikeClause %type TableLikeOptionList TableLikeOption -%type column_compression opt_column_compression +%type column_compression opt_column_compression column_storage opt_column_storage %type ColQualList %type ColConstraint ColConstraintElem ConstraintAttr %type key_match @@ -2537,13 +2537,13 @@ alter_table_cmd: $$ = (Node *) n; } /* ALTER TABLE ALTER [COLUMN] SET STORAGE */ - | ALTER opt_column ColId SET STORAGE ColId + | ALTER opt_column ColId SET column_storage { AlterTableCmd *n = makeNode(AlterTableCmd); n->subtype = AT_SetStorage; n->name = $3; - n->def = (Node *) makeString($6); + n->def = (Node *) makeString($5); $$ = (Node *) n; } /* ALTER TABLE ALTER [COLUMN] SET COMPRESSION */ @@ -3778,13 +3778,14 @@ TypedTableElement: | TableConstraint { $$ = $1; } ; -columnDef: ColId Typename opt_column_compression create_generic_options ColQualList +columnDef: ColId Typename opt_column_storage opt_column_compression create_generic_options ColQualList { ColumnDef *n = makeNode(ColumnDef); n->colname = $1; n->typeName = $2; - n->compression = $3; + n->storage_name = $3; + n->compression = $4; n->inhcount = 0; n->is_local = true; n->is_not_null = false; @@ -3793,8 +3794,8 @@ columnDef: ColId Typename opt_column_compression create_generic_options ColQualL n->raw_default = NULL; n->cooked_default = NULL; n->collOid = InvalidOid; - n->fdwoptions = $4; - SplitColQualList($5, &n->constraints, &n->collClause, + n->fdwoptions = $5; + SplitColQualList($6, &n->constraints, &n->collClause, yyscanner); n->location = @1; $$ = (Node *) n; @@ -3851,6 +3852,15 @@ opt_column_compression: | /*EMPTY*/ { $$ = NULL; } ; +column_storage: + STORAGE ColId { $$ = $2; } + ; + +opt_column_storage: + column_storage { $$ = $1; } + | /*EMPTY*/ { $$ = NULL; } + ; + ColQualList: ColQualList ColConstraint { $$ = lappend($1, $2); } | /*EMPTY*/ { $$ = NIL; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 0b6a7bb365..280bf03795 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -695,6 +695,7 @@ typedef struct ColumnDef bool is_not_null; /* NOT NULL constraint specified? */ bool is_from_type; /* column definition came from table type */ char storage; /* attstorage setting, or 0 for default */ + char *storage_name; /* attstorage setting name or NULL for default */ Node *raw_default; /* default value (untransformed parse tree) */ Node *cooked_default; /* default value (transformed expr tree) */ char identity; /* attidentity setting */ diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 5ede56d9b5..fa1ee9e9b6 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -2244,7 +2244,7 @@ alter table recur1 add column f2 int; alter table recur1 alter column f2 type recur2; -- fails ERROR: composite type recur1 cannot be made a member of itself -- SET STORAGE may need to add a TOAST table -create table test_storage (a text); +create table test_storage (a text, c text storage plain); alter table test_storage alter a set storage plain; alter table test_storage add b int default 0; -- rewrite table to remove its TOAST table alter table test_storage alter a set storage extended; -- re-add TOAST table @@ -2256,6 +2256,9 @@ where oid = 'test_storage'::regclass; t (1 row) +--check STORAGE correctness +create table test_storage_failed (a text, b int storage extended); +ERROR: column data type integer can only have storage PLAIN -- test that SET STORAGE propagates to index correctly create index test_storage_idx on test_storage (b, a); alter table test_storage alter column a set storage external; @@ -2264,6 +2267,7 @@ alter table test_storage alter column a set storage external; Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+---------+-----------+----------+---------+----------+--------------+------------- a | text | | | | external | | + c | text | | | | plain | | b | integer | | | 0 | plain | | Indexes: "test_storage_idx" btree (b, a) diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql index 52001e3135..534166501c 100644 --- a/src/test/regress/sql/alter_table.sql +++ b/src/test/regress/sql/alter_table.sql @@ -1527,7 +1527,7 @@ alter table recur1 add column f2 int; alter table recur1 alter column f2 type recur2; -- fails -- SET STORAGE may need to add a TOAST table -create table test_storage (a text); +create table test_storage (a text, c text storage plain); alter table test_storage alter a set storage plain; alter table test_storage add b int default 0; -- rewrite table to remove its TOAST table alter table test_storage alter a set storage extended; -- re-add TOAST table @@ -1536,6 +1536,9 @@ select reltoastrelid <> 0 as has_toast_table from pg_class where oid = 'test_storage'::regclass; +--check STORAGE correctness +create table test_storage_failed (a text, b int storage extended); + -- test that SET STORAGE propagates to index correctly create index test_storage_idx on test_storage (b, a); alter table test_storage alter column a set storage external; -- 2.36.1