From b443e11d175898a07b7bff2692e9b5ce769d3a12 Mon Sep 17 00:00:00 2001 From: Nikolay Samokhvalov Date: Tue, 27 May 2025 01:30:57 -0700 Subject: [PATCH] Add wal_compression parameter with method:level syntax - Implement single-parameter design replacing separate wal_compression and wal_compression_level parameters - Support 'off', 'pglz', 'lz4[:level]', 'zstd[:level]' syntax with comprehensive validation - Add performance optimizations, thread safety, and extensive test coverage --- configure.ac | 1 + doc/src/sgml/config.sgml | 86 +++++- src/backend/access/transam/xlog.c | 268 +++++++++++++++++- src/backend/access/transam/xloginsert.c | 108 +++++-- src/backend/utils/misc/guc_tables.c | 41 +-- src/backend/utils/misc/postgresql.conf.sample | 5 +- src/include/access/xlog.h | 23 +- src/include/utils/guc_hooks.h | 5 +- src/test/recovery/t/035_wal_compression.pl | 99 +++++++ src/test/regress/expected/wal_compression.out | 120 ++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/wal_compression.sql | 61 ++++ 12 files changed, 760 insertions(+), 59 deletions(-) create mode 100644 src/test/recovery/t/035_wal_compression.pl create mode 100644 src/test/regress/expected/wal_compression.out create mode 100644 src/test/regress/sql/wal_compression.sql diff --git a/configure.ac b/configure.ac index 4b8335dc613..ed105c403a1 100644 --- a/configure.ac +++ b/configure.ac @@ -1563,6 +1563,7 @@ fi PGAC_PATH_PROGS(LZ4, lz4) if test "$with_lz4" = yes; then AC_CHECK_HEADER(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])]) + AC_CHECK_HEADER(lz4hc.h, [AC_DEFINE(HAVE_LZ4HC_H, 1, [Define to 1 if you have the header file.])]) fi PGAC_PATH_PROGS(ZSTD, zstd) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index ca2a567b2b1..230715ab22d 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3387,7 +3387,7 @@ include_dir 'conf.d' - wal_compression (enum) + wal_compression (string) wal_compression configuration parameter @@ -3395,30 +3395,100 @@ include_dir 'conf.d' This parameter enables compression of WAL using the specified - compression method. + compression method and optional compression level. When enabled, the PostgreSQL server compresses full page images written to WAL when is on or during a base backup. A compressed page image will be decompressed during WAL replay. - The supported methods are pglz, - lz4 (if PostgreSQL - was compiled with ) and - zstd (if PostgreSQL - was compiled with ). - The default value is off. + + + + The parameter accepts values in the format method or + method:level. Supported values include: + + + off — no compression (default) + + + pglz — use PGLZ compression (no levels supported) + + + lz4 or lz4:level — use LZ4 compression + (if PostgreSQL was compiled with ). + Valid levels are 1-12. When no level is specified, lz4 defaults to level 1. + Levels 1-9 use standard LZ4, while levels 10-12 use LZ4HC for better compression at higher CPU cost. + Important: If built without LZ4HC support, compression + levels 10-12 are rejected at configuration time with a clear error message. + To use these levels, PostgreSQL must be compiled with LZ4HC headers available. + + + zstd or zstd:level — use ZSTD compression + (if PostgreSQL was compiled with ). + Valid levels are 1-22. When no level is specified, zstd defaults to level 3. + + + + + + Examples: 'off', 'pglz', + 'lz4', 'lz4:9', + 'zstd', 'zstd:15'. Only superusers and users with the appropriate SET privilege can change this setting. + + Values like '0', 'false', and 'no' + are recognized as aliases for 'off' to preserve backward compatibility. + + Enabling compression can reduce the WAL volume without increasing the risk of unrecoverable data corruption, but at the cost of some extra CPU spent on the compression during WAL logging and on the decompression during WAL replay. + Higher compression levels generally provide better compression + at the cost of increased CPU usage. + + + + Different compression levels provide varying trade-offs between compression + ratio and CPU usage. For zstd, higher levels can achieve + substantially better compression ratios, though with increased CPU overhead. + For lz4, levels 10-12 use the high-compression LZ4HC algorithm + and provide better compression than levels 1-9 (standard LZ4) at the cost of + higher CPU usage during compression. The optimal level depends on + your workload characteristics and the balance between I/O reduction + and CPU overhead in your environment. + + + + Performance Guidelines: For I/O-bound systems with + spare CPU capacity, higher compression levels (e.g., zstd:9 + or lz4:12 if LZ4HC is available) can significantly reduce + WAL volume and improve overall throughput. For CPU-bound systems, lower levels + (e.g., lz4:1 or zstd:3) provide + good compression with minimal CPU overhead. In high-transaction environments, + lz4 typically offers the best balance of compression + speed and CPU efficiency, while zstd excels when + maximum compression is needed and CPU resources are available. + + + + Higher compression levels increase CPU usage both during WAL generation + on the primary server and during WAL replay on standby servers. On + standby servers, decompression occurs in the single-threaded WAL replay + process, which can become a bottleneck if very high compression levels + are used. This is particularly important to consider in streaming + replication setups where standby lag could increase due to slower + WAL replay. Monitor standby lag when using higher compression levels + to ensure acceptable replication performance. Consider using lower + compression levels if standby lag becomes problematic. + wal_init_zero (boolean) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 1914859b2ee..cd064c97464 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -121,9 +121,21 @@ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; -int wal_compression = WAL_COMPRESSION_NONE; +char *wal_compression = NULL; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; + +/* + * Parsed values from wal_compression string - updated by assign_wal_compression() + * + * Thread safety: These variables are written only during GUC assignment, which + * is serialized by PostgreSQL's GUC mechanism. They are read during WAL insertion + * via GetWalCompressionMethod(). Since GUC changes are rare and atomic pointer + * reads/writes are guaranteed on all supported platforms, no additional locking + * is required. + */ +static WalCompression wal_compression_method = WAL_COMPRESSION_NONE; +static int wal_compression_level = WAL_COMPRESSION_LEVEL_NONE; bool wal_init_zero = true; bool wal_recycle = true; bool log_checkpoints = true; @@ -596,8 +608,8 @@ static ControlFileData *ControlFile = NULL; (((idx) == XLogCtl->XLogCacheBlck) ? 0 : ((idx) + 1)) /* - * XLogRecPtrToBufIdx returns the index of the WAL buffer that holds, or - * would hold if it was in cache, the page containing 'recptr'. + * XLogRecPtrToBufIdx returns the index of the WAL buffer that holds, + * or would hold if it was in cache, the page containing 'recptr'. */ #define XLogRecPtrToBufIdx(recptr) \ (((recptr) / XLOG_BLCKSZ) % (XLogCtl->XLogCacheBlck + 1)) @@ -2365,6 +2377,8 @@ check_max_slot_wal_keep_size(int *newval, void **extra, GucSource source) return true; } + + /* * At a checkpoint, how many WAL segments to recycle as preallocated future * XLOG segments? Returns the highest segment that should be preallocated. @@ -4984,6 +4998,254 @@ InitializeWalConsistencyChecking(void) } } +/* + * Helper function to parse compression level from colon-separated format + * Returns true if level was successfully parsed, false on error + */ +static bool +ParseCompressionLevel(const char *level_str, int *level) +{ + if (strlen(level_str) == 0) + return false; + + *level = atoi(level_str); + + /* Ensure level_str represents a numeric level explicitly (reject non-numeric inputs) */ + if (*level == 0 && strcmp(level_str, "0") != 0) + return false; + + return true; +} + +/* + * Helper function to determine compression method from method string + * Sets method and default level if no level was specified + */ +static bool +ParseCompressionMethod(const char *method_str, bool level_specified, + WalCompression *method, int *level) +{ + if (strcmp(method_str, "pglz") == 0) + { + *method = WAL_COMPRESSION_PGLZ; + if (!level_specified) + *level = WAL_COMPRESSION_LEVEL_NONE; + return true; + } + else if (strcmp(method_str, "lz4") == 0) + { + *method = WAL_COMPRESSION_LZ4; + if (!level_specified) + *level = LZ4_DEFAULT_COMPRESSION_LEVEL; + return true; + } + else if (strcmp(method_str, "zstd") == 0) + { + *method = WAL_COMPRESSION_ZSTD; + if (!level_specified) + *level = ZSTD_DEFAULT_COMPRESSION_LEVEL; + return true; + } + else if (strcmp(method_str, "on") == 0 || strcmp(method_str, "true") == 0 || + strcmp(method_str, "yes") == 0 || strcmp(method_str, "1") == 0) + { + *method = WAL_COMPRESSION_PGLZ; + *level = WAL_COMPRESSION_LEVEL_NONE; + return true; + } + else if (strcmp(method_str, "off") == 0 || strcmp(method_str, "false") == 0 || + strcmp(method_str, "no") == 0 || strcmp(method_str, "0") == 0) + { + *method = WAL_COMPRESSION_NONE; + *level = WAL_COMPRESSION_LEVEL_NONE; + return true; + } + + /* Unrecognized method */ + return false; +} + +/* + * Parse wal_compression string into method and level components + * + * Supports formats like: + * "off" -> WAL_COMPRESSION_NONE, level WAL_COMPRESSION_LEVEL_NONE + * "pglz" -> WAL_COMPRESSION_PGLZ, level WAL_COMPRESSION_LEVEL_NONE + * "lz4" -> WAL_COMPRESSION_LZ4, level 1 + * "lz4:9" -> WAL_COMPRESSION_LZ4, level 9 + * "zstd" -> WAL_COMPRESSION_ZSTD, level 3 + * "zstd:15" -> WAL_COMPRESSION_ZSTD, level 15 + * + * Returns true if parsing was successful, false if invalid format + */ +bool +ParseWalCompression(const char *compression_str, WalCompression *method, int *level) +{ + char *str_copy; + char *colon_pos; + bool level_specified = false; + bool method_valid; + + /* Set defaults */ + *method = WAL_COMPRESSION_NONE; + *level = WAL_COMPRESSION_LEVEL_NONE; + + /* Handle "off" and equivalent values */ + if (!compression_str || strcmp(compression_str, "off") == 0 || + strcmp(compression_str, "false") == 0 || strcmp(compression_str, "no") == 0 || + strcmp(compression_str, "0") == 0) + { + return true; + } + + /* Look for colon separator */ + colon_pos = strchr(compression_str, ':'); + if (colon_pos) + { + /* Check for malformed syntax like "method:" or ":level" */ + if (colon_pos == compression_str) + return false; /* Starts with colon */ + if (*(colon_pos + 1) == '\0') + return false; /* Ends with colon */ + + /* Extract method name and parse level */ + size_t method_len = colon_pos - compression_str; + str_copy = palloc(method_len + 1); + memcpy(str_copy, compression_str, method_len); + str_copy[method_len] = '\0'; + + level_specified = true; + if (!ParseCompressionLevel(colon_pos + 1, level)) + { + pfree(str_copy); + return false; /* Invalid level format */ + } + } + else + { + /* No colon, duplicate the string for consistent memory management */ + str_copy = pstrdup(compression_str); + } + + /* Parse method and set defaults */ + method_valid = ParseCompressionMethod(str_copy, level_specified, method, level); + + /* Clean up allocated memory */ + pfree(str_copy); + + return method_valid; +} + +/* + * GUC check_hook for wal_compression + */ +bool +check_wal_compression(char **newval, void **extra, GucSource source) +{ + WalCompression method; + int level; + + /* Parse the compression string */ + if (!ParseWalCompression(*newval, &method, &level)) + { + GUC_check_errdetail("Unrecognized compression method: \"%s\".", *newval); + return false; + } + + /* Validate the method is supported */ + switch (method) + { + case WAL_COMPRESSION_NONE: + case WAL_COMPRESSION_PGLZ: + /* Always supported */ + break; + + case WAL_COMPRESSION_LZ4: +#ifndef USE_LZ4 + GUC_check_errdetail("LZ4 compression is not supported by this build."); + return false; +#endif + /* Validate level range for LZ4 */ + if (level < LZ4_MIN_LEVEL || level > LZ4_MAX_LEVEL) + { + GUC_check_errdetail("LZ4 compression level must be between %d and %d.", + LZ4_MIN_LEVEL, LZ4_MAX_LEVEL); + return false; + } + /* Check LZ4HC availability for levels 10-12 */ +#ifndef HAVE_LZ4HC_H + if (level >= LZ4HC_MIN_LEVEL) + { + GUC_check_errdetail("LZ4HC (required for compression levels %d-%d) is not available in this PostgreSQL build.", + LZ4HC_MIN_LEVEL, LZ4_MAX_LEVEL); + GUC_check_errhint("Please rebuild PostgreSQL with LZ4HC support or select compression levels between %d and %d.", + LZ4_MIN_LEVEL, LZ4HC_MIN_LEVEL - 1); + return false; + } +#endif + break; + + case WAL_COMPRESSION_ZSTD: +#ifndef USE_ZSTD + GUC_check_errdetail("ZSTD compression is not supported by this build."); + return false; +#endif + /* Validate level range for ZSTD */ + if (level < ZSTD_MIN_LEVEL || level > ZSTD_MAX_LEVEL) + { + GUC_check_errdetail("ZSTD compression level must be between %d and %d.", + ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL); + return false; + } + break; + + default: + GUC_check_errdetail("Unrecognized compression method: \"%s\".", *newval); + return false; + } + + /* Methods that don't support levels should have level set to WAL_COMPRESSION_LEVEL_NONE */ + if ((method == WAL_COMPRESSION_PGLZ || method == WAL_COMPRESSION_NONE) && + level != WAL_COMPRESSION_LEVEL_NONE) + { + GUC_check_errdetail("Compression method \"%s\" does not support compression levels.", + method == WAL_COMPRESSION_PGLZ ? "pglz" : "off"); + return false; + } + + return true; +} + +/* + * GUC assign_hook for wal_compression + */ +void +assign_wal_compression(const char *newval, void *extra) +{ + /* + * Parse and store the compression method and level for fast access + * during compression operations. This avoids parsing on every compression. + */ + if (newval) + ParseWalCompression(newval, &wal_compression_method, &wal_compression_level); + else + { + wal_compression_method = WAL_COMPRESSION_NONE; + wal_compression_level = WAL_COMPRESSION_LEVEL_NONE; + } +} + +/* + * Get the parsed WAL compression method and level. + * This is used by xloginsert.c to avoid parsing on every compression. + */ +void +GetWalCompressionMethod(WalCompression *method, int *level) +{ + *method = wal_compression_method; + *level = wal_compression_level; +} + /* * GUC show_hook for archive_command */ diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 5ee9d0b028e..1b2d591935a 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -21,6 +21,9 @@ #ifdef USE_LZ4 #include +#ifdef HAVE_LZ4HC_H +#include +#endif #endif #ifdef USE_ZSTD @@ -685,13 +688,19 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, /* * Try to compress a block image if wal_compression is enabled */ - if (wal_compression != WAL_COMPRESSION_NONE) { - is_compressed = - XLogCompressBackupBlock(page, bimg.hole_offset, - cbimg.hole_length, - regbuf->compressed_page, - &compressed_len); + WalCompression method; + int level; + + GetWalCompressionMethod(&method, &level); + if (method != WAL_COMPRESSION_NONE) + { + is_compressed = + XLogCompressBackupBlock(page, bimg.hole_offset, + cbimg.hole_length, + regbuf->compressed_page, + &compressed_len); + } } /* @@ -726,7 +735,13 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, bimg.length = compressed_len; /* Set the compression method used for this block */ - switch ((WalCompression) wal_compression) + WalCompression method; + int level; + + /* Get the pre-parsed compression method and level (avoids re-parsing) */ + GetWalCompressionMethod(&method, &level); + + switch (method) { case WAL_COMPRESSION_PGLZ: bimg.bimg_info |= BKPIMAGE_COMPRESS_PGLZ; @@ -846,7 +861,7 @@ XLogRecordAssemble(RmgrId rmid, uint8 info, scratch += sizeof(replorigin_session_origin); } - /* followed by toplevel XID, if not already included in previous record */ + /* followed by toplevel XID, if not already included in previous record */ if (IsSubxactTopXidLogPending()) { TransactionId xid = GetTopTransactionIdIfAny(); @@ -968,16 +983,61 @@ XLogCompressBackupBlock(const PageData *page, uint16 hole_offset, uint16 hole_le else source = page; - switch ((WalCompression) wal_compression) + { + WalCompression method; + int wal_compression_level; + + /* Get the pre-parsed compression method and level (avoids re-parsing) */ + GetWalCompressionMethod(&method, &wal_compression_level); + + switch (method) { case WAL_COMPRESSION_PGLZ: + /* PGLZ doesn't use levels - warn if level is set but continue */ + if (wal_compression_level != WAL_COMPRESSION_LEVEL_NONE) + elog(WARNING, "PGLZ compression level %d ignored, PGLZ does not support compression levels", + wal_compression_level); len = pglz_compress(source, orig_len, dest, PGLZ_strategy_default); break; case WAL_COMPRESSION_LZ4: #ifdef USE_LZ4 - len = LZ4_compress_default(source, dest, orig_len, - COMPRESS_BUFSIZE); + { + /* Validate LZ4 compression level range */ + if (wal_compression_level < LZ4_MIN_LEVEL || wal_compression_level > LZ4_MAX_LEVEL) + { + elog(ERROR, "Invalid LZ4 compression level %d, must be between %d and %d", + wal_compression_level, LZ4_MIN_LEVEL, LZ4_MAX_LEVEL); + } + + if (wal_compression_level <= 9) + { + /* + * Use standard LZ4 for levels 1-9. + * Note: LZ4_compress_default() ignores the level parameter, + * so all levels 1-9 produce the same result but we accept + * them for user convenience and future compatibility. + */ + len = LZ4_compress_default(source, dest, orig_len, + COMPRESS_BUFSIZE); + } + else + { + /* Use LZ4HC for levels 10-12 for better compression */ +#ifdef HAVE_LZ4HC_H + len = LZ4_compress_HC(source, dest, orig_len, + COMPRESS_BUFSIZE, wal_compression_level); +#else + /* + * This should not happen since we validate LZ4HC availability + * at SET time in check_wal_compression(). If we reach here, + * it means there's a bug in the validation logic. + */ + elog(ERROR, "LZ4HC compression level %d requested but LZ4HC is not available", + wal_compression_level); +#endif + } + } if (len <= 0) len = -1; /* failure */ #else @@ -987,20 +1047,29 @@ XLogCompressBackupBlock(const PageData *page, uint16 hole_offset, uint16 hole_le case WAL_COMPRESSION_ZSTD: #ifdef USE_ZSTD - len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, - ZSTD_CLEVEL_DEFAULT); - if (ZSTD_isError(len)) - len = -1; /* failure */ + { + /* Validate ZSTD compression level range */ + if (wal_compression_level < ZSTD_MIN_LEVEL || wal_compression_level > ZSTD_MAX_LEVEL) + { + elog(ERROR, "Invalid ZSTD compression level %d, must be between %d and %d", + wal_compression_level, ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL); + } + + len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, wal_compression_level); + if (ZSTD_isError(len)) + len = -1; /* failure */ + } #else elog(ERROR, "zstd is not supported by this build"); #endif break; case WAL_COMPRESSION_NONE: - Assert(false); /* cannot happen */ + elog(ERROR, "WAL_COMPRESSION_NONE should not reach compression function"); break; /* no default case, so that compiler will warn */ } + } /* * We recheck the actual size even if compression reports success and see @@ -1011,8 +1080,15 @@ XLogCompressBackupBlock(const PageData *page, uint16 hole_offset, uint16 hole_le len + extra_bytes < orig_len) { *dlen = (uint16) len; /* successful compression */ + elog(DEBUG2, "WAL compression successful: %s, orig=%d, compressed=%d+%d=%d (%.1f%%)", + wal_compression, orig_len, len, extra_bytes, len + extra_bytes, + 100.0 * (orig_len - (len + extra_bytes)) / orig_len); return true; } + + elog(DEBUG3, "WAL compression failed: %s, orig=%d, compressed=%d+%d=%d", + wal_compression, orig_len, len >= 0 ? len : 0, extra_bytes, + len >= 0 ? len + extra_bytes : 0); return false; } diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2f8cbd86759..4b645ed19b1 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -464,24 +464,7 @@ static const struct config_enum_entry default_toast_compression_options[] = { {NULL, 0, false} }; -static const struct config_enum_entry wal_compression_options[] = { - {"pglz", WAL_COMPRESSION_PGLZ, false}, -#ifdef USE_LZ4 - {"lz4", WAL_COMPRESSION_LZ4, false}, -#endif -#ifdef USE_ZSTD - {"zstd", WAL_COMPRESSION_ZSTD, false}, -#endif - {"on", WAL_COMPRESSION_PGLZ, false}, - {"off", WAL_COMPRESSION_NONE, false}, - {"true", WAL_COMPRESSION_PGLZ, true}, - {"false", WAL_COMPRESSION_NONE, true}, - {"yes", WAL_COMPRESSION_PGLZ, true}, - {"no", WAL_COMPRESSION_NONE, true}, - {"1", WAL_COMPRESSION_PGLZ, true}, - {"0", WAL_COMPRESSION_NONE, true}, - {NULL, 0, false} -}; + static const struct config_enum_entry file_copy_method_options[] = { {"copy", FILE_COPY_METHOD_COPY, false}, @@ -2318,6 +2301,8 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + + { {"max_connections", PGC_POSTMASTER, CONN_AUTH_SETTINGS, gettext_noop("Sets the maximum number of concurrent connections."), @@ -4210,6 +4195,16 @@ struct config_string ConfigureNamesString[] = NULL, NULL, NULL }, + { + {"wal_compression", PGC_SUSET, WAL_SETTINGS, + gettext_noop("Compresses full-page writes written in WAL file with specified method and level."), + gettext_noop("Valid formats: 'off', 'pglz', 'lz4', 'lz4:level', 'zstd', 'zstd:level'.") + }, + &wal_compression, + "off", + check_wal_compression, assign_wal_compression, NULL + }, + { {"recovery_target_timeline", PGC_POSTMASTER, WAL_RECOVERY_TARGET, gettext_noop("Specifies the timeline to recover into."), @@ -5214,15 +5209,7 @@ struct config_enum ConfigureNamesEnum[] = NULL, assign_stats_fetch_consistency, NULL }, - { - {"wal_compression", PGC_SUSET, WAL_SETTINGS, - gettext_noop("Compresses full-page writes written in WAL file with specified method."), - NULL - }, - &wal_compression, - WAL_COMPRESSION_NONE, wal_compression_options, - NULL, NULL, NULL - }, + { {"wal_level", PGC_POSTMASTER, WAL_SETTINGS, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 63f991c4f93..818ea04d853 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -247,8 +247,9 @@ #full_page_writes = on # recover from partial page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) -#wal_compression = off # enables compression of full-page writes; - # off, pglz, lz4, zstd, or on +#wal_compression = 'off' # enables compression of full-page writes; + # 'off', 'pglz', 'lz4', 'lz4:level', + # 'zstd', 'zstd:level' #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index d313099c027..54e52717afb 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -46,7 +46,7 @@ extern PGDLLIMPORT char *XLogArchiveCommand; extern PGDLLIMPORT bool EnableHotStandby; extern PGDLLIMPORT bool fullPageWrites; extern PGDLLIMPORT bool wal_log_hints; -extern PGDLLIMPORT int wal_compression; +extern PGDLLIMPORT char *wal_compression; extern PGDLLIMPORT bool wal_init_zero; extern PGDLLIMPORT bool wal_recycle; extern PGDLLIMPORT bool *wal_consistency_checking; @@ -85,6 +85,27 @@ typedef enum WalCompression WAL_COMPRESSION_ZSTD, } WalCompression; +/* Special value to indicate compression level is not applicable */ +#define WAL_COMPRESSION_LEVEL_NONE (-1) + +/* Default compression levels for each algorithm */ +#define LZ4_DEFAULT_COMPRESSION_LEVEL 1 +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 3 + +/* Compression level ranges */ +#define LZ4_MIN_LEVEL 1 +#define LZ4_MAX_LEVEL 12 +#define LZ4HC_MIN_LEVEL 10 +#define ZSTD_MIN_LEVEL 1 +#define ZSTD_MAX_LEVEL 22 + +/* Function to parse wal_compression string into method and level */ +extern bool ParseWalCompression(const char *compression_str, + WalCompression *method, int *level); + +/* Get the parsed WAL compression method and level (avoids re-parsing) */ +extern void GetWalCompressionMethod(WalCompression *method, int *level); + /* Recovery states */ typedef enum RecoveryState { diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 799fa7ace68..51884fa9956 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -168,9 +168,12 @@ extern bool check_transaction_read_only(bool *newval, void **extra, GucSource so extern void assign_transaction_timeout(int newval, void *extra); extern const char *show_unix_socket_permissions(void); extern bool check_wal_buffers(int *newval, void **extra, GucSource source); +extern bool check_wal_compression(char **newval, void **extra, GucSource source); +extern void assign_wal_compression(const char *newval, void *extra); extern bool check_wal_consistency_checking(char **newval, void **extra, - GucSource source); + GucSource source); extern void assign_wal_consistency_checking(const char *newval, void *extra); + extern bool check_wal_segment_size(int *newval, void **extra, GucSource source); extern void assign_wal_sync_method(int new_wal_sync_method, void *extra); extern bool check_synchronized_standby_slots(char **newval, void **extra, diff --git a/src/test/recovery/t/035_wal_compression.pl b/src/test/recovery/t/035_wal_compression.pl new file mode 100644 index 00000000000..69691b45afa --- /dev/null +++ b/src/test/recovery/t/035_wal_compression.pl @@ -0,0 +1,99 @@ +# Test WAL compression functionality with new single-parameter design +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Test basic functionality of wal_compression parameter with method:level syntax +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init(allows_streaming => 1); + +# Enable WAL compression with ZSTD and compression level +$node->append_conf('postgresql.conf', qq( +wal_compression = 'zstd:9' +)); + +$node->start; + +# Test that the parameter is set correctly +my $result = $node->safe_psql('postgres', 'SHOW wal_compression;'); +is($result, 'zstd:9', 'wal_compression is set to zstd:9'); + +# Test that we can change the compression method and level +$node->safe_psql('postgres', 'ALTER SYSTEM SET wal_compression = \'lz4:5\';'); +$node->safe_psql('postgres', 'SELECT pg_reload_conf();'); + +$result = $node->safe_psql('postgres', 'SHOW wal_compression;'); +is($result, 'lz4:5', 'wal_compression changed to lz4:5'); + +# Test method without level (should use defaults) +$node->safe_psql('postgres', 'ALTER SYSTEM SET wal_compression = \'zstd\';'); +$node->safe_psql('postgres', 'SELECT pg_reload_conf();'); + +$result = $node->safe_psql('postgres', 'SHOW wal_compression;'); +is($result, 'zstd', 'wal_compression set to zstd without level'); + +# Test PGLZ (no levels supported) +$node->safe_psql('postgres', 'ALTER SYSTEM SET wal_compression = \'pglz\';'); +$node->safe_psql('postgres', 'SELECT pg_reload_conf();'); + +$result = $node->safe_psql('postgres', 'SHOW wal_compression;'); +is($result, 'pglz', 'wal_compression set to pglz'); + +# Test that invalid values are rejected +my ($ret, $stdout, $stderr) = $node->psql('postgres', 'SET wal_compression = \'lz4:25\';'); +isnt($ret, 0, 'Setting wal_compression to lz4:25 should fail'); +like($stderr, qr/LZ4 compression level must be between 1 and 12/, 'Error message mentions LZ4 valid range'); + +# Test invalid ZSTD level +($ret, $stdout, $stderr) = $node->psql('postgres', 'SET wal_compression = \'zstd:25\';'); +isnt($ret, 0, 'Setting wal_compression to zstd:25 should fail'); +like($stderr, qr/ZSTD compression level must be between 1 and 22/, 'Error message mentions ZSTD valid range'); + +# Test PGLZ with level (should fail) +($ret, $stdout, $stderr) = $node->psql('postgres', 'SET wal_compression = \'pglz:5\';'); +isnt($ret, 0, 'Setting wal_compression to pglz:5 should fail'); +like($stderr, qr/PGLZ compression does not support compression levels/, 'Error message mentions PGLZ no levels'); + +# Test invalid method +($ret, $stdout, $stderr) = $node->psql('postgres', 'SET wal_compression = \'invalid\';'); +isnt($ret, 0, 'Setting wal_compression to invalid method should fail'); + +# Generate some WAL to ensure compression is working +$node->safe_psql('postgres', qq( + CREATE TABLE compression_test (id int, data text); + INSERT INTO compression_test SELECT i, repeat('test', 100) FROM generate_series(1, 1000) i; + UPDATE compression_test SET data = repeat('updated', 100) WHERE id % 10 = 0; +)); + +# Test various valid combinations +my @valid_settings = ( + 'off', + 'pglz', + 'lz4', + 'lz4:1', + 'lz4:12', + 'zstd', + 'zstd:1', + 'zstd:22' +); + +foreach my $setting (@valid_settings) { + $node->safe_psql('postgres', "ALTER SYSTEM SET wal_compression = '$setting';"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf();'); + + $result = $node->safe_psql('postgres', 'SHOW wal_compression;'); + is($result, $setting, "wal_compression successfully set to $setting"); +} + +# Reset to default +$node->safe_psql('postgres', 'ALTER SYSTEM SET wal_compression = \'off\';'); +$node->safe_psql('postgres', 'SELECT pg_reload_conf();'); + +$result = $node->safe_psql('postgres', 'SHOW wal_compression;'); +is($result, 'off', 'wal_compression reset to off'); + +$node->stop; + +done_testing(); diff --git a/src/test/regress/expected/wal_compression.out b/src/test/regress/expected/wal_compression.out new file mode 100644 index 00000000000..110ebb0cff1 --- /dev/null +++ b/src/test/regress/expected/wal_compression.out @@ -0,0 +1,120 @@ +-- +-- Test WAL compression functionality with method:level syntax +-- +-- Test basic parameter existence and default value +SHOW wal_compression; + wal_compression +----------------- + off +(1 row) + +-- Test setting compression methods without levels +SET wal_compression = 'off'; +SHOW wal_compression; + wal_compression +----------------- + off +(1 row) + +SET wal_compression = 'pglz'; +SHOW wal_compression; + wal_compression +----------------- + pglz +(1 row) + +SET wal_compression = 'lz4'; +SHOW wal_compression; + wal_compression +----------------- + lz4 +(1 row) + +SET wal_compression = 'zstd'; +SHOW wal_compression; + wal_compression +----------------- + zstd +(1 row) + +-- Test setting compression methods with levels +SET wal_compression = 'lz4:1'; +SHOW wal_compression; + wal_compression +----------------- + lz4:1 +(1 row) + +SET wal_compression = 'lz4:9'; +SHOW wal_compression; + wal_compression +----------------- + lz4:9 +(1 row) + +-- Note: LZ4HC levels (10-12) are not tested here because their availability +-- depends on build configuration (HAVE_LZ4HC_H). On builds without LZ4HC, +-- these levels are rejected at SET time with appropriate error messages. +SET wal_compression = 'zstd:1'; +SHOW wal_compression; + wal_compression +----------------- + zstd:1 +(1 row) + +SET wal_compression = 'zstd:9'; +SHOW wal_compression; + wal_compression +----------------- + zstd:9 +(1 row) + +SET wal_compression = 'zstd:22'; +SHOW wal_compression; + wal_compression +----------------- + zstd:22 +(1 row) + +-- Test invalid compression levels (should fail) +SET wal_compression = 'lz4:0'; +ERROR: invalid value for parameter "wal_compression": "lz4:0" +DETAIL: LZ4 compression level must be between 1 and 12. +SET wal_compression = 'lz4:13'; +ERROR: invalid value for parameter "wal_compression": "lz4:13" +DETAIL: LZ4 compression level must be between 1 and 12. +SET wal_compression = 'zstd:0'; +ERROR: invalid value for parameter "wal_compression": "zstd:0" +DETAIL: ZSTD compression level must be between 1 and 22. +SET wal_compression = 'zstd:23'; +ERROR: invalid value for parameter "wal_compression": "zstd:23" +DETAIL: ZSTD compression level must be between 1 and 22. +-- Test PGLZ with levels (should fail) +SET wal_compression = 'pglz:1'; +ERROR: invalid value for parameter "wal_compression": "pglz:1" +DETAIL: Compression method "pglz" does not support compression levels. +-- Test invalid compression methods (should fail) +SET wal_compression = 'invalid'; +ERROR: invalid value for parameter "wal_compression": "invalid" +DETAIL: Unrecognized compression method: "invalid". +SET wal_compression = 'gzip'; +ERROR: invalid value for parameter "wal_compression": "gzip" +DETAIL: Unrecognized compression method: "gzip". +-- Test malformed syntax (should fail) +SET wal_compression = 'lz4:'; +ERROR: invalid value for parameter "wal_compression": "lz4:" +DETAIL: Unrecognized compression method: "lz4:". +SET wal_compression = 'lz4:abc'; +ERROR: invalid value for parameter "wal_compression": "lz4:abc" +DETAIL: Unrecognized compression method: "lz4:abc". +SET wal_compression = ':5'; +ERROR: invalid value for parameter "wal_compression": ":5" +DETAIL: Unrecognized compression method: ":5". +-- Reset to default +SET wal_compression = 'off'; +SHOW wal_compression; + wal_compression +----------------- + off +(1 row) + diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index a424be2a6bf..c2b5f324a54 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -102,7 +102,7 @@ test: publication subscription # Another group of parallel tests # select_views depends on create_view # ---------- -test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock indirect_toast equivclass +test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combocid tsearch tsdicts foreign_data window xmlmap functional_deps advisory_lock indirect_toast equivclass wal_compression # ---------- # Another group of parallel tests (JSON related) diff --git a/src/test/regress/sql/wal_compression.sql b/src/test/regress/sql/wal_compression.sql new file mode 100644 index 00000000000..f4c13c07d69 --- /dev/null +++ b/src/test/regress/sql/wal_compression.sql @@ -0,0 +1,61 @@ +-- +-- Test WAL compression functionality with method:level syntax +-- + +-- Test basic parameter existence and default value +SHOW wal_compression; + +-- Test setting compression methods without levels +SET wal_compression = 'off'; +SHOW wal_compression; + +SET wal_compression = 'pglz'; +SHOW wal_compression; + +SET wal_compression = 'lz4'; +SHOW wal_compression; + +SET wal_compression = 'zstd'; +SHOW wal_compression; + +-- Test setting compression methods with levels +SET wal_compression = 'lz4:1'; +SHOW wal_compression; + +SET wal_compression = 'lz4:9'; +SHOW wal_compression; + +-- Note: LZ4HC levels (10-12) are not tested here because their availability +-- depends on build configuration (HAVE_LZ4HC_H). On builds without LZ4HC, +-- these levels are rejected at SET time with appropriate error messages. + +SET wal_compression = 'zstd:1'; +SHOW wal_compression; + +SET wal_compression = 'zstd:9'; +SHOW wal_compression; + +SET wal_compression = 'zstd:22'; +SHOW wal_compression; + +-- Test invalid compression levels (should fail) +SET wal_compression = 'lz4:0'; +SET wal_compression = 'lz4:13'; +SET wal_compression = 'zstd:0'; +SET wal_compression = 'zstd:23'; + +-- Test PGLZ with levels (should fail) +SET wal_compression = 'pglz:1'; + +-- Test invalid compression methods (should fail) +SET wal_compression = 'invalid'; +SET wal_compression = 'gzip'; + +-- Test malformed syntax (should fail) +SET wal_compression = 'lz4:'; +SET wal_compression = 'lz4:abc'; +SET wal_compression = ':5'; + +-- Reset to default +SET wal_compression = 'off'; +SHOW wal_compression; -- GitLab