On 2020-01-15 01:40, Masahiko Sawada wrote:
Could you rebase the main patch that adds base backup client as auxiliary backend process since the previous version patch (v3) conflicts with the current HEAD?
attached -- Peter Eisentraut http://www.2ndQuadrant.com/ PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
From 892ba431956c7d936555f758efc874f58b3679e8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut <pe...@eisentraut.org> Date: Wed, 15 Jan 2020 16:15:06 +0100 Subject: [PATCH v4] Base backup client as auxiliary backend process Discussion: https://www.postgresql.org/message-id/flat/61b8d18d-c922-ac99-b990-a31ba63cd...@2ndquadrant.com --- doc/src/sgml/protocol.sgml | 12 +- doc/src/sgml/ref/initdb.sgml | 17 + src/backend/access/transam/xlog.c | 102 +++-- src/backend/bootstrap/bootstrap.c | 9 + src/backend/postmaster/pgstat.c | 6 + src/backend/postmaster/postmaster.c | 114 ++++- src/backend/replication/basebackup.c | 70 +++ .../libpqwalreceiver/libpqwalreceiver.c | 400 ++++++++++++++++++ src/backend/replication/repl_gram.y | 9 +- src/backend/replication/repl_scanner.l | 1 + src/backend/storage/file/fd.c | 36 +- src/bin/initdb/initdb.c | 39 +- src/bin/pg_resetwal/pg_resetwal.c | 6 +- src/include/access/xlog.h | 8 +- src/include/miscadmin.h | 2 + src/include/pgstat.h | 1 + src/include/replication/basebackup.h | 2 + src/include/replication/walreceiver.h | 4 + src/include/storage/fd.h | 2 +- src/include/utils/guc.h | 2 +- src/test/recovery/t/018_basebackup.pl | 29 ++ 21 files changed, 783 insertions(+), 88 deletions(-) create mode 100644 src/test/recovery/t/018_basebackup.pl diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 80275215e0..f54b820edf 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2466,7 +2466,7 @@ <title>Streaming Replication Protocol</title> </varlistentry> <varlistentry> - <term><literal>BASE_BACKUP</literal> [ <literal>LABEL</literal> <replaceable>'label'</replaceable> ] [ <literal>PROGRESS</literal> ] [ <literal>FAST</literal> ] [ <literal>WAL</literal> ] [ <literal>NOWAIT</literal> ] [ <literal>MAX_RATE</literal> <replaceable>rate</replaceable> ] [ <literal>TABLESPACE_MAP</literal> ] [ <literal>NOVERIFY_CHECKSUMS</literal> ] + <term><literal>BASE_BACKUP</literal> [ <literal>LABEL</literal> <replaceable>'label'</replaceable> ] [ <literal>PROGRESS</literal> ] [ <literal>FAST</literal> ] [ <literal>WAL</literal> ] [ <literal>NOWAIT</literal> ] [ <literal>MAX_RATE</literal> <replaceable>rate</replaceable> ] [ <literal>TABLESPACE_MAP</literal> ] [ <literal>NOVERIFY_CHECKSUMS</literal> ] [ <literal>EXCLUDE_CONF</literal> ] <indexterm><primary>BASE_BACKUP</primary></indexterm> </term> <listitem> @@ -2576,6 +2576,16 @@ <title>Streaming Replication Protocol</title> </para> </listitem> </varlistentry> + + <varlistentry> + <term><literal>EXCLUDE_CONF</literal></term> + <listitem> + <para> + Do not copy configuration files, that is, files that end in + <filename>.conf</filename>. + </para> + </listitem> + </varlistentry> </variablelist> </para> <para> diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index da5c8f5307..1261e02d59 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -286,6 +286,23 @@ <title>Options</title> </listitem> </varlistentry> + <varlistentry> + <term><option>-r</option></term> + <term><option>--replica</option></term> + <listitem> + <para> + Initialize a data directory for a physical replication replica. The + data directory will not be initialized with a full database system, + but will instead only contain a minimal set of files. A server that + is started on this data directory will first fetch a base backup and + then switch to standby mode. The connection information for the base + backup has to be configured by setting <xref + linkend="guc-primary-conninfo"/>, and other parameters as desired, + before the server is started. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>-S</option></term> <term><option>--sync-only</option></term> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 7f4f784c0e..36c6cdef82 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -903,8 +903,6 @@ static void CheckRecoveryConsistency(void); static XLogRecord *ReadCheckpointRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int whichChkpt, bool report); static bool rescanLatestTimeLine(void); -static void WriteControlFile(void); -static void ReadControlFile(void); static char *str_time(pg_time_t tnow); static bool CheckForStandbyTrigger(void); @@ -4494,7 +4492,7 @@ rescanLatestTimeLine(void) * ReadControlFile() verifies they are correct. We could split out the * I/O and compatibility-check functions, but there seems no need currently. */ -static void +void WriteControlFile(void) { int fd; @@ -4585,7 +4583,7 @@ WriteControlFile(void) XLOG_CONTROL_FILE))); } -static void +void ReadControlFile(void) { pg_crc32c crc; @@ -5075,6 +5073,41 @@ XLOGShmemInit(void) InitSharedLatch(&XLogCtl->recoveryWakeupLatch); } +void +InitControlFile(uint64 sysidentifier) +{ + char mock_auth_nonce[MOCK_AUTH_NONCE_LEN]; + + /* + * Generate a random nonce. This is used for authentication requests that + * will fail because the user does not exist. The nonce is used to create + * a genuine-looking password challenge for the non-existent user, in lieu + * of an actual stored password. + */ + if (!pg_strong_random(mock_auth_nonce, MOCK_AUTH_NONCE_LEN)) + ereport(PANIC, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate secret authorization token"))); + + memset(ControlFile, 0, sizeof(ControlFileData)); + /* Initialize pg_control status fields */ + ControlFile->system_identifier = sysidentifier; + memcpy(ControlFile->mock_authentication_nonce, mock_auth_nonce, MOCK_AUTH_NONCE_LEN); + ControlFile->state = DB_SHUTDOWNED; + ControlFile->unloggedLSN = FirstNormalUnloggedLSN; + + /* Set important parameter values for use when replaying WAL */ + ControlFile->MaxConnections = MaxConnections; + ControlFile->max_worker_processes = max_worker_processes; + ControlFile->max_wal_senders = max_wal_senders; + ControlFile->max_prepared_xacts = max_prepared_xacts; + ControlFile->max_locks_per_xact = max_locks_per_xact; + ControlFile->wal_level = wal_level; + ControlFile->wal_log_hints = wal_log_hints; + ControlFile->track_commit_timestamp = track_commit_timestamp; + ControlFile->data_checksum_version = bootstrap_data_checksum_version; +} + /* * This func must be called ONCE on system install. It creates pg_control * and the initial XLOG segment. @@ -5090,7 +5123,6 @@ BootStrapXLOG(void) char *recptr; bool use_existent; uint64 sysidentifier; - char mock_auth_nonce[MOCK_AUTH_NONCE_LEN]; struct timeval tv; pg_crc32c crc; @@ -5111,17 +5143,6 @@ BootStrapXLOG(void) sysidentifier |= ((uint64) tv.tv_usec) << 12; sysidentifier |= getpid() & 0xFFF; - /* - * Generate a random nonce. This is used for authentication requests that - * will fail because the user does not exist. The nonce is used to create - * a genuine-looking password challenge for the non-existent user, in lieu - * of an actual stored password. - */ - if (!pg_strong_random(mock_auth_nonce, MOCK_AUTH_NONCE_LEN)) - ereport(PANIC, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate secret authorization token"))); - /* First timeline ID is always 1 */ ThisTimeLineID = 1; @@ -5229,30 +5250,12 @@ BootStrapXLOG(void) openLogFile = -1; /* Now create pg_control */ - - memset(ControlFile, 0, sizeof(ControlFileData)); - /* Initialize pg_control status fields */ - ControlFile->system_identifier = sysidentifier; - memcpy(ControlFile->mock_authentication_nonce, mock_auth_nonce, MOCK_AUTH_NONCE_LEN); - ControlFile->state = DB_SHUTDOWNED; + InitControlFile(sysidentifier); ControlFile->time = checkPoint.time; ControlFile->checkPoint = checkPoint.redo; ControlFile->checkPointCopy = checkPoint; - ControlFile->unloggedLSN = FirstNormalUnloggedLSN; - - /* Set important parameter values for use when replaying WAL */ - ControlFile->MaxConnections = MaxConnections; - ControlFile->max_worker_processes = max_worker_processes; - ControlFile->max_wal_senders = max_wal_senders; - ControlFile->max_prepared_xacts = max_prepared_xacts; - ControlFile->max_locks_per_xact = max_locks_per_xact; - ControlFile->wal_level = wal_level; - ControlFile->wal_log_hints = wal_log_hints; - ControlFile->track_commit_timestamp = track_commit_timestamp; - ControlFile->data_checksum_version = bootstrap_data_checksum_version; /* some additional ControlFile fields are set in WriteControlFile() */ - WriteControlFile(); /* Bootstrap the commit log, too */ @@ -6297,24 +6300,31 @@ StartupXLOG(void) */ ValidateXLOGDirectoryStructure(); - /*---------- + /* * If we previously crashed, perform a couple of actions: + * * - The pg_wal directory may still include some temporary WAL segments - * used when creating a new segment, so perform some clean up to not - * bloat this path. This is done first as there is no point to sync this - * temporary data. - * - There might be data which we had written, intending to fsync it, - * but which we had not actually fsync'd yet. Therefore, a power failure - * in the near future might cause earlier unflushed writes to be lost, - * even though more recent data written to disk from here on would be - * persisted. To avoid that, fsync the entire data directory. - *--------- + * used when creating a new segment, so perform some clean up to not + * bloat this path. This is done first as there is no point to sync + * this temporary data. + * + * - There might be data which we had written, intending to fsync it, but + * which we had not actually fsync'd yet. Therefore, a power failure + * in the near future might cause earlier unflushed writes to be lost, + * even though more recent data written to disk from here on would be + * persisted. To avoid that, fsync the entire data directory. Errors + * are logged but not considered fatal. Aborting on error would result + * in failure to start for harmless cases such as read-only files in + * the data directory, and that's not good either. + * + * Note that if we previously crashed due to a PANIC on fsync(), we'll + * be rewriting all changes again during recovery. */ if (ControlFile->state != DB_SHUTDOWNED && ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY) { RemoveTempXlogFiles(); - SyncDataDirectory(); + SyncDataDirectory(true, LOG); } /* diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index bfc629c753..c4d2bff0b3 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -36,6 +36,7 @@ #include "postmaster/bgwriter.h" #include "postmaster/startup.h" #include "postmaster/walwriter.h" +#include "replication/basebackup.h" #include "replication/walreceiver.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" @@ -326,6 +327,9 @@ AuxiliaryProcessMain(int argc, char *argv[]) case StartupProcess: statmsg = pgstat_get_backend_desc(B_STARTUP); break; + case BaseBackupProcess: + statmsg = pgstat_get_backend_desc(B_BASE_BACKUP); + break; case BgWriterProcess: statmsg = pgstat_get_backend_desc(B_BG_WRITER); break; @@ -451,6 +455,11 @@ AuxiliaryProcessMain(int argc, char *argv[]) StartupProcessMain(); proc_exit(1); /* should never return */ + case BaseBackupProcess: + /* don't set signals, basebackup has its own agenda */ + BaseBackupMain(); + proc_exit(1); /* should never return */ + case BgWriterProcess: /* don't set signals, bgwriter has its own agenda */ BackgroundWriterMain(); diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index 51c486bebd..f4bb4192b7 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -2927,6 +2927,9 @@ pgstat_bestart(void) case StartupProcess: lbeentry.st_backendType = B_STARTUP; break; + case BaseBackupProcess: + lbeentry.st_backendType = B_BASE_BACKUP; + break; case BgWriterProcess: lbeentry.st_backendType = B_BG_WRITER; break; @@ -4285,6 +4288,9 @@ pgstat_get_backend_desc(BackendType backendType) case B_BG_WORKER: backendDesc = "background worker"; break; + case B_BASE_BACKUP: + backendDesc = "base backup"; + break; case B_BG_WRITER: backendDesc = "background writer"; break; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 7a92dac525..07555a55d7 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -116,6 +116,7 @@ #include "postmaster/postmaster.h" #include "postmaster/syslogger.h" #include "replication/logicallauncher.h" +#include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/fd.h" #include "storage/ipc.h" @@ -248,6 +249,7 @@ bool restart_after_crash = true; /* PIDs of special child processes; 0 when not running */ static pid_t StartupPID = 0, + BaseBackupPID = 0, BgWriterPID = 0, CheckpointerPID = 0, WalWriterPID = 0, @@ -539,6 +541,7 @@ static void ShmemBackendArrayRemove(Backend *bn); #endif /* EXEC_BACKEND */ #define StartupDataBase() StartChildProcess(StartupProcess) +#define StartBaseBackup() StartChildProcess(BaseBackupProcess) #define StartBackgroundWriter() StartChildProcess(BgWriterProcess) #define StartCheckpointer() StartChildProcess(CheckpointerProcess) #define StartWalWriter() StartChildProcess(WalWriterProcess) @@ -572,6 +575,8 @@ PostmasterMain(int argc, char *argv[]) bool listen_addr_saved = false; int i; char *output_config_variable = NULL; + struct stat stat_buf; + bool basebackup_signal_file_found = false; InitProcessGlobals(); @@ -886,12 +891,27 @@ PostmasterMain(int argc, char *argv[]) /* Verify that DataDir looks reasonable */ checkDataDir(); - /* Check that pg_control exists */ - checkControlFile(); - /* And switch working directory into it */ ChangeToDataDir(); + if (stat(BASEBACKUP_SIGNAL_FILE, &stat_buf) == 0) + { + int fd; + + fd = BasicOpenFilePerm(STANDBY_SIGNAL_FILE, O_RDWR | PG_BINARY, + S_IRUSR | S_IWUSR); + if (fd >= 0) + { + (void) pg_fsync(fd); + close(fd); + } + basebackup_signal_file_found = true; + } + + /* Check that pg_control exists */ + if (!basebackup_signal_file_found) + checkControlFile(); + /* * Check for invalid combinations of GUC settings. */ @@ -970,7 +990,8 @@ PostmasterMain(int argc, char *argv[]) * processes will inherit the correct function pointer and not need to * repeat the test. */ - LocalProcessControlFile(false); + if (!basebackup_signal_file_found) + LocalProcessControlFile(false); /* * Initialize SSL library, if specified. @@ -1386,6 +1407,39 @@ PostmasterMain(int argc, char *argv[]) */ AddToDataDirLockFile(LOCK_FILE_LINE_PM_STATUS, PM_STATUS_STARTING); + if (basebackup_signal_file_found) + { + BaseBackupPID = StartBaseBackup(); + + /* + * Wait until done. Start WAL receiver in the meantime, once base + * backup has received the starting position. + */ + while (BaseBackupPID != 0) + { + PG_SETMASK(&UnBlockSig); + pg_usleep(1000000L); + PG_SETMASK(&BlockSig); + MaybeStartWalReceiver(); + } + + /* + * XXX Shut down WAL receiver. It will be restarted later in xlog.c, + * and that will complain if it's already running. + */ + ShutdownWalRcv(); + + /* + * Base backup done, now signal standby mode. + */ + durable_rename(BASEBACKUP_SIGNAL_FILE, STANDBY_SIGNAL_FILE, FATAL); + + /* + * Reread the control file that came in with the base backup. + */ + ReadControlFile(); + } + /* * We're ready to rock and roll... */ @@ -2665,6 +2719,8 @@ SIGHUP_handler(SIGNAL_ARGS) SignalChildren(SIGHUP); if (StartupPID != 0) signal_child(StartupPID, SIGHUP); + if (BaseBackupPID != 0) + signal_child(BaseBackupPID, SIGHUP); if (BgWriterPID != 0) signal_child(BgWriterPID, SIGHUP); if (CheckpointerPID != 0) @@ -2824,6 +2880,8 @@ pmdie(SIGNAL_ARGS) if (StartupPID != 0) signal_child(StartupPID, SIGTERM); + if (BaseBackupPID != 0) + signal_child(BaseBackupPID, SIGTERM); if (BgWriterPID != 0) signal_child(BgWriterPID, SIGTERM); if (WalReceiverPID != 0) @@ -3062,6 +3120,23 @@ reaper(SIGNAL_ARGS) continue; } + /* + * Was it the base backup process? + */ + if (pid == BaseBackupPID) + { + BaseBackupPID = 0; + if (EXIT_STATUS_0(exitstatus)) + ; + else if (EXIT_STATUS_1(exitstatus)) + ereport(FATAL, + (errmsg("base backup failed"))); + else + HandleChildCrash(pid, exitstatus, + _("base backup process")); + continue; + } + /* * Was it the bgwriter? Normal exit can be ignored; we'll start a new * one at the next iteration of the postmaster's main loop, if @@ -3583,6 +3658,18 @@ HandleChildCrash(int pid, int exitstatus, const char *procname) StartupStatus = STARTUP_SIGNALED; } + /* Take care of the base backup process too */ + if (pid == BaseBackupPID) + BaseBackupPID = 0; + else if (BaseBackupPID != 0 && take_action) + { + ereport(DEBUG2, + (errmsg_internal("sending %s to process %d", + (SendStop ? "SIGSTOP" : "SIGQUIT"), + (int) BaseBackupPID))); + signal_child(BaseBackupPID, (SendStop ? SIGSTOP : SIGQUIT)); + } + /* Take care of the bgwriter too */ if (pid == BgWriterPID) BgWriterPID = 0; @@ -3817,6 +3904,7 @@ PostmasterStateMachine(void) if (CountChildren(BACKEND_TYPE_NORMAL | BACKEND_TYPE_WORKER) == 0 && StartupPID == 0 && WalReceiverPID == 0 && + BaseBackupPID == 0 && BgWriterPID == 0 && (CheckpointerPID == 0 || (!FatalError && Shutdown < ImmediateShutdown)) && @@ -3911,6 +3999,7 @@ PostmasterStateMachine(void) /* These other guys should be dead already */ Assert(StartupPID == 0); Assert(WalReceiverPID == 0); + Assert(BaseBackupPID == 0); Assert(BgWriterPID == 0); Assert(CheckpointerPID == 0); Assert(WalWriterPID == 0); @@ -4094,6 +4183,8 @@ TerminateChildren(int signal) if (signal == SIGQUIT || signal == SIGKILL) StartupStatus = STARTUP_SIGNALED; } + if (BaseBackupPID != 0) + signal_child(BgWriterPID, signal); if (BgWriterPID != 0) signal_child(BgWriterPID, signal); if (CheckpointerPID != 0) @@ -4919,6 +5010,7 @@ SubPostmasterMain(int argc, char *argv[]) strcmp(argv[1], "--forkavlauncher") == 0 || strcmp(argv[1], "--forkavworker") == 0 || strcmp(argv[1], "--forkboot") == 0 || + strcmp(argv[1], "--forkbasebackup") == 0 || strncmp(argv[1], "--forkbgworker=", 15) == 0) PGSharedMemoryReAttach(); else @@ -4958,7 +5050,8 @@ SubPostmasterMain(int argc, char *argv[]) * (re-)read control file, as it contains config. The postmaster will * already have read this, but this process doesn't know about that. */ - LocalProcessControlFile(false); + if (strcmp(argv[1], "--forkbasebackup") != 0) + LocalProcessControlFile(false); /* * Reload any libraries that were preloaded by the postmaster. Since we @@ -5019,7 +5112,8 @@ SubPostmasterMain(int argc, char *argv[]) /* And run the backend */ BackendRun(&port); /* does not return */ } - if (strcmp(argv[1], "--forkboot") == 0) + if (strcmp(argv[1], "--forkboot") == 0 || + strcmp(argv[1], "--forkbasebackup") == 0) { /* Restore basic shared memory pointers */ InitShmemAccess(UsedShmemSegAddr); @@ -5431,7 +5525,7 @@ StartChildProcess(AuxProcType type) av[ac++] = "postgres"; #ifdef EXEC_BACKEND - av[ac++] = "--forkboot"; + av[ac++] = (type == BaseBackupProcess) ? "--forkbasebackup" : "--forkboot"; av[ac++] = NULL; /* filled in by postmaster_forkexec */ #endif @@ -5475,6 +5569,10 @@ StartChildProcess(AuxProcType type) ereport(LOG, (errmsg("could not fork startup process: %m"))); break; + case BaseBackupProcess: + ereport(LOG, + (errmsg("could not fork base backup process: %m"))); + break; case BgWriterProcess: ereport(LOG, (errmsg("could not fork background writer process: %m"))); @@ -5616,7 +5714,7 @@ static void MaybeStartWalReceiver(void) { if (WalReceiverPID == 0 && - (pmState == PM_STARTUP || pmState == PM_RECOVERY || + (pmState == PM_INIT || pmState == PM_STARTUP || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || pmState == PM_WAIT_READONLY) && Shutdown == NoShutdown) { diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index dea8aab45e..e2f88a2f11 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -29,6 +29,7 @@ #include "port.h" #include "postmaster/syslogger.h" #include "replication/basebackup.h" +#include "replication/walreceiver.h" #include "replication/walsender.h" #include "replication/walsender_private.h" #include "storage/bufpage.h" @@ -38,6 +39,7 @@ #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" +#include "utils/guc.h" #include "utils/ps_status.h" #include "utils/relcache.h" #include "utils/timestamp.h" @@ -121,6 +123,9 @@ static long long int total_checksum_failures; /* Do not verify checksums. */ static bool noverify_checksums = false; +/* Do not copy config files. */ +static bool exclude_conf = false; + /* * The contents of these directories are removed or recreated during server * start so they are not included in backups. The directories themselves are @@ -639,6 +644,7 @@ parse_basebackup_options(List *options, basebackup_options *opt) bool o_maxrate = false; bool o_tablespace_map = false; bool o_noverify_checksums = false; + bool o_exclude_conf = false; MemSet(opt, 0, sizeof(*opt)); foreach(lopt, options) @@ -727,6 +733,15 @@ parse_basebackup_options(List *options, basebackup_options *opt) noverify_checksums = true; o_noverify_checksums = true; } + else if (strcmp(defel->defname, "exclude_conf") == 0) + { + if (o_exclude_conf) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("duplicate option \"%s\"", defel->defname))); + exclude_conf = true; + o_exclude_conf = true; + } else elog(ERROR, "option \"%s\" not recognized", defel->defname); @@ -1136,6 +1151,18 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, continue; } + if (exclude_conf) + { + char *dot = strrchr(de->d_name, '.'); + if (dot && strcmp(dot, ".conf") == 0) + { + elog(DEBUG2, + "configuration file \"%s\" excluded from backup", + de->d_name); + continue; + } + } + snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name); /* Skip pg_control here to back up it last */ @@ -1730,3 +1757,46 @@ throttle(size_t increment) */ throttled_last = GetCurrentTimestamp(); } + + +/* + * base backup worker process (client) main function + */ +void +BaseBackupMain(void) +{ + WalReceiverConn *wrconn = NULL; + char *err; + TimeLineID primaryTLI; + uint64 primary_sysid; + + /* Load the libpq-specific functions */ + load_file("libpqwalreceiver", false); + if (WalReceiverFunctions == NULL) + elog(ERROR, "libpqwalreceiver didn't initialize correctly"); + + /* Establish the connection to the primary */ + wrconn = walrcv_connect(PrimaryConnInfo, false, cluster_name[0] ? cluster_name : "basebackup", &err); + if (!wrconn) + ereport(ERROR, + (errmsg("could not connect to the primary server: %s", err))); + + /* + * Get the remote sysid and stick it into the local control file, so that + * the walreceiver is happy. The control file will later be overwritten + * by the base backup. + */ + primary_sysid = strtoull(walrcv_identify_system(wrconn, &primaryTLI), NULL, 10); + InitControlFile(primary_sysid); + WriteControlFile(); + + walrcv_base_backup(wrconn); + + walrcv_disconnect(wrconn); + + SyncDataDirectory(false, ERROR); + + ereport(LOG, + (errmsg("base backup completed"))); + proc_exit(0); +} diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index e4fd1f9bb6..52819d504c 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -17,20 +17,29 @@ #include "postgres.h" #include <unistd.h> +#include <sys/stat.h> #include <sys/time.h> +#ifdef USE_SYSTEMD +#include <systemd/sd-daemon.h> +#endif + #include "access/xlog.h" #include "catalog/pg_type.h" +#include "common/string.h" #include "funcapi.h" #include "libpq-fe.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "pgstat.h" +#include "pgtar.h" #include "pqexpbuffer.h" #include "replication/walreceiver.h" #include "utils/builtins.h" +#include "utils/guc.h" #include "utils/memutils.h" #include "utils/pg_lsn.h" +#include "utils/ps_status.h" #include "utils/tuplestore.h" PG_MODULE_MAGIC; @@ -61,6 +70,7 @@ static int libpqrcv_server_version(WalReceiverConn *conn); static void libpqrcv_readtimelinehistoryfile(WalReceiverConn *conn, TimeLineID tli, char **filename, char **content, int *len); +static void libpqrcv_base_backup(WalReceiverConn *conn); static bool libpqrcv_startstreaming(WalReceiverConn *conn, const WalRcvStreamOptions *options); static void libpqrcv_endstreaming(WalReceiverConn *conn, @@ -89,6 +99,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { libpqrcv_identify_system, libpqrcv_server_version, libpqrcv_readtimelinehistoryfile, + libpqrcv_base_backup, libpqrcv_startstreaming, libpqrcv_endstreaming, libpqrcv_receive, @@ -358,6 +369,395 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * XXX copied from pg_basebackup.c + */ + +unsigned long long totaldone; +unsigned long long totalsize_kb; +int tablespacenum; +int tablespacecount; + +static void +base_backup_report_progress(void) +{ + int percent; + char *progress; + + percent = totalsize_kb ? (int) ((totaldone / 1024) * 100 / totalsize_kb) : 0; + + /* + * Avoid overflowing past 100% or the full size. This may make the total + * size number change as we approach the end of the backup (the estimate + * will always be wrong if WAL is included), but that's better than having + * the done column be bigger than the total. + */ + if (percent > 100) + percent = 100; + if (totaldone / 1024 > totalsize_kb) + totalsize_kb = totaldone / 1024; + + /* Note: no translation of ps status */ + progress = psprintf((tablespacecount == 1 ? + "%llu/%llu kB (%d%%), %d/%d tablespace" : + "%llu/%llu kB (%d%%), %d/%d tablespaces"), + totaldone / 1024, + totalsize_kb, + percent, + tablespacenum, + tablespacecount); + + set_ps_display(progress, false); +#ifdef USE_SYSTEMD + sd_pid_notifyf(PostmasterPid, 0, "STATUS=base backup %s", progress); +#endif + + pfree(progress); +} + +static void +ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res) +{ + char current_path[MAXPGPATH]; + char filename[MAXPGPATH]; + pgoff_t current_len_left = 0; + int current_padding = 0; + char *copybuf = NULL; + FILE *file = NULL; + off_t flush_offset; + + strlcpy(current_path, DataDir, sizeof(current_path)); + + /* + * Get the COPY data + */ + res = PQgetResult(conn); + if (PQresultStatus(res) != PGRES_COPY_OUT) + ereport(ERROR, + (errmsg("could not get COPY data stream: %s", + PQerrorMessage(conn)))); + + while (1) + { + int r; + + if (copybuf != NULL) + { + PQfreemem(copybuf); + copybuf = NULL; + } + + r = PQgetCopyData(conn, ©buf, 0); + + if (r == -1) + { + /* + * End of chunk + */ + if (file) + fclose(file); + + break; + } + else if (r == -2) + { + ereport(ERROR, + (errmsg("could not read COPY data: %s", + PQerrorMessage(conn)))); + } + + if (file == NULL) + { + int filemode; + + /* + * No current file, so this must be the header for a new file + */ + if (r != 512) + ereport(ERROR, + (errmsg("invalid tar block header size: %d", r))); + + current_len_left = read_tar_number(©buf[124], 12); + + /* Set permissions on the file */ + filemode = read_tar_number(©buf[100], 8); + + /* + * All files are padded up to 512 bytes + */ + current_padding = + ((current_len_left + 511) & ~511) - current_len_left; + + /* + * First part of header is zero terminated filename + */ + snprintf(filename, sizeof(filename), "%s/%s", current_path, + copybuf); + if (filename[strlen(filename) - 1] == '/') + { + /* + * Ends in a slash means directory or symlink to directory + */ + if (copybuf[156] == '5') + { + /* + * Directory + */ + filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ + if (MakePGDirectory(filename) != 0) + { + if (errno != EEXIST) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", + filename))); + } +#ifndef WIN32 + if (chmod(filename, (mode_t) filemode)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not set permissions on directory \"%s\": %m", + filename))); +#endif + } + /* + * Symbolic link + */ + else if (copybuf[156] == '2') + { + /* TODO: tablespace mapping */ + const char *tblspc_path = ©buf[157]; + + filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ + + if (symlink(tblspc_path, filename) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create symbolic link from \"%s\" to \"%s\": %m", + filename, tblspc_path))); + } + else + { + ereport(ERROR, + (errmsg("unrecognized link indicator \"%c\"", + copybuf[156]))); + } + continue; /* directory or link handled */ + } + + /* + * regular file + */ + file = fopen(filename, "wb"); + if (!file) + ereport(ERROR, + (errcode_for_file_access(), + (errmsg("could not create file \"%s\": %m", filename)))); + + flush_offset = 0; + +#ifndef WIN32 + if (chmod(filename, (mode_t) filemode)) + ereport(ERROR, + (errcode_for_file_access(), + (errmsg("could not set permissions on file \"%s\": %m", + filename)))); +#endif + + if (current_len_left == 0) + { + /* + * Done with this file, next one will be a new tar header + */ + fclose(file); + file = NULL; + continue; + } + } /* new file */ + else + { + /* + * Continuing blocks in existing file + */ + if (current_len_left == 0 && r == current_padding) + { + /* + * Received the padding block for this file, ignore it and + * close the file, then move on to the next tar header. + */ + fclose(file); + file = NULL; + continue; + } + + if (fwrite(copybuf, r, 1, file) != 1) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write to file \"%s\": %m", filename))); + + pg_flush_data(fileno(file), flush_offset, r); + flush_offset += r; + totaldone += r; + base_backup_report_progress(); + + current_len_left -= r; + if (current_len_left == 0 && current_padding == 0) + { + /* + * Received the last block, and there is no padding to be + * expected. Close the file and move on to the next tar + * header. + */ + fclose(file); + file = NULL; + continue; + } + } /* continuing data in existing file */ + } /* loop over all data blocks */ + base_backup_report_progress(); + + if (file != NULL) + ereport(ERROR, + (errmsg("COPY stream ended before last file was finished"))); + + if (copybuf != NULL) + PQfreemem(copybuf); +} + +/* + * Make base backup from remote and write to local disk. + */ +static void +libpqrcv_base_backup(WalReceiverConn *conn) +{ + StringInfoData stmt; + PGresult *res; + char xlogstart[64]; + TimeLineID starttli; + XLogRecPtr recptr; + bool error; + + ereport(LOG, + (errmsg("initiating base backup, waiting for remote checkpoint to complete"))); + set_ps_display("waiting for checkpoint", false); + + initStringInfo(&stmt); + appendStringInfo(&stmt, "BASE_BACKUP PROGRESS NOWAIT EXCLUDE_CONF"); + if (cluster_name && cluster_name[0]) + appendStringInfo(&stmt, " LABEL %s", quote_literal_cstr(cluster_name)); + + if (PQsendQuery(conn->streamConn, stmt.data) == 0) + ereport(ERROR, + (errmsg("could not start base backup on remote server: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + + /* + * First result set: WAL start position and timeline ID + */ + res = PQgetResult(conn->streamConn); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + ereport(ERROR, + (errmsg("could not start base backup on remote server: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + } + if (PQntuples(res) != 1) + { + PQclear(res); + ereport(ERROR, + (errmsg("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2))); + } + + ereport(LOG, + (errmsg("remote checkpoint completed"))); + + strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart)); + starttli = atoi(PQgetvalue(res, 0, 1)); + PQclear(res); + elog(DEBUG1, "write-ahead log start point: %s on timeline %u", + xlogstart, starttli); + recptr = pg_lsn_in_internal(xlogstart, &error); + if (error) + elog(ERROR, "invalid LSN received: %s", xlogstart); + + /* + * Second result set: tablespace information + */ + res = PQgetResult(conn->streamConn); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + ereport(ERROR, + (errmsg("could not get backup header: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + } + if (PQntuples(res) < 1) + { + PQclear(res); + ereport(ERROR, + (errmsg("no data returned from server"))); + } + + totalsize_kb = totaldone = 0; + tablespacecount = PQntuples(res); + for (int i = 0; i < PQntuples(res); i++) + { + totalsize_kb += atol(PQgetvalue(res, i, 2)); + } + + RequestXLogStreaming(starttli, recptr, PrimaryConnInfo, PrimarySlotName); + + /* + * Start receiving chunks + */ + for (int i = 0; i < PQntuples(res); i++) + { + tablespacenum = i; + ReceiveAndUnpackTarFile(conn->streamConn, res); + } + tablespacenum++; + base_backup_report_progress(); + + PQclear(res); + + /* + * Final result set: WAL end position and timeline ID + */ + res = PQgetResult(conn->streamConn); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + ereport(ERROR, + (errmsg("could not get write-ahead log end position from server: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + } + if (PQntuples(res) != 1) + { + PQclear(res); + ereport(ERROR, + (errmsg("no write-ahead log end position returned from server"))); + } + PQclear(res); + + res = PQgetResult(conn->streamConn); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE); + + if (sqlstate && + strcmp(sqlstate, "XX001" /*ERRCODE_DATA_CORRUPTED*/) == 0) + ereport(ERROR, + (errmsg("checksum error occurred"))); + else + ereport(ERROR, + (errmsg("final receive failed: %s", + pchomp(PQerrorMessage(conn->streamConn))))); + } + PQclear(res); +} + /* * Start streaming WAL data from given streaming options. * diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 2d96567409..0a679ea0ee 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -78,6 +78,7 @@ static SQLCmd *make_sqlcmd(void); %token K_WAL %token K_TABLESPACE_MAP %token K_NOVERIFY_CHECKSUMS +%token K_EXCLUDE_CONF %token K_TIMELINE %token K_PHYSICAL %token K_LOGICAL @@ -154,8 +155,7 @@ var_name: IDENT { $$ = $1; } ; /* - * BASE_BACKUP [LABEL '<label>'] [PROGRESS] [FAST] [WAL] [NOWAIT] - * [MAX_RATE %d] [TABLESPACE_MAP] [NOVERIFY_CHECKSUMS] + * BASE_BACKUP [option]... */ base_backup: K_BASE_BACKUP base_backup_opt_list @@ -214,6 +214,11 @@ base_backup_opt: $$ = makeDefElem("noverify_checksums", (Node *)makeInteger(true), -1); } + | K_EXCLUDE_CONF + { + $$ = makeDefElem("exclude_conf", + (Node *)makeInteger(true), -1); + } ; create_replication_slot: diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 14c9a1e798..c2e2aced7d 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -93,6 +93,7 @@ MAX_RATE { return K_MAX_RATE; } WAL { return K_WAL; } TABLESPACE_MAP { return K_TABLESPACE_MAP; } NOVERIFY_CHECKSUMS { return K_NOVERIFY_CHECKSUMS; } +EXCLUDE_CONF { return K_EXCLUDE_CONF; } TIMELINE { return K_TIMELINE; } START_REPLICATION { return K_START_REPLICATION; } CREATE_REPLICATION_SLOT { return K_CREATE_REPLICATION_SLOT; } diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index fa79b45f63..6dd7e2f938 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -3154,21 +3154,14 @@ looks_like_temp_rel_name(const char *name) * Other symlinks are presumed to point at files we're not responsible * for fsyncing, and might not have privileges to write at all. * - * Errors are logged but not considered fatal; that's because this is used - * only during database startup, to deal with the possibility that there are - * issued-but-unsynced writes pending against the data directory. We want to - * ensure that such writes reach disk before anything that's done in the new - * run. However, aborting on error would result in failure to start for - * harmless cases such as read-only files in the data directory, and that's - * not good either. - * - * Note that if we previously crashed due to a PANIC on fsync(), we'll be - * rewriting all changes again during recovery. + * If pre_sync is true, issue flush requests to the kernel before starting the + * actual fsync calls. This can be skipped if the caller has already done it + * itself. * * Note we assume we're chdir'd into PGDATA to begin with. */ void -SyncDataDirectory(void) +SyncDataDirectory(bool pre_sync, int loglevel) { bool xlog_is_symlink; @@ -3187,7 +3180,7 @@ SyncDataDirectory(void) struct stat st; if (lstat("pg_wal", &st) < 0) - ereport(LOG, + ereport(loglevel, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", "pg_wal"))); @@ -3201,15 +3194,18 @@ SyncDataDirectory(void) /* * If possible, hint to the kernel that we're soon going to fsync the data - * directory and its contents. Errors in this step are even less + * directory and its contents. Errors in this step are less * interesting than normal, so log them only at DEBUG1. */ + if (pre_sync) + { #ifdef PG_FLUSH_DATA_WORKS - walkdir(".", pre_sync_fname, false, DEBUG1); - if (xlog_is_symlink) - walkdir("pg_wal", pre_sync_fname, false, DEBUG1); - walkdir("pg_tblspc", pre_sync_fname, true, DEBUG1); + walkdir(".", pre_sync_fname, false, DEBUG1); + if (xlog_is_symlink) + walkdir("pg_wal", pre_sync_fname, false, DEBUG1); + walkdir("pg_tblspc", pre_sync_fname, true, DEBUG1); #endif + } /* * Now we do the fsync()s in the same order. @@ -3220,10 +3216,10 @@ SyncDataDirectory(void) * in pg_tblspc, they'll get fsync'd twice. That's not an expected case * so we don't worry about optimizing it. */ - walkdir(".", datadir_fsync_fname, false, LOG); + walkdir(".", datadir_fsync_fname, false, loglevel); if (xlog_is_symlink) - walkdir("pg_wal", datadir_fsync_fname, false, LOG); - walkdir("pg_tblspc", datadir_fsync_fname, true, LOG); + walkdir("pg_wal", datadir_fsync_fname, false, loglevel); + walkdir("pg_tblspc", datadir_fsync_fname, true, loglevel); } /* diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index ec6d0bdf8e..fc8ca5a65b 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -136,6 +136,7 @@ static char *pwfilename = NULL; static char *superuser_password = NULL; static const char *authmethodhost = NULL; static const char *authmethodlocal = NULL; +static bool replica = false; static bool debug = false; static bool noclean = false; static bool do_sync = true; @@ -2935,6 +2936,22 @@ initialize_data_directory(void) /* Now create all the text config files */ setup_config(); + /* + * If data directory for replica requested, write basebackup.signal, and + * then we are done here. + */ + if (replica) + { + char *path; + char *lines[1] = {NULL}; + + path = psprintf("%s/basebackup.signal", pg_data); + writefile(path, lines); + free(path); + + return; + } + /* Bootstrap template1 */ bootstrap_template1(); @@ -3026,6 +3043,7 @@ main(int argc, char *argv[]) {"wal-segsize", required_argument, NULL, 12}, {"data-checksums", no_argument, NULL, 'k'}, {"allow-group-access", no_argument, NULL, 'g'}, + {"replica", no_argument, NULL, 'r'}, {NULL, 0, NULL, 0} }; @@ -3067,7 +3085,7 @@ main(int argc, char *argv[]) /* process command-line options */ - while ((c = getopt_long(argc, argv, "dD:E:kL:nNU:WA:sST:X:g", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "dD:E:kL:nNrU:WA:sST:X:g", long_options, &option_index)) != -1) { switch (c) { @@ -3113,6 +3131,9 @@ main(int argc, char *argv[]) case 'N': do_sync = false; break; + case 'r': + replica = true; + break; case 'S': sync_only = true; break; @@ -3334,9 +3355,19 @@ main(int argc, char *argv[]) /* translator: This is a placeholder in a shell command. */ appendPQExpBuffer(start_db_cmd, " -l %s start", _("logfile")); - printf(_("\nSuccess. You can now start the database server using:\n\n" - " %s\n\n"), - start_db_cmd->data); + if (!replica) + { + printf(_("\nSuccess. You can now start the database server using:\n\n" + " %s\n\n"), + start_db_cmd->data); + } + else + { + printf(_("\nSo far so good. Now configure the replication connection in\n" + "postgresql.conf, and then start the database server using:\n\n" + " %s\n\n"), + start_db_cmd->data); + } destroyPQExpBuffer(start_db_cmd); diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index f9cfeae264..c9edeb54d3 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -76,7 +76,7 @@ static int WalSegSz; static int set_wal_segsize; static void CheckDataVersion(void); -static bool ReadControlFile(void); +static bool read_controlfile(void); static void GuessControlValues(void); static void PrintControlValues(bool guessed); static void PrintNewControlValues(void); @@ -393,7 +393,7 @@ main(int argc, char *argv[]) /* * Attempt to read the existing pg_control file */ - if (!ReadControlFile()) + if (!read_controlfile()) GuessControlValues(); /* @@ -578,7 +578,7 @@ CheckDataVersion(void) * to the current format. (Currently we don't do anything of the sort.) */ static bool -ReadControlFile(void) +read_controlfile(void) { int fd; int len; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 98b033fc20..e56d85a96d 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -127,8 +127,8 @@ extern char *archiveCleanupCommand; extern bool recoveryTargetInclusive; extern int recoveryTargetAction; extern int recovery_min_apply_delay; -extern char *PrimaryConnInfo; -extern char *PrimarySlotName; +extern PGDLLIMPORT char *PrimaryConnInfo; +extern PGDLLIMPORT char *PrimarySlotName; /* indirectly set via GUC system */ extern TransactionId recoveryTargetXid; @@ -298,6 +298,9 @@ extern Size XLOGShmemSize(void); extern void XLOGShmemInit(void); extern void BootStrapXLOG(void); extern void LocalProcessControlFile(bool reset); +extern void InitControlFile(uint64 sysidentifier); +extern void WriteControlFile(void); +extern void ReadControlFile(void); extern void StartupXLOG(void); extern void ShutdownXLOG(int code, Datum arg); extern void InitXLOGAccess(void); @@ -354,6 +357,7 @@ extern void register_persistent_abort_backup_handler(void); extern SessionBackupState get_backup_status(void); /* File path names (all relative to $PGDATA) */ +#define BASEBACKUP_SIGNAL_FILE "basebackup.signal" #define RECOVERY_SIGNAL_FILE "recovery.signal" #define STANDBY_SIGNAL_FILE "standby.signal" #define BACKUP_LABEL_FILE "backup_label" diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 62d64aa0a1..38311b05a1 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -396,6 +396,7 @@ typedef enum CheckerProcess = 0, BootstrapProcess, StartupProcess, + BaseBackupProcess, BgWriterProcess, CheckpointerProcess, WalWriterProcess, @@ -408,6 +409,7 @@ extern AuxProcType MyAuxProcType; #define AmBootstrapProcess() (MyAuxProcType == BootstrapProcess) #define AmStartupProcess() (MyAuxProcType == StartupProcess) +#define AmBaseBackupProcess() (MyAuxProcType == BaseBackupProcess) #define AmBackgroundWriterProcess() (MyAuxProcType == BgWriterProcess) #define AmCheckpointerProcess() (MyAuxProcType == CheckpointerProcess) #define AmWalWriterProcess() (MyAuxProcType == WalWriterProcess) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 36b530bc27..2f87ef63a6 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -721,6 +721,7 @@ typedef enum BackendType B_AUTOVAC_LAUNCHER, B_AUTOVAC_WORKER, B_BACKEND, + B_BASE_BACKUP, B_BG_WORKER, B_BG_WRITER, B_CHECKPOINTER, diff --git a/src/include/replication/basebackup.h b/src/include/replication/basebackup.h index 07ed281bd6..0764a7b67b 100644 --- a/src/include/replication/basebackup.h +++ b/src/include/replication/basebackup.h @@ -33,4 +33,6 @@ extern void SendBaseBackup(BaseBackupCmd *cmd); extern int64 sendTablespace(char *path, bool sizeonly); +extern void BaseBackupMain(void); + #endif /* _BASEBACKUP_H */ diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index e08afc6548..83fc4b3fb0 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -221,6 +221,7 @@ typedef void (*walrcv_readtimelinehistoryfile_fn) (WalReceiverConn *conn, TimeLineID tli, char **filename, char **content, int *size); +typedef void (*walrcv_base_backup_fn) (WalReceiverConn *conn); typedef bool (*walrcv_startstreaming_fn) (WalReceiverConn *conn, const WalRcvStreamOptions *options); typedef void (*walrcv_endstreaming_fn) (WalReceiverConn *conn, @@ -249,6 +250,7 @@ typedef struct WalReceiverFunctionsType walrcv_identify_system_fn walrcv_identify_system; walrcv_server_version_fn walrcv_server_version; walrcv_readtimelinehistoryfile_fn walrcv_readtimelinehistoryfile; + walrcv_base_backup_fn walrcv_base_backup; walrcv_startstreaming_fn walrcv_startstreaming; walrcv_endstreaming_fn walrcv_endstreaming; walrcv_receive_fn walrcv_receive; @@ -275,6 +277,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions; WalReceiverFunctions->walrcv_server_version(conn) #define walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) \ WalReceiverFunctions->walrcv_readtimelinehistoryfile(conn, tli, filename, content, size) +#define walrcv_base_backup(conn) \ + WalReceiverFunctions->walrcv_base_backup(conn) #define walrcv_startstreaming(conn, options) \ WalReceiverFunctions->walrcv_startstreaming(conn, options) #define walrcv_endstreaming(conn, next_tli) \ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index c6ce7eacf2..5b3153455c 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -148,7 +148,7 @@ extern void fsync_fname(const char *fname, bool isdir); extern int durable_rename(const char *oldfile, const char *newfile, int loglevel); extern int durable_unlink(const char *fname, int loglevel); extern int durable_link_or_rename(const char *oldfile, const char *newfile, int loglevel); -extern void SyncDataDirectory(void); +extern void SyncDataDirectory(bool pre_sync, int loglevel); extern int data_sync_elevel(int elevel); /* Filename components */ diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index ce93ace76c..c087c51dbe 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -264,7 +264,7 @@ extern int temp_file_limit; extern int num_temp_buffers; -extern char *cluster_name; +extern PGDLLIMPORT char *cluster_name; extern PGDLLIMPORT char *ConfigFileName; extern char *HbaFileName; extern char *IdentFileName; diff --git a/src/test/recovery/t/018_basebackup.pl b/src/test/recovery/t/018_basebackup.pl new file mode 100644 index 0000000000..99731fc388 --- /dev/null +++ b/src/test/recovery/t/018_basebackup.pl @@ -0,0 +1,29 @@ +# Test basebackup worker functionality +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 2; + +my $node1 = get_new_node('node1'); +$node1->init(allows_streaming => 1); +$node1->start; + +$node1->safe_psql('postgres', + "CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a"); + +my $node2 = get_new_node('node2'); +$node2->init(allows_streaming => 1, extra => [ '--replica' ]); +$node2->append_conf('postgresql.conf', "primary_conninfo = '" . $node1->connstr . "'"); +my $old_mtime = (stat($node2->data_dir . '/postgresql.conf'))[9]; +$node2->start; + +$node1->wait_for_catchup($node2, 'replay', $node1->lsn('insert')); + +is($node2->safe_psql('postgres', "SELECT count(*) FROM tab_int"), + qq(1000), + 'check content of standby'); + +my $new_mtime = (stat($node2->data_dir . '/postgresql.conf'))[9]; +is($new_mtime, $old_mtime, + 'configuration files were not copied'); base-commit: a166d408eb0b35023c169e765f4664c3b114b52e -- 2.24.1
