From c6115df297caf26a294911b30926f28dfb361e50 Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 25 Oct 2021 15:41:44 -0400 Subject: [PATCH v9 2/3] Support base backup targets. pg_basebackup now has a --target=TARGET[:DETAIL] option. If specfied, it is sent to the server as the value of the TARGET option to the BASE_BACKUP command. If DETAIL is included, it is sent as the value of the new TARGET_DETAIL option to the BASE_BACKUP command. If the target is anything other than 'client', pg_basebackup assumes that it will now be the server's job to write the backup in a location somehow defined by the target, and that it therefore needs to write nothing locally. However, the server will still send messages to the client for progress reporting purposes. On the server side, we now support two additional types of backup targets. There is a 'blackhole' target, which just throws away the backup data without doing anything at all with it. Naturally, this should only be used for testing and debugging purposes, since you will not actually have a backup when it finishes running. More usefully, there is also a 'server' target, so you can now use something like 'pg_basebackup -Xnone -t server:/SOME/PATH' to write a backup to some location on the server. Since WAL fetching is handled with separate client-side logic, it's not part of this mechanism; thus, backups with non-default targets, at least for now, must use -Xnone. Patch by me, with a bug fix by Jeevan Ladhe. --- doc/src/sgml/ref/pg_basebackup.sgml | 29 ++ src/backend/replication/Makefile | 1 + src/backend/replication/basebackup.c | 81 ++++- src/backend/replication/basebackup_copy.c | 21 +- src/backend/replication/basebackup_server.c | 302 ++++++++++++++++++ src/backend/replication/basebackup_throttle.c | 2 +- src/backend/utils/activity/wait_event.c | 6 + src/bin/pg_basebackup/pg_basebackup.c | 199 +++++++++--- src/include/replication/basebackup_sink.h | 3 +- src/include/utils/wait_event.h | 2 + 10 files changed, 587 insertions(+), 59 deletions(-) create mode 100644 src/backend/replication/basebackup_server.c diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 9e6807b457..90c366e8d3 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -224,6 +224,35 @@ PostgreSQL documentation + + + + + + + Specifies the target for the base backup. The default target is + client, which specifies that the backup should + be sent to the machine where pg_basebackup + is running. If the target is instead set to + server:/some/path, the backup will be stored on + the machine where the server is running in the + /some/path directory. Storing a backup on the + server requires superuser privileges. If the target is set to + blackhole causes the contents of the backup to be + discarded and not stored anywhere. This should only be used for + testing purposes, as you will not end up with an actual backup. + + + + Due to limitations of the implementation, WAL cannot be included + in backups with non-default targets; threfore, the use of + -Xnone is required when a non-default target + is specified. + + + + + diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile index 74b97cf126..a8f4757f0c 100644 --- a/src/backend/replication/Makefile +++ b/src/backend/replication/Makefile @@ -19,6 +19,7 @@ OBJS = \ basebackup.o \ basebackup_copy.o \ basebackup_progress.o \ + basebackup_server.o \ basebackup_sink.o \ basebackup_throttle.o \ repl_gram.o \ diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 096455ad02..ac1e0d8733 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -55,8 +55,10 @@ typedef enum { + BACKUP_TARGET_BLACKHOLE, BACKUP_TARGET_COMPAT, - BACKUP_TARGET_CLIENT + BACKUP_TARGET_CLIENT, + BACKUP_TARGET_SERVER } backup_target_type; typedef struct @@ -69,6 +71,7 @@ typedef struct uint32 maxrate; bool sendtblspcmapfile; backup_target_type target; + char *target_detail; backup_manifest_option manifest; pg_checksum_type manifest_checksum_type; } basebackup_options; @@ -691,6 +694,8 @@ parse_basebackup_options(List *options, basebackup_options *opt) bool o_manifest = false; bool o_manifest_checksums = false; bool o_target = false; + bool o_target_detail = false; + char *target_str = "compat"; /* placate compiler */ MemSet(opt, 0, sizeof(*opt)); opt->target = BACKUP_TARGET_COMPAT; @@ -836,25 +841,35 @@ parse_basebackup_options(List *options, basebackup_options *opt) } else if (strcmp(defel->defname, "target") == 0) { - char *optval = defGetString(defel); + target_str = defGetString(defel); if (o_target) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); - if (strcmp(optval, "client") == 0) + if (strcmp(target_str, "blackhole") == 0) + opt->target = BACKUP_TARGET_BLACKHOLE; + else if (strcmp(target_str, "client") == 0) opt->target = BACKUP_TARGET_CLIENT; + else if (strcmp(target_str, "server") == 0) + opt->target = BACKUP_TARGET_SERVER; else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized target: \"%s\"", optval))); + errmsg("unrecognized target: \"%s\"", target_str))); o_target = true; } - else - ereport(ERROR, - errcode(ERRCODE_SYNTAX_ERROR), - errmsg("option \"%s\" not recognized", - defel->defname)); + else if (strcmp(defel->defname, "target_detail") == 0) + { + char *optval = defGetString(defel); + + if (o_target_detail) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("duplicate option \"%s\"", defel->defname))); + opt->target_detail = optval; + o_target_detail = true; + } } if (opt->label == NULL) opt->label = "base backup"; @@ -866,6 +881,22 @@ parse_basebackup_options(List *options, basebackup_options *opt) errmsg("manifest checksums require a backup manifest"))); opt->manifest_checksum_type = CHECKSUM_TYPE_NONE; } + if (opt->target == BACKUP_TARGET_SERVER) + { + if (opt->target_detail == NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("target '%s' requires a target detail", + target_str))); + } + else + { + if (opt->target_detail != NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("target '%s' does not accept a target detail", + target_str))); + } } @@ -897,14 +928,38 @@ SendBaseBackup(BaseBackupCmd *cmd) /* * If the TARGET option was specified, then we can use the new copy-stream - * protocol. If not, we must fall back to the old and less capable - * copy-tablespace protocol. + * protocol. If the target is specifically 'client' then set up to stream + * the backup to the client; otherwise, it's being sent someplace else and + * should not be sent to the client. + * + * If the TARGET option was not specified, we must fall back to the older + * and less capable copy-tablespace protocol. */ - if (opt.target != BACKUP_TARGET_COMPAT) - sink = bbsink_copystream_new(); + if (opt.target == BACKUP_TARGET_CLIENT) + sink = bbsink_copystream_new(true); + else if (opt.target != BACKUP_TARGET_COMPAT) + sink = bbsink_copystream_new(false); else sink = bbsink_copytblspc_new(); + /* + * If a non-default backup target is in use, arrange to send the data + * wherever it needs to go. + */ + switch (opt.target) + { + case BACKUP_TARGET_BLACKHOLE: + /* Nothing to do, just discard data. */ + break; + case BACKUP_TARGET_COMPAT: + case BACKUP_TARGET_CLIENT: + /* Nothing to do, handling above is sufficient. */ + break; + case BACKUP_TARGET_SERVER: + sink = bbsink_server_new(sink, opt.target_detail); + break; + } + /* Set up network throttling, if client requested it */ if (opt.maxrate > 0) sink = bbsink_throttle_new(sink, opt.maxrate); diff --git a/src/backend/replication/basebackup_copy.c b/src/backend/replication/basebackup_copy.c index 57183f4d46..2e9058b041 100644 --- a/src/backend/replication/basebackup_copy.c +++ b/src/backend/replication/basebackup_copy.c @@ -44,6 +44,9 @@ typedef struct bbsink_copystream /* Common information for all types of sink. */ bbsink base; + /* Are we sending the archives to the client, or somewhere else? */ + bool send_to_client; + /* * Protocol message buffer. We assemble CopyData protocol messages by * setting the first character of this buffer to 'd' (archive or manifest @@ -131,11 +134,12 @@ const bbsink_ops bbsink_copytblspc_ops = { * Create a new 'copystream' bbsink. */ bbsink * -bbsink_copystream_new(void) +bbsink_copystream_new(bool send_to_client) { bbsink_copystream *sink = palloc0(sizeof(bbsink_copystream)); *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_copystream_ops; + sink->send_to_client = send_to_client; /* Set up for periodic progress reporting. */ sink->last_progress_report_time = GetCurrentTimestamp(); @@ -208,8 +212,12 @@ bbsink_copystream_archive_contents(bbsink *sink, size_t len) StringInfoData buf; uint64 targetbytes; - /* Send the archive content to the client (with leading type byte). */ - pq_putmessage('d', mysink->msgbuffer, len + 1); + /* Send the archive content to the client, if appropriate. */ + if (mysink->send_to_client) + { + /* Add one because we're also sending a leading type byte. */ + pq_putmessage('d', mysink->msgbuffer, len + 1); + } /* Consider whether to send a progress report to the client. */ targetbytes = mysink->bytes_done_at_last_time_check @@ -290,8 +298,11 @@ bbsink_copystream_manifest_contents(bbsink *sink, size_t len) { bbsink_copystream *mysink = (bbsink_copystream *) sink; - /* Send the manifest content to the client (with leading type byte). */ - pq_putmessage('d', mysink->msgbuffer, len + 1); + if (mysink->send_to_client) + { + /* Add one because we're also sending a leading type byte. */ + pq_putmessage('d', mysink->msgbuffer, len + 1); + } } /* diff --git a/src/backend/replication/basebackup_server.c b/src/backend/replication/basebackup_server.c new file mode 100644 index 0000000000..ce1b7b4797 --- /dev/null +++ b/src/backend/replication/basebackup_server.c @@ -0,0 +1,302 @@ +/*------------------------------------------------------------------------- + * + * basebackup_server.c + * store basebackup archives on the server + * + * IDENTIFICATION + * src/backend/replication/basebackup_server.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "miscadmin.h" +#include "replication/basebackup.h" +#include "replication/basebackup_sink.h" +#include "storage/fd.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + +typedef struct bbsink_server +{ + /* Common information for all types of sink. */ + bbsink base; + + /* Directory in which backup is to be stored. */ + char *pathname; + + /* Currently open file (or 0 if nothing open). */ + File file; + + /* Current file position. */ + off_t filepos; +} bbsink_server; + +static void bbsink_server_begin_archive(bbsink *sink, + const char *archive_name); +static void bbsink_server_archive_contents(bbsink *sink, size_t len); +static void bbsink_server_end_archive(bbsink *sink); +static void bbsink_server_begin_manifest(bbsink *sink); +static void bbsink_server_manifest_contents(bbsink *sink, size_t len); +static void bbsink_server_end_manifest(bbsink *sink); + +const bbsink_ops bbsink_server_ops = { + .begin_backup = bbsink_forward_begin_backup, + .begin_archive = bbsink_server_begin_archive, + .archive_contents = bbsink_server_archive_contents, + .end_archive = bbsink_server_end_archive, + .begin_manifest = bbsink_server_begin_manifest, + .manifest_contents = bbsink_server_manifest_contents, + .end_manifest = bbsink_server_end_manifest, + .end_backup = bbsink_forward_end_backup, + .cleanup = bbsink_forward_cleanup +}; + +/* + * Create a new 'server' bbsink. + */ +bbsink * +bbsink_server_new(bbsink *next, char *pathname) +{ + bbsink_server *sink = palloc0(sizeof(bbsink_server)); + + *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_server_ops; + sink->pathname = pathname; + sink->base.bbs_next = next; + + /* Replication permission is not sufficient in this case. */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to create server backup"))); + + /* + * It's not a good idea to store your backups in the same directory that + * you're backing up. If we allowed a relative path here, that could easily + * happen accidentally, so we don't. The user could still accomplish the + * same thing by including the absolute path to $PGDATA in the pathname, + * but that's likely an intentional bad decision rather than an accident. + */ + if (!is_absolute_path(pathname)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_NAME), + errmsg("relative path not allowed for server backup"))); + + switch (pg_check_dir(pathname)) + { + case 0: + /* + * Does not exist, so create it using the same permissions we'd use + * for a new subdirectory of the data directory itself. + */ + if (MakePGDirectory(pathname) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create directory \"%s\": %m", pathname))); + break; + + case 1: + /* Exists, empty. */ + break; + + case 2: + case 3: + case 4: + /* Exists, not empty. */ + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_FILE), + errmsg("directory \"%s\" exists but is not empty", + pathname))); + break; + + default: + /* Access problem. */ + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not access directory \"%s\": %m", + pathname))); + } + + return &sink->base; +} + +/* + * Open the correct output file for this archive. + */ +static void +bbsink_server_begin_archive(bbsink *sink, const char *archive_name) +{ + bbsink_server *mysink = (bbsink_server *) sink; + char *filename; + + Assert(mysink->file == 0); + Assert(mysink->filepos == 0); + + filename = psprintf("%s/%s", mysink->pathname, archive_name); + + mysink->file = PathNameOpenFile(filename, + O_CREAT | O_EXCL | O_WRONLY | PG_BINARY); + if (mysink->file <= 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", filename))); + + pfree(filename); + + bbsink_forward_begin_archive(sink, archive_name); +} + +/* + * Write the data to the output file. + */ +static void +bbsink_server_archive_contents(bbsink *sink, size_t len) +{ + bbsink_server *mysink = (bbsink_server *) sink; + int nbytes; + + nbytes = FileWrite(mysink->file, mysink->base.bbs_buffer, len, + mysink->filepos, WAIT_EVENT_BASEBACKUP_WRITE); + + if (nbytes != len) + { + if (nbytes < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + FilePathName(mysink->file)), + errhint("Check free disk space."))); + /* short write: complain appropriately */ + ereport(ERROR, + (errcode(ERRCODE_DISK_FULL), + errmsg("could not write file \"%s\": wrote only %d of %d bytes at offset %u", + FilePathName(mysink->file), + nbytes, (int) len, (unsigned) mysink->filepos), + errhint("Check free disk space."))); + } + + mysink->filepos += nbytes; + + bbsink_forward_archive_contents(sink, len); +} + +/* + * fsync and close the current output file. + */ +static void +bbsink_server_end_archive(bbsink *sink) +{ + bbsink_server *mysink = (bbsink_server *) sink; + + /* + * We intentionally don't use data_sync_elevel here, because the server + * shouldn't PANIC just because we can't guarantee the the backup has been + * written down to disk. Running recovery won't fix anything in this case + * anyway. + */ + if (FileSync(mysink->file, WAIT_EVENT_BASEBACKUP_SYNC) < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", + FilePathName(mysink->file)))); + + + /* We're done with this file now. */ + FileClose(mysink->file); + mysink->file = 0; + mysink->filepos = 0; + + bbsink_forward_end_archive(sink); +} + +/* + * Open the output file to which we will write the manifest. + * + * Just like pg_basebackup, we write the manifest first under a temporary + * name and then rename it into place after fsync. That way, if the manifest + * is there and under the correct name, the user can be sure that the backup + * completed. + */ +static void +bbsink_server_begin_manifest(bbsink *sink) +{ + bbsink_server *mysink = (bbsink_server *) sink; + char *tmp_filename; + + Assert(mysink->file == 0); + + tmp_filename = psprintf("%s/backup_manifest.tmp", mysink->pathname); + + mysink->file = PathNameOpenFile(tmp_filename, + O_CREAT | O_EXCL | O_WRONLY | PG_BINARY); + if (mysink->file <= 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", tmp_filename))); + + pfree(tmp_filename); + + bbsink_forward_begin_manifest(sink); +} + +/* + * Each chunk of manifest data is sent using a CopyData message. + */ +static void +bbsink_server_manifest_contents(bbsink *sink, size_t len) +{ + bbsink_server *mysink = (bbsink_server *) sink; + int nbytes; + + nbytes = FileWrite(mysink->file, mysink->base.bbs_buffer, len, + mysink->filepos, WAIT_EVENT_BASEBACKUP_WRITE); + + if (nbytes != len) + { + if (nbytes < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write file \"%s\": %m", + FilePathName(mysink->file)), + errhint("Check free disk space."))); + /* short write: complain appropriately */ + ereport(ERROR, + (errcode(ERRCODE_DISK_FULL), + errmsg("could not write file \"%s\": wrote only %d of %d bytes at offset %u", + FilePathName(mysink->file), + nbytes, (int) len, (unsigned) mysink->filepos), + errhint("Check free disk space."))); + } + + mysink->filepos += nbytes; + + bbsink_forward_manifest_contents(sink, len); +} + +/* + * fsync the backup manifest, close the file, and then rename it into place. + */ +static void +bbsink_server_end_manifest(bbsink *sink) +{ + bbsink_server *mysink = (bbsink_server *) sink; + char *tmp_filename; + char *filename; + + /* We're done with this file now. */ + FileClose(mysink->file); + mysink->file = 0; + + /* + * Rename it into place. This also fsyncs the temporary file, so we don't + * need to do that here. We don't use data_sync_elevel here for the same + * reasons as in bbsink_server_end_archive. + */ + tmp_filename = psprintf("%s/backup_manifest.tmp", mysink->pathname); + filename = psprintf("%s/backup_manifest", mysink->pathname); + durable_rename(tmp_filename, filename, ERROR); + pfree(filename); + pfree(tmp_filename); + + bbsink_forward_end_manifest(sink); +} diff --git a/src/backend/replication/basebackup_throttle.c b/src/backend/replication/basebackup_throttle.c index f163931f8a..f5202bae87 100644 --- a/src/backend/replication/basebackup_throttle.c +++ b/src/backend/replication/basebackup_throttle.c @@ -122,7 +122,7 @@ bbsink_throttle_manifest_contents(bbsink *sink, size_t len) { throttle((bbsink_throttle *) sink, len); - bbsink_forward_manifest_contents(sink->bbs_next, len); + bbsink_forward_manifest_contents(sink, len); } /* diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 4a5b7502f5..3b5e6b799a 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -510,6 +510,12 @@ pgstat_get_wait_io(WaitEventIO w) case WAIT_EVENT_BASEBACKUP_READ: event_name = "BaseBackupRead"; break; + case WAIT_EVENT_BASEBACKUP_SYNC: + event_name = "BaseBackupSync"; + break; + case WAIT_EVENT_BASEBACKUP_WRITE: + event_name = "BaseBackupWrite"; + break; case WAIT_EVENT_BUFFILE_READ: event_name = "BufFileRead"; break; diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index ffeb6a3117..e8f76d2eb6 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -109,7 +109,7 @@ typedef enum static char *basedir = NULL; static TablespaceList tablespace_dirs = {NULL, NULL}; static char *xlog_dir = NULL; -static char format = 'p'; /* p(lain)/t(ar) */ +static char format = '\0'; /* p(lain)/t(ar) */ static char *label = "pg_basebackup base backup"; static bool noclean = false; static bool checksum_failure = false; @@ -126,6 +126,7 @@ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ static char *replication_slot = NULL; static bool temp_replication_slot = true; +static char *backup_target = NULL; static bool create_slot = false; static bool no_slot = false; static bool verify_checksums = true; @@ -357,6 +358,8 @@ usage(void) printf(_("Usage:\n")); printf(_(" %s [OPTION]...\n"), progname); printf(_("\nOptions controlling the output:\n")); + printf(_(" -t, --target=TARGET[:DETAIL]\n" + " backup target (if other than client)\n")); printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n")); printf(_(" -F, --format=p|t output format (plain (default), tar)\n")); printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n" @@ -1219,15 +1222,22 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data) } /* - * Create an appropriate backup streamer. We know that - * recovery GUCs are supported, because this protocol can only - * be used on v15+. + * Create an appropriate backup streamer, unless a backup + * target was specified. In that case, it's up to the server + * to put the backup wherever it needs to go. */ - state->streamer = - CreateBackupStreamer(archive_name, - spclocation, - &state->manifest_inject_streamer, - true); + if (backup_target == NULL) + { + /* + * We know that recovery GUCs are supported, because this + * protocol can only be used on v15+. + */ + state->streamer = + CreateBackupStreamer(archive_name, + spclocation, + &state->manifest_inject_streamer, + true); + } break; } @@ -1299,24 +1309,32 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data) GetCopyDataEnd(r, copybuf, cursor); /* - * If we're supposed inject the manifest into the archive, we - * prepare to buffer it in memory; otherwise, we prepare to - * write it to a temporary file. + * If a backup target was specified, figuring out where to put + * the manifest is the server's problem. Otherwise, we need to + * deal with it. */ - if (state->manifest_inject_streamer != NULL) - state->manifest_buffer = createPQExpBuffer(); - else + if (backup_target == NULL) { - snprintf(state->manifest_filename, - sizeof(state->manifest_filename), - "%s/backup_manifest.tmp", basedir); - state->manifest_file = - fopen(state->manifest_filename, "wb"); - if (state->manifest_file == NULL) + /* + * If we're supposed inject the manifest into the archive, + * we prepare to buffer it in memory; otherwise, we + * prepare to write it to a temporary file. + */ + if (state->manifest_inject_streamer != NULL) + state->manifest_buffer = createPQExpBuffer(); + else { - pg_log_error("could not create file \"%s\": %m", - state->manifest_filename); - exit(1); + snprintf(state->manifest_filename, + sizeof(state->manifest_filename), + "%s/backup_manifest.tmp", basedir); + state->manifest_file = + fopen(state->manifest_filename, "wb"); + if (state->manifest_file == NULL) + { + pg_log_error("could not create file \"%s\": %m", + state->manifest_filename); + exit(1); + } } } break; @@ -1687,7 +1705,35 @@ BaseBackup(void) "MANIFEST_CHECKSUMS", manifest_checksums); } - if (serverMajor >= 1500) + if (backup_target != NULL) + { + char *colon; + + if (serverMajor < 1500) + { + pg_log_error("backup targets are not supported by this server version"); + exit(1); + } + + AppendPlainCommandOption(&buf, use_new_option_syntax, "TABLESPACE_MAP"); + + if ((colon = strchr(backup_target, ':')) == NULL) + { + AppendStringCommandOption(&buf, use_new_option_syntax, + "TARGET", backup_target); + } + else + { + char *target; + + target = pnstrdup(backup_target, colon - backup_target); + AppendStringCommandOption(&buf, use_new_option_syntax, + "TARGET", target); + AppendStringCommandOption(&buf, use_new_option_syntax, + "TARGET_DETAIL", colon + 1); + } + } + else if (serverMajor >= 1500) AppendStringCommandOption(&buf, use_new_option_syntax, "TARGET", "client"); @@ -1782,8 +1828,13 @@ BaseBackup(void) * Verify tablespace directories are empty. Don't bother with the * first once since it can be relocated, and it will be checked before * we do anything anyway. + * + * Note that this is skipped for tar format backups and backups that + * the server is storing to a target location, since in that case + * we won't be storing anything into these directories and thus should + * not create them. */ - if (format == 'p' && !PQgetisnull(res, i, 1)) + if (backup_target == NULL && format == 'p' && !PQgetisnull(res, i, 1)) { char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1))); @@ -1794,7 +1845,8 @@ BaseBackup(void) /* * When writing to stdout, require a single tablespace */ - writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0; + writing_to_stdout = format == 't' && basedir != NULL && + strcmp(basedir, "-") == 0; if (writing_to_stdout && PQntuples(res) > 1) { pg_log_error("can only write single tablespace to stdout, database has %d", @@ -1877,7 +1929,7 @@ BaseBackup(void) res = PQgetResult(conn); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - pg_log_error("could not get write-ahead log end position from server: %s", + pg_log_error("backup failed: %s", PQerrorMessage(conn)); exit(1); } @@ -2011,8 +2063,11 @@ BaseBackup(void) * synced after being completed. In plain format, all the data of the * base directory is synced, taking into account all the tablespaces. * Errors are not considered fatal. + * + * If, however, there's a backup target, we're not writing anything + * locally, so in that case we skip this step. */ - if (do_sync) + if (do_sync && backup_target == NULL) { if (verbose) pg_log_info("syncing data to disk ..."); @@ -2034,7 +2089,7 @@ BaseBackup(void) * without a backup_manifest file, decreasing the chances that a directory * we leave behind will be mistaken for a valid backup. */ - if (!writing_to_stdout && manifest) + if (!writing_to_stdout && manifest && backup_target == NULL) { char tmp_filename[MAXPGPATH]; char filename[MAXPGPATH]; @@ -2068,6 +2123,7 @@ main(int argc, char **argv) {"max-rate", required_argument, NULL, 'r'}, {"write-recovery-conf", no_argument, NULL, 'R'}, {"slot", required_argument, NULL, 'S'}, + {"target", required_argument, NULL, 't'}, {"tablespace-mapping", required_argument, NULL, 'T'}, {"wal-method", required_argument, NULL, 'X'}, {"gzip", no_argument, NULL, 'z'}, @@ -2118,7 +2174,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP", + while ((c = getopt_long(argc, argv, "CD:F:r:RS:t:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP", long_options, &option_index)) != -1) { switch (c) @@ -2159,6 +2215,9 @@ main(int argc, char **argv) case 2: no_slot = true; break; + case 't': + backup_target = pg_strdup(optarg); + break; case 'T': tablespace_list_append(optarg); break; @@ -2291,18 +2350,50 @@ main(int argc, char **argv) } /* - * Required arguments + * Setting the backup target to 'client' is equivalent to leaving out the + * option. This logic allows us to assume elsewhere that the backup is + * being stored locally if and only if backup_target == NULL. + */ + if (backup_target != NULL && strcmp(backup_target, "client") == 0) + { + pg_free(backup_target); + backup_target = NULL; + } + + /* + * Can't use --format with --target. Without --target, default format is + * tar. */ - if (basedir == NULL) + if (backup_target != NULL && format != '\0') { - pg_log_error("no target directory specified"); + pg_log_error("cannot specify both format and backup target"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } + if (format == '\0') + format = 'p'; /* - * Mutually exclusive arguments + * Either directory or backup target should be specified, but not both + */ + if (basedir == NULL && backup_target == NULL) + { + pg_log_error("must specify output directory or backup target"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } + if (basedir != NULL && backup_target != NULL) + { + pg_log_error("cannot specify both output directory and backup target"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } + + /* + * Compression doesn't make sense unless tar format is in use. */ if (format == 'p' && compresslevel != 0) { @@ -2312,6 +2403,16 @@ main(int argc, char **argv) exit(1); } + /* + * Sanity checks for WAL method. + */ + if (backup_target != NULL && includewal != NO_WAL) + { + pg_log_error("WAL cannot be included when a backup target is specified"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0) { pg_log_error("cannot stream write-ahead logs in tar mode to stdout"); @@ -2328,6 +2429,9 @@ main(int argc, char **argv) exit(1); } + /* + * Sanity checks for replication slot options. + */ if (no_slot) { if (replication_slot) @@ -2361,8 +2465,18 @@ main(int argc, char **argv) } } + /* + * Sanity checks on WAL directory. + */ if (xlog_dir) { + if (backup_target != NULL) + { + pg_log_error("WAL directory location cannot be specified along with a backup target"); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), + progname); + exit(1); + } if (format != 'p') { pg_log_error("WAL directory location can only be specified in plain mode"); @@ -2383,6 +2497,7 @@ main(int argc, char **argv) } #ifndef HAVE_LIBZ + /* Sanity checks for compression level. */ if (compresslevel != 0) { pg_log_error("this build does not support compression"); @@ -2390,6 +2505,9 @@ main(int argc, char **argv) } #endif + /* + * Sanity checks for progress reporting options. + */ if (showprogress && !estimatesize) { pg_log_error("%s and %s are incompatible options", @@ -2399,6 +2517,9 @@ main(int argc, char **argv) exit(1); } + /* + * Sanity checks for backup manifest options. + */ if (!manifest && manifest_checksums != NULL) { pg_log_error("%s and %s are incompatible options", @@ -2441,11 +2562,11 @@ main(int argc, char **argv) manifest = false; /* - * Verify that the target directory exists, or create it. For plaintext - * backups, always require the directory. For tar backups, require it - * unless we are writing to stdout. + * If an output directory was specified, verify that it exists, or create + * it. Note that for a tar backup, an output directory of "-" means we are + * writing to stdout, so do nothing in that case. */ - if (format == 'p' || strcmp(basedir, "-") != 0) + if (basedir != NULL && (format == 'p' || strcmp(basedir, "-") != 0)) verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata); /* determine remote server's xlog segment size */ diff --git a/src/include/replication/basebackup_sink.h b/src/include/replication/basebackup_sink.h index 36b9b76c5f..0e337a86f4 100644 --- a/src/include/replication/basebackup_sink.h +++ b/src/include/replication/basebackup_sink.h @@ -282,9 +282,10 @@ extern void bbsink_forward_end_backup(bbsink *sink, XLogRecPtr endptr, extern void bbsink_forward_cleanup(bbsink *sink); /* Constructors for various types of sinks. */ -extern bbsink *bbsink_copystream_new(void); +extern bbsink *bbsink_copystream_new(bool send_to_client); extern bbsink *bbsink_copytblspc_new(void); extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size); +extern bbsink *bbsink_server_new(bbsink *next, char *pathname); extern bbsink *bbsink_throttle_new(bbsink *next, uint32 maxrate); /* Extra interface functions for progress reporting. */ diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index c22142365f..0b20981614 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -153,6 +153,8 @@ typedef enum typedef enum { WAIT_EVENT_BASEBACKUP_READ = PG_WAIT_IO, + WAIT_EVENT_BASEBACKUP_SYNC, + WAIT_EVENT_BASEBACKUP_WRITE, WAIT_EVENT_BUFFILE_READ, WAIT_EVENT_BUFFILE_WRITE, WAIT_EVENT_BUFFILE_TRUNCATE, -- 2.24.3 (Apple Git-128)