Per discussionat the developer meeting in Brussels, here's a patch that
makes some updates to the backup APIs, to support non-exclusive backups
without using pg_basebackup. The idea is to fix at least three main issues
that are there today -- that you cannot run concurrent backups, that the
backup_label file is created in the data directory which makes it
impossible to distinguish between a cluster restored from backup and one
that crashed while a backup was running, and a cluster can get "stuck" in
backup mode if the backup script/software crashes.
To make this work, this patch:
* Introduces a new argument for pg_start_backup(), for "exclusive". It
defaults to true, which means that just calling pg_start_backup() like
before will behave exactly like before. If it's called with exclusive='f',
a non-exclusive backup is started, and the backup label is collected in
memory instead of in the backup_label file.
* If the client disconnects with a non-exclusive backup running, the backup
is automatically aborted. This is the same thing that pg_basebackup does.
To use these non-exclusive backups the backup software will have to
maintain a persistent connection to the database -- something that should
not be a problem for any of the modern ones out there (but would've been
slightly trickier back in the days when we suggested shellscripts)
* A new version of pg_stop_backup is created, taking an argument specifying
if it's exclusive or not. The current version of pg_stop_backup() continues
to work for exclusive backups, with no changes to behavior. The new
pg_stop_backup will return a record of three columns instead of just the
value -- the LSN (pglsn), the backup label file (text) and the tablespace
map file (text). If used to cancel an exclusive backup, backup label file
and tablespace map will be NULL. At this point it's up to the backup
software to write down the files in the location of the backup.
Attached patch currently doesn't include full documentation, since Bruce
was going to restructure that section of the docs (also based on the
devmeeting). I will write up the docs once that is done (I assume it will
be soon enough, or I'll go do it regardless), but I wanted to get some
review in on the code while waiting.
--
Magnus Hagander
Me: http://www.hagander.net/
Work: http://www.redpill-linpro.com/
*** a/doc/src/sgml/func.sgml
--- b/doc/src/sgml/func.sgml
***************
*** 16981,16987 **** SELECT set_config('log_statement_stats', 'off', false);
</row>
<row>
<entry>
! <literal><function>pg_start_backup(<parameter>label</> <type>text</> <optional>, <parameter>fast</> <type>boolean</> </optional>)</function></literal>
</entry>
<entry><type>pg_lsn</type></entry>
<entry>Prepare for performing on-line backup (restricted to superusers or replication roles)</entry>
--- 16981,16987 ----
</row>
<row>
<entry>
! <literal><function>pg_start_backup(<parameter>label</> <type>text</> <optional>, <parameter>fast</> <type>boolean</> <optional>, <parameter>exclusive</> <type>boolean</> </optional></optional>)</function></literal>
</entry>
<entry><type>pg_lsn</type></entry>
<entry>Prepare for performing on-line backup (restricted to superusers or replication roles)</entry>
***************
*** 16991,16997 **** SELECT set_config('log_statement_stats', 'off', false);
<literal><function>pg_stop_backup()</function></literal>
</entry>
<entry><type>pg_lsn</type></entry>
! <entry>Finish performing on-line backup (restricted to superusers or replication roles)</entry>
</row>
<row>
<entry>
--- 16991,17004 ----
<literal><function>pg_stop_backup()</function></literal>
</entry>
<entry><type>pg_lsn</type></entry>
! <entry>Finish performing exclusive on-line backup (restricted to superusers or replication roles)</entry>
! </row>
! <row>
! <entry>
! <literal><function>pg_stop_backup(<parameter>exclusive</> <type>boolean</>)</function></literal>
! </entry>
! <entry><type>setof record</type></entry>
! <entry>Finish performing exclusive or non-exclusive on-line backup (restricted to superusers or replication roles)</entry>
</row>
<row>
<entry>
*** a/src/backend/access/transam/xlog.c
--- b/src/backend/access/transam/xlog.c
***************
*** 9797,9804 **** XLogFileNameP(TimeLineID tli, XLogSegNo segno)
*/
XLogRecPtr
do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
! char **labelfile, DIR *tblspcdir, List **tablespaces,
! char **tblspcmapfile, bool infotbssize,
bool needtblspcmapfile)
{
bool exclusive = (labelfile == NULL);
--- 9797,9804 ----
*/
XLogRecPtr
do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
! StringInfo labelfile, DIR *tblspcdir, List **tablespaces,
! StringInfo tblspcmapfile, bool infotbssize,
bool needtblspcmapfile)
{
bool exclusive = (labelfile == NULL);
***************
*** 9812,9819 **** do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
XLogSegNo _logSegNo;
struct stat stat_buf;
FILE *fp;
- StringInfoData labelfbuf;
- StringInfoData tblspc_mapfbuf;
backup_started_in_recovery = RecoveryInProgress();
--- 9812,9817 ----
*** a/src/backend/access/transam/xlogfuncs.c
--- b/src/backend/access/transam/xlogfuncs.c
***************
*** 28,41 ****
--- 28,64 ----
#include "replication/walreceiver.h"
#include "storage/smgr.h"
#include "utils/builtins.h"
+ #include "utils/memutils.h"
#include "utils/numeric.h"
#include "utils/guc.h"
#include "utils/pg_lsn.h"
#include "utils/timestamp.h"
+ #include "utils/tuplestore.h"
#include "storage/fd.h"
+ #include "storage/ipc.h"
/*
+ * Store label file and tablespace map during non-exclusive backups.
+ */
+ static StringInfo label_file;
+ static StringInfo tblspc_map_file;
+ static bool exclusive_backup_running = false;
+ static bool nonexclusive_backup_running = false;
+
+ /*
+ * Called when the backend exits with a running non-exclusive base backup,
+ * to clean up state.
+ */
+ static void
+ nonexclusive_base_backup_cleanup(int code, Datum arg)
+ {
+ do_pg_abort_backup();
+ ereport(WARNING,
+ (errmsg("aborting backup due to backend exiting before pg_stop_backup was called")));
+ }
+
+ /*
* pg_start_backup: set up for taking an on-line backup dump
*
* Essentially what this does is to create a backup label file in $PGDATA,
***************
*** 49,54 **** pg_start_backup(PG_FUNCTION_ARGS)
--- 72,78 ----
{
text *backupid = PG_GETARG_TEXT_P(0);
bool fast = PG_GETARG_BOOL(1);
+ bool exclusive = PG_GETARG_BOOL(2);
char *backupidstr;
XLogRecPtr startpoint;
DIR *dir;
***************
*** 60,73 **** pg_start_backup(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("must be superuser or replication role to run a backup")));
/* Make sure we can open the directory with tablespaces in it */
dir = AllocateDir("pg_tblspc");
if (!dir)
ereport(ERROR,
(errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
! startpoint = do_pg_start_backup(backupidstr, fast, NULL, NULL,
! dir, NULL, NULL, false, true);
FreeDir(dir);
--- 84,125 ----
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("must be superuser or replication role to run a backup")));
+ if (exclusive_backup_running || nonexclusive_backup_running)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("a backup is already in progress in this session")));
+
/* Make sure we can open the directory with tablespaces in it */
dir = AllocateDir("pg_tblspc");
if (!dir)
ereport(ERROR,
(errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
! if (exclusive)
! {
! startpoint = do_pg_start_backup(backupidstr, fast, NULL, NULL,
! dir, NULL, NULL, false, true);
! exclusive_backup_running = true;
! }
! else
! {
! MemoryContext oldcontext;
!
! /*
! * Label file and tablespace map file need to be long-lived, since they
! * are read in pg_stop_backup.
! */
! oldcontext = MemoryContextSwitchTo(TopMemoryContext);
! label_file = makeStringInfo();
! tblspc_map_file = makeStringInfo();
! MemoryContextSwitchTo(oldcontext);
!
! startpoint = do_pg_start_backup(backupidstr, fast, NULL, label_file,
! dir, NULL, tblspc_map_file, false, true);
! nonexclusive_backup_running = true;
!
! before_shmem_exit(nonexclusive_base_backup_cleanup, (Datum) 0);
! }
FreeDir(dir);
***************
*** 86,91 **** pg_start_backup(PG_FUNCTION_ARGS)
--- 138,147 ----
* record for that and the file is for informational and debug purposes only.
*
* Note: different from CancelBackup which just cancels online backup mode.
+ *
+ * Note: this version is only called to stop an exclusive backup. The function
+ * pg_stop_backup2 (overloaded as pg_stop_backup in SQL) is called to
+ * stop non-exclusive backups.
*/
Datum
pg_stop_backup(PG_FUNCTION_ARGS)
***************
*** 97,107 **** pg_stop_backup(PG_FUNCTION_ARGS)
--- 153,277 ----
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
(errmsg("must be superuser or replication role to run a backup"))));
+ if (nonexclusive_backup_running)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("non-exclusive backup in progress"),
+ errhint("did you mean to use pg_stop_backup('f')?")));
+
+ /*
+ * Exclusive backups were typically started in a different connection,
+ * so don't try to verify that exclusive_backup_running is set in this one.
+ * Actual verification that an exclusive backup is in fact running is handled
+ * inside do_pg_stop_backup.
+ */
stoppoint = do_pg_stop_backup(NULL, true, NULL);
+ exclusive_backup_running = false;
+
PG_RETURN_LSN(stoppoint);
}
+
+ /*
+ * pg_stop_backup2: finish taking exclusive or nonexclusive on-line backup.
+ *
+ * Works the same as pg_stop_backup, except for non-exclusive backups it returns
+ * the backup label and tablespace map files as text fields in as part of the
+ * resultset.
+ */
+ Datum
+ pg_stop_backup2(PG_FUNCTION_ARGS)
+ {
+ ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+ TupleDesc tupdesc;
+ Tuplestorestate *tupstore;
+ MemoryContext per_query_ctx;
+ MemoryContext oldcontext;
+ Datum values[3];
+ bool nulls[3];
+
+ bool exclusive = PG_GETARG_BOOL(0);
+ XLogRecPtr stoppoint;
+
+ /* check to see if caller supports us returning a tuplestore */
+ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("set-valued function called in context that cannot accept a set")));
+ if (!(rsinfo->allowedModes & SFRM_Materialize))
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("materialize mode required, but it is not " \
+ "allowed in this context")));
+
+ if (!superuser() && !has_rolreplication(GetUserId()))
+ ereport(ERROR,
+ (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+ (errmsg("must be superuser or replication role to run a backup"))));
+
+ /* Build a tuple descriptor for our result type */
+ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+ elog(ERROR, "return type must be a row type");
+
+ per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+ oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+ tupstore = tuplestore_begin_heap(true, false, work_mem);
+ rsinfo->returnMode = SFRM_Materialize;
+ rsinfo->setResult = tupstore;
+ rsinfo->setDesc = tupdesc;
+
+ MemoryContextSwitchTo(oldcontext);
+
+ MemSet(values, 0, sizeof(values));
+ MemSet(nulls, 0, sizeof(nulls));
+
+ if (exclusive)
+ {
+ if (nonexclusive_backup_running)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("non-exclusive backup in progress"),
+ errhint("did you mean to use pg_stop_backup('f')?")));
+
+ stoppoint = do_pg_stop_backup(NULL, true, NULL);
+ exclusive_backup_running = false;
+
+ nulls[1] = true;
+ nulls[2] = true;
+ }
+ else
+ {
+ if (!nonexclusive_backup_running)
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("non-exclusive backup is not in progress"),
+ errhint("did you mean to use pg_stop_backup('t')?")));
+
+ stoppoint = do_pg_stop_backup(label_file->data, true, NULL);
+ nonexclusive_backup_running = false;
+ cancel_before_shmem_exit(nonexclusive_base_backup_cleanup, (Datum) 0);
+
+ values[1] = CStringGetTextDatum(label_file->data);
+ values[2] = CStringGetTextDatum(tblspc_map_file->data);
+
+ /* Free structures allocated in TopMemoryContext */
+ pfree(label_file->data);
+ pfree(label_file);
+ label_file = NULL;
+ pfree(tblspc_map_file->data);
+ pfree(tblspc_map_file);
+ tblspc_map_file = NULL;
+ }
+
+ values[0] = LSNGetDatum(stoppoint);
+ tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+ tuplestore_donestoring(typstore);
+
+ return (Datum) 0;
+ }
+
/*
* pg_switch_xlog: switch to next xlog file
*/
*** a/src/backend/catalog/system_views.sql
--- b/src/backend/catalog/system_views.sql
***************
*** 896,902 **** COMMENT ON FUNCTION ts_debug(text) IS
--
CREATE OR REPLACE FUNCTION
! pg_start_backup(label text, fast boolean DEFAULT false)
RETURNS pg_lsn STRICT VOLATILE LANGUAGE internal AS 'pg_start_backup';
-- legacy definition for compatibility with 9.3
--- 896,902 ----
--
CREATE OR REPLACE FUNCTION
! pg_start_backup(label text, fast boolean DEFAULT false, exclusive boolean DEFAULT true)
RETURNS pg_lsn STRICT VOLATILE LANGUAGE internal AS 'pg_start_backup';
-- legacy definition for compatibility with 9.3
*** a/src/backend/replication/basebackup.c
--- b/src/backend/replication/basebackup.c
***************
*** 117,124 **** perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
TimeLineID starttli;
XLogRecPtr endptr;
TimeLineID endtli;
! char *labelfile;
! char *tblspc_map_file = NULL;
int datadirpathlen;
List *tablespaces = NIL;
--- 117,124 ----
TimeLineID starttli;
XLogRecPtr endptr;
TimeLineID endtli;
! StringInfo labelfile;
! StringInfo tblspc_map_file = NULL;
int datadirpathlen;
List *tablespaces = NIL;
***************
*** 126,134 **** perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
backup_started_in_recovery = RecoveryInProgress();
startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
! &labelfile, tblspcdir, &tablespaces,
! &tblspc_map_file,
opt->progress, opt->sendtblspcmapfile);
/*
--- 126,137 ----
backup_started_in_recovery = RecoveryInProgress();
+ labelfile = makeStringInfo();
+ tblspc_map_file = makeStringInfo();
+
startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
! labelfile, tblspcdir, &tablespaces,
! tblspc_map_file,
opt->progress, opt->sendtblspcmapfile);
/*
***************
*** 206,212 **** perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
struct stat statbuf;
/* In the main tar, include the backup_label first... */
! sendFileWithContent(BACKUP_LABEL_FILE, labelfile);
/*
* Send tablespace_map file if required and then the bulk of
--- 209,215 ----
struct stat statbuf;
/* In the main tar, include the backup_label first... */
! sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data);
/*
* Send tablespace_map file if required and then the bulk of
***************
*** 214,220 **** perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
*/
if (tblspc_map_file && opt->sendtblspcmapfile)
{
! sendFileWithContent(TABLESPACE_MAP, tblspc_map_file);
sendDir(".", 1, false, tablespaces, false);
}
else
--- 217,223 ----
*/
if (tblspc_map_file && opt->sendtblspcmapfile)
{
! sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
sendDir(".", 1, false, tablespaces, false);
}
else
***************
*** 247,253 **** perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
}
PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
! endptr = do_pg_stop_backup(labelfile, !opt->nowait, &endtli);
if (opt->includewal)
{
--- 250,256 ----
}
PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
! endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
if (opt->includewal)
{
*** a/src/include/access/xlog.h
--- b/src/include/access/xlog.h
***************
*** 275,282 **** extern void assign_checkpoint_completion_target(double newval, void *extra);
* Starting/stopping a base backup
*/
extern XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast,
! TimeLineID *starttli_p, char **labelfile, DIR *tblspcdir,
! List **tablespaces, char **tblspcmapfile, bool infotbssize,
bool needtblspcmapfile);
extern XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive,
TimeLineID *stoptli_p);
--- 275,282 ----
* Starting/stopping a base backup
*/
extern XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast,
! TimeLineID *starttli_p, StringInfo labelfile, DIR *tblspcdir,
! List **tablespaces, StringInfo tblspcmapfile, bool infotbssize,
bool needtblspcmapfile);
extern XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive,
TimeLineID *stoptli_p);
*** a/src/include/access/xlog_fn.h
--- b/src/include/access/xlog_fn.h
***************
*** 15,20 ****
--- 15,21 ----
extern Datum pg_start_backup(PG_FUNCTION_ARGS);
extern Datum pg_stop_backup(PG_FUNCTION_ARGS);
+ extern Datum pg_stop_backup2(PG_FUNCTION_ARGS);
extern Datum pg_switch_xlog(PG_FUNCTION_ARGS);
extern Datum pg_create_restore_point(PG_FUNCTION_ARGS);
extern Datum pg_current_xlog_location(PG_FUNCTION_ARGS);
*** a/src/include/catalog/pg_proc.h
--- b/src/include/catalog/pg_proc.h
***************
*** 3069,3078 **** DATA(insert OID = 2171 ( pg_cancel_backend PGNSP PGUID 12 1 0 0 0 f f f f t f v
DESCR("cancel a server process' current query");
DATA(insert OID = 2096 ( pg_terminate_backend PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 16 "23" _null_ _null_ _null_ _null_ _null_ pg_terminate_backend _null_ _null_ _null_ ));
DESCR("terminate a server process");
! DATA(insert OID = 2172 ( pg_start_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 2 0 3220 "25 16" _null_ _null_ _null_ _null_ _null_ pg_start_backup _null_ _null_ _null_ ));
DESCR("prepare for taking an online backup");
DATA(insert OID = 2173 ( pg_stop_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 3220 "" _null_ _null_ _null_ _null_ _null_ pg_stop_backup _null_ _null_ _null_ ));
DESCR("finish taking an online backup");
DATA(insert OID = 3813 ( pg_is_in_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 16 "" _null_ _null_ _null_ _null_ _null_ pg_is_in_backup _null_ _null_ _null_ ));
DESCR("true if server is in online backup");
DATA(insert OID = 3814 ( pg_backup_start_time PGNSP PGUID 12 1 0 0 0 f f f f t f s s 0 0 1184 "" _null_ _null_ _null_ _null_ _null_ pg_backup_start_time _null_ _null_ _null_ ));
--- 3069,3080 ----
DESCR("cancel a server process' current query");
DATA(insert OID = 2096 ( pg_terminate_backend PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 16 "23" _null_ _null_ _null_ _null_ _null_ pg_terminate_backend _null_ _null_ _null_ ));
DESCR("terminate a server process");
! DATA(insert OID = 2172 ( pg_start_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 3 0 3220 "25 16 16" _null_ _null_ _null_ _null_ _null_ pg_start_backup _null_ _null_ _null_ ));
DESCR("prepare for taking an online backup");
DATA(insert OID = 2173 ( pg_stop_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 3220 "" _null_ _null_ _null_ _null_ _null_ pg_stop_backup _null_ _null_ _null_ ));
DESCR("finish taking an online backup");
+ DATA(insert OID = 2561 ( pg_stop_backup PGNSP PGUID 12 1 1 0 0 f f f f t t v s 1 0 2249 "16" "{16,3220,25,25}" "{i,o,o,o}" "{exclusive,lsn,labelfile,spcmapfile}" _null_ _null_ pg_stop_backup2 _null_ _null_ _null_ ));
+ DESCR("finish taking an online backup");
DATA(insert OID = 3813 ( pg_is_in_backup PGNSP PGUID 12 1 0 0 0 f f f f t f v s 0 0 16 "" _null_ _null_ _null_ _null_ _null_ pg_is_in_backup _null_ _null_ _null_ ));
DESCR("true if server is in online backup");
DATA(insert OID = 3814 ( pg_backup_start_time PGNSP PGUID 12 1 0 0 0 f f f f t f s s 0 0 1184 "" _null_ _null_ _null_ _null_ _null_ pg_backup_start_time _null_ _null_ _null_ ));
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
