I enclose a working set of context diff patches and new files to make
PITR archiving work, as of cvstip (NOW).
You'll see the new options in the postgresql.conf...though you may wish
to use archive_debug = true as well, when testing.
There is one bug: shutdown doesn't work quite right. I haven't fixed
this because I've spent too long trying to decipher how pgstat did a
clean shutdown, discovering now that it didn't and that has now been
patched...something similar is required for pgarch, but I'm out of time
now...leaving time for discussion of this lot...
I'm looking to have this lot committed asap, cos my fingers are starting
to catch the bitrot now. :)
I have a considerable amount still to learn about CVS, diff and patch,
so anybody wanting to spend 10-15 mins on the phone with me would
greatly enhance my chances of helping patch my patch, when the bugs roll
in.
If we get this done smoothly, I reckon I can have some PITR recovery
control done by beta freeze.
Best regards, Simon Riggs
? Makefile.global
? pitr_arch_v3.01.patch
? pitr_v3.1.patch
? backend/postgres
? backend/catalog/postgres.bki
? backend/catalog/postgres.description
? backend/postmaster/pgarch.c
? backend/postmaster/postmaster.c.new
? backend/utils/mb/conversion_procs/conversion_create.sql
? backend/utils/mb/conversion_procs/ascii_and_mic/libascii_and_mic.so.0.0
? backend/utils/mb/conversion_procs/cyrillic_and_mic/libcyrillic_and_mic.so.0.0
? backend/utils/mb/conversion_procs/euc_cn_and_mic/libeuc_cn_and_mic.so.0.0
? backend/utils/mb/conversion_procs/euc_jp_and_sjis/libeuc_jp_and_sjis.so.0.0
? backend/utils/mb/conversion_procs/euc_kr_and_mic/libeuc_kr_and_mic.so.0.0
? backend/utils/mb/conversion_procs/euc_tw_and_big5/libeuc_tw_and_big5.so.0.0
? backend/utils/mb/conversion_procs/latin2_and_win1250/liblatin2_and_win1250.so.0.0
? backend/utils/mb/conversion_procs/latin_and_mic/liblatin_and_mic.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_ascii/libutf8_and_ascii.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_big5/libutf8_and_big5.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_cyrillic/libutf8_and_cyrillic.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_euc_cn/libutf8_and_euc_cn.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_euc_jp/libutf8_and_euc_jp.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_euc_kr/libutf8_and_euc_kr.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_euc_tw/libutf8_and_euc_tw.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_gb18030/libutf8_and_gb18030.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_gbk/libutf8_and_gbk.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_iso8859/libutf8_and_iso8859.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_iso8859_1/libutf8_and_iso8859_1.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_johab/libutf8_and_johab.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_sjis/libutf8_and_sjis.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_tcvn/libutf8_and_tcvn.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_uhc/libutf8_and_uhc.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_win1250/libutf8_and_win1250.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_win1256/libutf8_and_win1256.so.0.0
? backend/utils/mb/conversion_procs/utf8_and_win874/libutf8_and_win874.so.0.0
? bin/initdb/initdb
? bin/initlocation/initlocation
? bin/ipcclean/ipcclean
? bin/pg_config/pg_config
? bin/pg_controldata/pg_controldata
? bin/pg_ctl/pg_ctl
? bin/pg_dump/pg_dump
? bin/pg_dump/pg_dumpall
? bin/pg_dump/pg_restore
? bin/pg_resetxlog/pg_resetxlog
? bin/psql/psql
? bin/scripts/clusterdb
? bin/scripts/createdb
? bin/scripts/createlang
? bin/scripts/createuser
? bin/scripts/dropdb
? bin/scripts/droplang
? bin/scripts/dropuser
? bin/scripts/vacuumdb
? include/pg_config.h
? include/pgarch.h
? include/stamp-h
? interfaces/ecpg/compatlib/libecpg_compat.so.1.1
? interfaces/ecpg/ecpglib/libecpg.so.4.2
? interfaces/ecpg/pgtypeslib/libpgtypes.so.1.2
? interfaces/ecpg/preproc/ecpg
? interfaces/libpq/libpq.so.3.2
? pl/plpgsql/src/libplpgsql.so.1.0
? port/pg_config_paths.h
? timezone/zic
Index: backend/access/nbtree/nbtsort.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/backend/access/nbtree/nbtsort.c,v
retrieving revision 1.82
diff -c -r1.82 nbtsort.c
*** backend/access/nbtree/nbtsort.c 2 Jun 2004 17:28:17 -0000 1.82
--- backend/access/nbtree/nbtsort.c 15 Jun 2004 15:19:48 -0000
***************
*** 67,72 ****
--- 67,73 ----
#include "miscadmin.h"
#include "storage/smgr.h"
#include "utils/tuplesort.h"
+ #include "access/xlog.h"
/*
***************
*** 220,235 ****
wstate.index = btspool->index;
/*
! * We need to log index creation in WAL iff WAL archiving is enabled
* AND it's not a temp index.
- *
- * XXX when WAL archiving is actually supported, this test will likely
- * need to change; and the hardwired extern is cruddy anyway ...
*/
{
! extern char XLOG_archive_dir[];
!
! wstate.btws_use_wal = XLOG_archive_dir[0] && !wstate.index->rd_istemp;
}
/* reserve the metapage */
wstate.btws_pages_alloced = BTREE_METAPAGE + 1;
--- 221,231 ----
wstate.index = btspool->index;
/*
! * We need to log index creation in WAL if WAL archiving is enabled
* AND it's not a temp index.
*/
{
! wstate.btws_use_wal = XLogArchiveMode && !wstate.index->rd_istemp;
}
/* reserve the metapage */
wstate.btws_pages_alloced = BTREE_METAPAGE + 1;
Index: backend/access/transam/xlog.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/backend/access/transam/xlog.c,v
retrieving revision 1.146
diff -c -r1.146 xlog.c
*** backend/access/transam/xlog.c 3 Jun 2004 02:08:00 -0000 1.146
--- backend/access/transam/xlog.c 15 Jun 2004 15:19:52 -0000
***************
*** 35,40 ****
--- 35,41 ----
#include "storage/proc.h"
#include "storage/sinval.h"
#include "storage/spin.h"
+ #include "storage/pmsignal.h"
#include "utils/builtins.h"
#include "utils/guc.h"
#include "utils/relcache.h"
***************
*** 84,95 ****
/* User-settable parameters */
int CheckPointSegments = 3;
int XLOGbuffers = 8;
char *XLOG_sync_method = NULL;
const char XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR;
- char XLOG_archive_dir[MAXPGPATH]; /* null string means
- * delete 'em */
#ifdef WAL_DEBUG
bool XLOG_DEBUG = false;
--- 85,98 ----
/* User-settable parameters */
+ bool XLogArchiveMode = false;
+ bool XLogArchiveDEBUG = false;
+ char *XLogArchiveDest;
+ char *XLogArchiveProgram;
int CheckPointSegments = 3;
int XLOGbuffers = 8;
char *XLOG_sync_method = NULL;
const char XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR;
#ifdef WAL_DEBUG
bool XLOG_DEBUG = false;
***************
*** 392,397 ****
--- 395,401 ----
/* File path names */
static char XLogDir[MAXPGPATH];
+ static char RLogDir[MAXPGPATH];
static char ControlFilePath[MAXPGPATH];
/*
***************
*** 433,438 ****
--- 437,444 ----
static bool InRedo = false;
+ static bool XLogArchiveNotify(uint32 log, uint32 seg);
+ static bool XLogArchiveDone(char xlog[MAXPGPATH]);
static bool AdvanceXLInsertBuffer(void);
static bool WasteXLInsertBuffer(void);
***************
*** 911,916 ****
--- 917,1041 ----
}
/*
+ * XLogArchive API calls
+ *
+ * Two calls implement the backend side of the XLogArchive API
+ * XLogArchiveNotify
+ * XLogArchiveDone
+ */
+
+ /*
+ * XLogArchiveNotify
+ *
+ * Writes an archive notification file to the RLogDir
+ *
+ * The name of the notification file is the message that will be picked up
+ * by the archiver, e.g. we write RLogDir/00000001000000C6.ready
+ * and the archiver then knows to archive XLogDir/00000001000000C6,
+ * then when complete, rename it to RLogDir/00000001000000C6.done
+ *
+ * Called only when in XLogArchiveMode by one backend process
+ */
+ static bool
+ XLogArchiveNotify(uint32 log, uint32 seg)
+ {
+ char rlog[32];
+ char rlogpath[MAXPGPATH];
+ FILE *rlogFD;
+
+ /* insert an otherwise empty file called <XLOG>.ready */
+ sprintf(rlog, "%08X%08X.ready", log, seg);
+ snprintf(rlogpath, MAXPGPATH, "%s/%s", RLogDir, rlog);
+ rlogFD = AllocateFile(rlogpath, "w");
+ if (rlogFD == NULL)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not write archive_status file \"%s\" ",
+ rlogpath)));
+ FreeFile(rlogFD);
+
+ /* the existence of this file is the message to the archiver to identify
+ * which files require archiving
+ *
+ * if this file is written OK, we then signal the ARCHIVER to do its thang
+ */
+
+ if (XLogArchiveDEBUG)
+ elog(LOG, "backend: written %s", rlogpath );
+
+ SendPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER);
+
+ return true;
+ }
+
+ /*
+ * XLogArchiveDone
+ *
+ * Searches for an archive notification file in RLogDir
+ *
+ * Reads RLogDir looking for a specific filename. If that filename ends with .done
+ * then we know that the filename refers to an xlog in XLogDir that is safe to
+ * recycle. If the filename ends .ready then thats OK, else we have an error.
+ *
+ * Called only when in XLogArchiveMode by bgwriter (when performing checkpoint)
+ *
+ */
+ static bool
+ XLogArchiveDone(char xlog[32])
+ {
+ char rlogpath[MAXPGPATH];
+ FILE *rlogFD;
+
+ if (XLogArchiveDEBUG)
+ elog(LOG, "chkpt: checking for log file %s",
+ xlog);
+
+ /* If <XLOG>.done exists then return XLA_ARCHIVE_OK */
+ snprintf(rlogpath, MAXPGPATH, "%s/%s.done", RLogDir, xlog);
+ rlogFD = AllocateFile(rlogpath, "r");
+ if (rlogFD != NULL) {
+ FreeFile(rlogFD);
+ if (XLogArchiveDEBUG)
+ elog(LOG, "chkpt: archiving done for log file %s",
+ xlog);
+
+ /* really ought to wait until after xlog recycled, in case there
+ * is an error between the two activities -- do that later
+ */
+ unlink(rlogpath);
+ return true;
+ }
+ else
+ {
+ /*
+ * else if <XLOG>.ready exists then return false and issue WARNING
+ * ...this indicates archiver is either not working at all or
+ * if it is, then its just way too slow or incorrectly configured
+ */
+ snprintf(rlogpath, MAXPGPATH, "%s/%s.ready", RLogDir, xlog);
+ rlogFD = AllocateFile(rlogpath, "r");
+ if (rlogFD != NULL) {
+ FreeFile(rlogFD);
+ elog(WARNING, "chkpt: archiving not yet started for log file %s",
+ xlog);
+ return false;
+ }
+ else
+ {
+ /* else issue a WARNING.... a notification file SHOULD exist...unless the
+ * database has just been restored in which case it may be absent, so
+ * issue a WARNING, not an error, then return
+ */
+ ereport(WARNING,
+ (errcode_for_file_access(),
+ errmsg("chkpt: cannot find archive_status file: %s ",
+ rlogpath)));
+ return false;
+ }
+ }
+ }
+
+ /*
* Advance the Insert state to the next buffer page, writing out the next
* buffer if it still contains unwritten data.
*
***************
*** 1259,1264 ****
--- 1384,1397 ----
{
issue_xlog_fsync();
LogwrtResult.Flush = LogwrtResult.Write; /* end of current page */
+
+ /*
+ * Notify xlog ready to archive?
+ */
+ if (XLogArchiveMode && !XLogArchiveNotify(openLogId, openLogSeg))
+ elog(WARNING, "could not set notify for archiver to read log file %u, segment %u",
+ openLogId, openLogSeg);
+
}
if (ispartialpage)
***************
*** 1761,1785 ****
errno = 0;
while ((xlde = readdir(xldir)) != NULL)
{
if (strlen(xlde->d_name) == 16 &&
strspn(xlde->d_name, "0123456789ABCDEF") == 16 &&
strcmp(xlde->d_name, lastoff) <= 0)
{
snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name);
! if (XLOG_archive_dir[0])
! {
! ereport(LOG,
! (errmsg("archiving transaction log file \"%s\"",
! xlde->d_name)));
! elog(WARNING, "archiving log files is not implemented");
! }
! else
{
/*
* Before deleting the file, see if it can be recycled as
* a future log segment. We allow recycling segments up
! * to XLOGfileslop segments beyond the current XLOG
! * location.
*/
if (InstallXLogFileSegment(endlogId, endlogSeg, path,
true, XLOGfileslop,
--- 1894,1919 ----
errno = 0;
while ((xlde = readdir(xldir)) != NULL)
{
+ /* if correct length and alphanumeric makeup of file looks correct
+ * use the alphanumeric sorting property of the filenames to decide
+ * which ones are earlier than the lastoff transaction log
+ * ...maybe should read lastwrite datetime of lastoff, then check that
+ * only files last written earlier than this are removed/recycled
+ */
if (strlen(xlde->d_name) == 16 &&
strspn(xlde->d_name, "0123456789ABCDEF") == 16 &&
strcmp(xlde->d_name, lastoff) <= 0)
{
snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name);
! if ( !XLogArchiveMode ||
! (XLogArchiveMode && XLogArchiveDone(xlde->d_name))
! )
{
/*
* Before deleting the file, see if it can be recycled as
* a future log segment. We allow recycling segments up
! * until there are XLOGfileslop segments beyond the
! * current XLOG location, otherwise they are removed.
*/
if (InstallXLogFileSegment(endlogId, endlogSeg, path,
true, XLOGfileslop,
***************
*** 1793,1799 ****
{
/* No need for any more future segments... */
ereport(LOG,
! (errmsg("removing transaction log file \"%s\"",
xlde->d_name)));
unlink(path);
}
--- 1927,1933 ----
{
/* No need for any more future segments... */
ereport(LOG,
! (errmsg("too many transaction log files, removing \"%s\"",
xlde->d_name)));
unlink(path);
}
***************
*** 2254,2259 ****
--- 2388,2394 ----
{
/* Init XLOG file paths */
snprintf(XLogDir, MAXPGPATH, "%s/pg_xlog", DataDir);
+ snprintf(RLogDir, MAXPGPATH, "%s/archive_status", XLogDir);
snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir);
}
Index: backend/postmaster/Makefile
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/backend/postmaster/Makefile,v
retrieving revision 1.15
diff -c -r1.15 Makefile
*** backend/postmaster/Makefile 29 May 2004 22:48:19 -0000 1.15
--- backend/postmaster/Makefile 15 Jun 2004 15:19:53 -0000
***************
*** 12,18 ****
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
! OBJS = postmaster.o bgwriter.o pgstat.o
all: SUBSYS.o
--- 12,18 ----
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
! OBJS = postmaster.o bgwriter.o pgstat.o pgarch.o
all: SUBSYS.o
Index: backend/postmaster/postmaster.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/backend/postmaster/postmaster.c,v
retrieving revision 1.404
diff -c -r1.404 postmaster.c
*** backend/postmaster/postmaster.c 14 Jun 2004 18:08:19 -0000 1.404
--- backend/postmaster/postmaster.c 15 Jun 2004 15:19:58 -0000
***************
*** 111,117 ****
#include "utils/ps_status.h"
#include "bootstrap/bootstrap.h"
#include "pgstat.h"
!
/*
* List of active backends (or child processes anyway; we don't actually
--- 111,117 ----
#include "utils/ps_status.h"
#include "bootstrap/bootstrap.h"
#include "pgstat.h"
! #include "pgarch.h"
/*
* List of active backends (or child processes anyway; we don't actually
***************
*** 812,817 ****
--- 812,824 ----
*/
whereToSendOutput = None;
+ /*
+ * Initialize and try to startup the archiver process
+ */
+ if (XLogArchiveMode) {
+ pgarch_start();
+ }
+
/*
* Initialize the statistics collector stuff
*/
***************
*** 1148,1153 ****
--- 1155,1164 ----
kill(BgWriterPID, SIGUSR2);
}
+ /* If we have lost the archiver, try to start a new one */
+ if (Shutdown == NoShutdown && XLogArchiveMode && !pgarch_is_running)
+ pgarch_start();
+
/* If we have lost the stats collector, try to start a new one */
if (PgStatPID == 0 &&
StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
***************
*** 1869,1874 ****
--- 1880,1898 ----
#endif /* WIN32 */
#endif /* HAVE_WAITPID */
+ /*
+ * Check if this child was the archiver. If so, try to
+ * start a new one. (If fail, we'll try again in future cycles of
+ * the main loop.)
+ */
+ if (XLogArchiveMode && pgarch_ispgarch(pid))
+ {
+ LogChildExit(LOG, gettext("archiver process"),
+ pid, exitstatus);
+ pgarch_start();
+ continue;
+ }
+
/*
* Check if this child was a startup process.
*/
***************
*** 2882,2887 ****
--- 2906,2925 ----
if (Shutdown <= SmartShutdown)
SignalChildren(SIGUSR1);
}
+
+ if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER))
+ {
+ /*
+ * Send SIGUSR1 to ARCHIVER process, to wake it up and begin
+ * archiving next transaction log file. Backend should only
+ * send if in XLogArchiveMode...
+ */
+ if (XLogArchiveMode && Shutdown == NoShutdown) {
+ if (XLogArchiveDEBUG)
+ elog(LOG, "postmaster: WAKEN_ARCHIVER received, sending SIGUSR1 to archiver");
+ SendArchiverSignal();
+ }
+ }
PG_SETMASK(&UnBlockSig);
Index: backend/utils/misc/guc.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/backend/utils/misc/guc.c,v
retrieving revision 1.211
diff -c -r1.211 guc.c
*** backend/utils/misc/guc.c 11 Jun 2004 03:54:54 -0000 1.211
--- backend/utils/misc/guc.c 15 Jun 2004 15:20:16 -0000
***************
*** 371,376 ****
--- 371,392 ----
static struct config_bool ConfigureNamesBool[] =
{
+ {
+ {"archive_mode", PGC_POSTMASTER, WAL_SETTINGS,
+ gettext_noop("Enable archiving of full transaction log files to a specified archival destination."),
+ NULL
+ },
+ &XLogArchiveMode,
+ false, NULL, NULL
+ },
+ {
+ {"archive_debug", PGC_SIGHUP, WAL_SETTINGS,
+ gettext_noop("Provide debug output for archive activities."),
+ NULL
+ },
+ &XLogArchiveDEBUG,
+ false, NULL, NULL
+ },
{
{"enable_seqscan", PGC_USERSET, QUERY_TUNING_METHOD,
gettext_noop("Enables the planner's use of sequential-scan plans."),
***************
*** 1400,1405 ****
--- 1416,1439 ----
static struct config_string ConfigureNamesString[] =
{
+ {
+ {"archive_dest", PGC_POSTMASTER, WAL_SETTINGS,
+ gettext_noop("Specifies where to archive WAL logs."),
+ gettext_noop("A directory or specific location for archiving transation log files from PostgreSQL")
+ },
+ &XLogArchiveDest,
+ "", NULL, NULL
+ },
+
+ {
+ {"archive_program", PGC_POSTMASTER, WAL_SETTINGS,
+ gettext_noop("Archive program"),
+ gettext_noop("The external program that will be called to execute the archival process")
+ },
+ &XLogArchiveProgram,
+ "", NULL, NULL
+ },
+
{
{"client_encoding", PGC_USERSET, CLIENT_CONN_LOCALE,
gettext_noop("Sets the client's character set encoding."),
Index: backend/utils/misc/postgresql.conf.sample
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/backend/utils/misc/postgresql.conf.sample,v
retrieving revision 1.113
diff -c -r1.113 postgresql.conf.sample
*** backend/utils/misc/postgresql.conf.sample 7 Apr 2004 05:05:50 -0000 1.113
--- backend/utils/misc/postgresql.conf.sample 15 Jun 2004 15:20:16 -0000
***************
*** 103,108 ****
--- 103,119 ----
#---------------------------------------------------------------------------
+ # ARCHIVING
+ #---------------------------------------------------------------------------
+
+ # - Settings -
+
+ #archive_mode = true # enables archiving of full txn log files
+ #archive_dest = '/tmp' # specifies destination of archive files
+ #archive_program = 'cp %s %s' # external archiving program command line
+
+
+ #---------------------------------------------------------------------------
# QUERY TUNING
#---------------------------------------------------------------------------
Index: bin/initdb/initdb.c
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/bin/initdb/initdb.c,v
retrieving revision 1.37
diff -c -r1.37 initdb.c
*** bin/initdb/initdb.c 10 Jun 2004 22:26:20 -0000 1.37
--- bin/initdb/initdb.c 15 Jun 2004 15:20:17 -0000
***************
*** 1785,1791 ****
char *pgdenv; /* PGDATA value got from sent to
* environment */
char *subdirs[] =
! {"global", "pg_xlog", "pg_clog", "base", "base/1"};
progname = get_progname(argv[0]);
set_pglocale_pgservice(argv[0], "initdb");
--- 1785,1791 ----
char *pgdenv; /* PGDATA value got from sent to
* environment */
char *subdirs[] =
! {"global", "pg_xlog", "pg_xlog/archive_status", "pg_clog", "base", "base/1"};
progname = get_progname(argv[0]);
set_pglocale_pgservice(argv[0], "initdb");
Index: include/access/xlog.h
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/include/access/xlog.h,v
retrieving revision 1.51
diff -c -r1.51 xlog.h
*** include/access/xlog.h 29 May 2004 22:48:22 -0000 1.51
--- include/access/xlog.h 15 Jun 2004 15:20:17 -0000
***************
*** 210,215 ****
--- 210,219 ----
extern int XLOGbuffers;
extern char *XLOG_sync_method;
extern const char XLOG_sync_method_default[];
+ extern bool XLogArchiveMode;
+ extern bool XLogArchiveDEBUG;
+ extern char *XLogArchiveDest;
+ extern char *XLogArchiveProgram;
#ifdef WAL_DEBUG
extern bool XLOG_DEBUG;
Index: include/storage/pmsignal.h
===================================================================
RCS file: /projects/cvsroot/pgsql-server/src/include/storage/pmsignal.h,v
retrieving revision 1.8
diff -c -r1.8 pmsignal.h
*** include/storage/pmsignal.h 29 May 2004 22:48:23 -0000 1.8
--- include/storage/pmsignal.h 15 Jun 2004 15:20:18 -0000
***************
*** 24,29 ****
--- 24,30 ----
{
PMSIGNAL_PASSWORD_CHANGE, /* pg_pwd file has changed */
PMSIGNAL_WAKEN_CHILDREN, /* send a SIGUSR1 signal to all backends */
+ PMSIGNAL_WAKEN_ARCHIVER, /* send a NOTIFY signal to ARCHIVER */
NUM_PMSIGNALS /* Must be last value of enum! */
} PMSignalReason;
/* ----------
* pgarch.h
*
* Definitions for the PostgreSQL archiver daemon.
*
* ----------
*/
/* ----------
* Timer definitions.
* ----------
*/
#define PGARCH_AUTOWAKE_INTERVAL 600 /* How often to wake and poll */
#define PGARCH_RESTART_INTERVAL 60 /* How often to attempt to restart */
/* a failed statistics collector; in seconds. */
#define NUM_ARCHIVE_RETRIES 3
/* ----------
* Other global variables
* ----------
*/
extern bool pgarch_is_running;
/* ----------
* Functions called from postmaster
* ----------
*/
extern void pgarch_start(void);
extern bool pgarch_ispgarch(int pid);
extern void SendArchiverSignal(void);
/* ----------
* pgarch.c
*
* PostgreSQL transaction log archiver
*
* All functions relating to archiver are included here
*
* - All functions executed by archiver process
*
* - Postmaster is forked from postmaster, and the two
* processes then communicate using signals. All functions
* executed by postmaster are included in this file.
*
* Simon Riggs [EMAIL PROTECTED]
*
* ----------
*/
#include "postgres.h"
#include <unistd.h>
#include <fcntl.h>
#include <sys/param.h>
#include <sys/time.h>
#include <sys/types.h>
#include <errno.h>
#include <signal.h>
#include <time.h>
#include "pgarch.h"
#include "storage/fd.h"
#include "miscadmin.h"
#include "access/xlog.h"
#include "libpq/pqsignal.h"
#include "storage/ipc.h"
#include "postmaster/postmaster.h"
#include "storage/pg_shmem.h"
#include "storage/pmsignal.h"
#include "utils/ps_status.h"
/* ----------
* GUC parameters
* ----------
*/
bool XLogArchiveMode;
bool XLogArchiveDEBUG;
char *XLogArchiveDest;
char *XLogArchiveProgram;
/* ----------
* Other global variables
* ----------
*/
bool pgarch_is_running = false;
/* ----------
* Local data
* ----------
*/
static char XLogDir[MAXPGPATH];
static char XLogArchiveStatusDir[MAXPGPATH];
static int pgarchPid;
static time_t last_pgarch_start_time;
static time_t last_pgarch_archivercopy_time;
static time_t curtime;
static bool archiving_in_progress = false;
static bool shutdowncalled = false;
/* ----------
* Local function forward declarations
* ----------
*/
NON_EXEC_STATIC void pgarch_Main(int argc, char *argv[]);
static void pgarch_die(SIGNAL_ARGS);
static void pgarch_shutdown(SIGNAL_ARGS);
static void pgarch_waken(SIGNAL_ARGS);
static void pgarch_MainWaitLoop(void);
static void pgarch_ArchiverCopyLoop(void);
static bool pgarch_archiveXlog(char *xlog);
static bool pgarch_readyXlog(char *xlog);
static bool pgarch_archiveDone(char *xlog);
/* ------------------------------------------------------------
* Public functions called from postmaster follow
* ------------------------------------------------------------
*/
/* ----------
* pgarch_start() -
*
* Called from postmaster at startup
* or after the previous archiver diedexisting collector died.
*
* Note: if fail, we will be called again from the postmaster main loop.
* ----------
*/
void
pgarch_start(void)
{
/*
* Do nothing if no archiver needed
*/
if (pgarch_is_running)
return;
/*
* Do nothing if too soon since last archiver start. This is a
* safety valve to protect against continuous respawn attempts if the
* archiver is dying immediately at launch. Note that since we will
* be re-called from the postmaster main loop, we will get another
* chance later.
*/
curtime = time(NULL);
if ((unsigned int) (curtime - last_pgarch_start_time) <
(unsigned int) PGARCH_RESTART_INTERVAL) {
return;
}
last_pgarch_start_time = curtime;
/*
* Okay, fork off the collector. Remember its PID for
* pgarch_ispgarch.
*/
fflush(stdout);
fflush(stderr);
#ifdef __BEOS__
/* Specific beos actions before backend startup */
beos_before_backend_startup();
#endif
#ifdef EXEC_BACKEND
switch ((pgarchPid = (int) pgarch_forkexec(STAT_PROC_BUFFER)))
#else
switch ((pgarchPid = (int) fork()))
#endif
{
case -1:
#ifdef __BEOS__
/* Specific beos actions */
beos_backend_startup_failed();
#endif
ereport(PANIC,
(errmsg("could not fork archiver")));
return;
#ifndef EXEC_BACKEND
case 0:
/* in postmaster child ... */
#ifdef __BEOS__
/* Specific beos actions after backend startup */
beos_backend_startup();
#endif
/* Close the postmaster's sockets */
ClosePostmasterPorts();
/* Drop our connection to postmaster's shared memory, as well */
PGSharedMemoryDetach();
pgarch_Main(0, NULL);
break;
#endif
default:
pgarch_is_running = true;
return;
}
}
/* ----------
* pgarch_ispgarch() -
*
* Called from postmaster to check if the terminated child process
* was the archiver
* ----------
*/
bool
pgarch_ispgarch(int pid)
{
if (!pgarch_is_running)
return false;
if (pgarchPid != pid)
return false;
/* Oh dear ... */
pgarch_is_running = false;
return true;
}
/* ----------
* SendArchiverSignal()
*
* Called from postmaster: Send SIGUSR1 to ARCHIVER process,
* to wake it up and begin archiving next transaction log file.
*
* ----------
*/
void
SendArchiverSignal(void)
{
kill(pgarchPid,SIGUSR1);
return;
}
/* ------------------------------------------------------------
* Local functions called by archiver follow
* ------------------------------------------------------------
*/
/* ----------
* pgarch_Main() -
*
*
* The argc/argv parameters are valid only in EXEC_BACKEND case.
* ----------
*/
NON_EXEC_STATIC void
pgarch_Main(int argc, char *argv[])
{
char testxlog[32] = "0000TEST0000TEST";
char testpath[MAXPGPATH];
FILE *rlogFD;
IsUnderPostmaster = true; /* we are a postmaster subprocess now */
MyProcPid = getpid(); /* reset MyProcPid */
/* Lose the postmaster's on-exit routines */
on_exit_reset();
/* Init XLOG file paths */
snprintf(XLogDir, MAXPGPATH, "%s/pg_xlog", DataDir);
snprintf(XLogArchiveStatusDir, MAXPGPATH, "%s/archive_status", XLogDir);
/*
* Test whether archive_program and archive_dest have been set
* correctly in postgresql.conf. This test does *not* look at
* archive_status, since it is a direct test of archival
*/
snprintf(testpath, MAXPGPATH, "%s/%s", XLogDir, testxlog);
unlink(testpath);
rlogFD = AllocateFile(testpath, "w");
if (rlogFD == NULL)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not write test file to pg_xlog \"%s\" ",
testpath)));
FreeFile(rlogFD);
if (!pgarch_archiveXlog(testxlog))
ereport(PANIC,
(errmsg("archive_program test failed...please correct this")));
unlink(testpath);
if (XLogArchiveDEBUG)
elog(LOG, "arch: archive_program test success");
/*
* Ignore all signals usually bound to some action in the postmaster,
*/
pqsignal(SIGHUP, SIG_IGN);
pqsignal(SIGTERM, pgarch_shutdown); /* smart shutdown */
pqsignal(SIGINT, pgarch_shutdown); /* fast shutdown */
pqsignal(SIGQUIT, pgarch_die); /* immediate shutdown */
pqsignal(SIGALRM, SIG_IGN);
pqsignal(SIGPIPE, SIG_IGN);
pqsignal(SIGUSR1, pgarch_waken);
pqsignal(SIGUSR2, SIG_IGN);
pqsignal(SIGCHLD, SIG_IGN);
pqsignal(SIGTTIN, SIG_DFL);
pqsignal(SIGTTOU, SIG_DFL);
pqsignal(SIGCONT, SIG_DFL);
pqsignal(SIGWINCH, SIG_DFL);
PG_SETMASK(&UnBlockSig);
#ifdef EXEC_BACKEND
pgstat_parseArgs(argc,argv);
#endif
/*
* Identify via ps
*/
init_ps_display("archiver process", "", "");
set_ps_display("");
elog(LOG, "archiver started");
/*
* When first started, check for outstanding archive files
* which may be present if we did a (normal) fast shutdown
* or if archiver died with some form of error
*/
pgarch_ArchiverCopyLoop();
pgarch_MainWaitLoop();
exit(0);
}
static void
pgarch_die(SIGNAL_ARGS)
{
PG_SETMASK(&BlockSig);
shutdowncalled = true;
exit(1);
}
static void
pgarch_shutdown(SIGNAL_ARGS)
{
shutdowncalled = true;
/*
* we do not block signals here, to allow a later,
* upgraded (fast/immediate) request for shutdown to
* override this behaviour
*/
if (archiving_in_progress) {
/* complete archiving, then exit
*/
return;
}
else {
/* This is a Smart or Fast Shutdown,
* so try archiving one last time. This is consistent
* with behaviour of a fast shutdown, in that we still
* write a shutdown checkpoint and try to recycle the
* log files - so before we do this, we try to archive
* away the last few .ready xlogs
*
* possible TODO: at this point
* archive partial xlog file, even though not full
* though we'd have to remember which one was next to
* do that
*/
pgarch_ArchiverCopyLoop();
return;
}
}
static void
pgarch_waken(SIGNAL_ARGS)
{
if (XLogArchiveDEBUG)
elog(LOG, "arch: archiver woken by SIGUSR1");
pgarch_ArchiverCopyLoop();
return;
}
/* ----------
* pgarch_MainWaitLoop() -
*
* Main wait loop for archiver
* ----------
*/
static void
pgarch_MainWaitLoop(void)
{
/*
* There shouldn't be anything for the archiver to do except
* to wait, so we could use pause(3) here...
* ...however, the archiver exists to protect our data, so
* she wakes up occaisionally to allow herself to be proactive.
* This shouldn't be required, but our data is important
* and this won't hurt to be cautious
*/
do {
/*
* Sleep for a while, hoping to be interrupted by signal
* if no signal, then check anyway....just to be sure
*/
sleep(PGARCH_AUTOWAKE_INTERVAL);
curtime = time(NULL);
if ((unsigned int) (curtime - last_pgarch_archivercopy_time) >=
(unsigned int) PGARCH_AUTOWAKE_INTERVAL) {
pgarch_ArchiverCopyLoop();
}
last_pgarch_archivercopy_time = curtime;
} while (PostmasterIsAlive(true));
return;
}
/* ----------
* pgarch_ArchiverCopyLoop() -
*
* Archives all outstanding xlogs then exits
* ----------
*/
static void
pgarch_ArchiverCopyLoop(void)
{
char xlog[32];
int try = 1;
/*
* We continue to trap for all signals, except for the
* one that brought us here in the first place. We
* loop through all transaction log files that require
* archiving, so no need to be interrupted to continue
* that task
*/
pqsignal(SIGUSR1, SIG_IGN);
archiving_in_progress = true;
/*
* loop through all xlogs with archive_status of .ready
* then archive them...mostly we expect this to be a single
* file, though not just a simple loop because we may add new
* files onto the list of those that need archiving while we
* are still copying earlier archives
*/
if (XLogArchiveDEBUG) {
elog(LOG, "arch: starting archive loop...");
}
while (pgarch_readyXlog(xlog) && try <= NUM_ARCHIVE_RETRIES)
{
if (pgarch_archiveXlog(xlog)) {
/*
* then update archive_status to show completion
*/
if (!pgarch_archiveDone(xlog)) {
ereport(LOG,(errmsg("arch: archive_status completion error")));
}
} else {
ereport(LOG,(errmsg("arch: archive copy error")));
}
/* if we have copied one file, we do not wait:
immediately loop back round and check to see if another is there.
Hopefully, we're quick enough....so we fall out and sleep again
*/
try++;
}
/*
* Reset & return OR exit
*/
if (shutdowncalled) {
exit(1);
}
else {
archiving_in_progress = false;
pqsignal(SIGUSR1, pgarch_waken);
}
return;
}
/*
* pgarch_archiveXlog
*
* Invokes system(3) to copy one archive file to XLogArchiveDest
* We assume xlog is a correct filename and that both
* XLogArchiveProgram and XLogArchiveDest are set correctly
*/
static bool
pgarch_archiveXlog(char *xlog)
{
char xlogarchcmd[MAXPGPATH];
char xlogpath[MAXPGPATH];
int rc;
snprintf(xlogpath, MAXPGPATH, "%s/%s", XLogDir,xlog);
/*
* set the string for the program and its parameters
* XLogArchiveProgram should contain 2 positional parameters
* xlog must be a full path to xlog
*/
snprintf(xlogarchcmd, MAXPGPATH, XLogArchiveProgram, xlogpath, XLogArchiveDest);
if (XLogArchiveDEBUG)
elog(LOG, "arch: system(%s)", xlogarchcmd);
rc = system(xlogarchcmd);
if (rc != 0 ) {
elog(LOG, "arch: system rc=%i", rc);
return false;
}
if (XLogArchiveDEBUG)
elog(LOG, "arch: archive success: %s", xlog);
return true;
}
/*
* XLogArchiveXlogs
*
* Return name of the oldest xlog file that has not yet been archived.
* No notification is set that file archiving is now in progress, [so
* this would need to be extended if multiple concurrent archival
* tasks were created]. If a failure occurs, we would completely
* re-copy the file at the next available opportunity.
*
* It is important that we return the oldest, so that we archive xlogs
* in order that they were written, for two reasons:
* 1) to maintain the sequential chain of xlogs required for recovery
* 2) because the oldest ones will sooner become candidates for
* recycling at time of checkpoint
*
*/
static bool
pgarch_readyXlog(char *xlog)
{
/*
* open XLogArchive directory and read through list of
* rlogs that have the .ready suffix, looking for earliest file.
* It is possible to optimise this code, though only a single
* file is expected on the vast majority of calls, so....
*/
char newxlog[32];
char emptystr[32] = "\0";
DIR *rldir;
struct dirent *rlde;
bool firstfile;
rldir = AllocateDir(XLogArchiveStatusDir);
if (rldir == NULL)
elog(PANIC, "cannot access archive_status");
firstfile = true;
while ((rlde = readdir(rldir)) != NULL)
{
if (strlen(rlde->d_name) == 22 &&
strspn(rlde->d_name, "0123456789ABCDEF") == 16 &&
strcmp(rlde->d_name + 16, ".ready") == 0)
{
if (firstfile) {
strcpy(newxlog, rlde->d_name);
firstfile = false;
} else {
if (strcmp(rlde->d_name, newxlog) <= 0)
strcpy(newxlog, rlde->d_name);
}
}
}
FreeDir(rldir);
if (firstfile) {
return false;
}
else {
if (XLogArchiveDEBUG)
elog(LOG, "arch: found archive_status file...%s", newxlog);
strcpy(xlog, emptystr);
strncat(xlog, newxlog, 16);
return true;
}
}
/*
* pgarch_archiveDone
*
* Write notification that an xlog has now been successfully archived
*/
static bool
pgarch_archiveDone(char *xlog)
{
char rlogready[MAXPGPATH];
char rlogdone[MAXPGPATH];
int rc;
snprintf(rlogready, MAXPGPATH, "%s/%s.ready", XLogArchiveStatusDir, xlog);
snprintf(rlogdone, MAXPGPATH, "%s/%s.done", XLogArchiveStatusDir, xlog);
rc = rename(rlogready, rlogdone);
if (rc < 0) {
ereport(WARNING,(errcode_for_file_access(),
errmsg("could not update archive_status for %s",
rlogready)));
return false;
}
return true;
}
---------------------------(end of broadcast)---------------------------
TIP 3: if posting/reading through Usenet, please send an appropriate
subscribe-nomail command to [EMAIL PROTECTED] so that your
message can get through to the mailing list cleanly
