Hi, I tried to backport Simon Riggs patches for 7.4.x series to 7.3.x with addition of python implementation of pg_arch.
It can be of interest if you consider to continue using 7.3 series of postgresql. src/backend/access/transam/xlog.c: calls to "ereport" was replaced with call to "elog". src/backend/utils/misc/guc.c : format of "wal_archive" was converted to 7.3 style src/bin/Makefile: was removed Attached patch can be applied to 7.3.4 and 7.3.5 During this week I'll test it in our development enviroment. Best regards -- Igor Poteryaev
pg_arch.py
Description: application/python
*** src/backend/utils/misc/guc.c.orig 2004-04-27 23:08:24.000000000 +0100 --- src/backend/utils/misc/guc.c 2004-03-23 22:52:31.000000000 +0000 *************** *** 352,357 **** --- 352,363 ---- {"fsync", PGC_SIGHUP}, &enableFsync, true, NULL, NULL }, + + { + {"wal_archive", PGC_SUSET}, &XLogArchivePolicy, + true, NULL, NULL + }, + { {"zero_damaged_pages", PGC_SUSET}, &zero_damaged_pages, false, NULL, NULL *** src/backend/access/transam/xlog.c.orig 2004-04-27 22:53:35.000000000 +0100 --- src/backend/access/transam/xlog.c 2004-04-27 22:01:15.000000000 +0100 *************** *** 84,96 **** /* User-settable parameters */ int CheckPointSegments = 3; int XLOGbuffers = 8; int XLOG_DEBUG = 0; char *XLOG_sync_method = NULL; const char XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR; - char XLOG_archive_dir[MAXPGPATH]; /* null string means - * delete 'em */ /* * XLOGfileslop is used in the code as the allowed "fuzz" in the number of --- 84,95 ---- /* User-settable parameters */ + bool XLogArchivePolicy = false; int CheckPointSegments = 3; int XLOGbuffers = 8; int XLOG_DEBUG = 0; char *XLOG_sync_method = NULL; const char XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR; /* * XLOGfileslop is used in the code as the allowed "fuzz" in the number of *************** *** 397,402 **** --- 397,403 ---- /* File path names */ static char XLogDir[MAXPGPATH]; + static char RLogDir[MAXPGPATH]; static char ControlFilePath[MAXPGPATH]; /* *************** *** 438,443 **** --- 438,451 ---- static bool InRedo = false; + static bool XLogArchiveNotify(uint32 log, uint32 seg); + #define XLA_NOTIFY_OK true + #define XLA_NOTIFY_ERROR false + + static bool XLogArchiveBusy(char xlog[MAXPGPATH]); + #define XLA_BUSY true + #define XLA_ARCHIVE_OK false + #define XLA_BUSY_ERROR 2 static bool AdvanceXLInsertBuffer(void); static void XLogWrite(XLogwrtRqst WriteRqst); *************** *** 772,778 **** MyProc->logRec = RecPtr; } ! if (XLOG_DEBUG) { char buf[8192]; --- 780,786 ---- MyProc->logRec = RecPtr; } ! if (XLOG_DEBUG==16) { char buf[8192]; *************** *** 878,883 **** --- 886,1021 ---- } /* + * XLogArchive API calls + * + * Two calls implement the PostgreSQL side of the XLogArchive API + * XLogArchiveNotify + * XLogArchiveBusy + */ + + /* + * XLogArchiveNotify + * + * Writes an archive notification file to the RLogDir + * + * The name of the notification file is the message that will be picked up + * by the archiver, e.g. we write RLogDir/00000001000000C6.full + * and the archiver then knows to archive XLOgDir/00000001000000C6, + * while it is doing so it will rename RLogDir/00000001000000C6.full + * to RLogDir/00000001000000C6.busy, then when complete, rename it again + * to RLogDir/00000001000000C6.done + * + * Called only when in wal_archive mode + */ + static bool + XLogArchiveNotify(uint32 log, uint32 seg) + { + char rlog[32]; + char rlogpath[MAXPGPATH]; + FILE *rlogFD; + + if (XLOG_DEBUG) + elog(LOG, "XLogArchiveNotify writing notification for log file %u, segment %u", + log, seg); + + /* insert an otherwise empty file called <XLOG>.full */ + sprintf(rlog, "%08X%08X.full", log, seg); + snprintf(rlogpath, MAXPGPATH, "%s/%s", RLogDir, rlog); + rlogFD = AllocateFile(rlogpath, "w"); + if (rlogFD == NULL) + elog(ERROR, + "could not write XLogArchive API Notify file \"%s\" ", + rlogpath); + FreeFile(rlogFD); + /* the existence of this file is the message to the archiver to begin archiving */ + /* the file <XLOG> from the pg_xlog directory */ + + if (XLOG_DEBUG) + elog(LOG, "XLogArchiveNotify written: %s", rlogpath ); + + return XLA_NOTIFY_OK; + } + + /* + * XLogArchiveBusy + * + * Searches for an archive notification file in RLogDir + * + * Reads RLogDir looking for a specific filename. If that filename ends with .done + * then we know that the filename refers to an xlog in XLogDir that is safe to + * recycle. If the filename ends .full or .busy then thats OK, if neither then + * we have an error. + * + * Called only when in wal_archive mode + * + */ + static bool + XLogArchiveBusy(char xlog[32]) + { + char rlogpath[MAXPGPATH]; + FILE *rlogFD; + + if (XLOG_DEBUG) + elog(LOG, "XLogArchiveBusy check for log file %s", + xlog); + + /* If <XLOG>.done exists then return XLA_ARCHIVE_OK */ + snprintf(rlogpath, MAXPGPATH, "%s/%s.done", RLogDir, xlog); + rlogFD = AllocateFile(rlogpath, "r"); + if (rlogFD != NULL) { + FreeFile(rlogFD); + if (XLOG_DEBUG) + elog(LOG, "XLogArchiveBusy shows archiving done for log file %s", + xlog); + /* really ought to wait until after xlog recycled, in case there */ + /* is an error between the two activities -- do that later */ + unlink(rlogpath); + return XLA_ARCHIVE_OK; + } + else + { + /* else if <XLOG>.busy exists then return XLA_BUSY */ + snprintf(rlogpath, MAXPGPATH, "%s/%s.busy", RLogDir, xlog); + rlogFD = AllocateFile(rlogpath, "r"); + if (rlogFD != NULL) { + FreeFile(rlogFD); + elog(LOG, "XLogArchive shows archiving still busy for log file %s", + xlog); + /* Do something clever here to avoid systematic or one-off */ + /* time delays from effecting server stability */ + return XLA_BUSY; + } + else + { + /* + * else if <XLOG>.full exists then return XLA_BUSY and issue WARNING + * ...this indicates archiver is either not working at all or + * if it is, then its just way too slow or incorrectly configured + */ + snprintf(rlogpath, MAXPGPATH, "%s/%s.full", RLogDir, xlog); + rlogFD = AllocateFile(rlogpath, "r"); + if (rlogFD != NULL) { + FreeFile(rlogFD); + elog(WARNING, "XLogArchive shows archiving not yet started for log file %s", + xlog); + return XLA_BUSY; + } + else + { + /* else issue a WARNING.... a notification file SHOULD exist...unless #Bug2..the + * database has just been restored in which case it may be absent, so + * issue a WARNING, not an error, then return + */ + elog(WARNING, + "cannot find XLogArchive notification file: %s ", + rlogpath); + return XLA_ARCHIVE_OK; + } + } + } + } + + /* * Advance the Insert state to the next buffer page, writing out the next * buffer if it still contains unwritten data. * *************** *** 1126,1131 **** --- 1268,1278 ---- { issue_xlog_fsync(); LogwrtResult.Flush = LogwrtResult.Write; /* end of current page */ + + /* Notify xlog ready to archive?*/ + if (XLogArchivePolicy && !XLogArchiveNotify(openLogId, openLogSeg)) + elog(WARNING, "could not set notify for archiver to read log file %u, segment %u", + openLogId, openLogSeg); } if (ispartialpage) *************** *** 1204,1210 **** XLogRecPtr WriteRqstPtr; XLogwrtRqst WriteRqst; ! if (XLOG_DEBUG) { elog(LOG, "XLogFlush%s%s: request %X/%X; write %X/%X; flush %X/%X", (IsBootstrapProcessingMode()) ? "(bootstrap)" : "", --- 1351,1357 ---- XLogRecPtr WriteRqstPtr; XLogwrtRqst WriteRqst; ! if (XLOG_DEBUG==16) { elog(LOG, "XLogFlush%s%s: request %X/%X; write %X/%X; flush %X/%X", (IsBootstrapProcessingMode()) ? "(bootstrap)" : "", *************** *** 1607,1630 **** errno = 0; while ((xlde = readdir(xldir)) != NULL) { if (strlen(xlde->d_name) == 16 && strspn(xlde->d_name, "0123456789ABCDEF") == 16 && strcmp(xlde->d_name, lastoff) <= 0) { snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name); ! if (XLOG_archive_dir[0]) ! { ! elog(LOG, "archiving transaction log file %s", ! xlde->d_name); ! elog(WARNING, "archiving log files is not implemented!"); ! } ! else { /* * Before deleting the file, see if it can be recycled as * a future log segment. We allow recycling segments up ! * to XLOGfileslop segments beyond the current XLOG ! * location. */ if (InstallXLogFileSegment(endlogId, endlogSeg, path, true, XLOGfileslop, --- 1754,1779 ---- errno = 0; while ((xlde = readdir(xldir)) != NULL) { + /* if correct length and alphanumeric makeup of file looks correct */ + /* use the alphanumeric sorting property of the filenames to decide */ + /* which ones are earlier than the lastoff transaction log */ + /* ...maybe should read lastwrite datetime of lastoff, then check that */ + /* only files last written earlier than this are removed/recycled */ if (strlen(xlde->d_name) == 16 && strspn(xlde->d_name, "0123456789ABCDEF") == 16 && strcmp(xlde->d_name, lastoff) <= 0) { snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name); ! ! if ( !XLogArchivePolicy || ! (XLogArchivePolicy && !XLogArchiveBusy(xlde->d_name)) ! ) { /* * Before deleting the file, see if it can be recycled as * a future log segment. We allow recycling segments up ! * until there are XLOGfileslop segments beyond the ! * current XLOG location, otherwise they are removed. */ if (InstallXLogFileSegment(endlogId, endlogSeg, path, true, XLOGfileslop, *************** *** 1636,1645 **** else { /* No need for any more future segments... */ ! elog(LOG, "removing transaction log file %s", xlde->d_name); unlink(path); } } } errno = 0; --- 1784,1797 ---- else { /* No need for any more future segments... */ ! elog(LOG, "too many transaction log files, removing %s", xlde->d_name); unlink(path); } + /* + if (XLogArchivePolicy && !XLogArchiveBusy(xlde->d_name)) + XLogArchiveUnNotify(xlde->d_name); + */ } } errno = 0; *************** *** 2067,2072 **** --- 2219,2225 ---- { /* Init XLOG file paths */ snprintf(XLogDir, MAXPGPATH, "%s/pg_xlog", DataDir); + snprintf(RLogDir, MAXPGPATH, "%s/pg_rlog", DataDir); snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir); } *************** *** 2624,2630 **** ShmemVariableCache->nextXid = record->xl_xid; TransactionIdAdvance(ShmemVariableCache->nextXid); } ! if (XLOG_DEBUG) { char buf[8192]; --- 2777,2783 ---- ShmemVariableCache->nextXid = record->xl_xid; TransactionIdAdvance(ShmemVariableCache->nextXid); } ! if (XLOG_DEBUG==16) { char buf[8192]; *** src/include/access/xlog.h.orig 2004-04-27 23:05:29.000000000 +0100 --- src/include/access/xlog.h 2004-04-27 23:05:37.000000000 +0100 *************** *** 188,194 **** extern int XLOG_DEBUG; extern char *XLOG_sync_method; extern const char XLOG_sync_method_default[]; ! extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata); extern void XLogFlush(XLogRecPtr RecPtr); --- 188,194 ---- extern int XLOG_DEBUG; extern char *XLOG_sync_method; extern const char XLOG_sync_method_default[]; ! extern bool XLogArchivePolicy; extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata); extern void XLogFlush(XLogRecPtr RecPtr);
---------------------------(end of broadcast)--------------------------- TIP 6: Have you searched our list archives? http://archives.postgresql.org