Full set of patches, including both earlier reported bugs fixed.

This note was originally posted at Tue, 27 Apr 2004 23:30:39 +0100, but
for some reason hasn't yet appeared on the [HACKERS] list, so I am now
reposting this. Trying PATCHES list now...

Full usage instructions (for TESTING only!)

Patches:
guc.c.patch goes to src/backend/utils/misc/guc.c
xlog.c.patch goes to src/backend/access/transam/xlog.c
xlog.h.patch goes to src/include/access/xlog.h
Makefile.patch goes to src/bin/Makefile
pgarch.tar adds directory and files for src/bin/pg_arch 

Build:
pg_arch builds as part of the full source tree...

General Execution
1. create pg_rlog directory beneath PGDATA
2. add (optional) line to postgresql.conf
wal_debug=1
3. add (required) line to postgresql.conf
wal_archive=true
4. execute pg_arch /your-archive-dir $PGDATA
5. startup new postmaster
6. execute workload of choice to force the creation of xlogs
7. watch the execution...

Recovery Test:
1. Work out ways of testing the logical validity of data
2. While postmaster is UP: take a full physical backup of PGDATA
3. wait for some time while transaction archives get posted
4. perform your choice of destructive action on postgresql
5. mv backup copy to a restore location, possibly where it just was
6. mv ALL transaction logs taken SINCE full backup
7. startup postmaster and watch
8. execute tests from 1, to ensure recovery successful

...my patch building experience is less than some might
expect so there are various possible annoyances here. I am on hand to
help and to learn by my mistakes.

Report bugs to me at [EMAIL PROTECTED] and/or to the [HACKERS] list
[EMAIL PROTECTED]

Thanks,

Best Regards

Simon Riggs
2nd Quadrant
http://www.2ndquadrant.com

Attachment: pgarch.tar
Description: Unix tar archive

*** guc.c.orig	2004-04-27 23:08:24.000000000 +0100
--- ./guc.c	2004-03-23 22:52:31.000000000 +0000
***************
*** 458,463 ****
--- 459,474 ----
  		&enableFsync,
  		true, NULL, NULL
  	},
+ 
+ 	{
+ 		{"wal_archive", PGC_SUSET, WAL_SETTINGS,
+ 			gettext_noop("Enabes archiving of WAL logs."),
+ 			gettext_noop("When enabled, PostgreSQL notifies an external archiver using the XLogArchive API")
+ 	},
+ 		&XLogArchivePolicy,
+ 		true, NULL, NULL
+ 	},
+ 
  	{
  		{"zero_damaged_pages", PGC_SUSET, DEVELOPER_OPTIONS,
  			gettext_noop("Continues processing past damaged page headers."),
*** Makefile1.43	2004-04-24 09:56:30.000000000 +0100
--- Makefile	2004-04-24 09:59:02.000000000 +0100
***************
*** 13,19 ****
  top_builddir = ../..
  include $(top_builddir)/src/Makefile.global
  
! DIRS := initdb initlocation ipcclean pg_ctl pg_dump \
  	psql scripts pg_config pg_controldata pg_resetxlog
  
  all install installdirs uninstall depend distprep:
--- 13,19 ----
  top_builddir = ../..
  include $(top_builddir)/src/Makefile.global
  
! DIRS := initdb initlocation ipcclean pg_ctl pg_dump pg_arch\
  	psql scripts pg_config pg_controldata pg_resetxlog
  
  all install installdirs uninstall depend distprep:
*** xlog.c.orig	2004-04-27 22:53:35.000000000 +0100
--- ./xlog.c	2004-04-27 22:01:15.000000000 +0100
***************
*** 82,94 ****
  
  
  /* User-settable parameters */
  int			CheckPointSegments = 3;
  int			XLOGbuffers = 8;
  int			XLOG_DEBUG = 0;
  char	   *XLOG_sync_method = NULL;
  const char	XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR;
- char		XLOG_archive_dir[MAXPGPATH];		/* null string means
- 												 * delete 'em */
  
  /*
   * XLOGfileslop is used in the code as the allowed "fuzz" in the number of
--- 82,93 ----
  
  
  /* User-settable parameters */
+ bool			XLogArchivePolicy = false;
  int			CheckPointSegments = 3;
  int			XLOGbuffers = 8;
  int			XLOG_DEBUG = 0;
  char	   *XLOG_sync_method = NULL;
  const char	XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR;
  
  /*
   * XLOGfileslop is used in the code as the allowed "fuzz" in the number of
***************
*** 396,401 ****
--- 395,401 ----
  
  /* File path names */
  static char XLogDir[MAXPGPATH];
+ static char RLogDir[MAXPGPATH];
  static char ControlFilePath[MAXPGPATH];
  
  /*
***************
*** 437,442 ****
--- 437,450 ----
  
  static bool InRedo = false;
  
+ static bool XLogArchiveNotify(uint32 log, uint32 seg);
+ #define XLA_NOTIFY_OK	 true
+ #define XLA_NOTIFY_ERROR false
+ 
+ static bool XLogArchiveBusy(char xlog[MAXPGPATH]);
+ #define XLA_BUSY	 true
+ #define XLA_ARCHIVE_OK	 false
+ #define XLA_BUSY_ERROR	 2
  
  static bool AdvanceXLInsertBuffer(void);
  static void XLogWrite(XLogwrtRqst WriteRqst);
***************
*** 770,776 ****
  		MyProc->logRec = RecPtr;
  	}
  
! 	if (XLOG_DEBUG)
  	{
  		char		buf[8192];
  
--- 778,784 ----
  		MyProc->logRec = RecPtr;
  	}
  
! 	if (XLOG_DEBUG==16)
  	{
  		char		buf[8192];
  
***************
*** 876,881 ****
--- 884,1021 ----
  }
  
  /*
+  * XLogArchive API calls
+  *
+  * Two calls implement the PostgreSQL side of the XLogArchive API
+  *  XLogArchiveNotify
+  *  XLogArchiveBusy
+  */
+ 
+ /*
+  * XLogArchiveNotify
+  *
+  * Writes an archive notification file to the RLogDir
+  *
+  * The name of the notification file is the message that will be picked up
+  * by the archiver, e.g. we write RLogDir/00000001000000C6.full 
+  * and the archiver then knows to archive XLOgDir/00000001000000C6,
+  * while it is doing so it will rename RLogDir/00000001000000C6.full
+  * to RLogDir/00000001000000C6.busy, then when complete, rename it again
+  * to RLogDir/00000001000000C6.done
+  *
+  * Called only when in wal_archive mode
+  */
+ static bool 
+ XLogArchiveNotify(uint32 log, uint32 seg)
+ {
+ 	char		rlog[32];
+ 	char		rlogpath[MAXPGPATH];
+ 	FILE	   	*rlogFD;
+ 
+ 	if (XLOG_DEBUG)
+ 		elog(LOG, "XLogArchiveNotify writing notification for log file %u, segment %u",
+ 						   log, seg);
+ 
+ /* insert an otherwise empty file called <XLOG>.full */
+ 	sprintf(rlog, "%08X%08X.full", log, seg);
+ 	snprintf(rlogpath, MAXPGPATH, "%s/%s", RLogDir, rlog);
+ 	rlogFD = AllocateFile(rlogpath, "w");
+ 	if (rlogFD == NULL)
+ 		ereport(ERROR,
+ 				(errcode_for_file_access(),
+ 			errmsg("could not write XLogArchive API Notify file \"%s\" ",
+ 				   rlogpath)));
+ 	FreeFile(rlogFD);
+ /* the existence of this file is the message to the archiver to begin archiving */
+ /* the file <XLOG> from the pg_xlog directory					*/
+ 
+ 	if (XLOG_DEBUG)
+ 		elog(LOG, "XLogArchiveNotify written: %s", rlogpath );
+ 
+ 	return XLA_NOTIFY_OK;
+ }
+ 
+ /*
+  * XLogArchiveBusy
+  *
+  * Searches for an archive notification file in RLogDir
+  * 
+  * Reads RLogDir looking for a specific filename. If that filename ends with .done
+  * then we know that the filename refers to an xlog in XLogDir that is safe to
+  * recycle. If the filename ends .full or .busy then thats OK, if neither then
+  * we have an error.
+  * 
+  * Called only when in wal_archive mode
+  *
+  */
+ static bool 
+ XLogArchiveBusy(char xlog[32])
+ {
+ 	char		rlogpath[MAXPGPATH];
+ 	FILE	   	*rlogFD;
+ 
+ 	if (XLOG_DEBUG)
+ 		elog(LOG, "XLogArchiveBusy check for log file %s",
+ 						   xlog);
+ 
+ /* If <XLOG>.done exists then return XLA_ARCHIVE_OK		*/
+ 	snprintf(rlogpath, MAXPGPATH, "%s/%s.done", RLogDir, xlog);
+ 	rlogFD = AllocateFile(rlogpath, "r");
+ 	if (rlogFD != NULL) {
+ 		FreeFile(rlogFD);
+ 		if (XLOG_DEBUG)
+ 			elog(LOG, "XLogArchiveBusy shows archiving done for log file %s",
+ 						   xlog);
+ 		/* really ought to wait until after xlog recycled, in case there */
+ 		/* is an error between the two activities -- do that later	 */
+ 		unlink(rlogpath);
+ 		return XLA_ARCHIVE_OK;
+ 	} 
+ 	else
+ 	{
+ /* else if <XLOG>.busy exists then return XLA_BUSY		*/
+ 		snprintf(rlogpath, MAXPGPATH, "%s/%s.busy", RLogDir, xlog);
+ 		rlogFD = AllocateFile(rlogpath, "r");
+ 		if (rlogFD != NULL) {
+ 			FreeFile(rlogFD);
+ 			elog(LOG, "XLogArchive shows archiving still busy for log file %s",
+ 						xlog);
+ 			/* Do something clever here to avoid systematic or one-off	*/
+ 			/* time delays from effecting server stability			*/
+ 			return XLA_BUSY;
+ 		}
+ 		else
+ 		{
+ /*
+  * else if <XLOG>.full exists then return XLA_BUSY and issue WARNING
+  * ...this indicates archiver is either not working at all or
+  * if it is, then its just way too slow or incorrectly configured
+  */
+ 			snprintf(rlogpath, MAXPGPATH, "%s/%s.full", RLogDir, xlog);
+ 			rlogFD = AllocateFile(rlogpath, "r");
+ 			if (rlogFD != NULL) {
+ 			    FreeFile(rlogFD);
+ 		 	    elog(WARNING, "XLogArchive shows archiving not yet started for log file %s", 
+ 						xlog);
+ 			    return XLA_BUSY;
+ 			}
+ 			else
+ 			{
+ /* else issue a WARNING.... a notification file SHOULD exist...unless #Bug2..the 
+  * database has just been restored in which case it may be absent, so
+  * issue a WARNING, not an error, then return
+  */ 
+ 			    ereport(WARNING,
+ 				(errcode_for_file_access(),
+ 			 	errmsg("cannot find XLogArchive notification file: %s ",
+ 						rlogpath)));
+ 			    return XLA_ARCHIVE_OK;
+ 			}
+ 		}
+ 	}
+ }
+ 
+ /*
   * Advance the Insert state to the next buffer page, writing out the next
   * buffer if it still contains unwritten data.
   *
***************
*** 1130,1135 ****
--- 1272,1282 ----
  		{
  			issue_xlog_fsync();
  			LogwrtResult.Flush = LogwrtResult.Write;	/* end of current page */
+ 
+ 			/* Notify xlog ready to archive?*/
+ 			if (XLogArchivePolicy && !XLogArchiveNotify(openLogId, openLogSeg))
+ 				elog(WARNING, "could not set notify for archiver to read log file %u, segment %u",
+ 					   openLogId, openLogSeg);
  		}
  
  		if (ispartialpage)
***************
*** 1218,1224 ****
  	if (XLByteLE(record, LogwrtResult.Flush))
  		return;
  
! 	if (XLOG_DEBUG)
  		elog(LOG, "xlog flush request %X/%X; write %X/%X; flush %X/%X",
  			 record.xlogid, record.xrecoff,
  			 LogwrtResult.Write.xlogid, LogwrtResult.Write.xrecoff,
--- 1365,1371 ----
  	if (XLByteLE(record, LogwrtResult.Flush))
  		return;
  
! 	if (XLOG_DEBUG==16)
  		elog(LOG, "xlog flush request %X/%X; write %X/%X; flush %X/%X",
  			 record.xlogid, record.xrecoff,
  			 LogwrtResult.Write.xlogid, LogwrtResult.Write.xrecoff,
***************
*** 1627,1651 ****
  	errno = 0;
  	while ((xlde = readdir(xldir)) != NULL)
  	{
  		if (strlen(xlde->d_name) == 16 &&
  			strspn(xlde->d_name, "0123456789ABCDEF") == 16 &&
  			strcmp(xlde->d_name, lastoff) <= 0)
  		{
  			snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name);
! 			if (XLOG_archive_dir[0])
! 			{
! 				ereport(LOG,
! 						(errmsg("archiving transaction log file \"%s\"",
! 								xlde->d_name)));
! 				elog(WARNING, "archiving log files is not implemented");
! 			}
! 			else
  			{
  				/*
  				 * Before deleting the file, see if it can be recycled as
  				 * a future log segment.  We allow recycling segments up
! 				 * to XLOGfileslop segments beyond the current XLOG
! 				 * location.
  				 */
  				if (InstallXLogFileSegment(endlogId, endlogSeg, path,
  										   true, XLOGfileslop,
--- 1774,1799 ----
  	errno = 0;
  	while ((xlde = readdir(xldir)) != NULL)
  	{
+ 		/* if correct length and alphanumeric makeup of file looks correct 	*/
+ 		/* use the alphanumeric sorting property of the filenames to decide 	*/
+ 		/* which ones are earlier than the lastoff transaction log		*/
+ 		/* ...maybe should read lastwrite datetime of lastoff, then check that  */
+ 		/* only files last written earlier than this are removed/recycled	*/
  		if (strlen(xlde->d_name) == 16 &&
  			strspn(xlde->d_name, "0123456789ABCDEF") == 16 &&
  			strcmp(xlde->d_name, lastoff) <= 0)
  		{
  			snprintf(path, MAXPGPATH, "%s/%s", XLogDir, xlde->d_name);
! 
! 			if ( !XLogArchivePolicy ||
! 				(XLogArchivePolicy && !XLogArchiveBusy(xlde->d_name))
! 			   )
  			{
  				/*
  				 * Before deleting the file, see if it can be recycled as
  				 * a future log segment.  We allow recycling segments up
! 				 * until there are XLOGfileslop segments beyond the
! 				 * current XLOG location, otherwise they are removed.
  				 */
  				if (InstallXLogFileSegment(endlogId, endlogSeg, path,
  										   true, XLOGfileslop,
***************
*** 1659,1668 ****
  				{
  					/* No need for any more future segments... */
  					ereport(LOG,
! 						  (errmsg("removing transaction log file \"%s\"",
  								  xlde->d_name)));
  					unlink(path);
  				}
  			}
  		}
  		errno = 0;
--- 1807,1820 ----
  				{
  					/* No need for any more future segments... */
  					ereport(LOG,
! 						  (errmsg("too many transaction log files, removing \"%s\"",
  								  xlde->d_name)));
  					unlink(path);
  				}
+ 				/*
+ 				if (XLogArchivePolicy && !XLogArchiveBusy(xlde->d_name))
+ 					XLogArchiveUnNotify(xlde->d_name);
+ 				 */
  			}
  		}
  		errno = 0;
***************
*** 2113,2118 ****
--- 2265,2271 ----
  {
  	/* Init XLOG file paths */
  	snprintf(XLogDir, MAXPGPATH, "%s/pg_xlog", DataDir);
+ 	snprintf(RLogDir, MAXPGPATH, "%s/pg_rlog", DataDir);
  	snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir);
  }
  
***************
*** 2740,2746 ****
  					ShmemVariableCache->nextXid = record->xl_xid;
  					TransactionIdAdvance(ShmemVariableCache->nextXid);
  				}
! 				if (XLOG_DEBUG)
  				{
  					char		buf[8192];
  
--- 2893,2899 ----
  					ShmemVariableCache->nextXid = record->xl_xid;
  					TransactionIdAdvance(ShmemVariableCache->nextXid);
  				}
! 				if (XLOG_DEBUG==16)
  				{
  					char		buf[8192];
  
*** xlog.h.orig	2004-04-27 23:05:29.000000000 +0100
--- ./xlog.h	2004-04-27 23:05:37.000000000 +0100
***************
*** 192,198 ****
  extern int	XLOG_DEBUG;
  extern char *XLOG_sync_method;
  extern const char XLOG_sync_method_default[];
! 
  
  extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata);
  extern void XLogFlush(XLogRecPtr RecPtr);
--- 192,198 ----
  extern int	XLOG_DEBUG;
  extern char *XLOG_sync_method;
  extern const char XLOG_sync_method_default[];
! extern bool	XLogArchivePolicy;
  
  extern XLogRecPtr XLogInsert(RmgrId rmid, uint8 info, XLogRecData *rdata);
  extern void XLogFlush(XLogRecPtr RecPtr);
---------------------------(end of broadcast)---------------------------
TIP 5: Have you checked our extensive FAQ?

               http://www.postgresql.org/docs/faqs/FAQ.html

Reply via email to