On 9/6/16 10:25 PM, Michael Paquier wrote:
>
On Wed, Sep 7, 2016 at 12:16 AM, David Steele <da...@pgmasters.net> wrote:
Attached is a new patch that adds sgml documentation. I can expand on each
directory individually if you think that's necessary, but thought it was
better to lump them into a few categories.
+ be ommitted from the backup as they will be initialized on postmaster
+ startup. If the <xref linkend="GUC-STATS-TEMP-DIRECTORY"> is set and is
+ under the database cluster directory then the contents of the directory
+ specified by <xref linkend="GUC-STATS-TEMP-DIRECTORY"> can also
be ommitted.
s/ommitted/omitted/
Fixed.
+#define EXCLUDE_DIR_MAX 8
+#define EXCLUDE_DIR_STAT_TMP 0
+
+const char *excludeDirContents[EXCLUDE_DIR_MAX] =
+{
+ /*
+ * Skip temporary statistics files. The first array position will be
+ * filled with the value of pgstat_stat_directory relative to PGDATA.
+ * PG_STAT_TMP_DIR must be skipped even when stats_temp_directory is set
+ * because PGSS_TEXT_FILE is always created there.
+ */
+ NULL,
I find that ugly. I'd rather use an array with undefined size for the
fixed elements finishing by NULL, remove EXCLUDE_DIR_MAX and
EXCLUDE_DIR_STAT_TMP and use a small routine to do the work done on
_tarWriteHeader...
Done. Also writing out pg_xlog with the new routine.
--
-David
da...@pgmasters.net
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 0f09d82..a8daa07 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1090,6 +1090,22 @@ SELECT pg_stop_backup();
</para>
<para>
+ The contents of the <filename>pg_dynshmem/</>, <filename>pg_stat_tmp/</>,
+ <filename>pg_notify/</>, <filename>pg_serial/</>,
+ <filename>pg_snapshots/</>, and <filename>pg_subtrans/</> directories can
+ be omitted from the backup as they will be initialized on postmaster
+ startup. If the <xref linkend="GUC-STATS-TEMP-DIRECTORY"> is set and is
+ under the database cluster directory then the contents of the directory
+ specified by <xref linkend="GUC-STATS-TEMP-DIRECTORY"> can also be omitted.
+ </para>
+
+ <para>
+ Any file or directory beginning with <filename>pgsql_tmp</filename> can be
+ omitted from the backup. These files are removed on postmaster start and
+ the directories will be recreated as needed.
+ </para>
+
+ <para>
The backup label
file includes the label string you gave to <function>pg_start_backup</>,
as well as the time at which <function>pg_start_backup</> was run, and
diff --git a/src/backend/replication/basebackup.c
b/src/backend/replication/basebackup.c
index da9b7a6..a441ae2 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -30,6 +30,7 @@
#include "replication/basebackup.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
+#include "storage/dsm_impl.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "utils/builtins.h"
@@ -57,6 +58,8 @@ static bool sendFile(char *readfilename, char *tarfilename,
static void sendFileWithContent(const char *filename, const char *content);
static void _tarWriteHeader(const char *filename, const char *linktarget,
struct stat * statbuf);
+static int64 _tarWriteDir(char *pathbuf, int basepathlen, struct stat *statbuf,
+ bool sizeonly);
static void send_int8_string(StringInfoData *buf, int64 intval);
static void SendBackupHeader(List *tablespaces);
static void base_backup_cleanup(int code, Datum arg);
@@ -95,6 +98,67 @@ static int64 elapsed_min_unit;
static int64 throttled_last;
/*
+ * The contents of these directories are removed or recreated during server
+ * start so they will not be included in the backup. The directory entry
+ * will be included to preserve permissions.
+ */
+const char *excludeDirContents[] =
+{
+ /*
+ * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
+ * when stats_temp_directory is set because PGSS_TEXT_FILE is always
created
+ * there.
+ */
+ PG_STAT_TMP_DIR,
+
+ /* Recreated on startup, see StartupReplicationSlots(). */
+ "pg_replslot",
+
+ /* Removed on startup, see dsm_cleanup_for_mmap(). */
+ PG_DYNSHMEM_DIR,
+
+ /* Recreated/zeroed on startup, see AsyncShmemInit(). */
+ "pg_notify",
+
+ /* Recreated on startup, see OldSerXidInit(). */
+ "pg_serial",
+
+ /* Removed on startup, see DeleteAllExportedSnapshotFiles(). */
+ "pg_snapshots",
+
+ /* Recreated/zeroed on startup, see StartupSUBTRANS(). */
+ "pg_subtrans",
+
+ /* Terminate list. */
+ NULL
+};
+
+/*
+ * Files that should not be included in the backup.
+ */
+const char *excludeFile[] =
+{
+ /* Skip auto conf temporary file. */
+ PG_AUTOCONF_FILENAME ".tmp",
+
+ /*
+ * If there's a backup_label or tablespace_map file, it belongs to a
backup
+ * started by the user with pg_start_backup(). It is *not* correct for
this
+ * backup, our backup_label/tablespace_map is injected into the tar
+ * separately.
+ */
+ BACKUP_LABEL_FILE,
+ TABLESPACE_MAP,
+
+ /* Skip postmaster.pid and postmaster.opts. */
+ "postmaster.pid",
+ "postmaster.opts",
+
+ /* Terminate list. */
+ NULL
+};
+
+/*
* Called when ERROR or FATAL happens in perform_base_backup() after
* we have started the backup - make sure we end it!
*/
@@ -889,6 +953,8 @@ sendDir(char *path, int basepathlen, bool sizeonly, List
*tablespaces,
char pathbuf[MAXPGPATH];
struct stat statbuf;
int64 size = 0;
+ int excludeIdx;
+ bool excludeFound;
dir = AllocateDir(path);
while ((de = ReadDir(dir, path)) != NULL)
@@ -903,22 +969,88 @@ sendDir(char *path, int basepathlen, bool sizeonly, List
*tablespaces,
strlen(PG_TEMP_FILE_PREFIX)) == 0)
continue;
- /* skip auto conf temporary file */
- if (strncmp(de->d_name,
- PG_AUTOCONF_FILENAME ".tmp",
- sizeof(PG_AUTOCONF_FILENAME) + 4) == 0)
+ /* Stat the file */
+ snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name);
+
+ if (lstat(pathbuf, &statbuf) != 0)
+ {
+ if (errno != ENOENT)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file or
directory \"%s\": %m",
+ pathbuf)));
+
+ /* If the file went away while scanning, it's not an
error. */
continue;
+ }
/*
- * If there's a backup_label or tablespace_map file, it belongs
to a
- * backup started by the user with pg_start_backup(). It is
*not*
- * correct for this backup, our backup_label/tablespace_map is
- * injected into the tar separately.
+ * Scan for files that should be excluded. See excludeFile[]
for info
+ * on exclusions.
*/
- if (strcmp(de->d_name, BACKUP_LABEL_FILE) == 0)
- continue;
+ excludeFound = false;
+ excludeIdx = 0;
+
+ while (excludeFile[excludeIdx] != NULL)
+ {
+ if (strcmp(pathbuf + 2, excludeFile[excludeIdx]) == 0)
+ {
+ ereport(DEBUG1,
+ (errmsg("file excluded from
backup: %s",
+
excludeFile[excludeIdx])));
- if (strcmp(de->d_name, TABLESPACE_MAP) == 0)
+ excludeFound = true;
+ break;
+ }
+
+ excludeIdx++;
+ }
+
+ /*
+ * Scan for directories whose contents should be excluded. See
+ * excludeDirContents[] for info on exclusions.
+ */
+ if (!excludeFound)
+ {
+ excludeIdx = 0;
+
+ while (excludeDirContents[excludeIdx] != NULL)
+ {
+ if (strcmp(pathbuf + 2,
excludeDirContents[excludeIdx]) == 0)
+ {
+ ereport(DEBUG1,
+ (errmsg("directory contents
excluded from backup: %s",
+
excludeDirContents[excludeIdx])));
+
+ size += _tarWriteDir(pathbuf,
basepathlen, &statbuf,
+
sizeonly);
+
+ excludeFound = true;
+ break;
+ }
+
+ excludeIdx++;
+ }
+ }
+
+ /*
+ * Exclude contents of directory specified by statrelpath if
not set
+ * to the default (pg_stat_tmp) which is caught in the loop
above.
+ */
+ if (!excludeFound && statrelpath != NULL &&
+ strcmp(pathbuf, statrelpath) == 0)
+ {
+ ereport(DEBUG1,
+ (errmsg("stats_temp_directory contents excluded
from backup: %s",
+ statrelpath)));
+
+ size += _tarWriteDir(pathbuf, basepathlen, &statbuf,
sizeonly);
+
+ excludeFound = true;
+ }
+
+ /* If file matched exclusion, continue. */
+ if (excludeFound)
continue;
/*
@@ -938,55 +1070,10 @@ sendDir(char *path, int basepathlen, bool sizeonly, List
*tablespaces,
"and should not be used. "
"Try taking another online
backup.")));
- snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name);
-
- /* Skip postmaster.pid and postmaster.opts in the data
directory */
- if (strcmp(pathbuf, "./postmaster.pid") == 0 ||
- strcmp(pathbuf, "./postmaster.opts") == 0)
- continue;
-
/* Skip pg_control here to back up it last */
if (strcmp(pathbuf, "./global/pg_control") == 0)
continue;
- if (lstat(pathbuf, &statbuf) != 0)
- {
- if (errno != ENOENT)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not stat file or
directory \"%s\": %m",
- pathbuf)));
-
- /* If the file went away while scanning, it's no error.
*/
- continue;
- }
-
- /*
- * Skip temporary statistics files. PG_STAT_TMP_DIR must be
skipped
- * even when stats_temp_directory is set because PGSS_TEXT_FILE
is
- * always created there.
- */
- if ((statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
||
- strncmp(de->d_name, PG_STAT_TMP_DIR, strlen(PG_STAT_TMP_DIR))
== 0)
- {
- if (!sizeonly)
- _tarWriteHeader(pathbuf + basepathlen + 1,
NULL, &statbuf);
- size += 512;
- continue;
- }
-
- /*
- * Skip pg_replslot, not useful to copy. But include it as an
empty
- * directory anyway, so we get permissions right.
- */
- if (strcmp(de->d_name, "pg_replslot") == 0)
- {
- if (!sizeonly)
- _tarWriteHeader(pathbuf + basepathlen + 1,
NULL, &statbuf);
- size += 512; /* Size of the header just
added */
- continue;
- }
-
/*
* We can skip pg_xlog, the WAL segments need to be fetched
from the
* WAL archive anyway. But include it as an empty directory
anyway, so
@@ -994,18 +1081,8 @@ sendDir(char *path, int basepathlen, bool sizeonly, List
*tablespaces,
*/
if (strcmp(pathbuf, "./pg_xlog") == 0)
{
- if (!sizeonly)
- {
- /* If pg_xlog is a symlink, write it as a
directory anyway */
-#ifndef WIN32
- if (S_ISLNK(statbuf.st_mode))
-#else
- if (pgwin32_is_junction(pathbuf))
-#endif
- statbuf.st_mode = S_IFDIR | S_IRWXU;
- _tarWriteHeader(pathbuf + basepathlen + 1,
NULL, &statbuf);
- }
- size += 512; /* Size of the header just
added */
+ /* If pg_xlog is a symlink, write it as a directory
anyway */
+ size += _tarWriteDir(pathbuf, basepathlen, &statbuf,
sizeonly);
/*
* Also send archive_status directory (by hackishly
reusing
@@ -1248,6 +1325,31 @@ _tarWriteHeader(const char *filename, const char
*linktarget,
}
/*
+ * Write tar header for a directory. If the entry in statbuf is a link then
+ * write it as a directory anyway.
+ */
+static int64
+_tarWriteDir(char *pathbuf, int basepathlen, struct stat *statbuf,
+ bool sizeonly)
+{
+ /* Include the directory entry to preserve permissions. */
+ if (!sizeonly)
+ {
+ /* If symlink, write it as a directory anyway */
+#ifndef WIN32
+ if (S_ISLNK(statbuf->st_mode))
+#else
+ if (pgwin32_is_junction(pathbuf))
+#endif
+ statbuf->st_mode = S_IFDIR | S_IRWXU;
+
+ _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf);
+ }
+
+ return 512;
+}
+
+/*
* Increment the network transfer counter by the given number of bytes,
* and sleep if necessary to comply with the requested network transfer
* rate.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
