On 2023-05-15 15:22, Michael Paquier wrote:
On Mon, May 15, 2023 at 03:16:46PM +0900, torikoshia wrote:
On 2023-05-15 09:18, Michael Paquier wrote:
How about plugging in some long options, and use something more
explicit like --clean-backup-history?
Agreed.
If you begin to implement that, it seems to me that this should be
shaped with a first separate patch that refactors the code to use
getopt_long(), and a second patch for the proposed feature that builds
on top of it.
Thanks for your advice, attached patches.
--
Regards,
--
Atsushi Torikoshi
NTT DATA CORPORATION
From 814b7351f14626f02c13b21d1a6737461117e5d0 Mon Sep 17 00:00:00 2001
From: Atsushi Torikoshi <torikos...@oss.nttdata.com>
Date: Mon, 22 May 2023 17:37:25 +0900
Subject: [PATCH v3 1/2] Introduce pg_archivecleanup into getopt_long
This patch is a preliminary step to add an easy-to-understand option
to delete backup history files, but it also adds long options to
the existing options.
---
doc/src/sgml/ref/pgarchivecleanup.sgml | 5 ++++-
src/bin/pg_archivecleanup/pg_archivecleanup.c | 20 ++++++++++++-------
2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/doc/src/sgml/ref/pgarchivecleanup.sgml b/doc/src/sgml/ref/pgarchivecleanup.sgml
index 635e7c7685..09991c2fcd 100644
--- a/doc/src/sgml/ref/pgarchivecleanup.sgml
+++ b/doc/src/sgml/ref/pgarchivecleanup.sgml
@@ -95,6 +95,7 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E"
<varlistentry>
<term><option>-d</option></term>
+ <term><option>--debug</option></term>
<listitem>
<para>
Print lots of debug logging output on <filename>stderr</filename>.
@@ -104,6 +105,7 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E"
<varlistentry>
<term><option>-n</option></term>
+ <term><option>--dry-run</option></term>
<listitem>
<para>
Print the names of the files that would have been removed on <filename>stdout</filename> (performs a dry run).
@@ -122,7 +124,8 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E"
</varlistentry>
<varlistentry>
- <term><option>-x</option> <replaceable>extension</replaceable></term>
+ <term><option>-x <replaceable class="parameter">extension</replaceable></option></term>
+ <term><option>--strip-extension=<replaceable class="parameter">extension</replaceable></option></term>
<listitem>
<para>
Provide an extension
diff --git a/src/bin/pg_archivecleanup/pg_archivecleanup.c b/src/bin/pg_archivecleanup/pg_archivecleanup.c
index 7726d05149..62914bdfa7 100644
--- a/src/bin/pg_archivecleanup/pg_archivecleanup.c
+++ b/src/bin/pg_archivecleanup/pg_archivecleanup.c
@@ -17,7 +17,7 @@
#include "access/xlog_internal.h"
#include "common/logging.h"
-#include "pg_getopt.h"
+#include "getopt_long.h"
const char *progname;
@@ -252,11 +252,11 @@ usage(void)
printf(_("Usage:\n"));
printf(_(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname);
printf(_("\nOptions:\n"));
- printf(_(" -d generate debug output (verbose mode)\n"));
- printf(_(" -n dry run, show the names of the files that would be removed\n"));
- printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -x EXT clean up files if they have this extension\n"));
- printf(_(" -?, --help show this help, then exit\n"));
+ printf(_(" -d, --debug generate debug output (verbose mode)\n"));
+ printf(_(" -n, --dry-run dry run, show the names of the files that would be removed\n"));
+ printf(_(" -V, --version output version information, then exit\n"));
+ printf(_(" -x --strip-extension=EXT clean up files if they have this extension\n"));
+ printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"For use as archive_cleanup_command in postgresql.conf:\n"
" archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n"
@@ -274,6 +274,12 @@ usage(void)
int
main(int argc, char **argv)
{
+ static struct option long_options[] = {
+ {"debug", no_argument, NULL, 'd'},
+ {"dry-run", no_argument, NULL, 'n'},
+ {"strip-extension", required_argument, NULL, 'x'},
+ {NULL, 0, NULL, 0}
+ };
int c;
pg_logging_init(argv[0]);
@@ -294,7 +300,7 @@ main(int argc, char **argv)
}
}
- while ((c = getopt(argc, argv, "dnx:")) != -1)
+ while ((c = getopt_long(argc, argv, "dnx:", long_options, NULL)) != -1)
{
switch (c)
{
base-commit: ac298d3cb56b015acd40d2e015e07a87d8aff124
--
2.39.2
From 6017a774691cf52f7f51b817dee26db3bc7879c0 Mon Sep 17 00:00:00 2001
From: Atsushi Torikoshi <torikos...@oss.nttdata.com>
Date: Mon, 22 May 2023 17:41:57 +0900
Subject: [PATCH v3 2/2] Allow pg_archivecleanup to remove backup history files
Backup history files are just few bytes, but it can be noisy for the
eye to see a large accumulation of history files mixed with the WAL
segments.
This patch adds a new option to remove files including backup
history files older oldestkeptwalfile.
---
doc/src/sgml/ref/pgarchivecleanup.sgml | 13 +++
src/bin/pg_archivecleanup/pg_archivecleanup.c | 88 +++++++++++--------
.../t/010_pg_archivecleanup.pl | 36 +++++---
3 files changed, 91 insertions(+), 46 deletions(-)
diff --git a/doc/src/sgml/ref/pgarchivecleanup.sgml b/doc/src/sgml/ref/pgarchivecleanup.sgml
index 09991c2fcd..e0efabd989 100644
--- a/doc/src/sgml/ref/pgarchivecleanup.sgml
+++ b/doc/src/sgml/ref/pgarchivecleanup.sgml
@@ -113,6 +113,19 @@ pg_archivecleanup: removing file "archive/00000001000000370000000E"
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-b</option></term>
+ <term><option>--clean-backup-history</option></term>
+ <listitem>
+ <para>
+ Remove files including backup history file.
+ For details about backup history file, please refer to the <xref linkend="backup-base-backup"/>.
+ Note that when <replaceable>oldestkeptwalfile</replaceable> is a backup history file,
+ specified file is kept and only preceding WAL files and backup history files are removed.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-V</option></term>
<term><option>--version</option></term>
diff --git a/src/bin/pg_archivecleanup/pg_archivecleanup.c b/src/bin/pg_archivecleanup/pg_archivecleanup.c
index 62914bdfa7..3fd4a441b2 100644
--- a/src/bin/pg_archivecleanup/pg_archivecleanup.c
+++ b/src/bin/pg_archivecleanup/pg_archivecleanup.c
@@ -23,6 +23,8 @@ const char *progname;
/* Options and defaults */
bool dryrun = false; /* are we performing a dry-run operation? */
+bool cleanBackupHistory = false; /* remove files including
++ * backup history files */
char *additional_ext = NULL; /* Extension to remove from filenames */
char *archiveLocation; /* where to find the archive? */
@@ -97,6 +99,8 @@ CleanupPriorWALFiles(void)
{
while (errno = 0, (xlde = readdir(xldir)) != NULL)
{
+ char WALFilePath[MAXPGPATH * 2]; /* the file path
+ * including archive */
/*
* Truncation is essentially harmless, because we skip names of
* length other than XLOG_FNAME_LEN. (In principle, one could use
@@ -106,6 +110,19 @@ CleanupPriorWALFiles(void)
TrimExtension(walfile, additional_ext);
/*
+ * Check file name.
+ *
+ * We skip files which are not WAL file or partial WAL file.
+ * Also we skip backup history files when --clean-backup-history
+ * is not specified.
+ */
+ if (!IsXLogFileName(walfile) && !IsPartialXLogFileName(walfile) &&
+ (!cleanBackupHistory || !IsBackupHistoryFileName(walfile)))
+ continue;
+
+ /*
+ * Check cutoff point.
+ *
* We ignore the timeline part of the XLOG segment identifiers in
* deciding whether a segment is still needed. This ensures that
* we won't prematurely remove a segment from a parent timeline.
@@ -118,39 +135,35 @@ CleanupPriorWALFiles(void)
* file. Note that this means files are not removed in the order
* they were originally written, in case this worries you.
*/
- if ((IsXLogFileName(walfile) || IsPartialXLogFileName(walfile)) &&
- strcmp(walfile + 8, exclusiveCleanupFileName + 8) < 0)
- {
- char WALFilePath[MAXPGPATH * 2]; /* the file path
- * including archive */
+ if (strcmp(walfile + 8, exclusiveCleanupFileName + 8) >= 0)
+ continue;
+ /*
+ * Use the original file name again now, including any
+ * extension that might have been chopped off before testing
+ * the sequence.
+ */
+ snprintf(WALFilePath, sizeof(WALFilePath), "%s/%s",
+ archiveLocation, xlde->d_name);
+
+ if (dryrun)
+ {
/*
- * Use the original file name again now, including any
- * extension that might have been chopped off before testing
- * the sequence.
+ * Prints the name of the file to be removed and skips the
+ * actual removal. The regular printout is so that the
+ * user can pipe the output into some other program.
*/
- snprintf(WALFilePath, sizeof(WALFilePath), "%s/%s",
- archiveLocation, xlde->d_name);
-
- if (dryrun)
- {
- /*
- * Prints the name of the file to be removed and skips the
- * actual removal. The regular printout is so that the
- * user can pipe the output into some other program.
- */
- printf("%s\n", WALFilePath);
- pg_log_debug("file \"%s\" would be removed", WALFilePath);
- continue;
- }
-
- pg_log_debug("removing file \"%s\"", WALFilePath);
-
- rc = unlink(WALFilePath);
- if (rc != 0)
- pg_fatal("could not remove file \"%s\": %m",
- WALFilePath);
+ printf("%s\n", WALFilePath);
+ pg_log_debug("file \"%s\" would be removed", WALFilePath);
+ continue;
}
+
+ pg_log_debug("removing file \"%s\"", WALFilePath);
+
+ rc = unlink(WALFilePath);
+ if (rc != 0)
+ pg_fatal("could not remove file \"%s\": %m",
+ WALFilePath);
}
if (errno)
@@ -252,11 +265,12 @@ usage(void)
printf(_("Usage:\n"));
printf(_(" %s [OPTION]... ARCHIVELOCATION OLDESTKEPTWALFILE\n"), progname);
printf(_("\nOptions:\n"));
- printf(_(" -d, --debug generate debug output (verbose mode)\n"));
- printf(_(" -n, --dry-run dry run, show the names of the files that would be removed\n"));
- printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -x --strip-extension=EXT clean up files if they have this extension\n"));
- printf(_(" -?, --help show this help, then exit\n"));
+ printf(_(" -d, --debug generate debug output (verbose mode)\n"));
+ printf(_(" -n, --dry-run dry run, show the names of the files that would be removed\n"));
+ printf(_(" -b, --clean-backup-history clean up files including backup history files\n"));
+ printf(_(" -V, --version output version information, then exit\n"));
+ printf(_(" -x --strip-extension=EXT clean up files if they have this extension\n"));
+ printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"For use as archive_cleanup_command in postgresql.conf:\n"
" archive_cleanup_command = 'pg_archivecleanup [OPTION]... ARCHIVELOCATION %%r'\n"
@@ -275,6 +289,7 @@ int
main(int argc, char **argv)
{
static struct option long_options[] = {
+ {"clean-backup-history", no_argument, NULL, 'b'},
{"debug", no_argument, NULL, 'd'},
{"dry-run", no_argument, NULL, 'n'},
{"strip-extension", required_argument, NULL, 'x'},
@@ -300,10 +315,13 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "dnx:", long_options, NULL)) != -1)
+ while ((c = getopt_long(argc, argv, "bdnx:", long_options, NULL)) != -1)
{
switch (c)
{
+ case 'b': /* Remove backup history files too */
+ cleanBackupHistory = true;
+ break;
case 'd': /* Debug mode */
pg_logging_increase_verbosity();
break;
diff --git a/src/bin/pg_archivecleanup/t/010_pg_archivecleanup.pl b/src/bin/pg_archivecleanup/t/010_pg_archivecleanup.pl
index cc3386d146..544762ae4f 100644
--- a/src/bin/pg_archivecleanup/t/010_pg_archivecleanup.pl
+++ b/src/bin/pg_archivecleanup/t/010_pg_archivecleanup.pl
@@ -14,6 +14,7 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
my @walfiles = (
'00000001000000370000000C.gz', '00000001000000370000000D',
+ '00000001000000370000000D.00000028.backup',
'00000001000000370000000E', '00000001000000370000000F.partial',);
sub create_files
@@ -59,7 +60,7 @@ command_fails_like(
my $stderr;
my $result =
IPC::Run::run [ 'pg_archivecleanup', '-d', '-n', $tempdir,
- $walfiles[2] ],
+ $walfiles[3] ],
'2>', \$stderr;
ok($result, "pg_archivecleanup dry run: exit code 0");
like(
@@ -76,32 +77,45 @@ sub run_check
{
local $Test::Builder::Level = $Test::Builder::Level + 1;
- my ($suffix, $test_name) = @_;
+ my ($suffix, $test_name, @options) = @_;
create_files();
command_ok(
[
- 'pg_archivecleanup', '-x', '.gz', $tempdir,
- $walfiles[2] . $suffix
+ 'pg_archivecleanup', @options, $tempdir,
+ $walfiles[3] . $suffix
],
"$test_name: runs");
- ok(!-f "$tempdir/$walfiles[0]",
- "$test_name: first older WAL file was cleaned up");
+ if (grep {$_ eq '-x.gz'} @options) {
+ ok(!-f "$tempdir/$walfiles[0]",
+ "$test_name: first older WAL file with .gz was cleaned up");
+ } else {
+ ok(-f "$tempdir/$walfiles[0]",
+ "$test_name: first older WAL file with .gz was not cleaned up");
+ }
ok(!-f "$tempdir/$walfiles[1]",
"$test_name: second older WAL file was cleaned up");
- ok(-f "$tempdir/$walfiles[2]",
- "$test_name: restartfile was not cleaned up");
+ if (grep {$_ eq '-b'} @options) {
+ ok(!-f "$tempdir/$walfiles[2]",
+ "$test_name: Backup history file was cleaned up");
+ } else {
+ ok(-f "$tempdir/$walfiles[2]",
+ "$test_name: Backup history file was not cleaned up");
+ }
ok(-f "$tempdir/$walfiles[3]",
+ "$test_name: restartfile was not cleaned up");
+ ok(-f "$tempdir/$walfiles[4]",
"$test_name: newer WAL file was not cleaned up");
ok(-f "$tempdir/unrelated_file",
"$test_name: unrelated file was not cleaned up");
return;
}
-run_check('', 'pg_archivecleanup');
-run_check('.partial', 'pg_archivecleanup with .partial file');
-run_check('.00000020.backup', 'pg_archivecleanup with .backup file');
+run_check('', 'pg_archivecleanup', '-x.gz');
+run_check('.partial', 'pg_archivecleanup with .partial file', '-x.gz');
+run_check('.00000020.backup', 'pg_archivecleanup with .backup file', '-x.gz');
+run_check('', 'pg_archivecleanup with --clean-backup-history', '-b');
done_testing();
--
2.39.2
