From 4735db532aa818a9e3958ccc79229044fdfc7069 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Date: Mon, 6 Dec 2021 11:39:36 +0000
Subject: [PATCH v1] background cleaner to offload checkpoint tasks

---
 src/backend/postmaster/Makefile             |   1 +
 src/backend/postmaster/bgcleaner.c          | 415 ++++++++++++++++++++
 src/backend/postmaster/postmaster.c         |  34 +-
 src/backend/replication/logical/logical.c   |  40 ++
 src/backend/replication/logical/snapbuild.c |   8 +
 src/backend/utils/activity/wait_event.c     |   3 +
 src/backend/utils/init/miscinit.c           |   3 +
 src/backend/utils/misc/guc.c                |  37 +-
 src/include/miscadmin.h                     |   1 +
 src/include/postmaster/bgcleaner.h          |  32 ++
 src/include/replication/logical.h           |   6 +
 src/include/utils/wait_event.h              |   1 +
 12 files changed, 579 insertions(+), 2 deletions(-)
 create mode 100644 src/backend/postmaster/bgcleaner.c
 create mode 100644 src/include/postmaster/bgcleaner.h

diff --git a/src/backend/postmaster/Makefile b/src/backend/postmaster/Makefile
index 787c6a2c3b..f55903dd1a 100644
--- a/src/backend/postmaster/Makefile
+++ b/src/backend/postmaster/Makefile
@@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
 OBJS = \
 	autovacuum.o \
 	auxprocess.o \
+	bgcleaner.o \
 	bgworker.o \
 	bgwriter.o \
 	checkpointer.o \
diff --git a/src/backend/postmaster/bgcleaner.c b/src/backend/postmaster/bgcleaner.c
new file mode 100644
index 0000000000..14d98e48eb
--- /dev/null
+++ b/src/backend/postmaster/bgcleaner.c
@@ -0,0 +1,415 @@
+/*-------------------------------------------------------------------------
+ *
+ * bgcleaner.c
+ *
+ * The background cleaner (bgcleaner) process removes unneeded replication
+ * slot files (.snap). This is to offload the checkpoint responsibility so
+ * that the checkpoint (and so the recovery) can be faster.
+ *
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ *	  src/backend/postmaster/bgcleaner.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgcleaner.h"
+#include "postmaster/fork_process.h"
+#include "postmaster/interrupt.h"
+#include "postmaster/postmaster.h"
+#include "replication/logical.h"
+#include "storage/dsm.h"
+#include "storage/fd.h"
+#include "storage/pg_shmem.h"
+#include "utils/guc.h"
+#include "utils/ps_status.h"
+
+/*
+ * GUC parameters
+ */
+bool	BgCleanerEnable = true;
+bool	BgCleanerStopProcessingFiles = false;
+int		BgCleanerDelay = 180;
+
+#ifdef EXEC_BACKEND
+static pid_t bgcleaner_forkexec(void);
+#endif
+
+NON_EXEC_STATIC void BackgroundCleanerMain(int argc, char *argv[]) pg_attribute_noreturn();
+
+static void CheckForBgCleanerInterrupts(void);
+static void ProcessSnapshotCutoffFiles(void);
+static int 	RemoveSnapShotFiles(XLogRecPtr cutoff);
+
+#ifdef EXEC_BACKEND
+/*
+ * bgcleaner_forkexec() -
+ *
+ * Format up the arglist for, then fork and exec, bgcleaner process
+ */
+static pid_t
+bgcleaner_forkexec(void)
+{
+	char	*av[10];
+	int		ac = 0;
+
+	av[ac++] = "postgres";
+	av[ac++] = "--forkbgcleaner";
+	av[ac++] = NULL;			/* filled in by postmaster_forkexec */
+
+	av[ac] = NULL;
+	Assert(ac < lengthof(av));
+
+	return postmaster_forkexec(ac, av);
+}
+#endif							/* EXEC_BACKEND */
+
+/*
+ *	Called from postmaster at startup or after an existing bgcleaner died.
+ *	Attempt to fire up a fresh bgcleaner.
+ *
+ *	Returns PID of child process, or 0 if fail.
+ *
+ *	Note: if fail, we will be called again from the postmaster main loop.
+ */
+int
+BgCleanerStart(void)
+{
+	pid_t	pid;
+
+#ifdef EXEC_BACKEND
+	switch ((pid = bgcleaner_forkexec()))
+#else
+	switch ((pid = fork_process()))
+#endif
+	{
+		case -1:
+			ereport(LOG,
+					(errmsg("could not fork background cleaner: %m")));
+			return 0;
+
+#ifndef EXEC_BACKEND
+		case 0:
+			/* in postmaster child ... */
+			InitPostmasterChild();
+
+			/* Close the postmaster's sockets */
+			ClosePostmasterPorts(false);
+
+			/* Drop our connection to postmaster's shared memory, as well */
+			dsm_detach_all();
+			PGSharedMemoryDetach();
+
+			BackgroundCleanerMain(0, NULL);
+			break;
+#endif
+
+		default:
+			return (int) pid;
+	}
+
+	/* shouldn't get here */
+	return 0;
+}
+
+/*
+ * Main entry point for bgcleaner process
+ *
+ * argc/argv parameters are valid only in EXEC_BACKEND case.
+ */
+NON_EXEC_STATIC void
+BackgroundCleanerMain(int argc, char *argv[])
+{
+	int		n_wait = 0;
+	bool	msg_logged = false;
+
+	/*
+	 * Ignore all signals usually bound to some action in the postmaster,
+	 * except SIGHUP, SIGTERM and SIGQUIT.  Note we don't need a SIGUSR1
+	 * handler to support latch operations, because we only use a local latch.
+	 */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGINT, SIG_IGN);
+	pqsignal(SIGTERM, SignalHandlerForShutdownRequest);
+	pqsignal(SIGQUIT, SignalHandlerForShutdownRequest);
+	pqsignal(SIGALRM, SIG_IGN);
+	pqsignal(SIGPIPE, SIG_IGN);
+	pqsignal(SIGUSR1, SIG_IGN);
+	pqsignal(SIGUSR2, SIG_IGN);
+	pqsignal(SIGCHLD, SIG_DFL);
+	pqsignal(SIGTTIN, SIG_DFL);
+	pqsignal(SIGTTOU, SIG_DFL);
+	pqsignal(SIGCONT, SIG_DFL);
+	pqsignal(SIGWINCH, SIG_DFL);
+	/*
+	 * Unblock signals (they were blocked when the postmaster forked us)
+	 */
+	PG_SETMASK(&UnBlockSig);
+
+	MyBackendType = B_BG_CLEANER;
+	init_ps_display(NULL);
+
+	/*
+	 * Loop until we get SIGQUIT, SIGTERM or detect ungraceful death of
+	 * parent postmaster.
+	 */
+	for (;;)
+	{
+		int		rc;
+
+		/* Clear any already-pending wakeups */
+		ResetLatch(MyLatch);
+
+		CheckForBgCleanerInterrupts();
+
+		/* Do the main work */
+		if (!BgCleanerStopProcessingFiles)
+		{
+			if (msg_logged)
+			{
+				/* we were told to start processing files */
+				elog(LOG, "background cleaner started file processing as parameter \"%s\" is set to off",
+					 "bgcleaner_stop_processing_files");
+				msg_logged = false;
+			}
+
+			ProcessSnapshotCutoffFiles();
+		}
+		else if (BgCleanerStopProcessingFiles && !msg_logged)
+		{
+			/* we were told to stop processing files */
+			elog(LOG, "background cleaner stopped file processing as parameter \"%s\" is set to on",
+				"bgcleaner_stop_processing_files");
+			msg_logged = true;
+		}
+
+		/*
+		 * Sleep until we are signaled or BgCleanerDelay has elapsed.
+		 */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   BgCleanerDelay * 1000L /* convert to ms */ ,
+					   WAIT_EVENT_BGCLEANER_MAIN);
+
+		/*
+		 * Emergency bailout if postmaster has died.  This is to avoid the
+		 * necessity for manual cleanup of all postmaster children.
+		 */
+		if (rc & WL_POSTMASTER_DEATH)
+			break;
+
+		n_wait++;
+
+		/*
+		 * Emit a log message after every 5 rounds of sleep to indicate the
+		 * process is active.
+		 */
+		if (n_wait == 5 && !BgCleanerStopProcessingFiles)
+		{
+			elog(LOG, "background cleaner is running with %d seconds of sleep time between rounds",
+				 BgCleanerDelay);
+
+			n_wait = 0;
+		}
+	}
+
+	exit(0);
+}
+
+/*
+ * Read snapshot cutoff file names written by checkpointer to get the cutoff
+ * LSN and remove the unneded snapshot files.
+ */
+static void
+ProcessSnapshotCutoffFiles(void)
+{
+	DIR *dir;
+	struct dirent *cutoff_de;
+
+	dir = AllocateDir("pg_logical");
+	while ((cutoff_de = ReadDir(dir, "pg_logical")) != NULL)
+	{
+		char	path[MAXPGPATH + 11];
+		uint32	hi;
+		uint32	lo;
+		XLogRecPtr	cutoff = InvalidXLogRecPtr;
+		XLogRecPtr	prev_cutoff = InvalidXLogRecPtr;
+		struct stat	statbuf;
+		int	res;
+
+		CheckForBgCleanerInterrupts();
+
+		/* see if we were told to stop processing files */
+		if (BgCleanerStopProcessingFiles)
+		{
+			elog(LOG, "background cleaner is stopping file processing at cutoff LSN: %X/%X as parameter \"%s\" is set to on",
+				 LSN_FORMAT_ARGS(prev_cutoff),
+				 "bgcleaner_stop_processing_files");
+			return;
+		}
+
+		if (strcmp(cutoff_de->d_name, ".") == 0 ||
+			strcmp(cutoff_de->d_name, "..") == 0)
+			continue;
+
+		snprintf(path, sizeof(path), "pg_logical/%s", cutoff_de->d_name);
+
+		if (lstat(path, &statbuf) == 0 && !S_ISREG(statbuf.st_mode))
+		{
+			elog(DEBUG2, "only regular files expected: %s", cutoff_de->d_name);
+			continue;
+		}
+
+		/*
+		 * We just log a message if a file doesn't fit the pattern, it's
+		 * probably some editors lock/state file or similar...
+		 */
+		if (sscanf(cutoff_de->d_name, "snapshot_cutoff_%X-%X", &hi, &lo) != 2)
+		{
+			elog(DEBUG2, "could not parse file name: %s", cutoff_de->d_name);
+			continue;
+		}
+
+		prev_cutoff = cutoff;
+		cutoff = ((uint64) hi) << 32 | lo;
+		elog(DEBUG2, "replication slots cutoff LSN: %X/%X",
+			 LSN_FORMAT_ARGS(cutoff));
+
+		res = RemoveSnapShotFiles(cutoff);
+
+		if (res == 0)
+		{
+			/* remove the file */
+			if (unlink(path) < 0)
+			{
+				elog(LOG, "could not remove snapshot cutoff file: %s",
+					 cutoff_de->d_name);
+				continue;
+			}
+
+			elog(DEBUG1, "removed snapshot cutoff file: %s", cutoff_de->d_name);
+		}
+		else if (res < 0)
+		{
+			elog(DEBUG2, "retained snapshot cutoff file: %s", cutoff_de->d_name);
+			continue;
+		}
+  }
+  FreeDir(dir);
+}
+
+/*
+ * Remove unneded snapshot files i.e. files with LSN < cutoff LSN.
+ *
+ * Return value -1 indicates that the caller can not remove the snapshot file.
+ *
+ * Return value 0 indicates that the caller can delete the snapshot cutoff
+ * file.
+ */
+static int
+RemoveSnapShotFiles(XLogRecPtr cutoff)
+{
+	DIR *dir;
+	struct dirent *snap_de;
+	int res = 0;
+	uint32	snap_files_deleted = 0;
+
+	dir = AllocateDir("pg_logical/snapshots");
+	while ((snap_de = ReadDir(dir, "pg_logical/snapshots")) != NULL)
+	{
+		char	path[MAXPGPATH + 21];
+		uint32	hi;
+		uint32	lo;
+		XLogRecPtr	lsn;
+		struct stat	statbuf;
+
+		CheckForBgCleanerInterrupts();
+
+		/* see if we were told to stop processing files */
+		if (BgCleanerStopProcessingFiles)
+		{
+			elog(LOG, "background cleaner is stopping file processing at cutoff LSN: %X/%X as parameter \"%s\" is set to on",
+				 LSN_FORMAT_ARGS(cutoff),
+				 "bgcleaner_stop_processing_files");
+			return -1;
+		}
+
+		if (strcmp(snap_de->d_name, ".") == 0 ||
+			strcmp(snap_de->d_name, "..") == 0)
+			continue;
+
+		snprintf(path, sizeof(path), "pg_logical/snapshots/%s", snap_de->d_name);
+
+		if (lstat(path, &statbuf) == 0 && !S_ISREG(statbuf.st_mode))
+			continue;
+
+		/*
+		 * We just log a message if a file doesn't fit the pattern, it's
+		 * probably some editors lock/state file or similar...
+		 */
+		if (sscanf(snap_de->d_name, "%X-%X.snap", &hi, &lo) != 2)
+		{
+			elog(DEBUG2, "could not parse file name: %s", snap_de->d_name);
+			continue;
+		}
+
+		lsn = ((uint64) hi) << 32 | lo;
+
+		/* check whether we still need it */
+		if (lsn < cutoff || cutoff == InvalidXLogRecPtr)
+		{
+			/* remove the file */
+			if (unlink(path) < 0)
+			{
+				elog(LOG, "could not remove snapshot file: %s",
+					 snap_de->d_name);
+				res = -1;
+				continue;
+			}
+
+			snap_files_deleted++;
+			elog(DEBUG1, "removed snapshot file: %s", snap_de->d_name);
+		}
+		else
+		{
+			elog(DEBUG2, "retained snapshot file: %s", snap_de->d_name);
+			continue;
+		}
+	}
+	FreeDir(dir);
+
+	if (snap_files_deleted > 0)
+		ereport(LOG,
+				(errmsg_plural("removed %u snapshot file with cutoff LSN %X/%X",
+							   "removed %u snapshot files with cutoff LSN %X/%X",
+							   snap_files_deleted,
+							   snap_files_deleted,
+							   LSN_FORMAT_ARGS(cutoff))));
+	return res;
+}
+
+static void
+CheckForBgCleanerInterrupts(void)
+{
+	if (ShutdownRequestPending)
+		exit(0);
+
+	if (ConfigReloadPending)
+	{
+		ConfigReloadPending = false;
+		ProcessConfigFile(PGC_SIGHUP);
+	}
+}
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 328ecafa8c..49325570e5 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -110,6 +110,7 @@
 #include "port/pg_bswap.h"
 #include "postmaster/autovacuum.h"
 #include "postmaster/auxprocess.h"
+#include "postmaster/bgcleaner.h"
 #include "postmaster/bgworker_internals.h"
 #include "postmaster/fork_process.h"
 #include "postmaster/interrupt.h"
@@ -255,7 +256,8 @@ static pid_t StartupPID = 0,
 			AutoVacPID = 0,
 			PgArchPID = 0,
 			PgStatPID = 0,
-			SysLoggerPID = 0;
+			SysLoggerPID = 0,
+			BgCleanerPID = 0;
 
 /* Startup process's status */
 typedef enum
@@ -1459,6 +1461,8 @@ PostmasterMain(int argc, char *argv[])
 		CheckpointerPID = StartCheckpointer();
 	if (BgWriterPID == 0)
 		BgWriterPID = StartBackgroundWriter();
+	if (BgCleanerPID == 0 && BgCleanerEnable)
+		BgCleanerPID = BgCleanerStart();
 
 	/*
 	 * We're ready to rock and roll...
@@ -1828,6 +1832,8 @@ ServerLoop(void)
 				CheckpointerPID = StartCheckpointer();
 			if (BgWriterPID == 0)
 				BgWriterPID = StartBackgroundWriter();
+			if (BgCleanerPID == 0 && BgCleanerEnable)
+				BgCleanerPID = BgCleanerStart();
 		}
 
 		/*
@@ -2794,6 +2800,8 @@ SIGHUP_handler(SIGNAL_ARGS)
 			signal_child(SysLoggerPID, SIGHUP);
 		if (PgStatPID != 0)
 			signal_child(PgStatPID, SIGHUP);
+		if (BgCleanerPID != 0)
+			signal_child(BgCleanerPID, SIGHUP);
 
 		/* Reload authentication config files too */
 		if (!load_hba())
@@ -3111,6 +3119,8 @@ reaper(SIGNAL_ARGS)
 				CheckpointerPID = StartCheckpointer();
 			if (BgWriterPID == 0)
 				BgWriterPID = StartBackgroundWriter();
+			if (BgCleanerPID == 0 && BgCleanerEnable)
+				BgCleanerPID = BgCleanerStart();
 			if (WalWriterPID == 0)
 				WalWriterPID = StartWalWriter();
 
@@ -3225,6 +3235,22 @@ reaper(SIGNAL_ARGS)
 			continue;
 		}
 
+		/*
+		 * Was it the bgcleaner?  If so, just try to start a new one; no need
+		 * to force reset of the rest of the system.  (If fail, we'll try again
+		 * in future cycles of the main loop.)
+		 */
+		if (pid == BgCleanerPID)
+		{
+			BgCleanerPID = 0;
+			if (!EXIT_STATUS_0(exitstatus))
+				LogChildExit(LOG, _("background cleaner process"),
+							 pid, exitstatus);
+			if (BgCleanerPID == 0 && BgCleanerEnable)
+				BgCleanerPID = BgCleanerStart();
+			continue;
+		}
+
 		/*
 		 * Was it the wal receiver?  If exit status is zero (normal) or one
 		 * (FATAL exit), we assume everything is all right just like normal
@@ -5175,6 +5201,12 @@ SubPostmasterMain(int argc, char *argv[])
 
 		SysLoggerMain(argc, argv);	/* does not return */
 	}
+	if (strcmp(argv[1], "--forkbgcleaner") == 0)
+	{
+		/* Do not want to attach to shared memory */
+
+		BackgroundCleanerMain(argc, argv);	/* does not return */
+	}
 
 	abort();					/* shouldn't get here */
 }
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 10cbdea124..7b947a7428 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -28,6 +28,8 @@
 
 #include "postgres.h"
 
+#include <unistd.h>
+
 #include "access/xact.h"
 #include "access/xlog_internal.h"
 #include "fmgr.h"
@@ -1842,3 +1844,41 @@ UpdateDecodingStats(LogicalDecodingContext *ctx)
 	rb->totalTxns = 0;
 	rb->totalBytes = 0;
 }
+
+void
+CreateReplicationCleanupFile(ReplCleanupFileKind kind, XLogRecPtr cutoff_lsn)
+{
+	int 	fd;
+	/*
+	 * 27 is size of the fixed path name "pg_logical/snapshot_cutoff_".
+	 * XXX: Dynamically allocate memory for the path variable, if at all, this
+	 * function is changed to deal with other kinds of files with differnt
+	 * fixed path names.
+	 */
+	char	path[MAXPGPATH + 27];
+
+	Assert(kind == REPL_CLEANUP_FILE_SNAPSHOT);
+
+	MemSet(path, '\0', sizeof(path));
+
+	/* create a file named snapshot_cutoff_<hi>-<lo> */
+	snprintf(path, sizeof(path), "pg_logical/snapshot_cutoff_%X-%X",
+			 LSN_FORMAT_ARGS(cutoff_lsn));
+
+	/*
+	 * We don't need O_EXCL flag as it might cause FATAL error if the file
+	 * already exists. It is the responsibility of the clean up prgoram to
+	 * delete the previous files.
+	 */
+	fd = BasicOpenFile(path, O_RDWR | O_CREAT);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not create file \"%s\": %m", path)));
+
+	/* make sure we persist */
+	fsync_fname(path, false);
+	fsync_fname("pg_logical", true);
+
+	close(fd);
+}
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index dbdc172a2b..3933923d52 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -125,6 +125,7 @@
 #include "access/xact.h"
 #include "miscadmin.h"
 #include "pgstat.h"
+#include "postmaster/bgcleaner.h"
 #include "replication/logical.h"
 #include "replication/reorderbuffer.h"
 #include "replication/snapbuild.h"
@@ -1941,6 +1942,13 @@ CheckPointSnapBuild(void)
 	if (redo < cutoff)
 		cutoff = redo;
 
+	/* do this only if there exists a background cleaner */
+	if (BgCleanerEnable && !BgCleanerStopProcessingFiles)
+	{
+		CreateReplicationCleanupFile(REPL_CLEANUP_FILE_SNAPSHOT, cutoff);
+		return;
+	}
+
 	snap_dir = AllocateDir("pg_logical/snapshots");
 	while ((snap_de = ReadDir(snap_dir, "pg_logical/snapshots")) != NULL)
 	{
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index 4d53f040e8..f0fc681e5f 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -215,6 +215,9 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_AUTOVACUUM_MAIN:
 			event_name = "AutoVacuumMain";
 			break;
+		case WAIT_EVENT_BGCLEANER_MAIN:
+			event_name = "BackgroundCleanerMain";
+			break;
 		case WAIT_EVENT_BGWRITER_HIBERNATE:
 			event_name = "BgWriterHibernate";
 			break;
diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c
index 88801374b5..f734c82229 100644
--- a/src/backend/utils/init/miscinit.c
+++ b/src/backend/utils/init/miscinit.c
@@ -264,6 +264,9 @@ GetBackendTypeDesc(BackendType backendType)
 		case B_BACKEND:
 			backendDesc = "client backend";
 			break;
+		case B_BG_CLEANER:
+			backendDesc = "background cleaner";
+			break;
 		case B_BG_WORKER:
 			backendDesc = "background worker";
 			break;
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ee6a838b3a..29eef8f155 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -68,6 +68,9 @@
 #include "parser/scansup.h"
 #include "pgstat.h"
 #include "postmaster/autovacuum.h"
+#ifdef AZURE_SERVICE_FABRIC
+#include "postmaster/bgcleaner.h"
+#endif
 #include "postmaster/bgworker_internals.h"
 #include "postmaster/bgwriter.h"
 #include "postmaster/postmaster.h"
@@ -1791,7 +1794,26 @@ static struct config_bool ConfigureNamesBool[] =
 		false,
 		NULL, NULL, NULL
 	},
-
+#ifdef AZURE_SERVICE_FABRIC
+	{
+		{"bgcleaner_enable", PGC_POSTMASTER, DEVELOPER_OPTIONS,
+			gettext_noop("Start a subprocess to remove unneeded replication slot snapshot files."),
+			NULL
+		},
+		&BgCleanerEnable,
+		true,
+		NULL, NULL, NULL
+	},
+	{
+		{"bgcleaner_stop_processing_files", PGC_SIGHUP, DEVELOPER_OPTIONS,
+			gettext_noop("Inform background cleaner to stop processing files."),
+			NULL
+		},
+		&BgCleanerStopProcessingFiles,
+		false,
+		NULL, NULL, NULL
+	},
+#endif
 #ifdef TRACE_SORT
 	{
 		{"trace_sort", PGC_USERSET, DEVELOPER_OPTIONS,
@@ -3065,6 +3087,19 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+#ifdef AZURE_SERVICE_FABRIC
+	{
+		{"bgcleaner_delay", PGC_SIGHUP, DEVELOPER_OPTIONS,
+			gettext_noop("Background cleaner sleep time between rounds."),
+			NULL,
+			GUC_UNIT_S
+		},
+		&BgCleanerDelay,
+		180, 60, 86400,
+		NULL, NULL, NULL
+	},
+#endif
+
 	{
 		{"effective_io_concurrency",
 			PGC_USERSET,
diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h
index 90a3016065..5449f07a80 100644
--- a/src/include/miscadmin.h
+++ b/src/include/miscadmin.h
@@ -326,6 +326,7 @@ typedef enum BackendType
 	B_AUTOVAC_LAUNCHER,
 	B_AUTOVAC_WORKER,
 	B_BACKEND,
+	B_BG_CLEANER,
 	B_BG_WORKER,
 	B_BG_WRITER,
 	B_CHECKPOINTER,
diff --git a/src/include/postmaster/bgcleaner.h b/src/include/postmaster/bgcleaner.h
new file mode 100644
index 0000000000..5bba615ec8
--- /dev/null
+++ b/src/include/postmaster/bgcleaner.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * bgcleaner.h
+ *	  Exports from postmaster/bgcleaner.c.
+ *
+ * The bgcleaner process removes unneeded replication slot files (.snap).
+ * This is to offload the checkpoint responsibility so that the checkpoint
+ * (and so the recovery) can be faster.
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ *
+ * src/include/postmaster/bgcleaner.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _BGCLEANER_H
+#define _BGCLEANER_H
+
+/*
+ * GUC parameters
+ */
+extern bool BgCleanerEnable;
+extern bool BgCleanerStopProcessingFiles;
+extern int	BgCleanerDelay;
+
+extern int	BgCleanerStart(void);
+
+#ifdef EXEC_BACKEND
+extern void BackgroundCleanerMain(int argc, char *argv[]) pg_attribute_noreturn();
+#endif
+
+#endif							/* _BGCLEANER_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index e0f513b773..9971dc81f7 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -16,6 +16,11 @@
 
 struct LogicalDecodingContext;
 
+typedef enum ReplCleanupFileKind
+{
+	REPL_CLEANUP_FILE_SNAPSHOT = 1
+} ReplCleanupFileKind;
+
 typedef void (*LogicalOutputPluginWriterWrite) (struct LogicalDecodingContext *lr,
 												XLogRecPtr Ptr,
 												TransactionId xid,
@@ -140,5 +145,6 @@ extern bool filter_prepare_cb_wrapper(LogicalDecodingContext *ctx,
 extern bool filter_by_origin_cb_wrapper(LogicalDecodingContext *ctx, RepOriginId origin_id);
 extern void ResetLogicalStreamingState(void);
 extern void UpdateDecodingStats(LogicalDecodingContext *ctx);
+extern void CreateReplicationCleanupFile(ReplCleanupFileKind kind, XLogRecPtr cutoff_lsn);
 
 #endif
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 8785a8e12c..be90cd0fe1 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -37,6 +37,7 @@ typedef enum
 {
 	WAIT_EVENT_ARCHIVER_MAIN = PG_WAIT_ACTIVITY,
 	WAIT_EVENT_AUTOVACUUM_MAIN,
+	WAIT_EVENT_BGCLEANER_MAIN,
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
-- 
2.25.1

