Hello, I`d like to show my implementation of SLRU file protection with
checksums.
It only has effect if checksums on database are enabled.
Detection of a checksum failure during a read normally causes PostgreSQL
to report an error. Setting ignore_slru_checksum_failure to on causes
the system to ignore the failure (but still report a warning), and
continue processing. It is similary like ignore_checksum_failure but for
some slru files. The default setting is true, and it can only be changed
by a database admin. For changes, use ALTER SYSTEM and reload
configuration:
ALTER SYSTEM SET ignore_slru_checksum_failure = off;
SELECT pg_reload_conf();
Impementation:
1) Checksum writes in last 2 bytes of every page
2) Checksum calculates and write down in page happens when page writes
on disk (same as relation does it)
3) Checking checksum happens same as relation does it, on
Read\PhysicalRead when GUC ignore_slru_checksum_failure = false
{default = true}.
I would like to hear your thoughts over my patch.
--
Ivan Kartyshov
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Companydiff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index aeda826..8501dd2 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -8394,6 +8394,29 @@ LOG: CleanUpLock: deleting: lock(0xb7acd844) id(24688,24696,0,0,0,1)
</listitem>
</varlistentry>
+ <varlistentry id="guc-ignore-slru-checksum-failure" xreflabel="ignore_slru_checksum_failure">
+ <term><varname>ignore_slru_checksum_failure</varname> (<type>boolean</type>)
+ <indexterm>
+ <primary><varname>ignore_slru_checksum_failure</> configuration parameter</primary>
+ </indexterm>
+ </term>
+ <listitem>
+ <para>
+ Only has effect if <xref linkend="app-initdb-data-checksums"> are enabled.
+ </para>
+ <para>
+ Detection of a checksum failure during a read normally causes
+ <productname>PostgreSQL</> to report an error. Setting <varname>ignore_slru_checksum_failure</>
+ to on causes the system to ignore the failure (but still report a warning), and
+ continue processing. Similary like <varname>ignore_checksum_failure</> but for
+ not relation files. The default setting is <literal>true</>, and it can only be
+ changed by a superuser. For changes, use ALTER SYSTEM and reload configuration:
+ ALTER SYSTEM SET ignore_slru_checksum_failure = off;
+ SELECT pg_reload_conf();
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="guc-zero-damaged-pages" xreflabel="zero_damaged_pages">
<term><varname>zero_damaged_pages</varname> (<type>boolean</type>)
<indexterm>
diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c
index a3e2b12..87ed09c 100644
--- a/src/backend/access/transam/clog.c
+++ b/src/backend/access/transam/clog.c
@@ -58,7 +58,7 @@
/* We need two bits per xact, so four xacts fit in a byte */
#define CLOG_BITS_PER_XACT 2
#define CLOG_XACTS_PER_BYTE 4
-#define CLOG_XACTS_PER_PAGE (BLCKSZ * CLOG_XACTS_PER_BYTE)
+#define CLOG_XACTS_PER_PAGE ((BLCKSZ - CHKSUMSZ) * CLOG_XACTS_PER_BYTE)
#define CLOG_XACT_BITMASK ((1 << CLOG_BITS_PER_XACT) - 1)
#define TransactionIdToPage(xid) ((xid) / (TransactionId) CLOG_XACTS_PER_PAGE)
diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c
index 7142ece..35c317e 100644
--- a/src/backend/access/transam/multixact.c
+++ b/src/backend/access/transam/multixact.c
@@ -106,7 +106,7 @@
*/
/* We need four bytes per offset */
-#define MULTIXACT_OFFSETS_PER_PAGE (BLCKSZ / sizeof(MultiXactOffset))
+#define MULTIXACT_OFFSETS_PER_PAGE ((BLCKSZ - CHKSUMSZ) / sizeof(MultiXactOffset))
#define MultiXactIdToOffsetPage(xid) \
((xid) / (MultiXactOffset) MULTIXACT_OFFSETS_PER_PAGE)
@@ -2039,7 +2039,6 @@ TrimMultiXact(void)
{
int slotno;
MultiXactOffset *offptr;
-
slotno = SimpleLruReadPage(MultiXactOffsetCtl, pageno, true, nextMXact);
offptr = (MultiXactOffset *) MultiXactOffsetCtl->shared->page_buffer[slotno];
offptr += entryno;
diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c
index 9dd7719..b7a154c 100644
--- a/src/backend/access/transam/slru.c
+++ b/src/backend/access/transam/slru.c
@@ -58,7 +58,18 @@
#include "storage/fd.h"
#include "storage/shmem.h"
#include "miscadmin.h"
+#include "utils/guc.h"
+#include "storage/checksum.h"
+#include "utils/memutils.h"
+
+/*
+ * GUC variable
+ * Set from backend:
+ * alter system set ignore_slru_checksum_failure = on/off;
+ * select pg_reload_conf();
+ */
+bool ignore_slru_checksum_failure = true;
#define SlruFileName(ctl, path, seg) \
snprintf(path, MAXPGPATH, "%s/%04X", (ctl)->Dir, seg)
@@ -376,6 +387,8 @@ SimpleLruReadPage(SlruCtl ctl, int pageno, bool write_ok,
TransactionId xid)
{
SlruShared shared = ctl->shared;
+ int checksum;
+ static char *pageCopy = NULL;
/* Outer loop handles restart if we must wait for someone else's I/O */
for (;;)
@@ -426,6 +439,22 @@ SimpleLruReadPage(SlruCtl ctl, int pageno, bool write_ok,
/* Do the read */
ok = SlruPhysicalReadPage(ctl, pageno, slotno);
+ if (pageCopy == NULL)
+ pageCopy = MemoryContextAlloc(TopMemoryContext, BLCKSZ);
+
+ memcpy(pageCopy, shared->page_buffer[slotno], BLCKSZ);
+
+ checksum = pg_getchecksum_slru_page(pageCopy);
+
+ if ( DataChecksumsEnabled() && (checksum != pg_checksum_slru_page(pageCopy) ))
+ {
+ elog(LOG, "CHECKSUM: Page Is not Verified.");
+ if (!ignore_slru_checksum_failure)
+ {
+ elog(ERROR, "CHECKSUM: ERROR ignore_slru_checksum_failure turned off.");
+ }
+ }
+
/* Set the LSNs for this newly read-in page to zero */
SimpleLruZeroLSNs(ctl, slotno);
@@ -539,6 +568,14 @@ SlruInternalWritePage(SlruCtl ctl, int slotno, SlruFlush fdata)
/* Release control lock while doing I/O */
LWLockRelease(shared->ControlLock);
+ /*
+ * Calculate checksum of the page and write in the last 2 bytes of the page
+ *
+ * NOTE: it`s all done in checksum_impl.h : pg_checksum_slru_page()
+ * you give there page and get page with updated checksum
+ */
+ pg_checksum_slru_page(shared->page_buffer[slotno]);
+
/* Do the write */
ok = SlruPhysicalWritePage(ctl, pageno, slotno, fdata);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ae22185..8d62e7e 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -116,6 +116,7 @@ extern int CommitSiblings;
extern char *default_tablespace;
extern char *temp_tablespaces;
extern bool ignore_checksum_failure;
+extern bool ignore_slru_checksum_failure;
extern bool synchronize_seqscans;
#ifdef TRACE_SYNCSCAN
@@ -1006,6 +1007,20 @@ static struct config_bool ConfigureNamesBool[] =
NULL, NULL, NULL
},
{
+ {"ignore_slru_checksum_failure", PGC_SUSET, DEVELOPER_OPTIONS,
+ gettext_noop("Continues processing after SLRU checksum failure."),
+ gettext_noop("Detection of a SLRU checksum failure normally causes PostgreSQL to "
+ "report an error. Setting ignore_slru_checksum_failure to true"
+ "causes the system to ignore the failure (but still report a warning), and"
+ "continue processing. This behavior could cause crashes or other serious"
+ "problems. Only has an effect if checksums are enabled."),
+ GUC_NOT_IN_SAMPLE
+ },
+ &ignore_slru_checksum_failure,
+ true,
+ NULL, NULL, NULL
+ },
+ {
{"zero_damaged_pages", PGC_SUSET, DEVELOPER_OPTIONS,
gettext_noop("Continues processing past damaged page headers."),
gettext_noop("Detection of a damaged page header normally causes PostgreSQL to "
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 0f2b8bd..57f7a1e 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -292,6 +292,9 @@ extern void assign_checkpoint_completion_target(double newval, void *extra);
* Routines to start, stop, and get status of a base backup.
*/
+/* Size of checksum in bytes default 2 bytes (uint16) */
+#define CHKSUMSZ 2
+
/*
* Session-level status of base backups
*
diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h
index b85f714..67535b0 100644
--- a/src/include/storage/checksum.h
+++ b/src/include/storage/checksum.h
@@ -21,4 +21,8 @@
*/
extern uint16 pg_checksum_page(char *page, BlockNumber blkno);
+extern uint16 pg_checksum_slru_page(char *page);
+
+extern uint16 pg_getchecksum_slru_page(char *page);
+
#endif /* CHECKSUM_H */
diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h
index bffd061..8e340db 100644
--- a/src/include/storage/checksum_impl.h
+++ b/src/include/storage/checksum_impl.h
@@ -106,6 +106,8 @@
#define N_SUMS 32
/* prime multiplier of FNV-1a hash */
#define FNV_PRIME 16777619
+/* Size of checksum in bytes default 2 bytes (uint16) */
+#define CHKSUMSZ 2
/*
* Base offsets to initialize each of the parallel FNV hashes into a
@@ -205,3 +207,51 @@ pg_checksum_page(char *page, BlockNumber blkno)
*/
return (checksum % 65535) + 1;
}
+
+/*
+ * Compute the checksum for a Postgres SLRU page. The page must be aligned on a
+ * 4-byte boundary.
+ *
+ * The checksum save itself to the last 2 bytes (CHKSUMSZ = 2 bytes) of the page
+ */
+uint16
+pg_checksum_slru_page(char *page)
+{
+ uint16 checksum;
+ uint8 bytes[2];
+
+ /* Set last 2 bytes to 0 */
+ page[BLCKSZ - 1] = 0;
+ page[BLCKSZ - 2] = 0;
+
+ checksum = (pg_checksum_block(page, BLCKSZ) % 65535) + 1;
+
+ bytes[0] = (checksum & 0X00FF); /* Least significant bit */
+ bytes[1] = (checksum & 0XFF00) >> 8; /* Most significant bit */
+
+
+ /* Set last 2 bytes to calculated checksum */
+ page[BLCKSZ - 1] = bytes[0];
+ page[BLCKSZ - 2] = bytes[1];
+
+ return checksum;
+}
+
+
+/*
+ * Get the checksum for a Postgres SLRU page. The page must be aligned on a
+ * 4-byte boundary.
+ *
+ * The checksum located at last 2 bytes (CHKSUMSZ = 2 bytes) of the page
+ */
+uint16
+pg_getchecksum_slru_page(char *page)
+{
+ uint16 checksum = 0X00000000;
+
+ checksum = checksum | page[BLCKSZ - 2];
+ checksum = checksum << 8 ;
+ checksum = checksum | (page[BLCKSZ - 1] & 0X00FF);
+
+ return checksum;
+}
