>> Based on the discussion and suggestions in this mail chain, following
>> features can be implemented:
>>
>> 1. To compute the value of max LSN in data pages based on user input whether
>> he wants it for an individual
>> file, a particular directory or whole database.
>>
>> 2a. To search the available WAL files for the latest checkpoint record and
>> prints the value.
>> 2b. To search the available WAL files for the latest checkpoint record and
>> recreates a pg_control file pointing at
>> that checkpoint.
>> I have kept both options to address different kind of corruption scenarios.
> I think I can see all of those things being potentially useful. There
> are a couple of pending patches that will revise the WAL format
> slightly; not sure how much those are likely to interfere with any
> development you might do on (2) in the meantime.
Based on above conclusion, I have prepared a patch which implements Option-1
To find the value of max LSN in data pages based on user input whether he wants
for
- An individual file
- A particular directory
- Whole database
Corresponding pg_resetxlog options are as follows
-p {file | dir} print max LSN from specified file or directory path
-P print max LSN from whole database
Note: in case of -p {file | dir} input path should be absolute path or relative
from data base directory.
These options are useful when pg_control, WAL files and data files are missing
or corrupted.
Using above options user can able to find the max LSN number and can be able to
compute the next redo log sequence number.
Sample output:
postgres@linux:> pg_resetxlog -P /home/postgres/installation/bin/data
Maximum LSN found is: 73325448, WAL segment file name (fileid, seg):
0000000000000004
Design:
Based on user option display max LSN.
1. Finding max LSN in an individual file [pg_resetxlog option: -p file-name]
A. Open the given file and check for the number of blocks;
B. Read page header and validate; if valid find the max lsn number; if invalid
log the page-id and filename and continue to next page.
2. Finding max LSN a folder (excluding sub directories) [pg_resetxlog option:
-p folder-name]
Note: Here we are not traversing through sub directories, as some times it
may possible to have recursive loops because of soft links
Read all the file in the given folder using ReadDir function
If file name / folder name start with pgsql_tmp ignore and continue to
next.
Find the max LSN in this file (refer 1. Finding max LSN in an
individual file)
3. Finding max LSN for whole database [pg_resetxlog option: -P]
A. Read the base directory
Format: pgDataDirecoty/base/databaseid/*
1. Skip the folder if name is equal to “0” or “1”; [skip
template database]
2. Form the new folder name as and call the function written in [2. Finding max
LSN a folder]
B. Read the global directory
pgDataDirecoty/global
Note: here need to exclude the files [pg_controldata, .. ] which
are taken care in folder reading function.
C. Read all table spaces
Folder structure: pg_tblspc/table space id/<CONSTANT PG VERSION
STRING>/Database ID/relfilenodes.
1. Read all table space names in pg_tblspc/*
1.1. For each folder form the path as
pg_tblspc/tblspc-folder-name/<CONSTANT PG VERSION
STRING>/
1.2. Read all the directories in pg_tblspc/table space id/<CONSTANT PG VERSION
STRING>/*
1.2.1. For each folder form the path as “pg_tblspc/
tblspc-folder-name /<CONSTANT-PG-VERSION STRING>/db-id-folder-name”
Comments/Objections?
With Regards,
Amit Kapila.
diff --git a/src/bin/pg_resetxlog/pg_resetxlog.c
b/src/bin/pg_resetxlog/pg_resetxlog.c
index d5d89ec..dcb62d1 100644
--- a/src/bin/pg_resetxlog/pg_resetxlog.c
+++ b/src/bin/pg_resetxlog/pg_resetxlog.c
@@ -54,6 +54,20 @@
#include "access/xlog_internal.h"
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
+#include "catalog/catalog.h"
+#include "storage/bufpage.h"
+#include "storage/fd.h"
+
+
+/* Page header size */
+#define PAGEHDRSZ (sizeof(PageHeaderData))
+
+
+/*
+ * relfile nodename validation allow only file name start with digit
+ */
+#define validateRelfilenodename(name) ((name[0] >= '0') && (name[0] <= '9'))
+
extern int optind;
extern char *optarg;
@@ -72,6 +86,9 @@ static void FindEndOfXLOG(void);
static void KillExistingXLOG(void);
static void KillExistingArchiveStatus(void);
static void WriteEmptyXLOG(void);
+static void FindMaxLSNinFile(char *filename, XLogRecPtr *maxlsn);
+static void FindMaxLSNinDir(char *path, XLogRecPtr *maxlsn);
+static void FindMaxLSNinPgData(XLogRecPtr *maxlsn);
static void usage(void);
@@ -92,6 +109,10 @@ main(int argc, char *argv[])
char *DataDir;
int fd;
char path[MAXPGPATH];
+ bool print_max_lsn = false;
+ bool print_pgdata_max_lsn = false;
+ char *LsnSearchPath;
+ uint64 maxLSN = 0;
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_resetxlog"));
@@ -112,7 +133,7 @@ main(int argc, char *argv[])
}
- while ((c = getopt(argc, argv, "fl:m:no:O:x:e:")) != -1)
+ while ((c = getopt(argc, argv, "fl:m:no:O:x:e:p:P")) != -1)
{
switch (c)
{
@@ -209,6 +230,15 @@ main(int argc, char *argv[])
XLogFromFileName(optarg, &minXlogTli,
&minXlogSegNo);
break;
+ case 'p':
+ print_max_lsn = true;
+ LsnSearchPath = optarg;
+ break;
+
+ case 'P':
+ print_pgdata_max_lsn = true;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more
information.\n"), progname);
exit(1);
@@ -270,6 +300,55 @@ main(int argc, char *argv[])
exit(1);
}
+
+ if (print_max_lsn || print_pgdata_max_lsn)
+ {
+ XLogSegNo logSegNo = 0;
+
+ if (print_pgdata_max_lsn)
+ {
+ FindMaxLSNinPgData(&maxLSN);
+ }
+ else
+ {
+ struct stat fst;
+
+ if (stat(LsnSearchPath, &fst) < 0)
+ {
+ if (errno == ENOENT)
+ {
+ fprintf(stderr, _("%s: file or
directory \"%s\" does not exists"),
+ progname,
LsnSearchPath);
+ }
+ else
+ {
+ fprintf(stderr, _("%s: file or
directory \"%s\" exists\n"
+ "\n"),
+ progname,
LsnSearchPath);
+ }
+ exit(1);
+ }
+
+ if (S_ISDIR(fst.st_mode))
+ {
+ FindMaxLSNinDir(LsnSearchPath, &maxLSN);
+ }
+ else
+ {
+ FindMaxLSNinFile(LsnSearchPath, &maxLSN);
+ }
+ }
+
+ XLByteToSeg(maxLSN, logSegNo);
+
+ printf ("Maximum LSN found is: " UINT64_FORMAT ", "
+ "WAL segment file name (fileid,seg):
%08X%08X\n",
+ maxLSN,
+ (uint32) ((logSegNo) / XLogSegmentsPerXLogId),
+ (uint32) ((logSegNo) % XLogSegmentsPerXLogId));
+ exit(0);
+ }
+
/*
* Attempt to read the existing pg_control file
*/
@@ -986,6 +1065,290 @@ WriteEmptyXLOG(void)
}
+/*
+ * PageHeaderIsValid: Check page is valid or not
+ */
+bool
+PageHeaderIsValid(PageHeader page)
+{
+ char *pagebytes;
+ int i;
+
+ /* Check normal case */
+ if (PageGetPageSize(page) == BLCKSZ &&
+ PageGetPageLayoutVersion(page) == PG_PAGE_LAYOUT_VERSION &&
+ (page->pd_flags & ~PD_VALID_FLAG_BITS) == 0 &&
+ page->pd_lower >= SizeOfPageHeaderData &&
+ page->pd_lower <= page->pd_upper &&
+ page->pd_upper <= page->pd_special &&
+ page->pd_special <= BLCKSZ &&
+ page->pd_special == MAXALIGN(page->pd_special))
+ return true;
+
+ /*
+ * Check all-zeroes till page header; this is used only to log the page
+ * details even we detect invalid page we will continue to nex pages
+ */
+ pagebytes = (char *) page;
+ for (i = 0; i < PAGEHDRSZ; i++)
+ {
+ if (pagebytes[i] != 0)
+ return false;
+ }
+ return true;
+}
+
+
+/*
+ * Read the maximum LSN number in the one of data file (relnode file).
+ *
+ */
+static void
+FindMaxLSNinFile(char *filename, XLogRecPtr *maxlsn)
+{
+ XLogRecPtr pagelsn;
+ off_t len,
+ seekpos;
+ uint32 nblocks,
+ blocknum;
+ char buffer[PAGEHDRSZ];
+ int nbytes;
+ int fd;
+
+ if ((fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
+ {
+ /*
+ * If file does not exist or or we can't read it. give error
+ */
+ fprintf(stderr, _("%s: could not open file \"%s\" for reading:
%s\n"),
+ progname, filename, strerror(errno));
+ return;
+ }
+
+ /* Calculate the number of pages in file */
+ len = lseek(fd, 0L, SEEK_END);
+ if (len < 0)
+ {
+ close(fd);
+ fprintf(stderr, _("%s: .. file \"%s\" for seeking: %s\n"),
+ progname, filename, strerror(errno));
+ return;
+ }
+
+ nblocks = (len / BLCKSZ);
+ if (nblocks > RELSEG_SIZE)
+ {
+ /*
+ * In one relfilenode file length can't be more that RELSEG_SIZE
+ */
+ close(fd);
+ fprintf(stderr, _("%s: .. file \"%s\" legth is more than
RELSEG_SIZE.\n"),
+ progname, filename);
+ return;
+ }
+
+ /*
+ * Read the only page header and validate; if we find invalid page log
the
+ * details of page and continue to next page.
+ */
+ seekpos = 0;
+ for (blocknum = 0; blocknum < nblocks; blocknum++)
+ {
+ len = lseek(fd, seekpos, SEEK_SET);
+ if (len != seekpos)
+ {
+ close(fd);
+ fprintf(stderr, _("%s: could not seek to next page
\"%s\": %s\n"),
+ progname, filename, strerror(errno));
+ return;
+ }
+
+ nbytes = read(fd, buffer, PAGEHDRSZ);
+ if (nbytes < 0)
+ {
+ close(fd);
+ fprintf(stderr, _("%s: could not read file \"%s\":
%s\n"),
+ progname, filename, strerror(errno));
+ return;
+ }
+
+ if (PageHeaderIsValid((PageHeader) buffer))
+ {
+ pagelsn = PageGetLSN(buffer);
+ if (XLByteLE(*maxlsn, pagelsn))
+ {
+ *maxlsn = pagelsn;
+ }
+ }
+ else
+ {
+ /*
+ * If page is invalid log the error and continue
+ */
+ fprintf(stderr, _("%s: Invalid page found in file
\"%s\" pagid:%d\n"),
+ progname, filename, blocknum);
+ }
+ seekpos += (off_t) BLCKSZ;
+ }
+
+ close(fd);
+ return;
+}
+
+/*
+ * Read the maximum LSN number in current directory; ignore sub directories.
+ */
+static void
+FindMaxLSNinDir(char *path, XLogRecPtr *maxlsn)
+{
+ DIR *dir;
+ struct dirent *de;
+ char pathbuf[MAXPGPATH];
+ struct stat statbuf;
+
+ dir = opendir(path);
+ if (NULL == dir)
+ {
+ fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"),
+ progname, path, strerror(errno));
+ return;
+ }
+
+ while ((de = readdir(dir)) != NULL)
+ {
+ /* Skip special stuff */
+ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") ==
0)
+ continue;
+
+ /* Skip temporary files */
+ if (strncmp(de->d_name,
+ PG_TEMP_FILE_PREFIX,
+ strlen(PG_TEMP_FILE_PREFIX)) == 0)
+ continue;
+
+ /*
+ * Skip all the local/global temporary files, and read and read
all
+ * reamining relfinenode files
+ */
+ if (!validateRelfilenodename(de->d_name))
+ continue;
+
+ snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name);
+
+ if (stat(pathbuf, &statbuf) != 0)
+ {
+ if (errno != ENOENT)
+ {
+ fprintf(stderr, "could not stat file or
directory \"%s\"",
+ pathbuf);
+ }
+ /* If the file went away while scanning, it's no error.
*/
+ continue;
+ }
+
+ /*
+ * Skip all directories
+ */
+ if (!S_ISDIR(statbuf.st_mode))
+ {
+ FindMaxLSNinFile(pathbuf, maxlsn);
+ }
+ }
+
+ closedir(dir);
+ return;
+}
+
+/*
+ * Read the maximum LSN number in the DATA directory.
+ */
+static void
+FindMaxLSNinPgData(XLogRecPtr *maxlsn)
+{
+ DIR *dir,
+ *dir2;
+ struct dirent *de,
+ *de2;
+ char pathbuf[MAXPGPATH],
+ pathbuf2[MAXPGPATH];
+
+ /*
+ * scan all the relfilenodes in global directory
+ */
+ FindMaxLSNinDir("global", maxlsn);
+
+ /*
+ * scan all the relfilenodes in base directory like base/<database-id>
+ */
+ dir = opendir("base");
+ if (NULL == dir)
+ {
+ fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"),
+ progname, "base", strerror(errno));
+ return;
+ }
+
+ while ((de = readdir(dir)) != NULL)
+ {
+ /* Skip special stuff */
+ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") ==
0)
+ continue;
+
+ /* Skip template database */
+ if (strcmp(de->d_name, "0") == 0 || strcmp(de->d_name, "1") ==
0)
+ continue;
+
+ snprintf(pathbuf, MAXPGPATH, "%s/%s", "base", de->d_name);
+
+ FindMaxLSNinDir(pathbuf, maxlsn);
+ }
+ closedir(dir);
+
+ /* Scan all tablespaces */
+ dir = opendir("pg_tblspc");
+ if (NULL == dir)
+ {
+ fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"),
+ progname, "pg_tblspc", strerror(errno));
+ return;
+ }
+
+ while ((de = readdir(dir)) != NULL)
+ {
+ /* Skip special stuff */
+ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") ==
0)
+ continue;
+
+ /*
+ * Scan all directories in tablespace
+ * ./pg_tblspc/tablespaceid/TABLESPACE_VERSION_DIRECTORY/ *
+ */
+ snprintf(pathbuf, MAXPGPATH, "%s/%s/%s", "pg_tblspc",
de->d_name, TABLESPACE_VERSION_DIRECTORY);
+ dir2 = opendir(pathbuf);
+ if (NULL == dir2)
+ {
+ fprintf(stderr, _("%s: could not open directory \"%s\":
%s\n"),
+ progname, pathbuf, strerror(errno));
+ return;
+ }
+
+ while ((de2 = readdir(dir2)) != NULL)
+ {
+ /* Skip special stuff */
+ if (strcmp(de2->d_name, ".") == 0 ||
strcmp(de2->d_name, "..") == 0)
+ continue;
+
+ snprintf(pathbuf2, MAXPGPATH, "%s/%s", pathbuf,
de2->d_name);
+
+ FindMaxLSNinDir(pathbuf2, maxlsn);
+ }
+ closedir(dir2);
+ }
+ closedir(dir);
+
+ return;
+}
+
static void
usage(void)
{
@@ -997,6 +1360,8 @@ usage(void)
printf(_(" -l xlogfile force minimum WAL starting location for
new transaction log\n"));
printf(_(" -m XID set next multitransaction ID\n"));
printf(_(" -n no update, just show extracted control
values (for testing)\n"));
+ printf(_(" -p {file | dir} print max LSN from specified file or
directory path\n"));
+ printf(_(" -P print max LSN from whole database\n"));
printf(_(" -o OID set next OID\n"));
printf(_(" -O OFFSET set next multitransaction offset\n"));
printf(_(" -V, --version output version information, then exit\n"));
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
