From dbc1862c4bfaab8c34b367ff57c8d7140063df40 Mon Sep 17 00:00:00 2001 From: Brian Weaver Date: Mon, 24 Sep 2012 21:24:15 -0400 Subject: [PATCH] Fix some inconsistencies in tar header generation The numeric group identity was one byte too far in its offset. The inclusion of the UStar magic and version fields were incorrect in the header. ustar was not terminated by a null byte and the '00' version was one byte too soon. Changed the emitting code to more rigorously follow the POSIX standard. Also switched from using sprintf to strncpy() and a custom octal conversion that works from the back of the buffer to the front. The octal conversion also checks for a numeric overflow now too. Refactored the common constants into their own header file. Used the constants (defines) in place of numeric values when possible to improved the reability of the code. --- src/backend/replication/basebackup.c | 307 ++++++++++++++++++++------------- src/bin/pg_basebackup/pg_basebackup.c | 115 +++++++++---- src/bin/pg_dump/pg_backup_tar.c | 291 ++++++++++++++++++++------------ src/bin/pg_dump/pg_backup_tar.h | 35 +---- src/include/portability/ustar.h | 97 +++++++++++ 5 files changed, 552 insertions(+), 293 deletions(-) create mode 100644 src/include/portability/ustar.h diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 4aaa9e3..9f456af 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -32,6 +32,7 @@ #include "utils/elog.h" #include "utils/memutils.h" #include "utils/ps_status.h" +#include "portability/ustar.h" typedef struct { @@ -44,11 +45,8 @@ typedef struct static int64 sendDir(char *path, int basepathlen, bool sizeonly); -static void sendFile(char *readfilename, char *tarfilename, - struct stat * statbuf); +static void sendFile(char *readfilename, char *tarfilename, struct stat * statbuf); static void sendFileWithContent(const char *filename, const char *content); -static void _tarWriteHeader(const char *filename, const char *linktarget, - struct stat * statbuf); static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); static void base_backup_cleanup(int code, Datum arg); @@ -56,6 +54,12 @@ static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir); static void parse_basebackup_options(List *options, basebackup_options *opt); static void SendXlogRecPtrResult(XLogRecPtr ptr); +static void _toOctal(unsigned int val, char *p, int length); +static unsigned int _tarChecksum(char const *header); +static void _tarWriteHeader(const char *filename, + const char *linktarget, + struct stat * statbuf); + /* * Size of each block sent into the tar stream for larger files. * @@ -133,12 +137,19 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir) fullpath))); continue; } + else if (rllen > TAR_LENGTH_LINKNAME) + { + ereport(WARNING, + (errmsg("symbolic link \"%s\" target is too long for the ustar archive format", + fullpath))); + continue; + } linkpath[rllen] = '\0'; ti = palloc(sizeof(tablespaceinfo)); ti->oid = pstrdup(de->d_name); ti->path = pstrdup(linkpath); - ti->size = opt->progress ? sendDir(linkpath, strlen(linkpath), true) : -1; + ti->size = opt->progress ? sendDir(linkpath, strnlen(linkpath,sizeof(linkpath)), true) : -1; tablespaces = lappend(tablespaces, ti); #else @@ -531,8 +542,10 @@ sendFileWithContent(const char *filename, const char *content) { struct stat statbuf; int pad, - len; + len, + off; + MemSet(&statbuf, 0, sizeof(statbuf)); len = strlen(content); /* @@ -552,14 +565,23 @@ sendFileWithContent(const char *filename, const char *content) statbuf.st_size = len; _tarWriteHeader(filename, NULL, &statbuf); + /* Send the contents as a CopyData message */ - pq_putmessage('d', content, len); + off = 0; + while ((len-off) > TAR_BLOCK_SIZE) + { + pq_putmessage('d', content+off, TAR_BLOCK_SIZE); + off += TAR_BLOCK_SIZE; + } + + if ((len-off) != 0) + pq_putmessage('d', content+off, len-off); /* Pad to 512 byte boundary, per tar format requirements */ - pad = ((len + 511) & ~511) - len; - if (pad > 0) + pad = TAR_BLKPAD_BYTES(len); + if (pad != 0) { - char buf[512]; + char buf[TAR_BLOCK_SIZE]; MemSet(buf, 0, pad); pq_putmessage('d', buf, pad); @@ -651,8 +673,8 @@ sendDir(char *path, int basepathlen, bool sizeonly) statbuf.st_mode = S_IFDIR | S_IRWXU; _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); } - size += 512; /* Size of the header just added */ - continue; /* don't recurse into pg_xlog */ + size += TAR_BLOCK_SIZE; /* Size of the header just added */ + continue; /* don't recurse into pg_xlog */ } /* Allow symbolic links in pg_tblspc only */ @@ -682,7 +704,7 @@ sendDir(char *path, int basepathlen, bool sizeonly) if (!sizeonly) _tarWriteHeader(pathbuf + basepathlen + 1, linkpath, &statbuf); - size += 512; /* Size of the header just added */ + size += TAR_BLOCK_SIZE; /* Size of the header just added */ #else /* @@ -704,7 +726,7 @@ sendDir(char *path, int basepathlen, bool sizeonly) */ if (!sizeonly) _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); - size += 512; /* Size of the header just added */ + size += TAR_BLOCK_SIZE; /* Size of the header just added */ /* call ourselves recursively for a directory */ size += sendDir(pathbuf, basepathlen, sizeonly); @@ -712,10 +734,10 @@ sendDir(char *path, int basepathlen, bool sizeonly) else if (S_ISREG(statbuf.st_mode)) { /* Add size, rounded up to 512byte block */ - size += ((statbuf.st_size + 511) & ~511); + size += TAR_BLKPAD_BYTES(statbuf.st_size); if (!sizeonly) sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf); - size += 512; /* Size of the header of the file */ + size += TAR_BLOCK_SIZE; /* Size of the header of the file */ } else ereport(WARNING, @@ -725,51 +747,18 @@ sendDir(char *path, int basepathlen, bool sizeonly) return size; } -/***** - * Functions for handling tar file format - * - * Copied from pg_dump, but modified to work with libpq for sending - */ - - -/* - * Utility routine to print possibly larger than 32 bit integers in a - * portable fashion. Filled with zeros. - */ -static void -print_val(char *s, uint64 val, unsigned int base, size_t len) -{ - int i; - - for (i = len; i > 0; i--) - { - int digit = val % base; - - s[i - 1] = '0' + digit; - val = val / base; - } -} - /* * Maximum file size for a tar member: The limit inherent in the * format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed * what we can represent in pgoff_t. + * + * There are some extensions to the tar format that would allow file + * sizes in excess of 2^33-1. Many recent implementions support base 256 + * encoding of the size if the Most Significant Bit (MSB) is set. That + * allows for signed integers of 2^87. */ #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1) -static int -_tarChecksum(char *header) -{ - int i, - sum; - - sum = 0; - for (i = 0; i < 512; i++) - if (i < 148 || i >= 156) - sum += 0xFF & header[i]; - return sum + 256; /* Assume 8 blanks in checksum field */ -} - /* Given the member, write the TAR header & send the file */ static void sendFile(char *readfilename, char *tarfilename, struct stat * statbuf) @@ -830,7 +819,7 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf) } /* Pad to 512 byte boundary, per tar format requirements */ - pad = ((len + 511) & ~511) - len; + pad = TAR_BLKPAD_BYTES(len); if (pad > 0) { MemSet(buf, 0, pad); @@ -841,89 +830,169 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf) } +/* + * convert a decimal number to octal, adding leading zeros + * as necessary. If the number overflows then the buffer + * is filled with all '7' characters. + */ static void -_tarWriteHeader(const char *filename, const char *linktarget, - struct stat * statbuf) -{ - char h[512]; - int lastSum = 0; - int sum; +_toOctal(unsigned int v, char *p, int length) { + char * q; - memset(h, 0, sizeof(h)); + /* write from the back to the front. The right shift + * introduces zero bits because v is unsigned + */ + q = p + (length-1); + do + { + *q = '0' + (v & 7); + v >>= 3; + } while(q-- != p); - /* Name 100 */ - sprintf(&h[0], "%.99s", filename); - if (linktarget != NULL || S_ISDIR(statbuf->st_mode)) + if (v) /* overflow */ { - /* - * We only support symbolic links to directories, and this is - * indicated in the tar format by adding a slash at the end of the - * name, the same as for regular directories. - */ - h[strlen(filename)] = '/'; - h[strlen(filename) + 1] = '\0'; + for (q = p; length != 0; length--) + *q++ = '7'; } +} - /* Mode 8 */ - sprintf(&h[100], "%07o ", (int) statbuf->st_mode); +static unsigned int +_tarChecksum(char const *header) +{ + int i; + unsigned int sum; + unsigned char const * p = (unsigned char *)header; + + sum = 256; /* Assume 8 blanks in checksum field */ + for (i = 0; i < TAR_OFFSET_CHECKSUM; i++) + sum += *p++; + + /* skip the checksum field, it's already in the calculation */ + p += TAR_LENGTH_CHECKSUM; + for (i += TAR_LENGTH_CHECKSUM; i < TAR_BLOCK_SIZE; i++) + sum += *p++; + + return sum; +} + +static void +_tarWriteHeader(const char *filename, const char *linktarget, struct stat * statbuf) +{ + char h[TAR_BLOCK_SIZE]; + int slen; + + /* set a link flag for ease of reference */ + const int islink = (linktarget != NULL || S_ISDIR(statbuf->st_mode)) ? 1 : 0; + + MemSet(h, 0, sizeof(h)); + + slen = strlen(filename); + if (islink) { + /* an extra character will be inserted at the end of the string + * to mark this as a directory. Add one to the length for determining + * if the string needs to be split into prefix+name + * + * The backup doesn't support links except to directories. Hardlinks + * are not supported at all. + */ + slen++; + } - /* User ID 8 */ - sprintf(&h[108], "%07o ", statbuf->st_uid); + /* do some basic error checking now, warn or error out depending on the + * type of failure + */ + if (slen > (TAR_LENGTH_NAME + TAR_LENGTH_PREFIX)) { + ereport(ERROR, + (errmsg("filename length \"%s\" is too long for the ustar archive format", + filename))); + } + else if (linktarget && (strlen(linktarget) > TAR_LENGTH_LINKNAME)) { + ereport(ERROR, + (errmsg("symbolic link \"%s\" target is too long for the ustar archive format", + linktarget))); + } - /* Group 8 */ - sprintf(&h[117], "%07o ", statbuf->st_gid); + if (slen > TAR_LENGTH_NAME) { + char const * p; - /* File size 12 - 11 digits, 1 space, no NUL */ - if (linktarget != NULL || S_ISDIR(statbuf->st_mode)) - /* Symbolic link or directory has size zero */ - print_val(&h[124], 0, 8, 11); - else - print_val(&h[124], statbuf->st_size, 8, 11); - sprintf(&h[135], " "); + /* start at a reasonable offset and work backward + * to determine a split point. Put as much as possible + * into the prefix + */ + p = filename + TAR_LENGTH_NAME; + if (slen > TAR_LENGTH_PREFIX) + p = filename + TAR_LENGTH_PREFIX; - /* Mod Time 12 */ - sprintf(&h[136], "%011o ", (int) statbuf->st_mtime); + while (p != filename && *p != '/') + --p; - /* Checksum 8 */ - sprintf(&h[148], "%06o ", lastSum); + if (strlen(p) > TAR_LENGTH_NAME) { + /* this is an error condition! Failed to find + * a good split point for the name+prefix + */ + ereport(ERROR, + (errmsg("filename length \"%s\" is too long for the ustar archive format", + filename))); + } + strncpy(&h[TAR_OFFSET_PREFIX], filename, (int)(p - filename)); + strncpy(&h[TAR_OFFSET_NAME], ++p, TAR_LENGTH_NAME); + } + else { + strncpy(&h[TAR_OFFSET_NAME], filename, TAR_LENGTH_NAME); + } - if (linktarget != NULL) - { - /* Type - Symbolic link */ - sprintf(&h[156], "2"); - sprintf(&h[157], "%.99s", linktarget); + if (islink) { + /* the buffer is already zero field so the name field is + * either full or null terminated already! + */ + h[TAR_OFFSET_NAME + strnlen(&h[TAR_OFFSET_NAME], TAR_LENGTH_NAME-1)] = '/'; } - else if (S_ISDIR(statbuf->st_mode)) - /* Type - directory */ - sprintf(&h[156], "5"); - else - /* Type - regular file */ - sprintf(&h[156], "0"); - /* Link tag 100 (NULL) */ + /* the POSIX spec states that each numeric field is termined by one or + * more space characters **OR** NUL characters. The buffer is already + * filled with NULL characters so no need to add them again after the + * octal conversion. + */ + _toOctal(statbuf->st_mode & TAR_MODE_MASK, + &h[TAR_OFFSET_MODE], + TAR_LENGTH_MODE-1); + + _toOctal(statbuf->st_uid, &h[TAR_OFFSET_UID], TAR_LENGTH_UID-1); + _toOctal(statbuf->st_uid, &h[TAR_OFFSET_GID], TAR_LENGTH_GID-1); - /* Magic 6 + Version 2 */ - sprintf(&h[257], "ustar00"); + _toOctal((islink ? 0 : statbuf->st_size), + &h[TAR_OFFSET_SIZE], + TAR_LENGTH_SIZE-1); + + _toOctal(statbuf->st_mtime, &h[TAR_OFFSET_MTIME], TAR_LENGTH_MTIME-1); - /* User 32 */ - /* XXX: Do we need to care about setting correct username? */ - sprintf(&h[265], "%.31s", "postgres"); + /* skip the checksum for now, the checksum algorith ignores the header value */ + + if (linktarget != NULL) { + h[TAR_OFFSET_TYPEFLAG] = TAR_TF_SYMLINK; + strncpy(&h[TAR_OFFSET_LINKNAME], linktarget, TAR_LENGTH_LINKNAME); + } + else if (S_ISDIR(statbuf->st_mode)) { + h[TAR_OFFSET_TYPEFLAG] = TAR_TF_DIR; + } + else { + h[TAR_OFFSET_TYPEFLAG] = TAR_TF_NORMAL; + } - /* Group 32 */ - /* XXX: Do we need to care about setting correct group name? */ - sprintf(&h[297], "%.31s", "postgres"); + /* UStar Magic and version */ + strncpy(&h[TAR_OFFSET_MAGIC], "ustar", TAR_LENGTH_MAGIC); + strncpy(&h[TAR_OFFSET_VERSION], "00", TAR_LENGTH_VERSION); - /* Maj Dev 8 */ - sprintf(&h[329], "%6o ", 0); + /* user name and group name */ + /* XXX: Do we need to care about setting correct username or group name? */ + strncpy(&h[TAR_OFFSET_UNAME], "postgres", TAR_LENGTH_UNAME); + strncpy(&h[TAR_OFFSET_GNAME], "postgres", TAR_LENGTH_GNAME); - /* Min Dev 8 */ - sprintf(&h[337], "%6o ", 0); + /* major and minor devices - set to zero */ + _toOctal(0, &h[TAR_OFFSET_DEVMAJOR], TAR_LENGTH_DEVMAJOR-1); + _toOctal(0, &h[TAR_OFFSET_DEVMINOR], TAR_LENGTH_DEVMINOR-1); - while ((sum = _tarChecksum(h)) != lastSum) - { - sprintf(&h[148], "%06o ", sum); - lastSum = sum; - } + _toOctal(_tarChecksum(h), &h[TAR_OFFSET_CHECKSUM], TAR_LENGTH_CHECKSUM-1); - pq_putmessage('d', h, 512); + pq_putmessage('d', h, sizeof(h)); } diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index ad994f4..e335bc9 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -34,6 +34,7 @@ #include "receivelog.h" #include "streamutil.h" +#include "portability/ustar.h" /* Global options */ @@ -82,6 +83,8 @@ static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished); +static unsigned int _fromOctal(char const *p, int length); + #ifdef HAVE_LIBZ static const char * get_gz_error(gzFile gzf) @@ -97,6 +100,19 @@ get_gz_error(gzFile gzf) } #endif +static +unsigned int _fromOctal(char const *p, int length) +{ + unsigned int v = 0; + while (length != 0 && *p && !(*p < '0' || *p > '7')) + { + v = (v << 3) + (*p++ - '0'); + --length; + } + + return v; +} + static void usage(void) { @@ -603,7 +619,7 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) * Also, write two completely empty blocks at the end of the tar * file, as required by some tar programs. */ - char zerobuf[1024]; + char zerobuf[TAR_BLOCK_SIZE * 2]; MemSet(zerobuf, 0, sizeof(zerobuf)); #ifdef HAVE_LIBZ @@ -714,6 +730,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) int current_padding = 0; char *copybuf = NULL; FILE *file = NULL; + int slen; if (PQgetisnull(res, rownum, 0)) strcpy(current_path, basedir); @@ -767,51 +784,68 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) /* * No current file, so this must be the header for a new file */ - if (r != 512) + if (r != TAR_BLOCK_SIZE) { fprintf(stderr, _("%s: invalid tar block header size: %d\n"), progname, r); disconnect_and_exit(1); } - totaldone += 512; + totaldone += TAR_BLOCK_SIZE; - if (sscanf(copybuf + 124, "%11o", ¤t_len_left) != 1) - { - fprintf(stderr, _("%s: could not parse file size\n"), - progname); - disconnect_and_exit(1); - } + current_len_left = + _fromOctal(copybuf + TAR_OFFSET_SIZE, TAR_LENGTH_SIZE); - /* Set permissions on the file */ - if (sscanf(©buf[100], "%07o ", &filemode) != 1) - { - fprintf(stderr, _("%s: could not parse file mode\n"), - progname); - disconnect_and_exit(1); - } + /* get the permissions on the file */ + filemode = _fromOctal(copybuf + TAR_OFFSET_MODE, TAR_LENGTH_MODE); /* * All files are padded up to 512 bytes */ - current_padding = - ((current_len_left + 511) & ~511) - current_len_left; + current_padding = TAR_BLKPAD_BYTES(current_len_left); /* - * First part of header is zero terminated filename + * First part of header is may be zero terminated filename + * The name and prefix fields are allowed to fully populate + * their space and may not be have a null terminator! + * + * there is an assumption that sizeof(filename) == MAXPGPATH + * which is greater than (TAR_LENGTH_PREFIX + TAR_LENGTH_NAME + 1) */ - snprintf(filename, sizeof(filename), "%s/%s", current_path, - copybuf); - if (filename[strlen(filename) - 1] == '/') + MemSet(filename, 0, sizeof(filename)); + slen = strlen(strncpy(filename, current_path, sizeof(filename))); + filename[slen++] = '/'; + + if (copybuf[TAR_OFFSET_PREFIX] != '\0') { + slen += strlen(strncpy(filename + slen, + copybuf + TAR_OFFSET_PREFIX, + TAR_LENGTH_PREFIX)); + + filename[slen++] = '/'; + + slen += strlen(strncpy(filename + slen, + copybuf + TAR_OFFSET_NAME, + TAR_LENGTH_NAME)); + } + else + { + slen += strlen(strncpy(filename + slen, + copybuf + TAR_OFFSET_NAME, + TAR_LENGTH_NAME)); + } + + if (filename[slen - 1] == '/') + { + + filename[--slen] = '\0'; /* Remove trailing slash */ /* * Ends in a slash means directory or symlink to directory */ - if (copybuf[156] == '5') + if (copybuf[TAR_OFFSET_TYPEFLAG] == TAR_TF_DIR) { /* * Directory */ - filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ if (mkdir(filename, S_IRWXU) != 0) { /* @@ -819,32 +853,39 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) * by the wal receiver process, so just ignore failure * on that. */ - if (!streamwal || strcmp(filename + strlen(filename) - 8, "/pg_xlog") != 0) + if (!streamwal || strncmp(filename + slen - 8, + "/pg_xlog", + sizeof(filename) - slen) != 0) { fprintf(stderr, - _("%s: could not create directory \"%s\": %s\n"), + _("%s: could not create directory \"%s\": %s\n"), progname, filename, strerror(errno)); disconnect_and_exit(1); } } #ifndef WIN32 - if (chmod(filename, (mode_t) filemode)) + if (chmod(filename, (mode_t) (filemode & (S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO)))) fprintf(stderr, _("%s: could not set permissions on directory \"%s\": %s\n"), progname, filename, strerror(errno)); #endif } - else if (copybuf[156] == '2') + else if (copybuf[TAR_OFFSET_TYPEFLAG] == TAR_TF_SYMLINK) { /* * Symbolic link */ - filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ - if (symlink(©buf[157], filename) != 0) + char linkname[TAR_LENGTH_LINKNAME+1]; + + MemSet(linkname, 0, sizeof(linkname)); + strncpy(linkname, copybuf + TAR_OFFSET_LINKNAME, TAR_LENGTH_LINKNAME); + + filename[--slen] = '\0'; /* Remove trailing slash */ + if (symlink(linkname, filename) != 0) { fprintf(stderr, _("%s: could not create symbolic link from \"%s\" to \"%s\": %s\n"), - progname, filename, ©buf[157], strerror(errno)); + progname, filename, linkname, strerror(errno)); disconnect_and_exit(1); } } @@ -852,12 +893,16 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) { fprintf(stderr, _("%s: unrecognized link indicator \"%c\"\n"), - progname, copybuf[156]); + progname, copybuf[TAR_OFFSET_TYPEFLAG]); disconnect_and_exit(1); } continue; /* directory or link handled */ } + /* XXX: should there be an assertion that the type is old normal (nul) + * or '0' which denotes a normal file? + */ + /* * regular file */ @@ -870,9 +915,13 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) } #ifndef WIN32 - if (chmod(filename, (mode_t) filemode)) + if (chmod(filename, (mode_t) (filemode & (S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO)))) fprintf(stderr, _("%s: could not set permissions on file \"%s\": %s\n"), progname, filename, strerror(errno)); + + /* XXX: Should the user and group be recovered from the user and group name fields? + * Alternately what about the user and group identifier fields? + */ #endif if (current_len_left == 0) diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index ced5c13..b1f93a6 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -115,7 +115,7 @@ static char *tarGets(char *buf, size_t len, TAR_MEMBER *th); static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 4))); static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th); -static int _tarChecksum(char *th); +static unsigned int _tarChecksum(char const *header); static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename); static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th); static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th); @@ -125,6 +125,9 @@ static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER * static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len); +static void _toOctal(unsigned int v, char *p, int length); +static unsigned int _fromOctal(char const *p, int length); + /* * Initializer */ @@ -883,7 +886,7 @@ _CloseArchive(ArchiveHandle *AH) tarClose(AH, th); /* Add a block of NULLs since it's de-rigeur. */ - for (i = 0; i < 512; i++) + for (i = 0; i < TAR_BLOCK_SIZE; i++) { if (fputc(0, ctx->tarFH) == EOF) exit_horribly(modulename, @@ -987,8 +990,6 @@ _EndBlobs(ArchiveHandle *AH, TocEntry *te) tarClose(AH, ctx->blobToc); } - - /*------------ * TAR Support *------------ @@ -1026,35 +1027,36 @@ tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) return cnt; } -static int -_tarChecksum(char *header) -{ - int i, - sum; - - sum = 0; - for (i = 0; i < 512; i++) - if (i < 148 || i >= 156) - sum += 0xFF & header[i]; - return sum + 256; /* Assume 8 blanks in checksum field */ -} - bool isValidTarHeader(char *header) { - int sum; - int chk = _tarChecksum(header); - - sscanf(&header[148], "%8o", &sum); - + int chk, + sum; + char fmtposix[TAR_LENGTH_MAGIC+TAR_LENGTH_MAGIC] = { + 'u', 's', 't', 'a', 'r', '\0', '0', '0' + }; + + chk = _tarChecksum(header); + sum = _fromOctal(&header[TAR_OFFSET_CHECKSUM], TAR_LENGTH_CHECKSUM); if (sum != chk) return false; /* POSIX format */ - if (strncmp(&header[257], "ustar00", 7) == 0) + if (memcmp(&header[TAR_OFFSET_MAGIC], &fmtposix[0], sizeof(fmtposix)) == 0) return true; - /* older format */ - if (strncmp(&header[257], "ustar ", 7) == 0) + + /* pg_dump wrote ustar00\0 at the offiset, which is incorrect. + * backend replication also had the same flaw + */ + if (strncmp(&header[TAR_OFFSET_MAGIC], + "ustar00", + TAR_LENGTH_MAGIC + TAR_LENGTH_VERSION - 1) == 0) + return true; + + /* GNU tar format */ + if (strncmp(&header[TAR_OFFSET_MAGIC], + "ustar ", + TAR_LENGTH_MAGIC + TAR_LENGTH_VERSION - 1) == 0) return true; return false; @@ -1131,7 +1133,7 @@ _tarPositionTo(ArchiveHandle *AH, const char *filename) lclContext *ctx = (lclContext *) AH->formatData; TAR_MEMBER *th = pg_calloc(1, sizeof(TAR_MEMBER)); char c; - char header[512]; + char header[TAR_BLOCK_SIZE]; size_t i, len, blks; @@ -1190,17 +1192,20 @@ _tarPositionTo(ArchiveHandle *AH, const char *filename) th->targetFile, filename); /* Header doesn't match, so read to next header */ - len = ((th->fileLen + 511) & ~511); /* Padded length */ - blks = len >> 9; /* # of 512 byte blocks */ + len = TAR_BLKPAD_BYTES(th->fileLen); /* padded length */ + blks = len >> 9; /* # of 512 byte blocks */ for (i = 0; i < blks; i++) - _tarReadRaw(AH, &header[0], 512, NULL, ctx->tarFH); + _tarReadRaw(AH, header, sizeof(header), NULL, ctx->tarFH); if (!_tarGetHeader(AH, th)) - exit_horribly(modulename, "could not find header for file \"%s\" in tar archive\n", filename); + exit_horribly(modulename, + "could not find header for file \"%s\" in tar archive\n", + filename); } - ctx->tarNextMember = ctx->tarFHpos + ((th->fileLen + 511) & ~511); + ctx->tarNextMember = ctx->tarFHpos + th->fileLen + + TAR_BLKPAD_BYTES(th->fileLen); th->pos = 0; return th; @@ -1211,14 +1216,14 @@ static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th) { lclContext *ctx = (lclContext *) AH->formatData; - char h[512]; - char tag[100]; + char h[TAR_BLOCK_SIZE]; + char tag[TAR_LENGTH_PREFIX+TAR_LENGTH_NAME+2]; int sum, chk; size_t len; - unsigned long ullen; pgoff_t hPos; bool gotBlock = false; + bool seeking = false; while (!gotBlock) { @@ -1240,11 +1245,11 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th) hPos = ctx->tarFHpos; /* Read a 512 byte block, return EOF, exit if short */ - len = _tarReadRaw(AH, h, 512, NULL, ctx->tarFH); + len = _tarReadRaw(AH, h, sizeof(h), NULL, ctx->tarFH); if (len == 0) /* EOF */ return 0; - if (len != 512) + if (len != sizeof(h)) exit_horribly(modulename, ngettext("incomplete tar header found (%lu byte)\n", "incomplete tar header found (%lu bytes)\n", @@ -1253,32 +1258,38 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th) /* Calc checksum */ chk = _tarChecksum(h); - sscanf(&h[148], "%8o", &sum); + sum = _fromOctal(&h[TAR_OFFSET_CHECKSUM], TAR_LENGTH_CHECKSUM); /* * If the checksum failed, see if it is a null block. If so, silently * continue to the next block. */ - if (chk == sum) + if (isValidTarHeader(h)) + { gotBlock = true; - else + seeking = false; + } + else if (!seeking) { - int i; - - for (i = 0; i < 512; i++) - { - if (h[i] != 0) - { - gotBlock = true; - break; - } - } + /* warn and search for the next valid header block */ + write_msg(modulename, + "WARNING: Invalid tar header block found at offset %llu, skipping...\n", + hPos); + seeking = true; } } - sscanf(&h[0], "%99s", tag); - sscanf(&h[124], "%12lo", &ullen); - len = (size_t) ullen; + MemSet(tag, 0, sizeof(tag)); + if (h[TAR_OFFSET_PREFIX] != '\0') { + int slen = strlen(strncpy(tag, &h[TAR_OFFSET_PREFIX], TAR_LENGTH_PREFIX)); + tag[slen++] = '/'; + strncpy(tag+slen, &h[TAR_OFFSET_NAME], TAR_LENGTH_NAME); + } + else { + strncpy(tag, &h[TAR_OFFSET_NAME], TAR_LENGTH_NAME); + } + + len = (size_t)_fromOctal(&h[TAR_OFFSET_SIZE], TAR_LENGTH_SIZE); { char buf[100]; @@ -1305,87 +1316,153 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th) return 1; } - /* - * Utility routine to print possibly larger than 32 bit integers in a - * portable fashion. Filled with zeros. + * convert a decimal number to octal, adding leading zeros + * as necessary. If the number overflows then the buffer + * is filled with all '7' characters. */ -static void -print_val(char *s, uint64 val, unsigned int base, size_t len) +static +void _toOctal(unsigned int v, char *p, int length) { - int i; + char * q; - for (i = len; i > 0; i--) + /* write from the back to the front. The right shift + * introduces zero bits because v is unsigned + */ + q = p + (length-1); + do { - int digit = val % base; + *q = '0' + (v & 7); + v >>= 3; + } while(q-- != p); - s[i - 1] = '0' + digit; - val = val / base; + if (v) /* overflow */ + { + for (q = p; length != 0; length--) + *q++ = '7'; } } +static +unsigned int _fromOctal(char const *p, int length) +{ + unsigned int v = 0; + while (length != 0 && *p && !(*p < '0' || *p > '7')) + { + v = (v << 3) + (*p++ - '0'); + --length; + } + + return v; +} -static void -_tarWriteHeader(TAR_MEMBER *th) +static +unsigned int _tarChecksum(char const *header) { - char h[512]; - int lastSum = 0; - int sum; + int i; + unsigned int sum; + unsigned char const * p = (unsigned char *)header; + + sum = 256; /* Assume 8 blanks in checksum field */ + for (i = 0; i < TAR_OFFSET_CHECKSUM; i++) + sum += *p++; + + /* skip the checksum field, it's already in the calculation */ + p += TAR_LENGTH_CHECKSUM; + for (i += TAR_LENGTH_CHECKSUM; i < TAR_BLOCK_SIZE; i++) + sum += *p++; + + return sum; +} - memset(h, 0, sizeof(h)); +static +void _tarWriteHeader(TAR_MEMBER *th) +{ + char h[TAR_BLOCK_SIZE]; + int slen; - /* Name 100 */ - sprintf(&h[0], "%.99s", th->targetFile); + MemSet(h, 0, sizeof(h)); - /* Mode 8 */ - sprintf(&h[100], "100600 "); + /* no link or directory support, only regular files */ + slen = strlen(th->targetFile); - /* User ID 8 */ - sprintf(&h[108], "004000 "); + /* do some basic error checking now, warn or error out depending on the + * type of failure + */ + if (slen > (TAR_LENGTH_NAME + TAR_LENGTH_PREFIX)) + { + exit_horribly(modulename, + "filename length \"%s\" is too long for the ustar archive format", + th->targetFile); + } - /* Group 8 */ - sprintf(&h[116], "002000 "); + if (slen > TAR_LENGTH_NAME) + { + char const * p; - /* File size 12 - 11 digits, 1 space, no NUL */ - print_val(&h[124], th->fileLen, 8, 11); - sprintf(&h[135], " "); + /* start at a reasonable offset and work backward + * to determine a split point. Put as much as possible + * into the prefix + */ + p = th->targetFile + TAR_LENGTH_NAME; + if (slen > TAR_LENGTH_PREFIX) + p = th->targetFile + TAR_LENGTH_PREFIX; - /* Mod Time 12 */ - sprintf(&h[136], "%011o ", (int) time(NULL)); + while (p != th->targetFile && *p != '/') + --p; - /* Checksum 8 */ - sprintf(&h[148], "%06o ", lastSum); + if (strlen(p) > TAR_LENGTH_NAME) + { + /* this is an error condition! Failed to find + * a good split point for the name+prefix + */ + exit_horribly(modulename, + "filename length \"%s\" is too long for the ustar archive format", + th->targetFile); + } + strncpy(&h[TAR_OFFSET_PREFIX], th->targetFile, (int)(p - th->targetFile)); + strncpy(&h[TAR_OFFSET_NAME], ++p, TAR_LENGTH_NAME); + } + else + { + strncpy(&h[TAR_OFFSET_NAME], th->targetFile, TAR_LENGTH_NAME); + } - /* Type - regular file */ - sprintf(&h[156], "0"); + /* the POSIX spec states that each numeric field is termined by one or + * more space characters **OR** NUL characters. The buffer is already + * filled with NULL characters so no need to add them again after the + * octal conversion. + */ + _toOctal(0100600, &h[TAR_OFFSET_MODE], TAR_LENGTH_MODE-1); + _toOctal(0004000, &h[TAR_OFFSET_UID], TAR_LENGTH_UID-1); + _toOctal(0002000, &h[TAR_OFFSET_GID], TAR_LENGTH_GID-1); - /* Link tag 100 (NULL) */ + _toOctal(th->fileLen, &h[TAR_OFFSET_SIZE], TAR_LENGTH_SIZE-1); + + _toOctal(time(NULL), &h[TAR_OFFSET_MTIME], TAR_LENGTH_MTIME-1); - /* Magic 6 + Version 2 */ - sprintf(&h[257], "ustar00"); + /* skip the checksum for now, the checksum algorith ignores the header value */ -#if 0 - /* User 32 */ - sprintf(&h[265], "%.31s", ""); /* How do I get username reliably? Do - * I need to? */ + h[TAR_OFFSET_TYPEFLAG] = TAR_TF_NORMAL; - /* Group 32 */ - sprintf(&h[297], "%.31s", ""); /* How do I get group reliably? Do I - * need to? */ + /* UStar Magic and version */ + strncpy(&h[TAR_OFFSET_MAGIC], "ustar", TAR_LENGTH_MAGIC); + strncpy(&h[TAR_OFFSET_VERSION], "00", TAR_LENGTH_VERSION); - /* Maj Dev 8 */ - sprintf(&h[329], "%6o ", 0); + /* user name and group name */ + /* XXX: Do we need to care about setting correct username or group name? */ + strncpy(&h[TAR_OFFSET_UNAME], "postgres", TAR_LENGTH_UNAME); + strncpy(&h[TAR_OFFSET_GNAME], "postgres", TAR_LENGTH_GNAME); - /* Min Dev 8 */ - sprintf(&h[337], "%6o ", 0); -#endif + /* major and minor devices - set to zero */ + _toOctal(0, &h[TAR_OFFSET_DEVMAJOR], TAR_LENGTH_DEVMAJOR-1); + _toOctal(0, &h[TAR_OFFSET_DEVMINOR], TAR_LENGTH_DEVMINOR-1); - while ((sum = _tarChecksum(h)) != lastSum) - { - sprintf(&h[148], "%06o ", sum); - lastSum = sum; - } + _toOctal(_tarChecksum(h), &h[TAR_OFFSET_CHECKSUM], TAR_LENGTH_CHECKSUM-1); - if (fwrite(h, 1, 512, th->tarFH) != 512) - exit_horribly(modulename, "could not write to output file: %s\n", strerror(errno)); + if (fwrite(h, sizeof(h), 1, th->tarFH) != 1) + exit_horribly(modulename, + "could not write to output file: %s\n", + strerror(errno)); } + diff --git a/src/bin/pg_dump/pg_backup_tar.h b/src/bin/pg_dump/pg_backup_tar.h index cb9be64..bcae7c2 100644 --- a/src/bin/pg_dump/pg_backup_tar.h +++ b/src/bin/pg_dump/pg_backup_tar.h @@ -1,34 +1 @@ -/* - * src/bin/pg_dump/pg_backup_tar.h - * - * TAR Header - * - * Offset Length Contents - * 0 100 bytes File name ('\0' terminated, 99 maximum length) - * 100 8 bytes File mode (in octal ascii) - * 108 8 bytes User ID (in octal ascii) - * 116 8 bytes Group ID (in octal ascii) - * 124 12 bytes File size (s) (in octal ascii) - * 136 12 bytes Modify time (in octal ascii) - * 148 8 bytes Header checksum (in octal ascii) - * 156 1 bytes Link flag - * 157 100 bytes Linkname ('\0' terminated, 99 maximum length) - * 257 8 bytes Magic ("ustar \0") - * 265 32 bytes User name ('\0' terminated, 31 maximum length) - * 297 32 bytes Group name ('\0' terminated, 31 maximum length) - * 329 8 bytes Major device ID (in octal ascii) - * 337 8 bytes Minor device ID (in octal ascii) - * 345 167 bytes Padding - * 512 (s+p)bytes File contents (s+p) := (((s) + 511) & ~511), round up to 512 bytes - */ - -/* The linkflag defines the type of file */ -#define LF_OLDNORMAL '\0' /* Normal disk file, Unix compatible */ -#define LF_NORMAL '0' /* Normal disk file */ -#define LF_LINK '1' /* Link to previously dumped file */ -#define LF_SYMLINK '2' /* Symbolic link */ -#define LF_CHR '3' /* Character special file */ -#define LF_BLK '4' /* Block special file */ -#define LF_DIR '5' /* Directory */ -#define LF_FIFO '6' /* FIFO special file */ -#define LF_CONTIG '7' /* Contiguous file */ +#include "portability/ustar.h" diff --git a/src/include/portability/ustar.h b/src/include/portability/ustar.h new file mode 100644 index 0000000..ac2b346 --- /dev/null +++ b/src/include/portability/ustar.h @@ -0,0 +1,97 @@ +#ifndef __PORTABILITY__USTAR_H__ +#define __PORTABILITY__USTAR_H__ + +/*---------- + * include/portability/ustar.h + * + * TAR Header + * + * Offset Length Contents + * 0 100 bytes File name ('\0' terminated, 99 maximum length) + * 100 8 bytes File mode (in octal ascii) + * 108 8 bytes User ID (in octal ascii) + * 116 8 bytes Group ID (in octal ascii) + * 124 12 bytes File size (s) (in octal ascii) + * 136 12 bytes Modify time (in octal ascii) + * 148 8 bytes Header checksum (in octal ascii) + * 156 1 bytes Link flag + * 157 100 bytes Linkname ('\0' terminated, 99 maximum length) + * 257 8 bytes Magic ("ustar \0") + * 265 32 bytes User name ('\0' terminated, 31 maximum length) + * 297 32 bytes Group name ('\0' terminated, 31 maximum length) + * 329 8 bytes Major device ID (in octal ascii) + * 337 8 bytes Minor device ID (in octal ascii) + * 345 167 bytes Padding + * 512 (s+p)bytes File contents (s+p) := (((s) + 511) & ~511), round up to 512 bytes + *---------- + */ + +#define TAR_BLOCK_SIZE 512 +#define TAR_BLKPAD_BYTES(_x) ( (((_x) + (TAR_BLOCK_SIZE-1)) & ~(TAR_BLOCK_SIZE-1)) - (_x) ) + +/* offsets defined in the POSIX specification */ +#define TAR_OFFSET_NAME 0 +#define TAR_OFFSET_MODE 100 +#define TAR_OFFSET_UID 108 +#define TAR_OFFSET_GID 116 +#define TAR_OFFSET_SIZE 124 +#define TAR_OFFSET_MTIME 136 +#define TAR_OFFSET_CHECKSUM 148 +#define TAR_OFFSET_TYPEFLAG 156 +#define TAR_OFFSET_LINKNAME 157 +#define TAR_OFFSET_MAGIC 257 +#define TAR_OFFSET_VERSION 263 +#define TAR_OFFSET_UNAME 265 +#define TAR_OFFSET_GNAME 297 +#define TAR_OFFSET_DEVMAJOR 329 +#define TAR_OFFSET_DEVMINOR 337 +#define TAR_OFFSET_PREFIX 345 +#define TAR_OFFSET_PADDING 500 + +/* lengths defined in the POSIX specifications */ +#define TAR_LENGTH_NAME 100 +#define TAR_LENGTH_MODE 8 +#define TAR_LENGTH_UID 8 +#define TAR_LENGTH_GID 8 +#define TAR_LENGTH_SIZE 12 +#define TAR_LENGTH_MTIME 12 +#define TAR_LENGTH_CHECKSUM 8 +#define TAR_LENGTH_TYPEFLAG 1 +#define TAR_LENGTH_LINKNAME 100 +#define TAR_LENGTH_MAGIC 6 +#define TAR_LENGTH_VERSION 2 +#define TAR_LENGTH_UNAME 32 +#define TAR_LENGTH_GNAME 32 +#define TAR_LENGTH_DEVMAJOR 8 +#define TAR_LENGTH_DEVMINOR 8 +#define TAR_LENGTH_PREFIX 155 +#define TAR_LENGTH_PADDING 12 +#define TAR_LENGTH_HEADER TAR_BLOCK_SIZE + +/* corresponds to the constants => (S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO) */ +#define TAR_MODE_MASK 06777U + +/* The linkflag defines the type of file */ +#define TAR_TF_OLDNORMAL '\0' /* Normal disk file, Unix compatible */ +#define TAR_TF_NORMAL '0' /* Normal disk file */ +#define TAR_TF_LINK '1' /* Link to previously dumped file */ +#define TAR_TF_SYMLINK '2' /* Symbolic link */ +#define TAR_TF_CHR '3' /* Character special file */ +#define TAR_TF_BLK '4' /* Block special file */ +#define TAR_TF_DIR '5' /* Directory */ +#define TAR_TF_FIFO '6' /* FIFO special file */ +#define TAR_TF_CONTIG '7' /* Contiguous file */ + + +/* a simple compiler check to avoid some problems in the + * code later. This allow some basic assumptions when + * checking and copying buffers in the code. + * + * MAXPGPATH is normally 1024 bytes which is far longer than + * the 255 bytes supported by the tar header + */ +#if (TAR_LENGTH_PREFIX + TAR_LENGTH_NAME +1) > MAXPGPATH +#error "MAXPGPATH must be larger than the sum of the UStar Name and Prefix fields" +#endif + +#endif /* __PORTABILITY__USTAR_H__ */ -- 1.7.7.5 (Apple Git-26)