On Fri, Dec 29, 2006 at 05:30:48PM +0100, Magnus Hagander wrote:
> On Tue, Dec 19, 2006 at 04:58:22PM +0100, Zeugswetter Andreas ADI SD wrote:
> >
> > > > > > MinGW has fseeko64 and ftello64 with off64_t.
> > > > > >
> > > > >
> > > > > Maybe we need separate macros for MSVC and MinGW. Given the other
> > > >
> > > > You mean something quick and dirty like this ? That would work.
> > >
> > > Yes, except does that actually work? If so you found the place in the
> > > headers to stick it without breaking things that I couldn't find ;-)
> >
> > Compiles clean without warnings on MinGW, but not tested, sorry also no
> > time.
>
> Does not compile on my MinGW - errors in the system headers (unistd.h,
> io.h) due to changing the argument format for chsize(). The change of
> off_t propagated into parts of the system headers, thus chaos was
> ensured.
>
> I still think we need to use a pgoff_t. Will look at combining these two
> approaches.
Here's a patch that tries this.
*needs more testing*. But built with this patch, I can dump and
restore a table at the end of a 10gb database without errors.
Does the method/patch seem reasonable? Anybody else who can run a couple
of tests on it?
//Magnus
Index: src/bin/pg_dump/pg_backup_archiver.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v
retrieving revision 1.141
diff -c -r1.141 pg_backup_archiver.c
*** src/bin/pg_dump/pg_backup_archiver.c 1 Feb 2007 19:10:28 -0000
1.141
--- src/bin/pg_dump/pg_backup_archiver.c 11 Feb 2007 15:00:55 -0000
***************
*** 1311,1334 ****
}
size_t
! WriteOffset(ArchiveHandle *AH, off_t o, int wasSet)
{
int off;
/* Save the flag */
(*AH->WriteBytePtr) (AH, wasSet);
! /* Write out off_t smallest byte first, prevents endian mismatch */
! for (off = 0; off < sizeof(off_t); off++)
{
(*AH->WriteBytePtr) (AH, o & 0xFF);
o >>= 8;
}
! return sizeof(off_t) + 1;
}
int
! ReadOffset(ArchiveHandle *AH, off_t *o)
{
int i;
int off;
--- 1311,1334 ----
}
size_t
! WriteOffset(ArchiveHandle *AH, pgoff_t o, int wasSet)
{
int off;
/* Save the flag */
(*AH->WriteBytePtr) (AH, wasSet);
! /* Write out pgoff_t smallest byte first, prevents endian mismatch */
! for (off = 0; off < sizeof(pgoff_t); off++)
{
(*AH->WriteBytePtr) (AH, o & 0xFF);
o >>= 8;
}
! return sizeof(pgoff_t) + 1;
}
int
! ReadOffset(ArchiveHandle *AH, pgoff_t *o)
{
int i;
int off;
***************
*** 1348,1355 ****
else if (i == 0)
return K_OFFSET_NO_DATA;
! /* Cast to off_t because it was written as an int. */
! *o = (off_t) i;
return K_OFFSET_POS_SET;
}
--- 1348,1355 ----
else if (i == 0)
return K_OFFSET_NO_DATA;
! /* Cast to pgoff_t because it was written as an int. */
! *o = (pgoff_t) i;
return K_OFFSET_POS_SET;
}
***************
*** 1379,1386 ****
*/
for (off = 0; off < AH->offSize; off++)
{
! if (off < sizeof(off_t))
! *o |= ((off_t) ((*AH->ReadBytePtr) (AH))) << (off * 8);
else
{
if ((*AH->ReadBytePtr) (AH) != 0)
--- 1379,1386 ----
*/
for (off = 0; off < AH->offSize; off++)
{
! if (off < sizeof(pgoff_t))
! *o |= ((pgoff_t) ((*AH->ReadBytePtr) (AH))) << (off *
8);
else
{
if ((*AH->ReadBytePtr) (AH) != 0)
***************
*** 1647,1653 ****
AH->createDate = time(NULL);
AH->intSize = sizeof(int);
! AH->offSize = sizeof(off_t);
if (FileSpec)
{
AH->fSpec = strdup(FileSpec);
--- 1647,1653 ----
AH->createDate = time(NULL);
AH->intSize = sizeof(int);
! AH->offSize = sizeof(pgoff_t);
if (FileSpec)
{
AH->fSpec = strdup(FileSpec);
***************
*** 2768,2778 ****
if (fseeko(fp, 0, SEEK_CUR) != 0)
return false;
! else if (sizeof(off_t) > sizeof(long))
/*
! * At this point, off_t is too large for long, so we return
based on
! * whether an off_t version of fseek is available.
*/
#ifdef HAVE_FSEEKO
return true;
--- 2768,2778 ----
if (fseeko(fp, 0, SEEK_CUR) != 0)
return false;
! else if (sizeof(pgoff_t) > sizeof(long))
/*
! * At this point, pgoff_t is too large for long, so we return
based on
! * whether an pgoff_t version of fseek is available.
*/
#ifdef HAVE_FSEEKO
return true;
Index: src/bin/pg_dump/pg_backup_archiver.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v
retrieving revision 1.74
diff -c -r1.74 pg_backup_archiver.h
*** src/bin/pg_dump/pg_backup_archiver.h 25 Jan 2007 03:30:43 -0000
1.74
--- src/bin/pg_dump/pg_backup_archiver.h 11 Feb 2007 15:01:10 -0000
***************
*** 199,205 ****
* format */
size_t lookaheadSize; /* Size of allocated buffer */
size_t lookaheadLen; /* Length of data in lookahead */
! off_t lookaheadPos; /* Current read position in lookahead
buffer */
ArchiveEntryPtr ArchiveEntryPtr; /* Called for each metadata
object */
StartDataPtr StartDataPtr; /* Called when table data is about to be
--- 199,205 ----
* format */
size_t lookaheadSize; /* Size of allocated buffer */
size_t lookaheadLen; /* Length of data in lookahead */
! pgoff_t lookaheadPos; /* Current read position in lookahead
buffer */
ArchiveEntryPtr ArchiveEntryPtr; /* Called for each metadata
object */
StartDataPtr StartDataPtr; /* Called when table data is about to be
***************
*** 332,339 ****
extern char *ReadStr(ArchiveHandle *AH);
extern size_t WriteStr(ArchiveHandle *AH, const char *s);
! int ReadOffset(ArchiveHandle *, off_t *);
! size_t WriteOffset(ArchiveHandle *, off_t, int);
extern void StartRestoreBlobs(ArchiveHandle *AH);
extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid);
--- 332,339 ----
extern char *ReadStr(ArchiveHandle *AH);
extern size_t WriteStr(ArchiveHandle *AH, const char *s);
! int ReadOffset(ArchiveHandle *, pgoff_t *);
! size_t WriteOffset(ArchiveHandle *, pgoff_t, int);
extern void StartRestoreBlobs(ArchiveHandle *AH);
extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid);
Index: src/bin/pg_dump/pg_backup_custom.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v
retrieving revision 1.36
diff -c -r1.36 pg_backup_custom.c
*** src/bin/pg_dump/pg_backup_custom.c 4 Oct 2006 00:30:05 -0000 1.36
--- src/bin/pg_dump/pg_backup_custom.c 11 Feb 2007 15:01:39 -0000
***************
*** 70,83 ****
char *zlibIn;
size_t inSize;
int hasSeek;
! off_t filePos;
! off_t dataStart;
} lclContext;
typedef struct
{
int dataState;
! off_t dataPos;
} lclTocEntry;
--- 70,83 ----
char *zlibIn;
size_t inSize;
int hasSeek;
! pgoff_t filePos;
! pgoff_t dataStart;
} lclContext;
typedef struct
{
int dataState;
! pgoff_t dataPos;
} lclTocEntry;
***************
*** 88,94 ****
static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
! static off_t _getFilePos(ArchiveHandle *AH, lclContext *ctx);
static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
static char *modulename = gettext_noop("custom archiver");
--- 88,94 ----
static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
! static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx);
static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
static char *modulename = gettext_noop("custom archiver");
***************
*** 791,797 ****
_CloseArchive(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
! off_t tpos;
if (AH->mode == archModeWrite)
{
--- 791,797 ----
_CloseArchive(ArchiveHandle *AH)
{
lclContext *ctx = (lclContext *) AH->formatData;
! pgoff_t tpos;
if (AH->mode == archModeWrite)
{
***************
*** 827,836 ****
/*
* Get the current position in the archive file.
*/
! static off_t
_getFilePos(ArchiveHandle *AH, lclContext *ctx)
{
! off_t pos;
if (ctx->hasSeek)
{
--- 827,836 ----
/*
* Get the current position in the archive file.
*/
! static pgoff_t
_getFilePos(ArchiveHandle *AH, lclContext *ctx)
{
! pgoff_t pos;
if (ctx->hasSeek)
{
***************
*** 841,847 ****
/*
* Prior to 1.7 (pg7.3) we relied on the internally
maintained
! * pointer. Now we rely on off_t always. pos =
ctx->filePos;
*/
}
}
--- 841,847 ----
/*
* Prior to 1.7 (pg7.3) we relied on the internally
maintained
! * pointer. Now we rely on pgoff_t always. pos =
ctx->filePos;
*/
}
}
Index: src/bin/pg_dump/pg_backup_files.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v
retrieving revision 1.30
diff -c -r1.30 pg_backup_files.c
*** src/bin/pg_dump/pg_backup_files.c 8 Feb 2007 11:10:27 -0000 1.30
--- src/bin/pg_dump/pg_backup_files.c 11 Feb 2007 15:01:57 -0000
***************
*** 51,57 ****
typedef struct
{
int hasSeek;
! off_t filePos;
FILE *blobToc;
} lclContext;
--- 51,57 ----
typedef struct
{
int hasSeek;
! pgoff_t filePos;
FILE *blobToc;
} lclContext;
Index: src/bin/pg_dump/pg_backup_tar.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v
retrieving revision 1.56
diff -c -r1.56 pg_backup_tar.c
*** src/bin/pg_dump/pg_backup_tar.c 1 Nov 2006 15:59:26 -0000 1.56
--- src/bin/pg_dump/pg_backup_tar.c 11 Feb 2007 15:02:36 -0000
***************
*** 67,96 ****
FILE *tmpFH;
char *targetFile;
char mode;
! off_t pos;
! off_t fileLen;
ArchiveHandle *AH;
} TAR_MEMBER;
/*
* Maximum file size for a tar member: The limit inherent in the
* format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
! * what we can represent by an off_t.
*/
#ifdef INT64_IS_BUSTED
#define MAX_TAR_MEMBER_FILELEN INT_MAX
#else
! #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(off_t)*8 - 1)) -
1)
#endif
typedef struct
{
int hasSeek;
! off_t filePos;
TAR_MEMBER *blobToc;
FILE *tarFH;
! off_t tarFHpos;
! off_t tarNextMember;
TAR_MEMBER *FH;
int isSpecialScript;
TAR_MEMBER *scriptTH;
--- 67,96 ----
FILE *tmpFH;
char *targetFile;
char mode;
! pgoff_t pos;
! pgoff_t fileLen;
ArchiveHandle *AH;
} TAR_MEMBER;
/*
* Maximum file size for a tar member: The limit inherent in the
* format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
! * what we can represent by an pgoff_t.
*/
#ifdef INT64_IS_BUSTED
#define MAX_TAR_MEMBER_FILELEN INT_MAX
#else
! #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1))
- 1)
#endif
typedef struct
{
int hasSeek;
! pgoff_t filePos;
TAR_MEMBER *blobToc;
FILE *tarFH;
! pgoff_t tarFHpos;
! pgoff_t tarNextMember;
TAR_MEMBER *FH;
int isSpecialScript;
TAR_MEMBER *scriptTH;
***************
*** 1048,1054 ****
FILE *tmp = th->tmpFH; /* Grab it for convenience */
char buf[32768];
size_t cnt;
! off_t len = 0;
size_t res;
size_t i,
pad;
--- 1048,1054 ----
FILE *tmp = th->tmpFH; /* Grab it for convenience */
char buf[32768];
size_t cnt;
! pgoff_t len = 0;
size_t res;
size_t i,
pad;
***************
*** 1061,1067 ****
/*
* Some compilers with throw a warning knowing this test can never be
true
! * because off_t can't exceed the compared maximum.
*/
if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
die_horribly(AH, modulename, "archive member too large for tar
format\n");
--- 1061,1067 ----
/*
* Some compilers with throw a warning knowing this test can never be
true
! * because pgoff_t can't exceed the compared maximum.
*/
if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
die_horribly(AH, modulename, "archive member too large for tar
format\n");
***************
*** 1197,1203 ****
chk;
size_t len;
unsigned long ullen;
! off_t hPos;
bool gotBlock = false;
while (!gotBlock)
--- 1197,1203 ----
chk;
size_t len;
unsigned long ullen;
! pgoff_t hPos;
bool gotBlock = false;
while (!gotBlock)
Index: src/bin/pg_dump/pg_dump.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/bin/pg_dump/pg_dump.h,v
retrieving revision 1.132
diff -c -r1.132 pg_dump.h
*** src/bin/pg_dump/pg_dump.h 23 Jan 2007 17:54:50 -0000 1.132
--- src/bin/pg_dump/pg_dump.h 11 Feb 2007 15:03:38 -0000
***************
*** 16,21 ****
--- 16,39 ----
#include "postgres_fe.h"
+ /*
+ * WIN32 does not provide 64-bit off_t, but does provide the functions
operating
+ * with 64-bit offsets.
+ */
+ #ifdef WIN32
+ #define pgoff_t __int64
+ #undef fseeko
+ #undef ftello
+ #ifdef WIN32_ONLY_COMPILER
+ #define fseeko(stream, offset, origin) _fseeki64(stream, offset, origin)
+ #define ftello(stream) _ftelli64(stream)
+ #else
+ #define fseeko(stream, offset, origin) fseeko64(stream, offset, origin)
+ #define ftello(stream) ftello64(stream)
+ #endif
+ #else
+ #define pgoff_t off_t
+ #endif
/*
* pg_dump uses two different mechanisms for identifying database objects:
---------------------------(end of broadcast)---------------------------
TIP 4: Have you searched our list archives?
http://archives.postgresql.org
