From 74096ed1fd32310f896cdf974460d1d4c9ce60c4 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Mon, 19 Feb 2007 15:05:06 +0000 Subject: [PATCH] Fix pg_dump on win32 to properly dump files larger than 2Gb when using binary dump formats. --- src/bin/pg_dump/pg_backup_archiver.c | 28 ++++++++++++++-------------- src/bin/pg_dump/pg_backup_archiver.h | 8 ++++---- src/bin/pg_dump/pg_backup_custom.c | 18 +++++++++--------- src/bin/pg_dump/pg_backup_files.c | 4 ++-- src/bin/pg_dump/pg_backup_tar.c | 22 +++++++++++----------- src/bin/pg_dump/pg_dump.h | 20 +++++++++++++++++++- 6 files changed, 59 insertions(+), 41 deletions(-) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index c64d68e380..3f014d9c62 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -15,7 +15,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.141 2007/02/01 19:10:28 momjian Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.142 2007/02/19 15:05:06 mha Exp $ * *------------------------------------------------------------------------- */ @@ -1311,24 +1311,24 @@ TocIDRequired(ArchiveHandle *AH, DumpId id, RestoreOptions *ropt) } size_t -WriteOffset(ArchiveHandle *AH, off_t o, int wasSet) +WriteOffset(ArchiveHandle *AH, pgoff_t o, int wasSet) { int off; /* Save the flag */ (*AH->WriteBytePtr) (AH, wasSet); - /* Write out off_t smallest byte first, prevents endian mismatch */ - for (off = 0; off < sizeof(off_t); off++) + /* Write out pgoff_t smallest byte first, prevents endian mismatch */ + for (off = 0; off < sizeof(pgoff_t); off++) { (*AH->WriteBytePtr) (AH, o & 0xFF); o >>= 8; } - return sizeof(off_t) + 1; + return sizeof(pgoff_t) + 1; } int -ReadOffset(ArchiveHandle *AH, off_t *o) +ReadOffset(ArchiveHandle *AH, pgoff_t *o) { int i; int off; @@ -1348,8 +1348,8 @@ ReadOffset(ArchiveHandle *AH, off_t *o) else if (i == 0) return K_OFFSET_NO_DATA; - /* Cast to off_t because it was written as an int. */ - *o = (off_t) i; + /* Cast to pgoff_t because it was written as an int. */ + *o = (pgoff_t) i; return K_OFFSET_POS_SET; } @@ -1379,8 +1379,8 @@ ReadOffset(ArchiveHandle *AH, off_t *o) */ for (off = 0; off < AH->offSize; off++) { - if (off < sizeof(off_t)) - *o |= ((off_t) ((*AH->ReadBytePtr) (AH))) << (off * 8); + if (off < sizeof(pgoff_t)) + *o |= ((pgoff_t) ((*AH->ReadBytePtr) (AH))) << (off * 8); else { if ((*AH->ReadBytePtr) (AH) != 0) @@ -1647,7 +1647,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->createDate = time(NULL); AH->intSize = sizeof(int); - AH->offSize = sizeof(off_t); + AH->offSize = sizeof(pgoff_t); if (FileSpec) { AH->fSpec = strdup(FileSpec); @@ -2768,11 +2768,11 @@ checkSeek(FILE *fp) if (fseeko(fp, 0, SEEK_CUR) != 0) return false; - else if (sizeof(off_t) > sizeof(long)) + else if (sizeof(pgoff_t) > sizeof(long)) /* - * At this point, off_t is too large for long, so we return based on - * whether an off_t version of fseek is available. + * At this point, pgoff_t is too large for long, so we return based on + * whether an pgoff_t version of fseek is available. */ #ifdef HAVE_FSEEKO return true; diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index 4a8cff3598..9e6e06daa8 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -17,7 +17,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.74 2007/01/25 03:30:43 momjian Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.75 2007/02/19 15:05:06 mha Exp $ * *------------------------------------------------------------------------- */ @@ -199,7 +199,7 @@ typedef struct _archiveHandle * format */ size_t lookaheadSize; /* Size of allocated buffer */ size_t lookaheadLen; /* Length of data in lookahead */ - off_t lookaheadPos; /* Current read position in lookahead buffer */ + pgoff_t lookaheadPos; /* Current read position in lookahead buffer */ ArchiveEntryPtr ArchiveEntryPtr; /* Called for each metadata object */ StartDataPtr StartDataPtr; /* Called when table data is about to be @@ -332,8 +332,8 @@ extern int ReadInt(ArchiveHandle *AH); extern char *ReadStr(ArchiveHandle *AH); extern size_t WriteStr(ArchiveHandle *AH, const char *s); -int ReadOffset(ArchiveHandle *, off_t *); -size_t WriteOffset(ArchiveHandle *, off_t, int); +int ReadOffset(ArchiveHandle *, pgoff_t *); +size_t WriteOffset(ArchiveHandle *, pgoff_t, int); extern void StartRestoreBlobs(ArchiveHandle *AH); extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid); diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 8d77af19c9..b859b04af9 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -19,7 +19,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.36 2006/10/04 00:30:05 momjian Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.37 2007/02/19 15:05:06 mha Exp $ * *------------------------------------------------------------------------- */ @@ -70,14 +70,14 @@ typedef struct char *zlibIn; size_t inSize; int hasSeek; - off_t filePos; - off_t dataStart; + pgoff_t filePos; + pgoff_t dataStart; } lclContext; typedef struct { int dataState; - off_t dataPos; + pgoff_t dataPos; } lclTocEntry; @@ -88,7 +88,7 @@ typedef struct static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id); static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te); static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te); -static off_t _getFilePos(ArchiveHandle *AH, lclContext *ctx); +static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx); static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush); static char *modulename = gettext_noop("custom archiver"); @@ -791,7 +791,7 @@ static void _CloseArchive(ArchiveHandle *AH) { lclContext *ctx = (lclContext *) AH->formatData; - off_t tpos; + pgoff_t tpos; if (AH->mode == archModeWrite) { @@ -827,10 +827,10 @@ _CloseArchive(ArchiveHandle *AH) /* * Get the current position in the archive file. */ -static off_t +static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx) { - off_t pos; + pgoff_t pos; if (ctx->hasSeek) { @@ -841,7 +841,7 @@ _getFilePos(ArchiveHandle *AH, lclContext *ctx) /* * Prior to 1.7 (pg7.3) we relied on the internally maintained - * pointer. Now we rely on off_t always. pos = ctx->filePos; + * pointer. Now we rely on pgoff_t always. pos = ctx->filePos; */ } } diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c index 72eedac2e4..f284df4f2b 100644 --- a/src/bin/pg_dump/pg_backup_files.c +++ b/src/bin/pg_dump/pg_backup_files.c @@ -20,7 +20,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.30 2007/02/08 11:10:27 petere Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.31 2007/02/19 15:05:06 mha Exp $ * *------------------------------------------------------------------------- */ @@ -51,7 +51,7 @@ static void _EndBlobs(ArchiveHandle *AH, TocEntry *te); typedef struct { int hasSeek; - off_t filePos; + pgoff_t filePos; FILE *blobToc; } lclContext; diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 71bf13c529..298a2fd0bf 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -16,7 +16,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.56 2006/11/01 15:59:26 tgl Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.57 2007/02/19 15:05:06 mha Exp $ * *------------------------------------------------------------------------- */ @@ -67,30 +67,30 @@ typedef struct FILE *tmpFH; char *targetFile; char mode; - off_t pos; - off_t fileLen; + pgoff_t pos; + pgoff_t fileLen; ArchiveHandle *AH; } TAR_MEMBER; /* * Maximum file size for a tar member: The limit inherent in the * format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed - * what we can represent by an off_t. + * what we can represent by an pgoff_t. */ #ifdef INT64_IS_BUSTED #define MAX_TAR_MEMBER_FILELEN INT_MAX #else -#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(off_t)*8 - 1)) - 1) +#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1) #endif typedef struct { int hasSeek; - off_t filePos; + pgoff_t filePos; TAR_MEMBER *blobToc; FILE *tarFH; - off_t tarFHpos; - off_t tarNextMember; + pgoff_t tarFHpos; + pgoff_t tarNextMember; TAR_MEMBER *FH; int isSpecialScript; TAR_MEMBER *scriptTH; @@ -1048,7 +1048,7 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th) FILE *tmp = th->tmpFH; /* Grab it for convenience */ char buf[32768]; size_t cnt; - off_t len = 0; + pgoff_t len = 0; size_t res; size_t i, pad; @@ -1061,7 +1061,7 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th) /* * Some compilers with throw a warning knowing this test can never be true - * because off_t can't exceed the compared maximum. + * because pgoff_t can't exceed the compared maximum. */ if (th->fileLen > MAX_TAR_MEMBER_FILELEN) die_horribly(AH, modulename, "archive member too large for tar format\n"); @@ -1197,7 +1197,7 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th) chk; size_t len; unsigned long ullen; - off_t hPos; + pgoff_t hPos; bool gotBlock = false; while (!gotBlock) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 9ba74aa98e..b584ab1fe0 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -6,7 +6,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.h,v 1.132 2007/01/23 17:54:50 tgl Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.h,v 1.133 2007/02/19 15:05:06 mha Exp $ * *------------------------------------------------------------------------- */ @@ -16,6 +16,24 @@ #include "postgres_fe.h" +/* + * WIN32 does not provide 64-bit off_t, but does provide the functions operating + * with 64-bit offsets. + */ +#ifdef WIN32 +#define pgoff_t __int64 +#undef fseeko +#undef ftello +#ifdef WIN32_ONLY_COMPILER +#define fseeko(stream, offset, origin) _fseeki64(stream, offset, origin) +#define ftello(stream) _ftelli64(stream) +#else +#define fseeko(stream, offset, origin) fseeko64(stream, offset, origin) +#define ftello(stream) ftello64(stream) +#endif +#else +#define pgoff_t off_t +#endif /* * pg_dump uses two different mechanisms for identifying database objects: -- GitLab