From d127c9afdff0710babd8db28d6b9870c537cbff7 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Wed, 25 Jun 2025 15:53:00 +0200 Subject: [PATCH v7 6/6] pg_dump compression API: read_func / gets_func read_func is defined to return the number of bytes read, zero for EOF and throw an error on all error conditions * lz4: Make sure to call throw an error and not return -1 * gzip: gzread returning zero cannot be assumed to indicate EOF as it is documented to return zero for some types of errors. * none: Don't call ferror on every fread unless it returne zero and we can expect there to be an error or EOF. gets_func is defined go return *s on success and NULL on error or if EOF is found before any char has been read. * lz4, zstd: Convert _read_internal functions to not automatically call pg_fatal on errors to be able to handle gets returning NULL on error. --- src/bin/pg_dump/compress_gzip.c | 15 +++++++++++++-- src/bin/pg_dump/compress_io.h | 2 +- src/bin/pg_dump/compress_lz4.c | 19 +++++++++++++++---- src/bin/pg_dump/compress_none.c | 2 +- src/bin/pg_dump/compress_zstd.c | 28 +++++++++++++++++++++++----- 5 files changed, 53 insertions(+), 13 deletions(-) diff --git a/src/bin/pg_dump/compress_gzip.c b/src/bin/pg_dump/compress_gzip.c index 2edcbffdd8d..4a067e1402c 100644 --- a/src/bin/pg_dump/compress_gzip.c +++ b/src/bin/pg_dump/compress_gzip.c @@ -258,10 +258,21 @@ Gzip_read(void *ptr, size_t size, CompressFileHandle *CFH) int gzret; gzret = gzread(gzfp, ptr, size); - if (gzret < 0) + + /* + * gzread returns zero on EOF as well as some error conditions, and less + * than zero on other error conditions, so we need to inspect for EOF on + * zero. + */ + if (gzret <= 0) { int errnum; - const char *errmsg = gzerror(gzfp, &errnum); + const char *errmsg; + + if (gzret == 0 && gzeof(gzfp)) + return 0; + + errmsg = gzerror(gzfp, &errnum); pg_fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index e4d98ca08d7..82f5c71f764 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -127,7 +127,7 @@ struct CompressFileHandle * 'ptr'. * * Returns number of bytes read (this might be less than 'size' if EOF was - * reached). Throws error for error conditions other than EOF. + * reached). Exits via pg_fatal all for error conditions. */ size_t (*read_func) (void *ptr, size_t size, CompressFileHandle *CFH); diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c index 700915fa90d..e2f7c468293 100644 --- a/src/bin/pg_dump/compress_lz4.c +++ b/src/bin/pg_dump/compress_lz4.c @@ -459,7 +459,11 @@ LZ4Stream_read_internal(LZ4State *state, void *ptr, int ptrsize, bool eol_flag) /* Lazy init */ if (!LZ4Stream_init(state, size, false /* decompressing */ )) + { + pg_log_error("unable to initialize LZ4 library: %s", + LZ4F_getErrorName(state->errcode)); return -1; + } /* No work needs to be done for a zero-sized output buffer */ if (size <= 0) @@ -486,7 +490,10 @@ LZ4Stream_read_internal(LZ4State *state, void *ptr, int ptrsize, bool eol_flag) rsize = fread(readbuf, 1, size, state->fp); if (rsize < size && !feof(state->fp)) + { + pg_log_error("could not read from input file: %m"); return -1; + } rp = (char *) readbuf; rend = (char *) readbuf + rsize; @@ -503,6 +510,8 @@ LZ4Stream_read_internal(LZ4State *state, void *ptr, int ptrsize, bool eol_flag) if (LZ4F_isError(status)) { state->errcode = status; + pg_log_error("could not read from input file: %s", + LZ4F_getErrorName(state->errcode)); return -1; } @@ -639,11 +648,13 @@ LZ4Stream_gets(char *ptr, int size, CompressFileHandle *CFH) int ret; ret = LZ4Stream_read_internal(state, ptr, size - 1, true); - if (ret < 0 || (ret == 0 && !LZ4Stream_eof(CFH))) - pg_fatal("could not read from input file: %s", LZ4Stream_get_error(CFH)); - /* Done reading */ - if (ret == 0) + /* + * LZ4Stream_read_internal returning 0 or -1 means that it was either an + * EOF or an error, but gets_func is defined to return NULL in either case + * so we can treat both the same here. + */ + if (ret <= 0) return NULL; /* diff --git a/src/bin/pg_dump/compress_none.c b/src/bin/pg_dump/compress_none.c index 9a24ef96cd0..c6234eec25c 100644 --- a/src/bin/pg_dump/compress_none.c +++ b/src/bin/pg_dump/compress_none.c @@ -90,7 +90,7 @@ read_none(void *ptr, size_t size, CompressFileHandle *CFH) size_t ret; ret = fread(ptr, 1, size, fp); - if (ferror(fp)) + if (ret == 0 && ferror(fp)) pg_fatal("could not read from input file: %m"); return ret; diff --git a/src/bin/pg_dump/compress_zstd.c b/src/bin/pg_dump/compress_zstd.c index 706c2427212..e24d45e1bbe 100644 --- a/src/bin/pg_dump/compress_zstd.c +++ b/src/bin/pg_dump/compress_zstd.c @@ -260,7 +260,7 @@ InitCompressorZstd(CompressorState *cs, */ static size_t -Zstd_read(void *ptr, size_t size, CompressFileHandle *CFH) +Zstd_read_internal(void *ptr, size_t size, CompressFileHandle *CFH, bool exit_on_error) { ZstdCompressorState *zstdcs = (ZstdCompressorState *) CFH->private_data; ZSTD_inBuffer *input = &zstdcs->input; @@ -278,7 +278,11 @@ Zstd_read(void *ptr, size_t size, CompressFileHandle *CFH) zstdcs->input.src = pg_malloc0(input_allocated_size); zstdcs->dstream = ZSTD_createDStream(); if (zstdcs->dstream == NULL) - pg_fatal("could not initialize compression library"); + { + if (exit_on_error) + pg_fatal("could not initialize compression library"); + return -1; + } } output->size = size; @@ -306,7 +310,11 @@ Zstd_read(void *ptr, size_t size, CompressFileHandle *CFH) { cnt = fread(unconstify(void *, input->src), 1, input_allocated_size, zstdcs->fp); if (ferror(zstdcs->fp)) - pg_fatal("could not read from input file: %m"); + { + if (exit_on_error) + pg_fatal("could not read from input file: %m"); + return -1; + } input->size = cnt; @@ -323,7 +331,11 @@ Zstd_read(void *ptr, size_t size, CompressFileHandle *CFH) res = ZSTD_decompressStream(zstdcs->dstream, output, input); if (ZSTD_isError(res)) - pg_fatal("could not decompress data: %s", ZSTD_getErrorName(res)); + { + if (exit_on_error) + pg_fatal("could not decompress data: %s", ZSTD_getErrorName(res)); + return -1; + } if (output->pos == output->size) break; /* No more room for output */ @@ -402,7 +414,7 @@ Zstd_gets(char *buf, int len, CompressFileHandle *CFH) */ for (i = 0; i < len - 1; ++i) { - if (CFH->read_func(&buf[i], 1, CFH) != 1) + if (Zstd_read_internal(&buf[i], 1, CFH, false) != 1) break; if (buf[i] == '\n') { @@ -414,6 +426,12 @@ Zstd_gets(char *buf, int len, CompressFileHandle *CFH) return i > 0 ? buf : NULL; } +static size_t +Zstd_read(void *ptr, size_t size, CompressFileHandle *CFH) +{ + return Zstd_read_internal(ptr, size, CFH, true); +} + static bool Zstd_close(CompressFileHandle *CFH) { -- 2.39.3 (Apple Git-146)