Hello, I would like to be able to retrieve the size of memory internally allocated by libpq for a result. The reason is that we have a Ruby wrapper that exposes libpq in Ruby. The problem is that Ruby's GC doesn't know how much memory has been allocated by libpq, so no pressure is applied to the GC when it should be. With this function we could instruct the GC about the memory usage associated to each result object.
This issue has already been discussed in the following thread, with the request to use custom malloc/realloc/free functions: https://www.postgresql.org/message-id/flat/20170828172834.GA71455%40TC.local#20170828172834.GA71455@TC.local Retrieving the allocated memory size is another approach to solve the same base issue. However since the relation between memory consumption and the particular result object is maintained, it can additionally be used to provide diagnostic information to each object. What do you think about adding such a function? -- Kind Regards, Lars
From d3ac8089a1b8c26d29d8d8e93c48a892cec75e53 Mon Sep 17 00:00:00 2001 From: Lars Kanis <l...@greiz-reinsdorf.de> Date: Sat, 23 Jun 2018 19:34:11 +0200 Subject: [PATCH] libpq: Add function PQresultMemsize() This function retrieves the number of bytes allocated for a given result. That can be used to instruct the GC about the memory consumed behind a wrapping object and for diagnosing memory consumtion. This is an alternative approach to customizable malloc/realloc/free functions as discussed here: https://www.postgresql.org/message-id/flat/20170828172834.GA71455%40TC.local#20170828172834.GA71455@TC.local --- src/interfaces/libpq/exports.txt | 1 + src/interfaces/libpq/fe-exec.c | 14 +++++++++++++- src/interfaces/libpq/libpq-fe.h | 1 + src/interfaces/libpq/libpq-int.h | 2 ++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt index d6a38d0df8..0b50dddbb7 100644 --- a/src/interfaces/libpq/exports.txt +++ b/src/interfaces/libpq/exports.txt @@ -172,3 +172,4 @@ PQsslAttribute 169 PQsetErrorContextVisibility 170 PQresultVerboseErrorMessage 171 PQencryptPasswordConn 172 +PQresultMemsize 173 diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index 4c0114c514..064c7a693c 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -166,6 +166,7 @@ PQmakeEmptyPGresult(PGconn *conn, ExecStatusType status) result->curBlock = NULL; result->curOffset = 0; result->spaceLeft = 0; + result->memsize = sizeof(PGresult); if (conn) { @@ -215,6 +216,12 @@ PQmakeEmptyPGresult(PGconn *conn, ExecStatusType status) return result; } +size_t +PQresultMemsize(const PGresult *res) +{ + return res->memsize; +} + /* * PQsetResultAttrs * @@ -567,9 +574,11 @@ pqResultAlloc(PGresult *res, size_t nBytes, bool isBinary) */ if (nBytes >= PGRESULT_SEP_ALLOC_THRESHOLD) { - block = (PGresult_data *) malloc(nBytes + PGRESULT_BLOCK_OVERHEAD); + size_t alloc_size = nBytes + PGRESULT_BLOCK_OVERHEAD; + block = (PGresult_data *) malloc(alloc_size); if (!block) return NULL; + res->memsize += alloc_size; space = block->space + PGRESULT_BLOCK_OVERHEAD; if (res->curBlock) { @@ -594,6 +603,7 @@ pqResultAlloc(PGresult *res, size_t nBytes, bool isBinary) block = (PGresult_data *) malloc(PGRESULT_DATA_BLOCKSIZE); if (!block) return NULL; + res->memsize += PGRESULT_DATA_BLOCKSIZE; block->next = res->curBlock; res->curBlock = block; if (isBinary) @@ -711,6 +721,7 @@ PQclear(PGresult *res) res->errFields = NULL; res->events = NULL; res->nEvents = 0; + res->memsize = 0; /* res->curBlock was zeroed out earlier */ /* Free the PGresult structure itself */ @@ -927,6 +938,7 @@ pqAddTuple(PGresult *res, PGresAttValue *tup, const char **errmsgp) realloc(res->tuples, newSize * sizeof(PGresAttValue *)); if (!newTuples) return false; /* malloc or realloc failed */ + res->memsize += (newSize - res->tupArrSize) * sizeof(PGresAttValue *); res->tupArrSize = newSize; res->tuples = newTuples; } diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index ed9c806861..4fd9a4fcda 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -491,6 +491,7 @@ extern int PQgetlength(const PGresult *res, int tup_num, int field_num); extern int PQgetisnull(const PGresult *res, int tup_num, int field_num); extern int PQnparams(const PGresult *res); extern Oid PQparamtype(const PGresult *res, int param_num); +extern size_t PQresultMemsize(const PGresult *res); /* Describe prepared statements and portals */ extern PGresult *PQdescribePrepared(PGconn *conn, const char *stmt); diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 9a586ff25a..37c9c3853d 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -208,6 +208,8 @@ struct pg_result PGresult_data *curBlock; /* most recently allocated block */ int curOffset; /* start offset of free space in block */ int spaceLeft; /* number of free bytes remaining in block */ + + size_t memsize; /* Number of bytes of all memory allocated for this result */ }; /* PGAsyncStatusType defines the state of the query-execution state machine */ -- 2.17.1
