AllocSet allocates memory for itself in blocks, which double in size up
to maxBlockSize. So, the current block (the last one malloc'd) may
represent half of the total memory allocated for the context itself.
The free space at the end of that block hasn't been touched at all, and
doesn't represent fragmentation or overhead. That means that the
"allocated" memory can be 2X the memory ever touched in the worst case.
Although that's technically correct, the purpose of
MemoryContextMemAllocated() is to give a "real" usage number so we know
when we're out of work_mem and need to spill (in particular, the disk-
based HashAgg work, but ideally other operators as well). This "real"
number should include fragmentation, freed-and-not-reused chunks, and
other overhead. But it should not include significant amounts of
allocated-but-never-touched memory, which says more about economizing
calls to malloc than it does about the algorithm's memory usage.
Attached is a patch that makes mem_allocated a method (rather than a
field) of MemoryContext, and allows each memory context type to track
the memory its own way. They all do the same thing as before
(increment/decrement a field), but AllocSet also subtracts out the free
space in the current block. For Slab and Generation, we could do
something similar, but it's not as much of a problem because there's no
doubling of the allocation size.
Although I think this still matches the word "allocation" in spirit,
it's not technically correct, so feel free to suggest a new name for
MemoryContextMemAllocated().
Regards,
Jeff Davis
diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c
index c0623f106d2..ccf78ffe0cb 100644
--- a/src/backend/utils/mmgr/aset.c
+++ b/src/backend/utils/mmgr/aset.c
@@ -132,6 +132,7 @@ typedef struct AllocSetContext
Size maxBlockSize; /* maximum block size */
Size nextBlockSize; /* next block size to allocate */
Size allocChunkLimit; /* effective chunk size limit */
+ Size memAllocated; /* track memory allocated for this context */
AllocBlock keeper; /* keep this block over resets */
/* freelist this context could be put in, or -1 if not a candidate: */
int freeListIndex; /* index in context_freelists[], or -1 */
@@ -272,6 +273,7 @@ static void *AllocSetRealloc(MemoryContext context, void *pointer, Size size);
static void AllocSetReset(MemoryContext context);
static void AllocSetDelete(MemoryContext context);
static Size AllocSetGetChunkSpace(MemoryContext context, void *pointer);
+static Size AllocSetMemAllocated(MemoryContext context);
static bool AllocSetIsEmpty(MemoryContext context);
static void AllocSetStats(MemoryContext context,
MemoryStatsPrintFunc printfunc, void *passthru,
@@ -291,6 +293,7 @@ static const MemoryContextMethods AllocSetMethods = {
AllocSetReset,
AllocSetDelete,
AllocSetGetChunkSpace,
+ AllocSetMemAllocated,
AllocSetIsEmpty,
AllocSetStats
#ifdef MEMORY_CONTEXT_CHECKING
@@ -464,8 +467,7 @@ AllocSetContextCreateInternal(MemoryContext parent,
parent,
name);
- ((MemoryContext) set)->mem_allocated =
- set->keeper->endptr - ((char *) set);
+ set->memAllocated = set->keeper->endptr - ((char *) set);
return (MemoryContext) set;
}
@@ -555,7 +557,7 @@ AllocSetContextCreateInternal(MemoryContext parent,
parent,
name);
- ((MemoryContext) set)->mem_allocated = firstBlockSize;
+ set->memAllocated = firstBlockSize;
return (MemoryContext) set;
}
@@ -617,7 +619,7 @@ AllocSetReset(MemoryContext context)
else
{
/* Normal case, release the block */
- context->mem_allocated -= block->endptr - ((char*) block);
+ set->memAllocated -= block->endptr - ((char*) block);
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
@@ -627,7 +629,7 @@ AllocSetReset(MemoryContext context)
block = next;
}
- Assert(context->mem_allocated == keepersize);
+ Assert(set->memAllocated == keepersize);
/* Reset block size allocation sequence, too */
set->nextBlockSize = set->initBlockSize;
@@ -703,7 +705,7 @@ AllocSetDelete(MemoryContext context)
AllocBlock next = block->next;
if (block != set->keeper)
- context->mem_allocated -= block->endptr - ((char *) block);
+ set->memAllocated -= block->endptr - ((char *) block);
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
@@ -715,7 +717,7 @@ AllocSetDelete(MemoryContext context)
block = next;
}
- Assert(context->mem_allocated == keepersize);
+ Assert(set->memAllocated == keepersize);
/* Finally, free the context header, including the keeper block */
free(set);
@@ -758,7 +760,7 @@ AllocSetAlloc(MemoryContext context, Size size)
if (block == NULL)
return NULL;
- context->mem_allocated += blksize;
+ set->memAllocated += blksize;
block->aset = set;
block->freeptr = block->endptr = ((char *) block) + blksize;
@@ -955,7 +957,7 @@ AllocSetAlloc(MemoryContext context, Size size)
if (block == NULL)
return NULL;
- context->mem_allocated += blksize;
+ set->memAllocated += blksize;
block->aset = set;
block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
@@ -1058,7 +1060,7 @@ AllocSetFree(MemoryContext context, void *pointer)
if (block->next)
block->next->prev = block->prev;
- context->mem_allocated -= block->endptr - ((char*) block);
+ set->memAllocated -= block->endptr - ((char*) block);
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->freeptr - ((char *) block));
@@ -1161,8 +1163,8 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size)
}
/* updated separately, not to underflow when (oldblksize > blksize) */
- context->mem_allocated -= oldblksize;
- context->mem_allocated += blksize;
+ set->memAllocated -= oldblksize;
+ set->memAllocated += blksize;
block->freeptr = block->endptr = ((char *) block) + blksize;
@@ -1337,6 +1339,24 @@ AllocSetGetChunkSpace(MemoryContext context, void *pointer)
return result;
}
+/*
+ * All memory currently allocated for this context (including fragmentation
+ * and freed chunks).
+ *
+ * Allocation sizes double (up to maxBlockSize), so the current block may
+ * represent half of the total space allocated to the context. Subtract away
+ * the free space at the tail of the current block, which may never have been
+ * touched.
+ */
+static Size
+AllocSetMemAllocated(MemoryContext context)
+{
+ AllocSet set = (AllocSet) context;
+ AllocBlock currentBlock = set->blocks;
+ Size tailSpace = currentBlock->endptr - currentBlock->freeptr;
+ return set->memAllocated - tailSpace;
+}
+
/*
* AllocSetIsEmpty
* Is an allocset empty of any allocated space?
@@ -1538,7 +1558,7 @@ AllocSetCheck(MemoryContext context)
name, block);
}
- Assert(total_allocated == context->mem_allocated);
+ Assert(total_allocated == set->memAllocated);
}
#endif /* MEMORY_CONTEXT_CHECKING */
diff --git a/src/backend/utils/mmgr/generation.c b/src/backend/utils/mmgr/generation.c
index 56651d06931..f0ef540a7c5 100644
--- a/src/backend/utils/mmgr/generation.c
+++ b/src/backend/utils/mmgr/generation.c
@@ -61,6 +61,7 @@ typedef struct GenerationContext
/* Generational context parameters */
Size blockSize; /* standard block size */
+ Size memAllocated; /* track memory allocated for this context */
GenerationBlock *block; /* current (most recently allocated) block */
dlist_head blocks; /* list of blocks */
@@ -152,6 +153,7 @@ static void *GenerationRealloc(MemoryContext context, void *pointer, Size size);
static void GenerationReset(MemoryContext context);
static void GenerationDelete(MemoryContext context);
static Size GenerationGetChunkSpace(MemoryContext context, void *pointer);
+static Size GenerationMemAllocated(MemoryContext context);
static bool GenerationIsEmpty(MemoryContext context);
static void GenerationStats(MemoryContext context,
MemoryStatsPrintFunc printfunc, void *passthru,
@@ -171,6 +173,7 @@ static const MemoryContextMethods GenerationMethods = {
GenerationReset,
GenerationDelete,
GenerationGetChunkSpace,
+ GenerationMemAllocated,
GenerationIsEmpty,
GenerationStats
#ifdef MEMORY_CONTEXT_CHECKING
@@ -258,6 +261,7 @@ GenerationContextCreate(MemoryContext parent,
/* Fill in GenerationContext-specific header fields */
set->blockSize = blockSize;
+ set->memAllocated = 0;
set->block = NULL;
dlist_init(&set->blocks);
@@ -297,7 +301,7 @@ GenerationReset(MemoryContext context)
dlist_delete(miter.cur);
- context->mem_allocated -= block->blksize;
+ set->memAllocated -= block->blksize;
#ifdef CLOBBER_FREED_MEMORY
wipe_mem(block, block->blksize);
@@ -354,7 +358,7 @@ GenerationAlloc(MemoryContext context, Size size)
if (block == NULL)
return NULL;
- context->mem_allocated += blksize;
+ set->memAllocated += blksize;
/* block with a single (used) chunk */
block->blksize = blksize;
@@ -411,7 +415,7 @@ GenerationAlloc(MemoryContext context, Size size)
if (block == NULL)
return NULL;
- context->mem_allocated += blksize;
+ set->memAllocated += blksize;
block->blksize = blksize;
block->nchunks = 0;
@@ -528,7 +532,7 @@ GenerationFree(MemoryContext context, void *pointer)
if (set->block == block)
set->block = NULL;
- context->mem_allocated -= block->blksize;
+ set->memAllocated -= block->blksize;
free(block);
}
@@ -666,6 +670,17 @@ GenerationGetChunkSpace(MemoryContext context, void *pointer)
return result;
}
+/*
+ * All memory currently allocated for this context (including fragmentation
+ * and freed chunks).
+ */
+static Size
+GenerationMemAllocated(MemoryContext context)
+{
+ GenerationContext *set = (GenerationContext *) context;
+ return set->memAllocated;
+}
+
/*
* GenerationIsEmpty
* Is a GenerationContext empty of any allocated space?
@@ -844,7 +859,7 @@ GenerationCheck(MemoryContext context)
name, nfree, block, block->nfree);
}
- Assert(total_allocated == context->mem_allocated);
+ Assert(total_allocated == gen->memAllocated);
}
#endif /* MEMORY_CONTEXT_CHECKING */
diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c
index 9e24fec72d6..e32e279c340 100644
--- a/src/backend/utils/mmgr/mcxt.c
+++ b/src/backend/utils/mmgr/mcxt.c
@@ -469,7 +469,7 @@ MemoryContextIsEmpty(MemoryContext context)
Size
MemoryContextMemAllocated(MemoryContext context, bool recurse)
{
- Size total = context->mem_allocated;
+ Size total = context->methods->mem_allocated(context);
AssertArg(MemoryContextIsValid(context));
@@ -760,7 +760,6 @@ MemoryContextCreate(MemoryContext node,
node->methods = methods;
node->parent = parent;
node->firstchild = NULL;
- node->mem_allocated = 0;
node->prevchild = NULL;
node->name = name;
node->ident = NULL;
diff --git a/src/backend/utils/mmgr/slab.c b/src/backend/utils/mmgr/slab.c
index c928476c479..63750fbc81f 100644
--- a/src/backend/utils/mmgr/slab.c
+++ b/src/backend/utils/mmgr/slab.c
@@ -67,6 +67,7 @@ typedef struct SlabContext
Size fullChunkSize; /* chunk size including header and alignment */
Size blockSize; /* block size */
Size headerSize; /* allocated size of context header */
+ Size memAllocated; /* track memory allocated for this context */
int chunksPerBlock; /* number of chunks per block */
int minFreeChunks; /* min number of free chunks in any block */
int nblocks; /* number of blocks allocated */
@@ -132,6 +133,7 @@ static void *SlabRealloc(MemoryContext context, void *pointer, Size size);
static void SlabReset(MemoryContext context);
static void SlabDelete(MemoryContext context);
static Size SlabGetChunkSpace(MemoryContext context, void *pointer);
+static Size SlabMemAllocated(MemoryContext context);
static bool SlabIsEmpty(MemoryContext context);
static void SlabStats(MemoryContext context,
MemoryStatsPrintFunc printfunc, void *passthru,
@@ -150,6 +152,7 @@ static const MemoryContextMethods SlabMethods = {
SlabReset,
SlabDelete,
SlabGetChunkSpace,
+ SlabMemAllocated,
SlabIsEmpty,
SlabStats
#ifdef MEMORY_CONTEXT_CHECKING
@@ -262,6 +265,7 @@ SlabContextCreate(MemoryContext parent,
slab->fullChunkSize = fullChunkSize;
slab->blockSize = blockSize;
slab->headerSize = headerSize;
+ slab->memAllocated = 0;
slab->chunksPerBlock = chunksPerBlock;
slab->minFreeChunks = 0;
slab->nblocks = 0;
@@ -286,6 +290,17 @@ SlabContextCreate(MemoryContext parent,
return (MemoryContext) slab;
}
+/*
+ * All memory currently allocated for this context (including fragmentation
+ * and freed chunks).
+ */
+static Size
+SlabMemAllocated(MemoryContext context)
+{
+ SlabContext *slab = (SlabContext *) context;
+ return slab->memAllocated;
+}
+
/*
* SlabReset
* Frees all memory which is allocated in the given set.
@@ -322,14 +337,14 @@ SlabReset(MemoryContext context)
#endif
free(block);
slab->nblocks--;
- context->mem_allocated -= slab->blockSize;
+ slab->memAllocated -= slab->blockSize;
}
}
slab->minFreeChunks = 0;
Assert(slab->nblocks == 0);
- Assert(context->mem_allocated == 0);
+ Assert(slab->memAllocated == 0);
}
/*
@@ -407,7 +422,7 @@ SlabAlloc(MemoryContext context, Size size)
slab->minFreeChunks = slab->chunksPerBlock;
slab->nblocks += 1;
- context->mem_allocated += slab->blockSize;
+ slab->memAllocated += slab->blockSize;
}
/* grab the block from the freelist (even the new block is there) */
@@ -501,7 +516,7 @@ SlabAlloc(MemoryContext context, Size size)
SlabAllocInfo(slab, chunk);
- Assert(slab->nblocks * slab->blockSize == context->mem_allocated);
+ Assert(slab->nblocks * slab->blockSize == slab->memAllocated);
return SlabChunkGetPointer(chunk);
}
@@ -578,13 +593,13 @@ SlabFree(MemoryContext context, void *pointer)
{
free(block);
slab->nblocks--;
- context->mem_allocated -= slab->blockSize;
+ slab->memAllocated -= slab->blockSize;
}
else
dlist_push_head(&slab->freelist[block->nfree], &block->node);
Assert(slab->nblocks >= 0);
- Assert(slab->nblocks * slab->blockSize == context->mem_allocated);
+ Assert(slab->nblocks * slab->blockSize == slab->memAllocated);
}
/*
@@ -804,7 +819,7 @@ SlabCheck(MemoryContext context)
}
}
- Assert(slab->nblocks * slab->blockSize == context->mem_allocated);
+ Assert(slab->nblocks * slab->blockSize == slab->memAllocated);
}
#endif /* MEMORY_CONTEXT_CHECKING */
diff --git a/src/include/nodes/memnodes.h b/src/include/nodes/memnodes.h
index c9f2bbcb367..b6296af7d5f 100644
--- a/src/include/nodes/memnodes.h
+++ b/src/include/nodes/memnodes.h
@@ -63,6 +63,7 @@ typedef struct MemoryContextMethods
void (*reset) (MemoryContext context);
void (*delete_context) (MemoryContext context);
Size (*get_chunk_space) (MemoryContext context, void *pointer);
+ Size (*mem_allocated) (MemoryContext context);
bool (*is_empty) (MemoryContext context);
void (*stats) (MemoryContext context,
MemoryStatsPrintFunc printfunc, void *passthru,
@@ -79,7 +80,6 @@ typedef struct MemoryContextData
/* these two fields are placed here to minimize alignment wastage: */
bool isReset; /* T = no space alloced since last reset */
bool allowInCritSection; /* allow palloc in critical section */
- Size mem_allocated; /* track memory allocated for this context */
const MemoryContextMethods *methods; /* virtual function table */
MemoryContext parent; /* NULL if no parent (toplevel context) */
MemoryContext firstchild; /* head of linked list of children */
