On Wed, 2014-08-06 at 11:43 -0400, Robert Haas wrote:
> Comparing the median times, that's about a 3% regression.  For this
> particular case, we might be able to recapture that by replacing the
> bespoke memory-tracking logic in tuplesort.c with use of this new
> facility.  I'm not sure whether there are other cases that we might
> also want to test; I think stuff that runs all on the server side is
> likely to show up problems more clearly than pgbench.  Maybe a
> PL/pgsql loop that does something allocation-intensive on each
> iteration, for example, like parsing a big JSON document.

I wasn't able to reproduce your results on my machine. At -s 300, with
maintenance_work_mem set high enough to do internal sort, it took about
40s and I heard some disk activity, so I didn't think it was a valid
result. I went down to -s 150, and it took around 5.3s on both master
and memory-accounting.

Either way, it's better to be conservative. Attached is a version of the
patch with opt-in memory usage tracking. Child contexts inherit the
setting from their parent.

Regards,
        Jeff Davis

*** a/src/backend/utils/mmgr/aset.c
--- b/src/backend/utils/mmgr/aset.c
***************
*** 242,247 **** typedef struct AllocChunkData
--- 242,249 ----
  #define AllocChunkGetPointer(chk)	\
  					((AllocPointer)(((char *)(chk)) + ALLOC_CHUNKHDRSZ))
  
+ static void update_allocation(MemoryContext context, int64 size);
+ 
  /*
   * These functions implement the MemoryContext API for AllocSet contexts.
   */
***************
*** 430,435 **** randomize_mem(char *ptr, size_t size)
--- 432,440 ----
   * minContextSize: minimum context size
   * initBlockSize: initial allocation block size
   * maxBlockSize: maximum allocation block size
+  *
+  * The flag determining whether this context tracks memory usage is inherited
+  * from the parent context.
   */
  MemoryContext
  AllocSetContextCreate(MemoryContext parent,
***************
*** 438,443 **** AllocSetContextCreate(MemoryContext parent,
--- 443,468 ----
  					  Size initBlockSize,
  					  Size maxBlockSize)
  {
+ 	return AllocSetContextCreateTracked(
+ 		parent, name, minContextSize, initBlockSize, maxBlockSize,
+ 		(parent == NULL) ? false : parent->track_mem);
+ }
+ 
+ /*
+  * AllocSetContextCreateTracked
+  *		Create a new AllocSet context.
+  *
+  * Implementation for AllocSetContextCreate, but also allows the caller to
+  * specify whether memory usage should be tracked or not.
+  */
+ MemoryContext
+ AllocSetContextCreateTracked(MemoryContext parent,
+ 							 const char *name,
+ 							 Size minContextSize,
+ 							 Size initBlockSize,
+ 							 Size maxBlockSize,
+ 							 bool track_mem)
+ {
  	AllocSet	context;
  
  	/* Do the type-independent part of context creation */
***************
*** 445,451 **** AllocSetContextCreate(MemoryContext parent,
  											 sizeof(AllocSetContext),
  											 &AllocSetMethods,
  											 parent,
! 											 name);
  
  	/*
  	 * Make sure alloc parameters are reasonable, and save them.
--- 470,477 ----
  											 sizeof(AllocSetContext),
  											 &AllocSetMethods,
  											 parent,
! 											 name,
! 											 track_mem);
  
  	/*
  	 * Make sure alloc parameters are reasonable, and save them.
***************
*** 500,505 **** AllocSetContextCreate(MemoryContext parent,
--- 526,534 ----
  					 errdetail("Failed while creating memory context \"%s\".",
  							   name)));
  		}
+ 
+ 		update_allocation((MemoryContext) context, blksize);
+ 
  		block->aset = context;
  		block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
  		block->endptr = ((char *) block) + blksize;
***************
*** 590,595 **** AllocSetReset(MemoryContext context)
--- 619,625 ----
  		else
  		{
  			/* Normal case, release the block */
+ 			update_allocation(context, -(block->endptr - ((char*) block)));
  #ifdef CLOBBER_FREED_MEMORY
  			wipe_mem(block, block->freeptr - ((char *) block));
  #endif
***************
*** 632,637 **** AllocSetDelete(MemoryContext context)
--- 662,668 ----
  	{
  		AllocBlock	next = block->next;
  
+ 		update_allocation(context, -(block->endptr - ((char*) block)));
  #ifdef CLOBBER_FREED_MEMORY
  		wipe_mem(block, block->freeptr - ((char *) block));
  #endif
***************
*** 678,683 **** AllocSetAlloc(MemoryContext context, Size size)
--- 709,717 ----
  					 errmsg("out of memory"),
  					 errdetail("Failed on request of size %zu.", size)));
  		}
+ 
+ 		update_allocation(context, blksize);
+ 
  		block->aset = set;
  		block->freeptr = block->endptr = ((char *) block) + blksize;
  
***************
*** 873,878 **** AllocSetAlloc(MemoryContext context, Size size)
--- 907,914 ----
  					 errdetail("Failed on request of size %zu.", size)));
  		}
  
+ 		update_allocation(context, blksize);
+ 
  		block->aset = set;
  		block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
  		block->endptr = ((char *) block) + blksize;
***************
*** 976,981 **** AllocSetFree(MemoryContext context, void *pointer)
--- 1012,1018 ----
  			set->blocks = block->next;
  		else
  			prevblock->next = block->next;
+ 		update_allocation(context, -(block->endptr - ((char*) block)));
  #ifdef CLOBBER_FREED_MEMORY
  		wipe_mem(block, block->freeptr - ((char *) block));
  #endif
***************
*** 1088,1093 **** AllocSetRealloc(MemoryContext context, void *pointer, Size size)
--- 1125,1131 ----
  		AllocBlock	prevblock = NULL;
  		Size		chksize;
  		Size		blksize;
+ 		Size		oldblksize;
  
  		while (block != NULL)
  		{
***************
*** 1105,1110 **** AllocSetRealloc(MemoryContext context, void *pointer, Size size)
--- 1143,1150 ----
  		/* Do the realloc */
  		chksize = MAXALIGN(size);
  		blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
+ 		oldblksize = block->endptr - ((char *)block);
+ 
  		block = (AllocBlock) realloc(block, blksize);
  		if (block == NULL)
  		{
***************
*** 1114,1119 **** AllocSetRealloc(MemoryContext context, void *pointer, Size size)
--- 1154,1160 ----
  					 errmsg("out of memory"),
  					 errdetail("Failed on request of size %zu.", size)));
  		}
+ 		update_allocation(context, blksize - oldblksize);
  		block->freeptr = block->endptr = ((char *) block) + blksize;
  
  		/* Update pointers since block has likely been moved */
***************
*** 1277,1282 **** AllocSetStats(MemoryContext context, int level)
--- 1318,1350 ----
  }
  
  
+ /*
+  * update_allocation
+  *
+  * Track newly-allocated or newly-freed memory (freed memory should be
+  * negative).
+  */
+ static void
+ update_allocation(MemoryContext context, int64 size)
+ {
+ 	MemoryContext parent;
+ 
+ 	if (!context->track_mem)
+ 		return;
+ 
+ 	context->self_allocated += size;
+ 
+ 	for (parent = context; parent != NULL; parent = parent->parent)
+ 	{
+ 		if (!parent->track_mem)
+ 			break;
+ 
+ 		parent->total_allocated += size;
+ 		Assert(parent->self_allocated >= 0);
+ 		Assert(parent->total_allocated >= 0);
+ 	}
+ }
+ 
  #ifdef MEMORY_CONTEXT_CHECKING
  
  /*
*** a/src/backend/utils/mmgr/mcxt.c
--- b/src/backend/utils/mmgr/mcxt.c
***************
*** 324,329 **** MemoryContextAllowInCriticalSection(MemoryContext context, bool allow)
--- 324,346 ----
  }
  
  /*
+  * MemoryContextGetAllocated
+  *
+  * Return memory allocated by the system to this context. If total is true,
+  * include child contexts. Context must have track_mem set.
+  */
+ int64
+ MemoryContextGetAllocated(MemoryContext context, bool total)
+ {
+ 	Assert(context->track_mem);
+ 
+ 	if (total)
+ 		return context->total_allocated;
+ 	else
+ 		return context->self_allocated;
+ }
+ 
+ /*
   * GetMemoryChunkSpace
   *		Given a currently-allocated chunk, determine the total space
   *		it occupies (including all memory-allocation overhead).
***************
*** 546,552 **** MemoryContext
  MemoryContextCreate(NodeTag tag, Size size,
  					MemoryContextMethods *methods,
  					MemoryContext parent,
! 					const char *name)
  {
  	MemoryContext node;
  	Size		needed = size + strlen(name) + 1;
--- 563,570 ----
  MemoryContextCreate(NodeTag tag, Size size,
  					MemoryContextMethods *methods,
  					MemoryContext parent,
! 					const char *name,
! 					bool track_mem)
  {
  	MemoryContext node;
  	Size		needed = size + strlen(name) + 1;
***************
*** 576,581 **** MemoryContextCreate(NodeTag tag, Size size,
--- 594,602 ----
  	node->firstchild = NULL;
  	node->nextchild = NULL;
  	node->isReset = true;
+ 	node->track_mem = track_mem;
+ 	node->total_allocated = 0;
+ 	node->self_allocated = 0;
  	node->name = ((char *) node) + size;
  	strcpy(node->name, name);
  
*** a/src/include/nodes/memnodes.h
--- b/src/include/nodes/memnodes.h
***************
*** 60,65 **** typedef struct MemoryContextData
--- 60,68 ----
  	MemoryContext nextchild;	/* next child of same parent */
  	char	   *name;			/* context name (just for debugging) */
  	bool		isReset;		/* T = no space alloced since last reset */
+ 	bool		track_mem;		/* whether to track memory usage */
+ 	int64		total_allocated; /* including child contexts */
+ 	int64		self_allocated; /* not including child contexts */
  #ifdef USE_ASSERT_CHECKING
  	bool		allowInCritSection;	/* allow palloc in critical section */
  #endif
*** a/src/include/utils/memutils.h
--- b/src/include/utils/memutils.h
***************
*** 96,101 **** extern void MemoryContextDeleteChildren(MemoryContext context);
--- 96,102 ----
  extern void MemoryContextResetAndDeleteChildren(MemoryContext context);
  extern void MemoryContextSetParent(MemoryContext context,
  					   MemoryContext new_parent);
+ extern int64 MemoryContextGetAllocated(MemoryContext context, bool total);
  extern Size GetMemoryChunkSpace(void *pointer);
  extern MemoryContext GetMemoryChunkContext(void *pointer);
  extern MemoryContext MemoryContextGetParent(MemoryContext context);
***************
*** 117,123 **** extern bool MemoryContextContains(MemoryContext context, void *pointer);
  extern MemoryContext MemoryContextCreate(NodeTag tag, Size size,
  					MemoryContextMethods *methods,
  					MemoryContext parent,
! 					const char *name);
  
  
  /*
--- 118,125 ----
  extern MemoryContext MemoryContextCreate(NodeTag tag, Size size,
  					MemoryContextMethods *methods,
  					MemoryContext parent,
! 					const char *name,
! 					bool track_mem);
  
  
  /*
***************
*** 130,135 **** extern MemoryContext AllocSetContextCreate(MemoryContext parent,
--- 132,143 ----
  					  Size minContextSize,
  					  Size initBlockSize,
  					  Size maxBlockSize);
+ extern MemoryContext AllocSetContextCreateTracked(MemoryContext parent,
+ 					  const char *name,
+ 					  Size minContextSize,
+ 					  Size initBlockSize,
+ 					  Size maxBlockSize,
+ 					  bool track_mem);
  
  /*
   * Recommended default alloc parameters, suitable for "ordinary" contexts
-- 
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

Reply via email to