On Thu, 4 Apr 2024 at 22:43, Tom Lane <t...@sss.pgh.pa.us> wrote: > > Matthias van de Meent <boekewurm+postg...@gmail.com> writes: > > On Mon, 25 Mar 2024 at 22:44, Tom Lane <t...@sss.pgh.pa.us> wrote: > >> Basically, I'm not happy with consuming the last reasonably-available > >> pattern for a memory context type that has little claim to being the > >> Last Context Type We Will Ever Want. Rather than making a further > >> dent in our ability to detect corrupted chunks, we should do something > >> towards restoring the expansibility that existed in the original > >> design. Then we can add bump contexts and whatever else we want. > > > So, would something like the attached make enough IDs available so > > that we can add the bump context anyway? > > > It extends memory context IDs to 5 bits (32 values), of which > > - 8 have glibc's malloc pattern of 001/010; > > - 1 is unused memory's 00000 > > - 1 is wipe_mem's 11111 > > - 4 are used by existing contexts (Aset/Generation/Slab/AlignedRedirect) > > - 18 are newly available. > > This seems like it would solve the problem for a good long time > to come; and if we ever need more IDs, we could steal one more bit > by requiring the offset to the block header to be a multiple of 8. > (Really, we could just about do that today at little or no cost ... > machines with MAXALIGN less than 8 are very thin on the ground.)
Hmm, it seems like a decent idea, but I didn't want to deal with the repercussions of that this late in the cycle when these 2 bits were still relatively easy to get hold of. > The only objection I can think of is that perhaps this would slow > things down a tad by requiring more complicated shifting/masking. > I wonder if we could redo the performance checks that were done > on the way to accepting the current design. I didn't do very extensive testing, but the light performance tests that I did with the palloc performance benchmark patch & script shared above indicate didn't measure an observable negative effect. An adapted version of the test that uses repalloc() to check performance differences in MCXT_METHOD() doesn't show a significant performance difference from master either. That test case is attached as repalloc-performance-test-function.patch.txt. The full set of patches would then accumulate to the attached v5 of the patchset. 0001 is an update of my patch from yesterday, in which I update MemoryContextMethodID infrastructure for more IDs, and use a new naming scheme for unused/reserved IDs. 0002 and 0003 are David's patches, with minor changes to work with 0001 (rebasing, and I moved the location around to keep function declaration in order with memctx ids) Kind regards, Matthias van de Meent
From b65320736cf63684b4710b3d63e243a0862d37c6 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <boekewurm+postg...@gmail.com>
Date: Fri, 5 Apr 2024 15:13:52 +0200
Subject: [PATCH] repalloc performance test function
---
src/backend/utils/adt/mcxtfuncs.c | 231 ++++++++++++++++++++++++++++++
src/include/catalog/pg_proc.dat | 12 ++
2 files changed, 243 insertions(+)
diff --git a/src/backend/utils/adt/mcxtfuncs.c
b/src/backend/utils/adt/mcxtfuncs.c
index 4d4a70915b..88de4bbcb5 100644
--- a/src/backend/utils/adt/mcxtfuncs.c
+++ b/src/backend/utils/adt/mcxtfuncs.c
@@ -15,8 +15,11 @@
#include "postgres.h"
+#include <time.h>
+
#include "funcapi.h"
#include "mb/pg_wchar.h"
+#include "miscadmin.h"
#include "storage/proc.h"
#include "storage/procarray.h"
#include "utils/builtins.h"
@@ -185,3 +188,231 @@ pg_log_backend_memory_contexts(PG_FUNCTION_ARGS)
PG_RETURN_BOOL(true);
}
+
+typedef struct AllocateTestNext
+{
+ struct AllocateTestNext *next; /* ptr to the next allocation */
+} AllocateTestNext;
+
+/* #define ALLOCATE_TEST_DEBUG */
+/*
+ * pg_allocate_memory_test
+ * Used to test the performance of a memory context types
+ */
+Datum
+pg_allocate_memory_test(PG_FUNCTION_ARGS)
+{
+ int32 chunk_size = PG_GETARG_INT32(0);
+ int64 keep_memory = PG_GETARG_INT64(1);
+ int64 total_alloc = PG_GETARG_INT64(2);
+ text *context_type_text = PG_GETARG_TEXT_PP(3);
+ char *context_type;
+ int64 curr_memory_use = 0;
+ int64 remaining_alloc_bytes = total_alloc;
+ MemoryContext context;
+ MemoryContext oldContext;
+ AllocateTestNext *next_free_ptr = NULL;
+ AllocateTestNext *last_alloc = NULL;
+ clock_t start, end;
+
+ if (chunk_size < sizeof(AllocateTestNext))
+ elog(ERROR, "chunk_size (%d) must be at least %ld bytes",
chunk_size,
+ sizeof(AllocateTestNext));
+ if (keep_memory > total_alloc)
+ elog(ERROR, "keep_memory (" INT64_FORMAT ") must be less than
total_alloc (" INT64_FORMAT ")",
+ keep_memory, total_alloc);
+
+ context_type = text_to_cstring(context_type_text);
+
+ start = clock();
+
+ if (strcmp(context_type, "generation") == 0)
+ context = GenerationContextCreate(CurrentMemoryContext,
+
"pg_allocate_memory_test",
+
ALLOCSET_DEFAULT_SIZES);
+ else if (strcmp(context_type, "aset") == 0)
+ context = AllocSetContextCreate(CurrentMemoryContext,
+
"pg_allocate_memory_test",
+
ALLOCSET_DEFAULT_SIZES);
+ else if (strcmp(context_type, "slab") == 0)
+ context = SlabContextCreate(CurrentMemoryContext,
+
"pg_allocate_memory_test",
+
ALLOCSET_DEFAULT_MAXSIZE,
+
chunk_size);
+ else
+ elog(ERROR, "context_type must be \"generation\", \"aset\" or
\"slab\"");
+
+ oldContext = MemoryContextSwitchTo(context);
+
+ while (remaining_alloc_bytes > 0)
+ {
+ AllocateTestNext *curr_alloc;
+
+ CHECK_FOR_INTERRUPTS();
+
+ /* Allocate the memory and update the counters */
+ curr_alloc = (AllocateTestNext *) palloc(chunk_size);
+ remaining_alloc_bytes -= chunk_size;
+ curr_memory_use += chunk_size;
+
+#ifdef ALLOCATE_TEST_DEBUG
+ elog(NOTICE, "alloc %p (curr_memory_use " INT64_FORMAT " bytes,
remaining_alloc_bytes " INT64_FORMAT ")", curr_alloc, curr_memory_use,
remaining_alloc_bytes);
+#endif
+
+ /*
+ * Point the last allocate to this one so that we can free
allocations
+ * starting with the oldest first.
+ */
+ curr_alloc->next = NULL;
+ if (last_alloc != NULL)
+ last_alloc->next = curr_alloc;
+
+ if (next_free_ptr == NULL)
+ {
+ /*
+ * Remember the first chunk to free. We will follow the
->next
+ * pointers to find the next chunk to free when freeing
memory
+ */
+ next_free_ptr = curr_alloc;
+ }
+
+ /*
+ * If the currently allocated memory has reached or exceeded
the amount
+ * of memory we want to keep allocated at once then we'd better
free
+ * some. Since all allocations are the same size we only need
to free
+ * one allocation per loop.
+ */
+ if (curr_memory_use >= keep_memory)
+ {
+ AllocateTestNext *next = next_free_ptr->next;
+
+ /* free the memory and update the current memory usage
*/
+ pfree(next_free_ptr);
+ curr_memory_use -= chunk_size;
+
+#ifdef ALLOCATE_TEST_DEBUG
+ elog(NOTICE, "free %p (curr_memory_use " INT64_FORMAT "
bytes, remaining_alloc_bytes " INT64_FORMAT ")", next_free_ptr,
curr_memory_use, remaining_alloc_bytes);
+#endif
+ /* get the next chunk to free */
+ next_free_ptr = next;
+ }
+
+ if (curr_memory_use > 0)
+ last_alloc = curr_alloc;
+ else
+ last_alloc = NULL;
+ }
+
+ /* cleanup loop -- pfree remaining memory */
+ while (next_free_ptr != NULL)
+ {
+ AllocateTestNext *next = next_free_ptr->next;
+
+ /* free the memory and update the current memory usage */
+ pfree(next_free_ptr);
+ curr_memory_use -= chunk_size;
+
+#ifdef ALLOCATE_TEST_DEBUG
+ elog(NOTICE, "free %p (curr_memory_use " INT64_FORMAT " bytes,
remaining_alloc_bytes " INT64_FORMAT ")", next_free_ptr, curr_memory_use,
remaining_alloc_bytes);
+#endif
+
+ next_free_ptr = next;
+ }
+
+ MemoryContextSwitchTo(oldContext);
+
+ end = clock();
+
+ PG_RETURN_FLOAT8((double) (end - start) / CLOCKS_PER_SEC);
+}
+
+Datum
+pg_allocate_memory_test_reset(PG_FUNCTION_ARGS)
+{
+ int32 chunk_size = PG_GETARG_INT32(0);
+ int64 keep_memory = PG_GETARG_INT64(1);
+ int64 total_alloc = PG_GETARG_INT64(2);
+ int64 n_ptrs;
+ text *context_type_text = PG_GETARG_TEXT_PP(3);
+ char *context_type;
+ int64 curr_memory_use = 0;
+ int64 remaining_alloc_bytes = total_alloc;
+ MemoryContext context;
+ MemoryContext oldContext;
+ clock_t start, end;
+ int64 **ptrs;
+ uint32 lcg_value = 0;
+ uint32 lcg_c = 1013904223;
+ uint32 lcg_a = 1664525;
+
+ if (chunk_size < 1)
+ elog(ERROR, "size of chunk must be above 0");
+ if (keep_memory > total_alloc)
+ elog(ERROR, "keep_memory (" INT64_FORMAT ") must be less than
total_alloc (" INT64_FORMAT ")",
+ keep_memory, total_alloc);
+
+ context_type = text_to_cstring(context_type_text);
+ n_ptrs = 1 << pg_leftmost_one_pos64(keep_memory / chunk_size);
+
+ ptrs = (int64 **) palloc0(n_ptrs * sizeof(int64 *));
+
+ start = clock();
+
+ if (strcmp(context_type, "generation") == 0)
+ context = GenerationContextCreate(CurrentMemoryContext,
+
"pg_allocate_memory_test",
+
ALLOCSET_DEFAULT_SIZES);
+ else if (strcmp(context_type, "aset") == 0)
+ context = AllocSetContextCreate(CurrentMemoryContext,
+
"pg_allocate_memory_test",
+
ALLOCSET_DEFAULT_SIZES);
+ else if (strcmp(context_type, "slab") == 0)
+ context = SlabContextCreate(CurrentMemoryContext,
+
"pg_allocate_memory_test",
+
ALLOCSET_DEFAULT_MAXSIZE,
+
chunk_size);
+// else if (strcmp(context_type, "bump") == 0)
+// context = BumpContextCreate(CurrentMemoryContext,
+//
"pg_allocate_memory_test",
+//
ALLOCSET_DEFAULT_SIZES);
+ else
+ elog(ERROR, "context_type must be \"generation\", \"aset\" or
\"slab\"");
+
+ oldContext = MemoryContextSwitchTo(context);
+
+ for (int i = 0; i < n_ptrs && remaining_alloc_bytes > 0;)
+ {
+ CHECK_FOR_INTERRUPTS();
+
+ /* Allocate the memory and update the counters */
+ ptrs[i++] = (int64 *) palloc(chunk_size);
+ remaining_alloc_bytes -= chunk_size;
+ curr_memory_use += chunk_size;
+ }
+
+ while (remaining_alloc_bytes > 0)
+ {
+ uint32 index;
+ CHECK_FOR_INTERRUPTS();
+ index = lcg_value % (uint32) n_ptrs;
+ lcg_value = lcg_value * lcg_a + lcg_c;
+ Assert(PointerIsValid(ptr));
+
+ /* reallocate, and update the counters */
+ ptrs[index] = (int64 *) repalloc(ptrs[index], chunk_size);
+
+ remaining_alloc_bytes -= chunk_size;
+ curr_memory_use += chunk_size;
+
+#ifdef ALLOCATE_TEST_DEBUG
+ elog(NOTICE, "alloc %p (curr_memory_use " INT64_FORMAT " bytes,
remaining_alloc_bytes " INT64_FORMAT ")", curr_alloc, curr_memory_use,
remaining_alloc_bytes);
+#endif
+ }
+
+ MemoryContextSwitchTo(oldContext);
+ MemoryContextDelete(context);
+
+ end = clock();
+
+ PG_RETURN_FLOAT8((double) (end - start) / CLOCKS_PER_SEC);
+}
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 153d816a05..15d191db8b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -8288,6 +8288,18 @@
prorettype => 'bool', proargtypes => 'int4',
prosrc => 'pg_log_backend_memory_contexts' },
+# just for testing memory context allocation speed
+{ oid => '9319', descr => 'for testing performance of allocation and freeing',
+ proname => 'pg_allocate_memory_test', provolatile => 'v',
+ prorettype => 'float8', proargtypes => 'int4 int8 int8 text',
+ prosrc => 'pg_allocate_memory_test' },
+
+# just for testing memory context allocation speed
+{ oid => '9320', descr => 'for testing performance of allocation and
resetting',
+ proname => 'pg_allocate_memory_test_reset', provolatile => 'v',
+ prorettype => 'float8', proargtypes => 'int4 int8 int8 text',
+ prosrc => 'pg_allocate_memory_test_reset' },
+
# non-persistent series generator
{ oid => '1066', descr => 'non-persistent series generator',
proname => 'generate_series', prorows => '1000',
--
2.40.1
v5-0002-Introduce-a-bump-memory-allocator.patch
Description: Binary data
v5-0001-Add-bitspace-for-more-memory-context-types-in-Mem.patch
Description: Binary data
v5-0003-Use-bump-memory-context-for-tuplesorts.patch
Description: Binary data
