Attached is a small patch that introduces a simple function,
AllocSetEstimateChunkSpace(), and uses it to improve the estimate for
the size of a hash entry for hash aggregation.
Getting reasonable estimates for the hash entry size (including the
TupleHashEntryData, the group key tuple, the per-group state, and by-
ref transition values) is important for multiple reasons:
* It helps the planner know when hash aggregation is likely to spill,
and how to cost it.
* It helps hash aggregation regulate itself by setting a group limit
(separate from the memory limit) to account for growing by-ref
transition values.
* It helps choose a good initial size for the hash table. Too large of
a hash table will crowd out space that could be used for the group keys
or the per-group state.
Each group requires up to three palloc chunks: one for the group key
tuple, one for the per-group states, and one for a by-ref transition
value. Each chunk can have a lot of overhead: in addition to the chunk
header (16 bytes overhead), it also rounds the value up to a power of
two (~25% overhead). So, it's important to account for this chunk
overhead.
I considered making it a method of a memory context, but the planner
will call this function before the hash agg memory context is created.
It seems to make more sense for this to just be an AllocSet-specific
function for now.
Regards,
Jeff Davis
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 2d94d7dc25e..7fa71ad2d65 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -1637,16 +1637,37 @@ find_hash_columns(AggState *aggstate)
/*
* Estimate per-hash-table-entry overhead.
+ *
+ * NB: This assumes the memory context at execution time will be an
+ * AllocSetContext.
*/
Size
-hash_agg_entry_size(int numAggs, Size tupleWidth, Size transitionSpace)
+hash_agg_entry_size(int numTrans, Size tupleWidth, Size transitionSpace)
{
+ Size tupleChunkSize;
+ Size pergroupChunkSize;
+ Size transitionChunkSize;
+ Size tupleSize = (MAXALIGN(SizeofMinimalTupleHeader) +
+ tupleWidth);
+ Size pergroupSize = numTrans * sizeof(AggStatePerGroupData);
+
+ tupleChunkSize = AllocSetEstimateChunkSpace(tupleSize);
+
+ if (pergroupSize > 0)
+ pergroupChunkSize = AllocSetEstimateChunkSpace(pergroupSize);
+ else
+ pergroupChunkSize = 0;
+
+ if (transitionSpace > 0)
+ transitionChunkSize = AllocSetEstimateChunkSpace(transitionSpace);
+ else
+ transitionChunkSize = 0;
+
return
- MAXALIGN(SizeofMinimalTupleHeader) +
- MAXALIGN(tupleWidth) +
- MAXALIGN(sizeof(TupleHashEntryData) +
- numAggs * sizeof(AggStatePerGroupData)) +
- transitionSpace;
+ sizeof(TupleHashEntryData) +
+ tupleChunkSize +
+ pergroupChunkSize +
+ transitionChunkSize;
}
/*
diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c
index c0623f106d2..678ddd77912 100644
--- a/src/backend/utils/mmgr/aset.c
+++ b/src/backend/utils/mmgr/aset.c
@@ -560,6 +560,30 @@ AllocSetContextCreateInternal(MemoryContext parent,
return (MemoryContext) set;
}
+/*
+ * Estimate total memory consumed for a chunk of the requested size.
+ */
+Size
+AllocSetEstimateChunkSpace(Size size)
+{
+ Size chunk_size;
+
+ if (size > ALLOC_CHUNK_LIMIT)
+ {
+ chunk_size = MAXALIGN(size);
+
+ return chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
+ }
+ else
+ {
+ int fidx = AllocSetFreeIndex(size);
+
+ chunk_size = (1 << ALLOC_MINBITS) << fidx;
+
+ return chunk_size + ALLOC_CHUNKHDRSZ;
+ }
+}
+
/*
* AllocSetReset
* Frees all memory which is allocated in the given set.
diff --git a/src/include/executor/nodeAgg.h b/src/include/executor/nodeAgg.h
index a5b8a004d1e..c2b55728bfa 100644
--- a/src/include/executor/nodeAgg.h
+++ b/src/include/executor/nodeAgg.h
@@ -314,7 +314,7 @@ extern AggState *ExecInitAgg(Agg *node, EState *estate, int eflags);
extern void ExecEndAgg(AggState *node);
extern void ExecReScanAgg(AggState *node);
-extern Size hash_agg_entry_size(int numAggs, Size tupleWidth,
+extern Size hash_agg_entry_size(int numTrans, Size tupleWidth,
Size transitionSpace);
extern void hash_agg_set_limits(double hashentrysize, uint64 input_groups,
int used_bits, Size *mem_limit,
diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h
index 909bc2e9888..67e53e4b977 100644
--- a/src/include/utils/memutils.h
+++ b/src/include/utils/memutils.h
@@ -155,6 +155,7 @@ extern MemoryContext AllocSetContextCreateInternal(MemoryContext parent,
Size minContextSize,
Size initBlockSize,
Size maxBlockSize);
+extern Size AllocSetEstimateChunkSpace(Size size);
/*
* This wrapper macro exists to check for non-constant strings used as context
