Author: Armin Rigo <ar...@tunes.org>
Branch: memory-accounting
Changeset: r93275:1cdb7e0ad36b
Date: 2017-12-05 19:11 +0100
http://bitbucket.org/pypy/pypy/changeset/1cdb7e0ad36b/
Log: add peak memory usage reporting
diff --git a/pypy/module/gc/app_referents.py b/pypy/module/gc/app_referents.py
--- a/pypy/module/gc/app_referents.py
+++ b/pypy/module/gc/app_referents.py
@@ -52,8 +52,10 @@
class GcStats(object):
def __init__(self, s):
self._s = s
- for item in ('total_gc_memory', 'jit_backend_used',
'total_memory_pressure',
- 'total_allocated_memory', 'jit_backend_allocated'):
+ for item in ('total_gc_memory', 'jit_backend_used',
+ 'total_memory_pressure',
+ 'total_allocated_memory', 'jit_backend_allocated',
+ 'peak_memory', 'peak_allocated_memory'):
setattr(self, item, self._format(getattr(self._s, item)))
self.memory_used_sum = self._format(self._s.total_gc_memory +
self._s.total_memory_pressure +
self._s.jit_backend_used)
@@ -68,21 +70,26 @@
def repr(self):
return """Total memory consumed:
-GC used: %s
+GC used: %s (peak: %s)
raw assembler used: %s
memory pressure: %s
-----------------------------
Total: %s
Total memory allocated:
-GC allocated: %s
+GC allocated: %s (peak: %s)
raw assembler allocated: %s
memory pressure: %s
-----------------------------
Total: %s
-""" % (self.total_gc_memory, self.jit_backend_used, self.total_memory_pressure,
+""" % (self.total_gc_memory, self.peak_memory,
+ self.jit_backend_used,
+ self.total_memory_pressure,
self.memory_used_sum,
- self.total_allocated_memory, self.jit_backend_allocated,
self.total_memory_pressure,
+
+ self.total_allocated_memory, self.peak_allocated_memory,
+ self.jit_backend_allocated,
+ self.total_memory_pressure,
self.memory_allocated_sum)
def get_stats():
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -176,6 +176,8 @@
self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
self.total_gc_memory = rgc.get_stats(rgc.TOTAL_MEMORY)
self.total_allocated_memory = rgc.get_stats(rgc.TOTAL_ALLOCATED_MEMORY)
+ self.peak_memory = rgc.get_stats(rgc.PEAK_MEMORY)
+ self.peak_allocated_memory = rgc.get_stats(rgc.PEAK_ALLOCATED_MEMORY)
self.jit_backend_allocated = jit_hooks.stats_asmmemmgr_allocated(None)
self.jit_backend_used = jit_hooks.stats_asmmemmgr_used(None)
@@ -184,6 +186,10 @@
cls=W_GcStats, wrapfn="newint"),
total_gc_memory=interp_attrproperty("total_gc_memory",
cls=W_GcStats, wrapfn="newint"),
+ peak_allocated_memory=interp_attrproperty("peak_allocated_memory",
+ cls=W_GcStats, wrapfn="newint"),
+ peak_memory=interp_attrproperty("peak_memory",
+ cls=W_GcStats, wrapfn="newint"),
total_allocated_memory=interp_attrproperty("total_allocated_memory",
cls=W_GcStats, wrapfn="newint"),
jit_backend_allocated=interp_attrproperty("jit_backend_allocated",
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -372,6 +372,7 @@
self.old_rawmalloced_objects = self.AddressStack()
self.raw_malloc_might_sweep = self.AddressStack()
self.rawmalloced_total_size = r_uint(0)
+ self.rawmalloced_peak_size = r_uint(0)
self.gc_state = STATE_SCANNING
#
@@ -997,6 +998,8 @@
# Record the newly allocated object and its full malloced size.
# The object is young or old depending on the argument.
self.rawmalloced_total_size += r_uint(allocsize)
+ self.rawmalloced_peak_size = max(self.rawmalloced_total_size,
+ self.rawmalloced_peak_size)
if alloc_young:
if not self.young_rawmalloced_objects:
self.young_rawmalloced_objects = self.AddressDict()
@@ -1189,6 +1192,19 @@
"""
return self.ac.total_memory_alloced + self.rawmalloced_total_size
+ def get_peak_memory_alloced(self):
+ """ Return the peak memory ever allocated. The peaks
+ can be at different times, but we just don't worry for now
+ """
+ return self.ac.peak_memory_alloced + self.rawmalloced_peak_size
+
+ def get_peak_memory_used(self):
+ """ Return the peak memory GC felt ever responsible for
+ """
+ mem_allocated = max(self.ac.peak_memory_used,
+ self.ac.total_memory_used)
+ return mem_allocated + self.rawmalloced_peak_size
+
def threshold_reached(self, extra=0):
return (self.next_major_collection_threshold -
float(self.get_total_memory_used())) < float(extra)
@@ -2161,6 +2177,8 @@
#
size_gc_header = self.gcheaderbuilder.size_gc_header
self.rawmalloced_total_size += r_uint(raw_malloc_usage(totalsize))
+ self.rawmalloced_peak_size = max(self.rawmalloced_total_size,
+ self.rawmalloced_peak_size)
self.old_rawmalloced_objects.append(arena + size_gc_header)
return arena
@@ -2929,6 +2947,10 @@
if stats_no == rgc.TOTAL_MEMORY:
return intmask(self.get_total_memory_used() + self.nursery_size)
+ elif stats_no == rgc.PEAK_MEMORY:
+ return intmask(self.get_peak_memory_used() + self.nursery_size)
+ elif stats_no == rgc.PEAK_ALLOCATED_MEMORY:
+ return intmask(self.get_peak_memory_alloced() + self.nursery_size)
elif stats_no == rgc.TOTAL_ALLOCATED_MEMORY:
return intmask(self.get_total_memory_alloced() + self.nursery_size)
elif stats_no == rgc.TOTAL_MEMORY_PRESSURE:
diff --git a/rpython/memory/gc/minimarkpage.py
b/rpython/memory/gc/minimarkpage.py
--- a/rpython/memory/gc/minimarkpage.py
+++ b/rpython/memory/gc/minimarkpage.py
@@ -140,7 +140,9 @@
# the total memory used, counting every block in use, without
# the additional bookkeeping stuff.
self.total_memory_used = r_uint(0)
+ self.peak_memory_used = r_uint(0)
self.total_memory_alloced = r_uint(0)
+ self.peak_memory_alloced = r_uint(0)
def _new_page_ptr_list(self, length):
@@ -295,6 +297,9 @@
# be a page-aligned address
arena_base = llarena.arena_malloc(self.arena_size, False)
self.total_memory_alloced += self.arena_size
+ self.peak_memory_alloced = max(self.total_memory_alloced,
+ self.peak_memory_alloced)
+
if not arena_base:
out_of_memory("out of memory: couldn't allocate the next arena")
arena_end = arena_base + self.arena_size
@@ -321,6 +326,8 @@
"""Prepare calls to mass_free_incremental(): moves the chained lists
into 'self.old_xxx'.
"""
+ self.peak_memory_used = max(self.peak_memory_used,
+ self.total_memory_used)
self.total_memory_used = r_uint(0)
#
size_class = self.small_request_threshold >> WORD_POWER_2
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -650,7 +650,8 @@
else:
return id(gcref._x)
-TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE = range(3)
+(TOTAL_MEMORY, TOTAL_ALLOCATED_MEMORY, TOTAL_MEMORY_PRESSURE,
+ PEAK_MEMORY, PEAK_ALLOCATED_MEMORY) = range(5)
@not_rpython
def get_stats(stat_no):
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
