Author: Armin Rigo <[email protected]>
Branch:
Changeset: r46116:e7121092d73f
Date: 2011-07-30 22:41 +0200
http://bitbucket.org/pypy/pypy/changeset/e7121092d73f/
Log: Fix the test. This gives a first approximation of the .NET
AddMemoryPressure(). This version is simpler than a counter that
needs to be carefully incremented and decremented by the exact same
amount. The idea is to use track_allocation=False to know when a
raw malloc is going to be attached to a GC object (good enough for
now). All such raw mallocs make the next major collection occur
earlier. So the major collection is triggered when
sum( GC object surviving minor collections + rawmallocs
with track_allocation=False ) > trigger
The raw mallocs are attached to GC objects with a __del__, which are
never allocated young, so they will survive at least until the major
collection. But if they survive for longer, they are ignored for
future major collections. This is again an approximation, but in
the "safe" way. Indeed, *not* ignoring them would simply mean
having a higher trigger, computed as (1.82*previous size).
diff --git a/pypy/rpython/memory/gc/minimark.py
b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -390,6 +390,11 @@
# initialize the threshold
self.min_heap_size = max(self.min_heap_size, self.nursery_size *
self.major_collection_threshold)
+ # the following two values are usually equal, but during raw mallocs
+ # of arrays, next_major_collection_threshold is decremented to make
+ # the next major collection arrive earlier.
+ # See translator/c/test/test_newgc, test_nongc_attached_to_gc
+ self.next_major_collection_initial = self.min_heap_size
self.next_major_collection_threshold = self.min_heap_size
self.set_major_threshold_from(0.0)
debug_stop("gc-set-nursery-size")
@@ -397,7 +402,7 @@
def set_major_threshold_from(self, threshold, reserving_size=0):
# Set the next_major_collection_threshold.
- threshold_max = (self.next_major_collection_threshold *
+ threshold_max = (self.next_major_collection_initial *
self.growth_rate_max)
if threshold > threshold_max:
threshold = threshold_max
@@ -412,6 +417,7 @@
else:
bounded = False
#
+ self.next_major_collection_initial = threshold
self.next_major_collection_threshold = threshold
return bounded
@@ -718,9 +724,18 @@
def set_max_heap_size(self, size):
self.max_heap_size = float(size)
if self.max_heap_size > 0.0:
+ if self.max_heap_size < self.next_major_collection_initial:
+ self.next_major_collection_initial = self.max_heap_size
if self.max_heap_size < self.next_major_collection_threshold:
self.next_major_collection_threshold = self.max_heap_size
+ def raw_malloc_varsize_hint(self, sizehint):
+ self.next_major_collection_threshold -= sizehint
+ if self.next_major_collection_threshold < 0:
+ # cannot trigger a full collection now, but we can ensure
+ # that one will occur very soon
+ self.nursery_free = self.nursery_top
+
def can_malloc_nonmovable(self):
return True
@@ -1600,7 +1615,7 @@
# Max heap size: gives an upper bound on the threshold. If we
# already have at least this much allocated, raise MemoryError.
if bounded and (float(self.get_total_memory_used()) + reserving_size >=
- self.next_major_collection_threshold):
+ self.next_major_collection_initial):
#
# First raise MemoryError, giving the program a chance to
# quit cleanly. It might still allocate in the nursery,
diff --git a/pypy/rpython/memory/gctransform/framework.py
b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -386,6 +386,18 @@
else:
self.malloc_varsize_nonmovable_ptr = None
+ if getattr(GCClass, 'raw_malloc_varsize_hint', False):
+ def raw_malloc_varsize_hint(length, itemsize):
+ totalmem = length * itemsize
+ if totalmem > 0:
+ gcdata.gc.raw_malloc_varsize_hint(totalmem)
+ #else: probably an overflow -- the following rawmalloc
+ # will fail then
+ self.raw_malloc_varsize_hint_ptr = getfn(
+ raw_malloc_varsize_hint,
+ [annmodel.SomeInteger(), annmodel.SomeInteger()],
+ annmodel.s_None, minimal_transform = False)
+
self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
[s_gc, s_gcref],
annmodel.SomeInteger(),
diff --git a/pypy/rpython/memory/gctransform/transform.py
b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -590,6 +590,16 @@
def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length,
c_const_size, c_item_size,
c_offset_to_length):
+ track_allocation = flags.get('track_allocation', True)
+ if not track_allocation:
+ # idea: raw mallocs with track_allocation=False correspond
+ # generally to raw mallocs of stuff that we store in GC objects.
+ # So we tell the GC about such raw mallocs, so that it can
+ # adjust its total size estimate.
+ if hasattr(self, 'raw_malloc_varsize_hint_ptr'):
+ hop.genop("direct_call",
+ [self.raw_malloc_varsize_hint_ptr,
+ v_length, c_item_size])
if c_offset_to_length is None:
if flags.get('zero'):
fnptr = self.raw_malloc_varsize_no_length_zero_ptr
@@ -605,7 +615,7 @@
[self.raw_malloc_varsize_ptr, v_length,
c_const_size, c_item_size, c_offset_to_length],
resulttype=llmemory.Address)
- if flags.get('track_allocation', True):
+ if track_allocation:
hop.genop("track_alloc_start", [v_raw])
return v_raw
diff --git a/pypy/translator/c/test/test_newgc.py
b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1403,15 +1403,20 @@
# allocate a total of ~77GB, but if the automatic gc'ing works,
# it should never need more than a few MBs at once
am1 = am2 = am3 = None
- for i in range(100000):
+ res = 0
+ for i in range(1, 100001):
+ if am3 is not None:
+ res += rffi.cast(lltype.Signed, am3.buf[0])
am3 = am2
am2 = am1
am1 = A(i * 4)
- return 42
+ am1.buf[0] = rffi.cast(rffi.INT, i-50000)
+ return res
return f
def test_nongc_attached_to_gc(self):
- self.run("nongc_attached_to_gc")
+ res = self.run("nongc_attached_to_gc")
+ assert res == -99997
# ____________________________________________________________________
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
