Author: Armin Rigo <ar...@tunes.org>
Branch:
Changeset: r46142:5282016589ed
Date: 2011-08-01 15:18 +0200
http://bitbucket.org/pypy/pypy/changeset/5282016589ed/
Log: Add an explicit flag 'add_memory_pressure=True' to the raw-flavored
malloc(). When we set it, the minimark GC will count the allocated
memory as part of the next major collection's threshold.
diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py
--- a/pypy/annotation/builtin.py
+++ b/pypy/annotation/builtin.py
@@ -416,7 +416,8 @@
from pypy.annotation.model import SomePtr
from pypy.rpython.lltypesystem import lltype
-def malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None):
+def malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None,
+ s_add_memory_pressure=None):
assert (s_n is None or s_n.knowntype == int
or issubclass(s_n.knowntype, pypy.rlib.rarithmetic.base_int))
assert s_T.is_constant()
@@ -432,6 +433,8 @@
else:
assert s_flavor.is_constant()
assert s_track_allocation is None or s_track_allocation.is_constant()
+ assert (s_add_memory_pressure is None or
+ s_add_memory_pressure.is_constant())
# not sure how to call malloc() for the example 'p' in the
# presence of s_extraargs
r = SomePtr(lltype.Ptr(s_T.const))
diff --git a/pypy/module/_rawffi/interp_rawffi.py
b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -268,7 +268,7 @@
self.ll_buffer = rffi.cast(rffi.VOIDP, address)
else:
self.ll_buffer = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw',
- zero=True)
+ zero=True, add_memory_pressure=True)
if tracker.DO_TRACING:
ll_buf = rffi.cast(lltype.Signed, self.ll_buffer)
tracker.trace_allocation(ll_buf, self)
diff --git a/pypy/module/array/interp_array.py
b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -226,7 +226,8 @@
some += size >> 3
self.allocated = size + some
new_buffer = lltype.malloc(mytype.arraytype,
- self.allocated, flavor='raw')
+ self.allocated, flavor='raw',
+ add_memory_pressure=True)
for i in range(min(size, self.len)):
new_buffer[i] = self.buffer[i]
else:
diff --git a/pypy/module/micronumpy/interp_numarray.py
b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -477,7 +477,8 @@
BaseArray.__init__(self)
self.size = size
self.storage = lltype.malloc(TP, size, zero=True,
- flavor='raw', track_allocation=False)
+ flavor='raw', track_allocation=False,
+ add_memory_pressure=True)
# XXX find out why test_zjit explodes with trackign of allocations
def get_concrete(self):
@@ -506,7 +507,7 @@
self._sliceloop2(start, stop, step, arr, self)
def __del__(self):
- lltype.free(self.storage, flavor='raw')
+ lltype.free(self.storage, flavor='raw', track_allocation=False)
def new_numarray(space, w_size_or_iterable):
l = space.listview(w_size_or_iterable)
diff --git a/pypy/rpython/lltypesystem/lltype.py
b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1939,7 +1939,7 @@
def malloc(T, n=None, flavor='gc', immortal=False, zero=False,
- track_allocation=True):
+ track_allocation=True, add_memory_pressure=False):
assert flavor in ('gc', 'raw')
if zero or immortal:
initialization = 'example'
diff --git a/pypy/rpython/memory/gc/minimark.py
b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -729,7 +729,7 @@
if self.max_heap_size < self.next_major_collection_threshold:
self.next_major_collection_threshold = self.max_heap_size
- def raw_malloc_varsize_hint(self, sizehint):
+ def raw_malloc_memory_pressure(self, sizehint):
self.next_major_collection_threshold -= sizehint
if self.next_major_collection_threshold < 0:
# cannot trigger a full collection now, but we can ensure
diff --git a/pypy/rpython/memory/gctransform/framework.py
b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -386,15 +386,15 @@
else:
self.malloc_varsize_nonmovable_ptr = None
- if getattr(GCClass, 'raw_malloc_varsize_hint', False):
- def raw_malloc_varsize_hint(length, itemsize):
+ if getattr(GCClass, 'raw_malloc_memory_pressure', False):
+ def raw_malloc_memory_pressure(length, itemsize):
totalmem = length * itemsize
if totalmem > 0:
- gcdata.gc.raw_malloc_varsize_hint(totalmem)
+ gcdata.gc.raw_malloc_memory_pressure(totalmem)
#else: probably an overflow -- the following rawmalloc
# will fail then
- self.raw_malloc_varsize_hint_ptr = getfn(
- raw_malloc_varsize_hint,
+ self.raw_malloc_memory_pressure_ptr = getfn(
+ raw_malloc_memory_pressure,
[annmodel.SomeInteger(), annmodel.SomeInteger()],
annmodel.s_None, minimal_transform = False)
diff --git a/pypy/rpython/memory/gctransform/transform.py
b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -590,15 +590,10 @@
def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length,
c_const_size, c_item_size,
c_offset_to_length):
- track_allocation = flags.get('track_allocation', True)
- if not track_allocation:
- # idea: raw mallocs with track_allocation=False correspond
- # generally to raw mallocs of stuff that we store in GC objects.
- # So we tell the GC about such raw mallocs, so that it can
- # adjust its total size estimate.
- if hasattr(self, 'raw_malloc_varsize_hint_ptr'):
+ if flags.get('add_memory_pressure', False):
+ if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
hop.genop("direct_call",
- [self.raw_malloc_varsize_hint_ptr,
+ [self.raw_malloc_memory_pressure_ptr,
v_length, c_item_size])
if c_offset_to_length is None:
if flags.get('zero'):
@@ -615,7 +610,7 @@
[self.raw_malloc_varsize_ptr, v_length,
c_const_size, c_item_size, c_offset_to_length],
resulttype=llmemory.Address)
- if track_allocation:
+ if flags.get('track_allocation', True):
hop.genop("track_alloc_start", [v_raw])
return v_raw
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -345,14 +345,17 @@
BUILTIN_TYPER[object.__init__] = rtype_object__init__
# annotation of low-level types
-def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None):
+def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None,
+ i_add_memory_pressure=None):
assert hop.args_s[0].is_constant()
vlist = [hop.inputarg(lltype.Void, arg=0)]
opname = 'malloc'
- v_flavor, v_zero, v_track_allocation = parse_kwds(hop,
+ v_flavor, v_zero, v_track_allocation, v_add_memory_pressure = parse_kwds(
+ hop,
(i_flavor, lltype.Void),
(i_zero, None),
- (i_track_allocation, None))
+ (i_track_allocation, None),
+ (i_add_memory_pressure, None))
flags = {'flavor': 'gc'}
if v_flavor is not None:
@@ -361,8 +364,11 @@
flags['zero'] = v_zero.value
if i_track_allocation is not None:
flags['track_allocation'] = v_track_allocation.value
+ if i_add_memory_pressure is not None:
+ flags['add_memory_pressure'] = v_add_memory_pressure.value
vlist.append(hop.inputconst(lltype.Void, flags))
-
+
+ assert 1 <= hop.nb_args <= 2
if hop.nb_args == 2:
vlist.append(hop.inputarg(lltype.Signed, arg=1))
opname += '_varsize'
diff --git a/pypy/translator/c/test/test_newgc.py
b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1396,9 +1396,10 @@
class A:
def __init__(self, n):
self.buf = lltype.malloc(ARRAY, n, flavor='raw',
- track_allocation=False)
+ add_memory_pressure=True)
def __del__(self):
- lltype.free(self.buf, flavor='raw', track_allocation=False)
+ lltype.free(self.buf, flavor='raw')
+ A(6)
def f():
# allocate a total of ~77GB, but if the automatic gc'ing works,
# it should never need more than a few MBs at once
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit
