Author: Stefan Beyer <[email protected]>
Branch: cpyext-gc-cycle
Changeset: r96723:0b73323d80cc
Date: 2019-05-30 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/0b73323d80cc/
Log: Fixed some issues with tracking/deallocating cpyext gc objects Added
rudimentary support for lists (gc issues, clear missing)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1169,6 +1169,11 @@
[rffi.VOIDP], lltype.Void,
compilation_info=eci,
_nowrapper=True)
+ state.C.PyObject_GC_Del = rffi.llexternal(
+ mangle_name(prefix, 'PyObject_GC_Del'),
+ [rffi.VOIDP], lltype.Void,
+ compilation_info=eci,
+ _nowrapper=True)
state.C.PyType_GenericAlloc = rffi.llexternal(
mangle_name(prefix, 'PyType_GenericAlloc'),
[PyTypeObjectPtr, Py_ssize_t], PyObject,
@@ -1186,6 +1191,9 @@
'_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
compilation_info=eci, _nowrapper=True)
from pypy.module.cpyext.typeobjectdefs import visitproc
+ state.C.PyObject_GC_Track = rffi.llexternal(
+ 'PyObject_GC_Track', [rffi.VOIDP], lltype.Void,
+ compilation_info=eci, _nowrapper=True)
state.C._PyPy_tuple_traverse = rffi.llexternal(
'_PyPy_tuple_traverse', [PyObject, visitproc, rffi.VOIDP],
rffi.INT, compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -8,7 +8,6 @@
from pypy.objspace.std.listobject import W_ListObject
from pypy.interpreter.error import oefmt
-
PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
@cpython_api([Py_ssize_t], PyObject)
@@ -30,6 +29,33 @@
w_list.convert_to_cpy_strategy(space)
return CPyListStrategy.unerase(w_list.lstorage)
+def list_traverse(space, w_list, visit, args):
+ from rpython.rlib.rawrefcount import PYOBJ_HDR_PTR
+ from pypy.module.cpyext.sequence import CPyListStrategy
+ from rpython.rtyper.lltypesystem import lltype, llmemory
+ from rpython.rlib.debug import debug_print
+ assert isinstance(w_list, W_ListObject)
+ obj = llmemory.cast_ptr_to_adr(w_list.lstorage)
+ objint = llmemory.cast_adr_to_int(obj, "forced")
+ if objint != 0:
+ cpy_strategy = space.fromcache(CPyListStrategy)
+ if not cpy_strategy.locked and cpy_strategy == w_list.strategy:
+ debug_print('rrc do traverse ', w_list)
+ index = 0
+ storage = CPyListStrategy.unerase(w_list.lstorage)
+ while index < storage._length:
+ pyobj = storage._elems[index]
+ pyobj_hdr = rffi.cast(PYOBJ_HDR_PTR, pyobj)
+ if pyobj_hdr:
+ vret = rffi.cast(lltype.Signed, visit(pyobj_hdr, args))
+ if vret:
+ return vret
+ index += 1
+ else:
+ debug_print('rrc no traverse', w_list, 'locked',
+ cpy_strategy.locked, 'strategy', w_list.strategy)
+ return 0
+
@cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void,
error=CANNOT_FAIL)
def PyList_SET_ITEM(space, w_list, index, py_item):
"""Form of PyList_SetItem() without error checking. This is normally
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -77,9 +77,9 @@
state = space.fromcache(State)
return state.C._PyPy_subtype_dealloc
- def get_free(self, space):
+ def get_free(self, space, gc):
state = space.fromcache(State)
- return state.C.PyObject_Free
+ return state.C.PyObject_GC_Del if gc else state.C.PyObject_Free
def has_traverse(self, space):
return False
@@ -156,7 +156,7 @@
return tp_dealloc
if tp_free:
- def get_free(self, space):
+ def get_free(self, space, gc):
return tp_free
if tp_attach:
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -281,6 +281,7 @@
erase, unerase = rerased.new_erasing_pair("cpylist")
erase = staticmethod(erase)
unerase = staticmethod(unerase)
+ locked = False
def _check_index(self, index, length):
if index < 0:
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -132,10 +132,17 @@
PyObject *
PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)
{
+ PyObject *obj;
+
if (PyType_IS_GC(type))
- return (PyObject*)_PyObject_GC_NewVar(type, nitems);
+ obj = (PyObject*)_PyObject_GC_NewVar(type, nitems);
else
- return (PyObject*)_PyObject_NewVar(type, nitems);
+ obj = (PyObject*)_PyObject_NewVar(type, nitems);
+
+ if (PyType_IS_GC(type))
+ _PyObject_GC_TRACK(obj);
+
+ return obj;
}
PyObject *
@@ -191,7 +198,7 @@
return (PyVarObject*)PyErr_NoMemory();
if (type->tp_itemsize == 0)
- return (PyVarObject*)PyObject_INIT(py_obj, type);
+ return PyObject_INIT(py_obj, type);
else
return PyObject_INIT_VAR((PyVarObject*)py_obj, type, nitems);
}
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -133,9 +133,10 @@
print 'dealloc_trigger DONE'
return "RETRY"
def tp_traverse(pyobj_ptr, callback, args):
- from pypy.module.cpyext.api import PyObject
+ from pypy.module.cpyext.api import PyObject, generic_cpy_call
from pypy.module.cpyext.typeobjectdefs import visitproc
- from pypy.module.cpyext.pyobject import cts
+ from pypy.module.cpyext.pyobject import cts, from_ref
+
# convert to pointers with correct types (PyObject)
callback_addr = llmemory.cast_ptr_to_adr(callback)
callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -148,9 +149,9 @@
print 'traverse PyObject', pyobj, 'of type', name
# now call tp_traverse (if possible)
- if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
- pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
- args)
+ if pto and pto.c_tp_traverse:
+ generic_cpy_call(space, pto.c_tp_traverse, pyobj,
+ callback_ptr, args)
rawrefcount.init(dealloc_trigger, tp_traverse)
else:
if space.config.translation.gc == "boehm":
@@ -176,9 +177,12 @@
(lambda w_obj: _clear_weakref_callbacks(w_obj))
def _tp_traverse(pyobj_ptr, callback, args):
- from pypy.module.cpyext.api import PyObject
+ from pypy.module.cpyext.api import PyObject, \
+ generic_cpy_call
from pypy.module.cpyext.typeobjectdefs import visitproc
- from pypy.module.cpyext.pyobject import cts
+ from pypy.module.cpyext.pyobject import cts, from_ref
+ from pypy.module.cpyext.listobject import list_traverse
+
# convert to pointers with correct types (PyObject)
callback_addr = llmemory.cast_ptr_to_adr(callback)
callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -208,9 +212,19 @@
debug_print(" " * i * 3, "unknown base")
pto2 = base
i += 1
+
+ if pyobj.c_ob_pypy_link != 0: # special traverse
+ w_obj = from_ref(space, pyobj)
+ w_obj_type = space.type(w_obj)
+ if space.is_w(w_obj_type, space.w_list): # list
+ debug_print('rrc list traverse ', pyobj)
+ list_traverse(space, w_obj, callback, args)
+ return
+
if pto and pto.c_tp_traverse:
debug_print("rrc do traverse", pyobj)
- pto.c_tp_traverse(pyobj, callback_ptr, args)
+ generic_cpy_call(space, pto.c_tp_traverse, pyobj,
+ callback_ptr, args)
self.tp_traverse = (lambda o, v, a:_tp_traverse(o, v, a))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -434,7 +434,7 @@
pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
elif space.issubtype_w(w_obj, space.w_tuple):
pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
- pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
+ pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC # TODO: remove? already done
elsewhere
elif space.issubtype_w(w_obj, space.w_list):
pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
elif space.issubtype_w(w_obj, space.w_dict):
@@ -610,6 +610,8 @@
elif space.is_w(w_type, space.w_tuple):
pto.c_tp_itemsize = rffi.sizeof(PyObject)
pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
+ elif space.is_w(w_type, space.w_list):
+ pto.c_tp_flags |= Py_TPFLAGS_HAVE_GC
state = space.fromcache(State)
pto.c_tp_alloc = state.C.PyType_GenericAlloc
@@ -646,13 +648,6 @@
# strange, but happens (ABCMeta)
pto.c_tp_dealloc = state.C._PyPy_subtype_dealloc
- # free
- if space.gettypeobject(w_type.layout.typedef) is w_type:
- # only for the exact type, like 'space.w_tuple' or 'space.w_list'
- pto.c_tp_free = typedescr.get_free(space)
- else:
- pto.c_tp_free = pto.c_tp_base.c_tp_free
-
from rpython.rtyper.lltypesystem import llmemory
from pypy.module.cpyext.typeobjectdefs import traverseproc
if typedescr.has_traverse(space):
@@ -685,6 +680,18 @@
if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
pto.c_tp_new = pto.c_tp_base.c_tp_new
decref(space, base_object_pyo)
+
+ # free
+ gc = pto.c_tp_flags & Py_TPFLAGS_HAVE_GC != 0
+ if space.gettypeobject(w_type.layout.typedef) is w_type:
+ # only for the exact type, like 'space.w_tuple' or 'space.w_list'
+ pto.c_tp_free = typedescr.get_free(space, gc)
+ else:
+ if gc:
+ pto.c_tp_free = state.C.PyObject_GC_Del
+ else:
+ pto.c_tp_free = state.C.PyObject_Free
+
pto.c_tp_flags |= Py_TPFLAGS_READY
return pto
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -248,9 +248,11 @@
cpy_strategy = self.space.fromcache(CPyListStrategy)
if self.strategy is cpy_strategy:
return
+ cpy_strategy.locked = True
lst = self.getitems()
self.strategy = cpy_strategy
self.lstorage = cpy_strategy.erase(CPyListStorage(space, lst))
+ cpy_strategy.locked = False
# ___________________________________________________
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3383,6 +3383,14 @@
#pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
#if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
# self._rrc_debug_check_list(pygchdr)
+
+ # TODO?
+ #pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+ #if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+ # next = pygchdr.c_gc_next
+ # next.c_gc_prev = pygchdr.c_gc_prev
+ # pygchdr.c_gc_prev.c_gc_next = next
+
self.rrc_dealloc_pending.append(pyobject)
# an object with refcnt == 0 cannot stay around waiting
# for its deallocator to be called. Some code (lxml)
@@ -3434,7 +3442,7 @@
# TODO: check again, if some new border objects have been marked and
# continue marking recursively... why needed? -> wrapper for
- # pypy-obj is no pygc-obj??? ...
+ # pypy-obj is no pygc-obj??? ...KI
self._rrc_debug_check_consistency(print_label="end-mark")
#self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc) #
TODO: remove?
@@ -3470,6 +3478,14 @@
self.visit_all_objects()
def rrc_major_collection_free(self):
+ if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+ self._rrc_debug_check_consistency()
+ self._rrc_clear_weakref_callbacks()
+ self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
+ self.rrc_pyobj_dead_list)
+ self._rrc_debug_check_consistency(print_label="before-sweep")
+
ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty
2")
length_estimate = self.rrc_p_dict.length()
self.rrc_p_dict.delete()
@@ -3489,14 +3505,6 @@
self.rrc_o_list_old.delete()
self.rrc_o_list_old = new_o_list
- if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
- if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
- self._rrc_debug_check_consistency()
- self._rrc_clear_weakref_callbacks()
- self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
- self.rrc_pyobj_dead_list)
- self._rrc_debug_check_consistency(print_label="before-sweep")
-
def _rrc_clear_weakref_callbacks(self):
# Look for any weakrefs within the trash cycle and remove the callback.
# This is only needed for weakrefs created from rawrefcounted objects
@@ -3564,13 +3572,13 @@
self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt)
pygchdr = pygchdr.c_gc_next
- self._rrc_debug_check_consistency(print_label="rc-initialized")
-
# For every object in this set, if it is marked, add 1 as a real
# refcount (p_list => pyobj stays alive if obj stays alive).
self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None)
+ self._rrc_debug_check_consistency(print_label="rc-initialized")
+
# Subtract all internal refcounts from the cyclic refcount
# of rawrefcounted objects
pygchdr = pygclist.c_gc_next
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
