Author: Stefan Beyer <[email protected]>
Branch: cpyext-gc-cycle
Changeset: r96623:074fd35b8463
Date: 2019-05-16 07:56 +0200
http://bitbucket.org/pypy/pypy/changeset/074fd35b8463/
Log: Fixed tp_clear call in cpyext (GIL handling) Fixed some inheritance
issues with cpyext objects Added debug output for rrc lists WIP:
implement gc support for cpyext tuples WIP: handling of untracked
rrc objects in incminimark (keep flag, etc)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1185,6 +1185,10 @@
state.C._PyPy_tuple_free = rffi.llexternal(
'_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
compilation_info=eci, _nowrapper=True)
+ from pypy.module.cpyext.typeobjectdefs import visitproc
+ state.C._PyPy_tuple_traverse = rffi.llexternal(
+ '_PyPy_tuple_traverse', [PyObject, visitproc, rffi.VOIDP],
+ rffi.INT, compilation_info=eci, _nowrapper=True)
state.C._PyPy_tuple_dealloc = rffi.llexternal(
'_PyPy_tuple_dealloc', [PyObject], lltype.Void,
compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/tupleobject.h
b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -19,6 +19,7 @@
PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
PyAPI_FUNC(void) _PyPy_tuple_free(void *);
+PyAPI_FUNC(int) _PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg);
/* defined in varargswrapper.c */
PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -81,6 +81,12 @@
state = space.fromcache(State)
return state.C.PyObject_Free
+ def has_traverse(self, space):
+ return False
+
+ def get_traverse(self, space):
+ return 0
+
def allocate(self, space, w_type, itemcount=0, immortal=False):
state = space.fromcache(State)
ob_type = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
@@ -129,6 +135,7 @@
tp_realize = kw.pop('realize', None)
tp_dealloc = kw.pop('dealloc', None)
tp_free = kw.pop('free', None)
+ tp_traverse = kw.pop('traverse', None)
assert not kw, "Extra arguments to make_typedescr"
null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void))
@@ -159,6 +166,17 @@
if tp_realize:
def realize(self, space, ref):
return tp_realize(space, ref)
+
+ if tp_traverse:
+ def get_traverse(self, space):
+ from rpython.rtyper.lltypesystem import llmemory
+ from pypy.module.cpyext.typeobjectdefs import traverseproc
+ ptr = rffi.cast(traverseproc, tp_traverse)
+ obj = llmemory.cast_ptr_to_adr(ptr)
+ return llmemory.cast_adr_to_int(obj, "symbolic")
+
+ def has_traverse(self, space):
+ return True
if typedef:
CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
diff --git a/pypy/module/cpyext/src/tupleobject.c
b/pypy/module/cpyext/src/tupleobject.c
--- a/pypy/module/cpyext/src/tupleobject.c
+++ b/pypy/module/cpyext/src/tupleobject.c
@@ -94,4 +94,15 @@
_PyPy_tuple_free(void *obj)
{
PyObject_GC_Del(obj);
+}
+
+int
+_PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg)
+{
+ PyTupleObject *o = (PyTupleObject *)ob;
+ Py_ssize_t i;
+
+ for (i = Py_SIZE(o); --i >= 0; )
+ Py_VISIT(o->ob_item[i]);
+ return 0;
}
\ No newline at end of file
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -6,6 +6,7 @@
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib.rdynload import DLLHANDLE
from rpython.rlib import rawrefcount
+from rpython.rlib.debug import debug_print
import sys
@@ -177,6 +178,7 @@
def _tp_traverse(pyobj_ptr, callback, args):
from pypy.module.cpyext.api import PyObject
from pypy.module.cpyext.typeobjectdefs import visitproc
+ from pypy.module.cpyext.pyobject import cts
# convert to pointers with correct types (PyObject)
callback_addr = llmemory.cast_ptr_to_adr(callback)
callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -184,9 +186,31 @@
pyobj_addr = llmemory.cast_ptr_to_adr(pyobj_ptr)
pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject)
# now call tp_traverse (if possible)
- if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
- pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
- args)
+ debug_print("rrc check traverse", pyobj)
+ pto = pyobj.c_ob_type
+ if pto and pto.c_tp_name:
+ tp_name = pto.c_tp_name
+ name = rffi.charp2str(cts.cast('char*', tp_name))
+ debug_print("rrc try traverse", pyobj, ": type", pto,
+ ": name", name)
+ pto2 = pto
+ i = 1
+ while pto2.c_tp_base:
+ base = pto2.c_tp_base
+ if base.c_tp_name:
+ tp_name = base.c_tp_name
+ name = rffi.charp2str(cts.cast('char*',
tp_name))
+ debug_print(" " * i * 3, "basetype",
+ base, ": name",
+ name, "traverse",
+ base.c_tp_traverse)
+ else:
+ debug_print(" " * i * 3, "unknown base")
+ pto2 = base
+ i += 1
+ if pto and pto.c_tp_traverse:
+ debug_print("rrc do traverse", pyobj)
+ pto.c_tp_traverse(pyobj, callback_ptr, args)
self.tp_traverse = (lambda o, v, a:_tp_traverse(o, v, a))
@@ -308,7 +332,8 @@
def _rawrefcount_perform(space):
from pypy.interpreter.baseobjspace import W_Root
from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
- finalize, from_ref)
+ finalize, from_ref, cts)
+ from pypy.module.cpyext.api import generic_cpy_call
while True:
py_obj = rawrefcount.next_dead(PyObject)
@@ -328,9 +353,15 @@
break
pyobj = rffi.cast(PyObject, py_obj)
adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
- if pyobj.c_ob_type.c_tp_clear:
+ pto = pyobj.c_ob_type
+ if pto.c_tp_clear:
incref(space, py_obj)
- pyobj.c_ob_type.c_tp_clear(pyobj)
+ if pto and pto.c_tp_name:
+ tp_name = pto.c_tp_name
+ name = rffi.charp2str(cts.cast('char*', tp_name))
+ debug_print("tp_clear", pyobj, ": type", pto,
+ ": name", name)
+ generic_cpy_call(space, pto.c_tp_clear, pyobj)
decref(space, py_obj)
head = rawrefcount.cyclic_garbage_head(PyObject)
if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
diff --git a/pypy/module/cpyext/tupleobject.py
b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -46,7 +46,8 @@
alloc=tuple_alloc,
dealloc=state.C._PyPy_tuple_dealloc,
realize=tuple_realize,
- free=state.C._PyPy_tuple_free)
+ free=state.C._PyPy_tuple_free,
+ traverse=state.C._PyPy_tuple_traverse)
PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -653,7 +653,13 @@
else:
pto.c_tp_free = pto.c_tp_base.c_tp_free
- # TODO: traverse (for tuple)
+ from rpython.rtyper.lltypesystem import llmemory
+ from pypy.module.cpyext.typeobjectdefs import traverseproc
+ if typedescr.has_traverse(space):
+ traverse = typedescr.get_traverse(space)
+ obj = llmemory.cast_int_to_adr(traverse)
+ traverse_ptr = llmemory.cast_adr_to_ptr(obj, traverseproc)
+ pto.c_tp_traverse = traverse_ptr
if builder.cpyext_type_init is not None:
builder.cpyext_type_init.append((pto, w_type))
@@ -723,6 +729,10 @@
pto.c_tp_init = base.c_tp_init
if not pto.c_tp_alloc:
pto.c_tp_alloc = base.c_tp_alloc
+ if not pto.c_tp_clear:
+ pto.c_tp_clear = base.c_tp_clear
+ if not pto.c_tp_traverse:
+ pto.c_tp_traverse = base.c_tp_traverse
# XXX check for correct GC flags!
if not pto.c_tp_free:
pto.c_tp_free = base.c_tp_free
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3119,6 +3119,7 @@
RAWREFCOUNT_FINALIZER_LEGACY = 2
RAWREFCOUNT_REFS_SHIFT = 1
RAWREFCOUNT_REFS_MASK_FINALIZED = 1
+ RAWREFCOUNT_REFS_UNTRACKED = -2
def _pyobj(self, pyobjaddr):
return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3379,6 +3380,9 @@
rc -= REFCNT_FROM_PYPY
self._pyobj(pyobject).c_ob_pypy_link = 0
if rc == 0:
+ #pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+ #if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+ # self._rrc_debug_check_list(pygchdr)
self.rrc_dealloc_pending.append(pyobject)
# an object with refcnt == 0 cannot stay around waiting
# for its deallocator to be called. Some code (lxml)
@@ -3393,6 +3397,7 @@
_rrc_free._always_inline_ = True
def rrc_major_collection_trace(self):
+ self._rrc_debug_check_consistency(print_label="begin-mark")
# Only trace and mark rawrefcounted object if we are not doing
# something special, like building gc.garbage.
if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
@@ -3401,38 +3406,57 @@
self._rrc_check_finalizer()
# collect all rawrefcounted roots
self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
+ self._rrc_debug_check_consistency(print_label="roots-marked")
# mark all objects reachable from rawrefcounted roots
self._rrc_mark_rawrefcount()
+ self._rrc_debug_check_consistency(print_label="before-fin")
self.rrc_state = self.RAWREFCOUNT_STATE_MARKING
if self._rrc_find_garbage(): # handle legacy finalizers
self._rrc_mark_garbage()
+ self._rrc_debug_check_consistency(print_label="end-legacy-fin")
self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
use_cylicrc = not self._rrc_find_finalizer() # modern finalizers
+ self._rrc_debug_check_consistency(print_label="end-mark-cyclic")
else:
use_cylicrc = False # don't sweep any objects in cyclic isolates
# now mark all pypy objects at the border, depending on the results
+ debug_print("use_cylicrc", use_cylicrc)
self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc)
- self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc)
+
+ # TODO: check again, if some new border objects have been marked and
+ # continue marking recursively... why needed? -> wrapper for
+ # pypy-obj is no pygc-obj??? ...
+
+ self._rrc_debug_check_consistency(print_label="end-mark")
+ #self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc) #
TODO: remove?
def _rrc_major_trace(self, pyobject, use_cylicrefcnt):
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
#
+ pyobj = self._pyobj(pyobject)
+ cyclic_rc = -42
if use_cylicrefcnt:
- pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+ pygchdr = self.rrc_pyobj_as_gc(pyobj)
if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+ #if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
+ cyclic_rc = rc
+ #else:
+ # rc = pyobj.c_ob_refcnt
else:
- rc = self._pyobj(pyobject).c_ob_refcnt
+ rc = pyobj.c_ob_refcnt
else:
- rc = self._pyobj(pyobject).c_ob_refcnt
+ rc = pyobj.c_ob_refcnt
if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0:
pass # the corresponding object may die
else:
# force the corresponding object to be alive
- intobj = self._pyobj(pyobject).c_ob_pypy_link
+ debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
+ cyclic_rc)
+ intobj = pyobj.c_ob_pypy_link
obj = llmemory.cast_int_to_adr(intobj)
self.objects_to_trace.append(obj)
self.visit_all_objects()
@@ -3445,7 +3469,7 @@
new_p_list = self.AddressStack()
while self.rrc_p_list_old.non_empty():
self._rrc_major_free(self.rrc_p_list_old.pop(), new_p_list,
- new_p_dict)
+ new_p_dict)
self.rrc_p_list_old.delete()
self.rrc_p_list_old = new_p_list
#
@@ -3453,15 +3477,17 @@
no_o_dict = self.null_address_dict()
while self.rrc_o_list_old.non_empty():
self._rrc_major_free(self.rrc_o_list_old.pop(), new_o_list,
- no_o_dict)
+ no_o_dict)
self.rrc_o_list_old.delete()
self.rrc_o_list_old = new_o_list
if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+ self._rrc_debug_check_consistency()
self._rrc_clear_weakref_callbacks()
self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
self.rrc_pyobj_dead_list)
+ self._rrc_debug_check_consistency(print_label="before-sweep")
def _rrc_clear_weakref_callbacks(self):
# Look for any weakrefs within the trash cycle and remove the callback.
@@ -3527,13 +3553,13 @@
refcnt -= REFCNT_FROM_PYPY_LIGHT
elif refcnt >= REFCNT_FROM_PYPY:
refcnt -= REFCNT_FROM_PYPY
- pygchdr.c_gc_refs = pygchdr.c_gc_refs & \
- self.RAWREFCOUNT_REFS_MASK_FINALIZED
- pygchdr.c_gc_refs = refcnt << self.RAWREFCOUNT_REFS_SHIFT
+ self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt)
pygchdr = pygchdr.c_gc_next
+ self._rrc_debug_check_consistency(print_label="rc-initialized")
+
# For every object in this set, if it is marked, add 1 as a real
- # refcount
+ # refcount (p_list => pyobj stays alive if obj stays alive).
self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None)
@@ -3548,13 +3574,31 @@
# now all rawrefcounted roots or live border objects have a
# refcount > 0
+ def _rrc_pyobj_gc_refcnt_set(self, pygchdr, refcnt):
+ pygchdr.c_gc_refs &= self.RAWREFCOUNT_REFS_MASK_FINALIZED
+ pygchdr.c_gc_refs |= refcnt << self.RAWREFCOUNT_REFS_SHIFT
+
def _rrc_obj_fix_refcnt(self, pyobject, ignore):
- intobj = self._pyobj(pyobject).c_ob_pypy_link
+ from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+ from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+ #
+ pyobj = self._pyobj(pyobject)
+ intobj = pyobj.c_ob_pypy_link
obj = llmemory.cast_int_to_adr(intobj)
- gchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+ gchdr = self.rrc_pyobj_as_gc(pyobj)
if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
- if self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
- gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT
+ #if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
+ # debug_print("gc obj not tracked", gchdr, ": obj", obj)
+ #else:
+ refcnt = pyobj.c_ob_refcnt
+ if refcnt >= REFCNT_FROM_PYPY_LIGHT:
+ refcnt -= REFCNT_FROM_PYPY_LIGHT
+ elif refcnt >= REFCNT_FROM_PYPY:
+ refcnt -= REFCNT_FROM_PYPY
+ if self.header(obj).tid & (GCFLAG_VISITED |
+ GCFLAG_NO_HEAP_PTRS):
+ refcnt += 1
+ self._rrc_pyobj_gc_refcnt_set(gchdr, refcnt)
def _rrc_mark_rawrefcount(self):
if self._rrc_gc_list_is_empty(self.rrc_pyobj_list):
@@ -3785,3 +3829,50 @@
def _rrc_gc_list_is_empty(self, pygclist):
return pygclist.c_gc_next == pygclist
+
+ def _rrc_debug_check_consistency(self, print_label=None):
+ if self.DEBUG:
+ should_print = print_label is not None
+ if should_print:
+ debug_start("rrc-lists " + print_label)
+ self._rrc_debug_check_list(self.rrc_pyobj_list,
+ should_print, "rrc_pyobj_list")
+ self._rrc_debug_check_list(self.rrc_pyobj_old_list,
+ should_print, "rrc_pyobj_old_list")
+ self._rrc_debug_check_list(self.rrc_pyobj_dead_list,
+ should_print, "rrc_pyobj_dead_list")
+ self._rrc_debug_check_list(self.rrc_pyobj_garbage_list,
+ should_print, "rrc_pyobj_garbage_list")
+ self._rrc_debug_check_list(self.rrc_pyobj_isolate_list,
+ should_print, "rrc_pyobj_isolate_list")
+ if should_print:
+ debug_stop("rrc-lists " + print_label)
+
+ def _rrc_debug_check_list(self, list, should_print, print_label):
+ if should_print:
+ debug_start(print_label)
+ gchdr = list.c_gc_next
+ prev = list
+ while gchdr <> list:
+ if should_print:
+ pyobj = self.rrc_gc_as_pyobj(gchdr)
+ intobj = pyobj.c_ob_pypy_link
+ debug_print("item", gchdr, ": pyobj", pyobj,
+ "cyclic refcnt",
+ gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT,
+ "refcnt", pyobj.c_ob_refcnt,
+ "link", intobj)
+ if intobj:
+ obj = llmemory.cast_int_to_adr(intobj)
+ marked = self.header(obj).tid & \
+ (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS)
+ debug_print(" linked obj", obj, ": marked", marked)
+
+ ll_assert(gchdr.c_gc_next != lltype.nullptr(self.PYOBJ_GC_HDR),
+ "gc_next is null")
+ ll_assert(gchdr.c_gc_prev == prev, "gc_prev is inconsistent")
+ prev = gchdr
+ gchdr = gchdr.c_gc_next
+ ll_assert(list.c_gc_prev == prev, "gc_prev is inconsistent")
+ if should_print:
+ debug_stop(print_label)
\ No newline at end of file
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
