[pypy-commit] pypy cpyext-gc-cycle: Added rrc gc tests to improve code coverage

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r98591:e8295913b0ef
Date: 2020-01-29 19:58 +0100
http://bitbucket.org/pypy/pypy/changeset/e8295913b0ef/

Log:Added rrc gc tests to improve code coverage

diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_4.dot
copy from rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot
copy to rpython/memory/gc/test/dot/free_cpython_weakref_simple_4.dot
--- a/rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_4.dot
@@ -1,7 +1,8 @@
 digraph G {
 "a" [type=C, alive=n];
 "b" [type=C, alive=n];
+"c" [type=C, alive=n];
 "a" -> "b";
-"b" -> "a";
+"b" -> "c";
 "a" -> "b" [weakref=y, callback=y, clear_callback=y];
 }
diff --git a/rpython/memory/gc/test/dot/free_finalizer_simple_1a.dot 
b/rpython/memory/gc/test/dot/free_finalizer_simple_2.dot
copy from rpython/memory/gc/test/dot/free_finalizer_simple_1a.dot
copy to rpython/memory/gc/test/dot/free_finalizer_simple_2.dot
--- a/rpython/memory/gc/test/dot/free_finalizer_simple_1a.dot
+++ b/rpython/memory/gc/test/dot/free_finalizer_simple_2.dot
@@ -1,10 +1,10 @@
 digraph G {
-"a" [type=P, alive=n];
-"b" [type=B, alive=n];
+"a" [type=C, alive=n];
+"b" [type=C, alive=n];
 "c" [type=C, alive=n, finalizer=modern];
 "d" [type=C, alive=n];
-"e" [type=B, alive=n];
-"f" [type=P, alive=n];
+"e" [type=C, alive=n];
+"f" [type=C, alive=n];
 "a" -> "b";
 "b" -> "c";
 "c" -> "d";
diff --git a/rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot 
b/rpython/memory/gc/test/dot/keep_cpython_inc_5.dot
copy from rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_5.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_5.dot
@@ -1,15 +1,10 @@
 digraph G {
 "a" [type=B, alive=y, ext_refcnt=1];
-"b" [type=P, alive=n];
-"c" [type=P, alive=y];
-"d" [type=B, alive=y];
-"e" [type=C, alive=y];
-"f" [type=C, alive=y, ext_refcnt=1, added=after_snap];
-"g" [type=B, alive=y, added=linked_after_snap, gc=n];
-"a" -> "b" [removed=after_snap];
+"b" [type=P, alive=y];
+"c" [type=B, alive=y, added=linked_after_snap];
+"d" [type=C, alive=y];
+"a" -> "b";
 "b" -> "c";
-"c" -> "d";
-"d" -> "e";
-"f" -> "g" [added=after_snap];
-"g" -> "c" [added=after_snap];
+"c" -> "d" [added=after_snap];
+"d" -> "a";
 }
diff --git a/rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot 
b/rpython/memory/gc/test/dot/keep_cpython_inc_6.dot
copy from rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_6.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_6.dot
@@ -1,15 +1,10 @@
 digraph G {
-"a" [type=B, alive=y, ext_refcnt=1];
-"b" [type=P, alive=n];
-"c" [type=P, alive=y];
-"d" [type=B, alive=y];
-"e" [type=C, alive=y];
-"f" [type=C, alive=y, ext_refcnt=1, added=after_snap];
-"g" [type=B, alive=y, added=linked_after_snap, gc=n];
-"a" -> "b" [removed=after_snap];
+"a" [type=C, alive=y, ext_refcnt=1];
+"b" [type=C, alive=y];
+"c" [type=C, alive=y, removed=after_snap];
+"d" [type=C, alive=n];
+"a" -> "b";
 "b" -> "c";
-"c" -> "d";
-"d" -> "e";
-"f" -> "g" [added=after_snap];
-"g" -> "c" [added=after_snap];
+"c" -> "d" [removed=after_snap];
+"d" -> "a";
 }
diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot 
b/rpython/memory/gc/test/dot/keep_cpython_weakref_simple_1.dot
copy from rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot
copy to rpython/memory/gc/test/dot/keep_cpython_weakref_simple_1.dot
--- a/rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_weakref_simple_1.dot
@@ -1,7 +1,10 @@
 digraph G {
-"a" [type=C, alive=n];
-"b" [type=C, alive=n];
+"a" [type=C, alive=y, ext_refcnt=1];
+"b" [type=C, alive=y];
+"c" [type=C, alive=n];
+"d" [type=C, alive=n];
 "a" -> "b";
-"b" -> "a";
-"a" -> "b" [weakref=y, callback=y, clear_callback=y];
+"c" -> "d";
+"d" -> "c";
+"b" -> "c" [weakref=y, callback=y, clear_callback=n];
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -25,18 +25,57 @@
  ('prev', lltype.Ptr(S)),
  ('next', lltype.Ptr(S
 
-
 class TestRawRefCount(BaseDirectGCTest):
 GCClass = IncMiniMark
 RRCGCClass = RawRefCountIncMarkGC
 #RRCGCClass = RawRefCountMarkGC
 
+# do cleanup after collection (clear all dead pyobjects)
+

[pypy-commit] pypy cpyext-gc-cycle: Fixed bug in rrc incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r98162:93669df5242d
Date: 2019-11-27 14:00 +0100
http://bitbucket.org/pypy/pypy/changeset/93669df5242d/

Log:Fixed bug in rrc incmark

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -238,8 +238,8 @@
 while pygchdr <> self.pyobj_list:
 self._pyobj_gc_refcnt_set(pygchdr, 1)
 pygchdr = pygchdr.c_gc_next
+# resurrect "dead" objects
 pygchdr = self.pyobj_old_list.c_gc_next
-# resurrect "dead" objects
 while pygchdr <> self.pyobj_old_list:
 self._pyobj_gc_refcnt_set(pygchdr, 1)
 pygchdr = pygchdr.c_gc_next
@@ -254,6 +254,7 @@
 start = time.time()
 
 if isolate_consistent:
+debug_print("isolate consistent")
 if not self._gc_list_is_empty(self.pyobj_isolate_old_list):
 self._gc_list_merge(self.pyobj_isolate_old_list,
 self.pyobj_list)
@@ -261,12 +262,14 @@
 self._gc_list_merge(self.pyobj_isolate_dead_list,
 self.pyobj_dead_list)
 else:
+debug_print("isolate inconsistent")
 # continue previous loop, keep objects alive
 pygchdr = pygchdr_continue_isolate
 while pygchdr <> self.pyobj_isolate_old_list:
 self._pyobj_gc_refcnt_set(pygchdr, 1)
 pygchdr = pygchdr.c_gc_next
 # resurrect "dead" objects
+pygchdr = self.pyobj_isolate_dead_list.c_gc_next
 while pygchdr <> self.pyobj_isolate_dead_list:
 self._pyobj_gc_refcnt_set(pygchdr, 1)
 pygchdr = pygchdr.c_gc_next
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Implemented incremental limit for non-rrc objects during rrc marking

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r98130:6a71ef8d2b2c
Date: 2019-11-21 15:50 +0100
http://bitbucket.org/pypy/pypy/changeset/6a71ef8d2b2c/

Log:Implemented incremental limit for non-rrc objects during rrc marking
Added flag to enable/disable tuple untracking for debugging

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -84,6 +84,8 @@
 RAWREFCOUNT_REFS_UNTRACKED = -2 << RAWREFCOUNT_REFS_SHIFT
 RAWREFCOUNT_REFS_REACHABLE = -3 << RAWREFCOUNT_REFS_SHIFT
 
+UNTRACK_TUPLES_DEFAULT = True
+
 def _pyobj(self, pyobjaddr):
 return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
 def _pygchdr(self, pygchdraddr):
@@ -124,6 +126,13 @@
 self.state = self.STATE_DEFAULT
 self.marking_state = 0
 inc_limit = env.read_uint_from_env('PYPY_RRC_GC_INCREMENT_STEP')
+untrack = env.read_uint_from_env('PYPY_RRC_GC_UNTRACK_TUPLES')
+if untrack == 1:
+self.untrack_tuples = True
+elif untrack == 2:
+self.untrack_tuples = False
+else:
+self.untrack_tuples = self.UNTRACK_TUPLES_DEFAULT
 if inc_limit > 0:
 self.inc_limit = inc_limit
 else:
@@ -469,17 +478,20 @@
 self._free(pyobject, True)
 
 def _untrack_tuples(self):
-gchdr = self.tuple_list.c_gc_next
-while gchdr <> self.tuple_list:
-gchdr_next = gchdr.c_gc_next
-pyobj = self.gc_as_pyobj(gchdr)
-result = self.tuple_maybe_untrack(pyobj)
-if result == 1: # contains gc objects -> promote to pyobj list
-next = gchdr.c_gc_next
-next.c_gc_prev = gchdr.c_gc_prev
-gchdr.c_gc_prev.c_gc_next = next
-self._gc_list_add(self.pyobj_list, gchdr)
-gchdr = gchdr_next
+if self.untrack_tuples:
+self._debug_check_consistency(print_label="before-untrack")
+gchdr = self.tuple_list.c_gc_next
+while gchdr <> self.tuple_list:
+gchdr_next = gchdr.c_gc_next
+pyobj = self.gc_as_pyobj(gchdr)
+result = self.tuple_maybe_untrack(pyobj)
+if result == 1: # contains gc objects -> promote to pyobj list
+next = gchdr.c_gc_next
+next.c_gc_prev = gchdr.c_gc_prev
+gchdr.c_gc_prev.c_gc_next = next
+self._gc_list_add(self.pyobj_list, gchdr)
+gchdr = gchdr_next
+self._debug_check_consistency(print_label="after-untrack")
 
 def _find_garbage(self, use_dict):
 found_garbage = False
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.memory.gc.rrc.base import RawRefCountBaseGC
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+from rpython.rlib.rarithmetic import intmask
 import time
 
 class RawRefCountIncMarkGC(RawRefCountBaseGC):
@@ -367,9 +368,22 @@
 snapobj.refcnt += 1
 self._mark_rawrefcount_obj(snapobj)
 simple_limit += 1
-if simple_limit > self.inc_limit: # TODO: add test
+if simple_limit > intmask(self.inc_limit): # TODO: add test
 reached_limit = True
-self.gc.visit_all_objects()  # TODO: implement sane limit
+
+if not reached_limit:
+# "split" the limit between rrc and non-rrc
+# if 25% of rrc limit have been used, now use max 75% or 
non-rrc limit
+if simple_limit > 0:
+estimate = intmask(self.gc.gc_increment_step) * 
intmask(self.inc_limit) / simple_limit
+else:
+estimate = intmask(self.gc.gc_increment_step)
+remaining = self.gc.visit_all_objects_step(estimate)
+if remaining == 0:
+reached_limit = True
+else:
+# if 25% of the non-rrc limit have been used, add 25% of 
the rrc limit to the counter
+simple_limit += intmask(self.inc_limit) * remaining / 
estimate
 first = False
 return not reached_limit # are there any objects left?
 
diff --git a/rpython/memory/gc/rrc/simple.py b/rpython/memory/gc/rrc/simple.py
--- a/rpython/memory/gc/rrc/simple.py
+++ b/rpython/memory/gc/rrc/simple.py
@@ -2,6 +2,12 @@
 
 class RawRefCountSimpleGC(RawRefCountBaseGC):
 
+UNTRACK_TUPLES_DEFAULT = False
+
 def major_collection_trace_step(self):
+self._untrack_tuples()
 self.p_list_old.foreach(self._major_trace, (False, False))
-return True
\ No newline at end of file
+return 

[pypy-commit] pypy cpyext-gc-cycle: Implemented working set for linked objects during rrc marking

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r98006:a3a571806246
Date: 2019-11-09 18:01 +0100
http://bitbucket.org/pypy/pypy/changeset/a3a571806246/

Log:Implemented working set for linked objects during rrc marking
Removed debug output

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2136,6 +2136,12 @@
 #
 hdr.tid |= GCFLAG_VISITED
 #
+if self.rrc_enabled: # TODO: is this safe here?
+if self.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING:
+self.rrc_gc.visit_pyobj(obj)
+elif self.rrc_gc.state == 
RawRefCountBaseGC.STATE_GARBAGE_MARKING:
+hdr.tid |= GCFLAG_GARBAGE
+#
 self.surviving_pinned_objects.append(
 llarena.getfakearenaaddress(obj - size_gc_header))
 self.pinned_objects_in_nursery += 1
@@ -2403,11 +2409,6 @@
 self.collect_nonstack_roots()
 self.visit_all_objects()
 #
-# If enabled, do a major collection step for rrc objects.
-# TODO: move up before "if remaining >= estimate // 2" to
-#   improve pause times, issues:
-# - (non-inc) mark expects all objects to be marked
-# - both do not rescan nonstack-roots
 if self.rrc_enabled:
 debug_print("starting rrc state:", self.rrc_gc.state)
 debug_print("starting marking_state:", 
self.rrc_gc.marking_state)
@@ -2742,9 +2743,11 @@
 # to also set TRACK_YOUNG_PTRS here, for the write barrier.
 hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
-if self.rrc_enabled and \
-self.rrc_gc.state == RawRefCountBaseGC.STATE_GARBAGE_MARKING:
-hdr.tid |= GCFLAG_GARBAGE
+if self.rrc_enabled:
+if self.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING:
+self.rrc_gc.visit_pyobj(obj)
+elif self.rrc_gc.state == RawRefCountBaseGC.STATE_GARBAGE_MARKING:
+hdr.tid |= GCFLAG_GARBAGE
 
 if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
 #
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -143,7 +143,7 @@
 if not self.gc.is_young_object(obj):
 lst = self.p_list_old
 if self.state == self.STATE_MARKING:
-debug_print("added p_list", pyobject)
+#debug_print("added p_list", pyobject)
 self.p_list_old_added.append(pyobject)
 lst.append(pyobject)
 dct.setitem(obj, pyobject)
@@ -307,7 +307,7 @@
 if surviving:
 surviving_list.append(pyobject)
 if working_set:
-debug_print("added p_list", pyobject)
+#debug_print("added p_list", pyobject)
 self.p_list_old_added.append(pyobject)
 else:
 self._free(pyobject)
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -25,6 +25,7 @@
 
 self.state = self.STATE_MARKING
 self.marking_state = 0
+self.pyobj_to_trace = self.gc.AddressStack()
 return False
 
 if self.state == self.STATE_MARKING:
@@ -38,7 +39,6 @@
 return False
 elif self.marking_state == 1:
 # initialize working set from roots, then pause
-self.pyobj_to_trace = self.gc.AddressStack()
 for i in range(0, self.total_objs):
 obj = self.snapshot_objs[i]
 self._mark_rawrefcount_obj(obj)
@@ -104,6 +104,27 @@
 debug_print("time mark p_list_old", time.time() - start)
 return True
 
+def visit_pyobj(self, gcobj):
+# if there is a pyobj, add it to the working set
+if self.gc.is_in_nursery(gcobj):
+dct = self.p_dict_nurs # is this even possible?
+else:
+dct = self.p_dict
+pyobject = dct.get(gcobj)
+if pyobject <> llmemory.NULL:
+pyobj = self._pyobj(pyobject)
+gchdr = self.pyobj_as_gc(pyobj)
+if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
+if gchdr.c_gc_refs != self.RAWREFCOUNT_REFS_REACHABLE and \
+gchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:  # 
object is in snapshot
+pass
+c_gc_refs = self._pyobj_gc_refcnt_get(gchdr)
+index = c_gc_refs - 1
+snapobj = self.snapshot_objs[index]
+if snapobj.refcnt 

[pypy-commit] pypy cpyext-gc-cycle: Increased incremental limit of rrc incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97967:1fd7c207819e
Date: 2019-11-06 10:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1fd7c207819e/

Log:Increased incremental limit of rrc incmark Changed default rrc
implementation Added simple rrc implementation (to mimic old
implementation)

diff --git a/rpython/config/translationoption.py 
b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -106,8 +106,8 @@
 ("translation.backend", "c")],
 }),
 ChoiceOption("rrcgc", "Garbage Collection Strategy for raw refcounted 
objects in cpyext",
- ["mark", "incmark", "none"],
- default="mark",
+ ["simple", "mark", "incmark"],
+ default="incmark",
  requires={
 "mark": [("translation.gc", "incminimark")],
 "incmark": [("translation.gc", "incminimark")],
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -5,7 +5,8 @@
 
 def choose_rrc_gc_from_config(config):
 if config.translation.rrcgc:
-classes = {"mark": "mark.RawRefCountMarkGC",
+classes = {"simple": "simple.RawRefCountSimpleGC",
+   "mark": "mark.RawRefCountMarkGC",
"incmark": "incmark.RawRefCountIncMarkGC",
}
 try:
@@ -126,7 +127,7 @@
 if inc_limit > 0:
 self.inc_limit = inc_limit
 else:
-self.inc_limit = 1000
+self.inc_limit = 5
 self.cycle_enabled = True
 
 def create_link_pypy(self, gcobj, pyobject):
diff --git a/rpython/memory/gc/rrc/simple.py b/rpython/memory/gc/rrc/simple.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/rrc/simple.py
@@ -0,0 +1,7 @@
+from rpython.memory.gc.rrc.base import RawRefCountBaseGC
+
+class RawRefCountSimpleGC(RawRefCountBaseGC):
+
+def major_collection_trace_step(self):
+self.p_list_old.foreach(self._major_trace, (False, False))
+return True
\ No newline at end of file
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed performance issues in incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97853:4dc7f472f3df
Date: 2019-10-24 21:22 +0200
http://bitbucket.org/pypy/pypy/changeset/4dc7f472f3df/

Log:Fixed performance issues in incmark Added debugging code

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -97,6 +97,8 @@
  self.GCFLAG_NO_HEAP_PTRS, self.GCFLAG_GARBAGE) = gc_flags
 self.p_list_young = self.gc.AddressStack()
 self.p_list_old = self.gc.AddressStack()
+self.p_list_old_added = self.gc.AddressStack()
+self.p_dict_old_free = self.gc.AddressDict()
 self.o_list_young = self.gc.AddressStack()
 self.o_list_old = self.gc.AddressStack()
 self.p_dict = self.gc.AddressDict()  # non-nursery keys only
@@ -139,6 +141,9 @@
 dct = self.p_dict
 if not self.gc.is_young_object(obj):
 lst = self.p_list_old
+if self.state == self.STATE_MARKING:
+debug_print("added p_list", pyobject)
+self.p_list_old_added.append(pyobject)
 lst.append(pyobject)
 dct.setitem(obj, pyobject)
 
@@ -259,14 +264,15 @@
 ll_assert(self.p_dict_nurs.length() == 0,
   "p_dict_nurs not empty 1")
 lst = self.p_list_young
+working_set = self.state == self.STATE_MARKING
 while lst.non_empty():
-self._minor_free(lst.pop(), self.p_list_old, self.p_dict)
+self._minor_free(lst.pop(), self.p_list_old, self.p_dict, 
working_set)
 lst = self.o_list_young
 no_o_dict = self.gc.null_address_dict()
 while lst.non_empty():
-self._minor_free(lst.pop(), self.o_list_old, no_o_dict)
+self._minor_free(lst.pop(), self.o_list_old, no_o_dict, False)
 
-def _minor_free(self, pyobject, surviving_list, surviving_dict):
+def _minor_free(self, pyobject, surviving_list, surviving_dict, 
working_set):
 intobj = self._pyobj(pyobject).c_ob_pypy_link
 obj = llmemory.cast_int_to_adr(intobj)
 if self.gc.is_in_nursery(obj):
@@ -299,6 +305,9 @@
 #
 if surviving:
 surviving_list.append(pyobject)
+if working_set:
+debug_print("added p_list", pyobject)
+self.p_list_old_added.append(pyobject)
 else:
 self._free(pyobject)
 
@@ -309,6 +318,7 @@
 rc = self._pyobj(pyobject).c_ob_refcnt
 if rc >= REFCNT_FROM_PYPY_LIGHT:
 rc -= REFCNT_FROM_PYPY_LIGHT
+debug_print("major_free", "free", pyobject, "refcnt", rc, "light")
 if rc == 0:
 pygchdr = self.pyobj_as_gc(self._pyobj(pyobject))
 if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
@@ -325,6 +335,7 @@
 ll_assert(rc < int(REFCNT_FROM_PYPY_LIGHT * 0.99),
   "refcount underflow from REFCNT_FROM_PYPY_LIGHT?")
 rc -= REFCNT_FROM_PYPY
+debug_print("major_free", "free", pyobject, "refcnt", rc, 
"nonlight")
 self._pyobj(pyobject).c_ob_pypy_link = 0
 if rc == 0:
 self.dealloc_pending.append(pyobject)
@@ -376,13 +387,15 @@
 pass  # the corresponding object may die
 else:
 # force the corresponding object to be alive
-debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
-cyclic_rc)
 if use_dict:
 obj = self.pypy_link_dict.get(pyobject)
 else:
 intobj = pyobj.c_ob_pypy_link
+if intobj == 0:
+return
 obj = llmemory.cast_int_to_adr(intobj)
+debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
+cyclic_rc)
 self.gc.objects_to_trace.append(obj)
 self.gc.visit_all_objects() # TODO: execute incrementally?
 
@@ -444,12 +457,14 @@
 #  * GCFLAG_NO_HEAP_PTRS: immortal object never traced (so far)
 intobj = self._pyobj(pyobject).c_ob_pypy_link
 obj = llmemory.cast_int_to_adr(intobj)
-if self.gc.header(obj).tid & \
+if intobj != 0 and self.gc.header(obj).tid & \
 (self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS):
+debug_print("major_free", "survive", pyobject)
 surviving_list.append(pyobject)
 if surviving_dict:
 surviving_dict.insertclean(obj, pyobject)
 else:
+debug_print("major_free", "free", pyobject)
 self._free(pyobject, True)
 
 def _untrack_tuples(self):
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi
 from rpython.memory.gc.rrc.base import 

[pypy-commit] pypy cpyext-gc-cycle: Improved stats

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97852:4f393184be0e
Date: 2019-10-23 19:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4f393184be0e/

Log:Improved stats

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -161,7 +161,7 @@
 use_bytecode_counter=True)
 else:
 module = space.builtin_modules['gc']
-attribute = space.newtext('cpyext_durations')
+attribute = space.newtext('cpyext_stat')
 space.setattr(module, attribute, space.newlist([]))
 
 pyobj_dealloc_action = PyObjDeallocAction(space)
@@ -321,26 +321,33 @@
  finalize, from_ref, cts)
 from pypy.module.cpyext.api import generic_cpy_call
 
-
+#debug_print("rrc perform")
 if rawrefcount.check_state():
 start = time.time()
+dead_calls = 0
+cyclic_isolate_calls = 0
+cyclic_garbage_calls = 0
+garbage_calls = 0
 
 while True:
 py_obj = rawrefcount.next_dead(PyObject)
 if not py_obj:
 break
+dead_calls += 1
 decref(space, py_obj)
 
 while True:
 py_obj = rawrefcount.next_cyclic_isolate(PyObject)
 if not py_obj:
 break
+cyclic_isolate_calls += 1
 finalize(space, py_obj)
 
 while True:
 py_obj = rawrefcount.cyclic_garbage_head(PyObject)
 if not py_obj:
 break
+cyclic_garbage_calls += 1
 pyobj = rffi.cast(PyObject, py_obj)
 adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
 pto = pyobj.c_ob_type
@@ -357,27 +364,38 @@
 if adr_int == 
llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
 
-rawrefcount.begin_garbage()
-w_list = space.newlist([])
-while True:
-w_obj = rawrefcount.next_garbage_pypy(W_Root)
-if not py_obj:
-break
-w_list.append(w_obj)
-while True:
-w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
-if not w_pyobj:
-break
-w_obj = from_ref(space, w_pyobj)
-w_list.append(w_obj)
-space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
-  w_list)
-rawrefcount.end_garbage()
+# TODO: instead of building up this list every time, only do it in 
case something changed
+if False: # TODO add check, if something changed
+rawrefcount.begin_garbage()
+w_list = space.newlist([]) # append to existing list???
+while True:
+w_obj = rawrefcount.next_garbage_pypy(W_Root)
+if not w_obj:
+break
+garbage_calls += 1
+debug_print("append garbage pypy", w_obj)
+w_list.append(w_obj)
+while True:
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
+if not w_pyobj:
+break
+garbage_calls += 1
+w_obj = from_ref(space, w_pyobj)
+debug_print("append garbage pyobj", w_obj)
+w_list.append(w_obj)
+space.setattr(space.builtin_modules['gc'],
+  space.newtext('garbage'), w_list)
+rawrefcount.end_garbage()
 
 duration = time.time() - start
+list = space.newlist([space.newfloat(duration),
+  space.newint(dead_calls),
+  space.newint(cyclic_isolate_calls),
+  space.newint(cyclic_garbage_calls),
+  space.newint(garbage_calls)])
 module = space.builtin_modules['gc']
-durations = space.getattr(module, space.newtext('cpyext_durations'))
-durations.append(space.newfloat(duration))
+durations = space.getattr(module, space.newtext('cpyext_stat'))
+durations.append(list)
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2026,7 +2026,7 @@
 #
 # Check that the flags are correct: we must not have
 # GCFLAG_TRACK_YOUNG_PTRS so far.
-# TODO: fix
+# TODO: fix (with rrc incmark, seems to work with rrc mark)
 #ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0,
 #  "old_objects_pointing_to_young contains obj with "
 #  

[pypy-commit] pypy cpyext-gc-cycle: Check state during cpyext rrc gc processing (incorrect state after garbage handling)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97756:30c564b06dd8
Date: 2019-10-10 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/30c564b06dd8/

Log:Check state during cpyext rrc gc processing (incorrect state after
garbage handling) Handle non-gc objects in o_list correctly Fixed
handling of gc_refs in rrc incmark Fixed issue with pypy_link of
freed objects in p_list in rrc incmark

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -321,61 +321,63 @@
  finalize, from_ref, cts)
 from pypy.module.cpyext.api import generic_cpy_call
 
-start = time.time()
 
-while True:
-py_obj = rawrefcount.next_dead(PyObject)
-if not py_obj:
-break
-decref(space, py_obj)
+if rawrefcount.check_state():
+start = time.time()
 
-while True:
-py_obj = rawrefcount.next_cyclic_isolate(PyObject)
-if not py_obj:
-break
-finalize(space, py_obj)
+while True:
+py_obj = rawrefcount.next_dead(PyObject)
+if not py_obj:
+break
+decref(space, py_obj)
 
-while True:
-py_obj = rawrefcount.cyclic_garbage_head(PyObject)
-if not py_obj:
-break
-pyobj = rffi.cast(PyObject, py_obj)
-adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
-pto = pyobj.c_ob_type
-if pto.c_tp_clear:
-incref(space, py_obj)
-#if pto and pto.c_tp_name:
-#tp_name = pto.c_tp_name
-#name = rffi.charp2str(cts.cast('char*', tp_name))
-#debug_print("tp_clear", pyobj, ": type", pto,
-#": name", name)
-generic_cpy_call(space, pto.c_tp_clear, pyobj)
-decref(space, py_obj)
-head = rawrefcount.cyclic_garbage_head(PyObject)
-if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
-rawrefcount.cyclic_garbage_remove()
+while True:
+py_obj = rawrefcount.next_cyclic_isolate(PyObject)
+if not py_obj:
+break
+finalize(space, py_obj)
 
-rawrefcount.begin_garbage()
-w_list = space.newlist([])
-while True:
-w_obj = rawrefcount.next_garbage_pypy(W_Root)
-if not py_obj:
-break
-w_list.append(w_obj)
-while True:
-w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
-if not w_pyobj:
-break
-w_obj = from_ref(space, w_pyobj)
-w_list.append(w_obj)
-space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
-  w_list)
-rawrefcount.end_garbage()
+while True:
+py_obj = rawrefcount.cyclic_garbage_head(PyObject)
+if not py_obj:
+break
+pyobj = rffi.cast(PyObject, py_obj)
+adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
+pto = pyobj.c_ob_type
+if pto.c_tp_clear:
+incref(space, py_obj)
+#if pto and pto.c_tp_name:
+#tp_name = pto.c_tp_name
+#name = rffi.charp2str(cts.cast('char*', tp_name))
+#debug_print("tp_clear", pyobj, ": type", pto,
+#": name", name)
+generic_cpy_call(space, pto.c_tp_clear, pyobj)
+decref(space, py_obj)
+head = rawrefcount.cyclic_garbage_head(PyObject)
+if adr_int == 
llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
+rawrefcount.cyclic_garbage_remove()
 
-duration = time.time() - start
-module = space.builtin_modules['gc']
-durations = space.getattr(module, space.newtext('cpyext_durations'))
-durations.append(space.newfloat(duration))
+rawrefcount.begin_garbage()
+w_list = space.newlist([])
+while True:
+w_obj = rawrefcount.next_garbage_pypy(W_Root)
+if not py_obj:
+break
+w_list.append(w_obj)
+while True:
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
+if not w_pyobj:
+break
+w_obj = from_ref(space, w_pyobj)
+w_list.append(w_obj)
+space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
+  w_list)
+rawrefcount.end_garbage()
+
+duration = time.time() - start
+module = space.builtin_modules['gc']
+durations = space.getattr(module, space.newtext('cpyext_durations'))
+durations.append(space.newfloat(duration))
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
diff --git 

[pypy-commit] pypy cpyext-gc-cycle: Fixed issues with rrc tuples

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97728:6e21fe036218
Date: 2019-10-05 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/6e21fe036218/

Log:Fixed issues with rrc tuples Implemented cpyext statistics

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -8,6 +8,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib.debug import debug_print
 import sys
+import time
 
 
 # Keep track of exceptions raised in cpyext for a particular execution
@@ -159,6 +160,10 @@
 space.actionflag.register_periodic_action(action,
 use_bytecode_counter=True)
 else:
+module = space.builtin_modules['gc']
+attribute = space.newtext('cpyext_durations')
+space.setattr(module, attribute, space.newlist([]))
+
 pyobj_dealloc_action = PyObjDeallocAction(space)
 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
@@ -310,12 +315,14 @@
 return True
 
 
-def _rawrefcount_perform(space):
+def _rawrefcount_perform(space): # TODO: measure time spent, make incremental??
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
  finalize, from_ref, cts)
 from pypy.module.cpyext.api import generic_cpy_call
 
+start = time.time()
+
 while True:
 py_obj = rawrefcount.next_dead(PyObject)
 if not py_obj:
@@ -337,11 +344,11 @@
 pto = pyobj.c_ob_type
 if pto.c_tp_clear:
 incref(space, py_obj)
-if pto and pto.c_tp_name:
-tp_name = pto.c_tp_name
-name = rffi.charp2str(cts.cast('char*', tp_name))
-debug_print("tp_clear", pyobj, ": type", pto,
-": name", name)
+#if pto and pto.c_tp_name:
+#tp_name = pto.c_tp_name
+#name = rffi.charp2str(cts.cast('char*', tp_name))
+#debug_print("tp_clear", pyobj, ": type", pto,
+#": name", name)
 generic_cpy_call(space, pto.c_tp_clear, pyobj)
 decref(space, py_obj)
 head = rawrefcount.cyclic_garbage_head(PyObject)
@@ -365,6 +372,11 @@
   w_list)
 rawrefcount.end_garbage()
 
+duration = time.time() - start
+module = space.builtin_modules['gc']
+durations = space.getattr(module, space.newtext('cpyext_durations'))
+durations.append(space.newfloat(duration))
+
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
 """
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -258,7 +258,8 @@
 addr = self.snapshot_refs[obj.refs_index + j]
 obj_ref = llmemory.cast_adr_to_ptr(addr,
self.PYOBJ_SNAPSHOT_OBJ_PTR)
-obj_ref.refcnt -= 1
+if obj_ref != lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ):
+obj_ref.refcnt -= 1
 
 # now all rawrefcounted roots or live border objects have a
 # refcount > 0
@@ -299,7 +300,8 @@
 addr = self.snapshot_refs[snapobj.refs_index + j]
 obj_ref = llmemory.cast_adr_to_ptr(addr,
self.PYOBJ_SNAPSHOT_OBJ_PTR)
-obj_ref.refcnt += 1
+if obj_ref != lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ):
+obj_ref.refcnt += 1
 # mark recursively, if it is a pypyobj
 if snapobj.pypy_link <> 0:
 intobj = snapobj.pypy_link
@@ -320,6 +322,10 @@
 total_refcnt += self._take_snapshot_count_gc(pygchdr)
 total_objs += 1
 pygchdr = pygchdr.c_gc_next
+pygchdr = self.tuple_list.c_gc_next
+while pygchdr <> self.tuple_list:
+total_refcnt += self._take_snapshot_count_gc(pygchdr)
+pygchdr = pygchdr.c_gc_next
 pygchdr = self.pyobj_isolate_old_list.c_gc_next
 while pygchdr <> self.pyobj_isolate_old_list:
 total_refcnt += self._take_snapshot_count_gc(pygchdr)
@@ -364,7 +370,10 @@
 pygchdr = self.pyobj_as_gc(pyobj)
 if (pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and
 pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
-obj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+if pygchdr.c_gc_refs > 0:
+obj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+else:
+obj = lltype.nullptr(self.PYOBJ_SNAPSHOT_OBJ)
 else:
 obj = 

[pypy-commit] pypy cpyext-gc-cycle: Removed obsolete code and TODOs

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97597:5172b6ed9d6c
Date: 2019-09-24 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/5172b6ed9d6c/

Log:Removed obsolete code and TODOs

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -162,20 +162,6 @@
 pyobj_dealloc_action = PyObjDeallocAction(space)
 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
-def _clear_weakref_callbacks(gcref):
-from pypy.module._weakref.interp__weakref import \
-W_Weakref, W_CallableProxy
-from pypy.module.gc.referents import \
-try_cast_gcref_to_w_root
-w_obj = try_cast_gcref_to_w_root(gcref)
-if type(w_obj) is W_Weakref:
-w_obj.w_callable = None
-elif type(w_obj) is W_CallableProxy:
-w_obj.w_callable = None
-
-self.clear_weakref_callbacks = \
-(lambda w_obj: _clear_weakref_callbacks(w_obj))
-
 def _tp_traverse(pyobj_ptr, callback, args):
 from pypy.module.cpyext.api import PyObject, \
 generic_cpy_call
@@ -248,8 +234,6 @@
 pypyobj_list, pypyobj_tuple_list,
 self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc,
 self.C._PyPy_finalizer_type,
-llhelper(rawrefcount.RAWREFCOUNT_CLEAR_WR_TYPE,
- self.clear_weakref_callbacks),
 self.C._PyTuple_MaybeUntrack)
 self.builder.attach_all(space)
 
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3094,7 +3094,6 @@
 RAWREFCOUNT_GC_AS_PYOBJ = RawRefCountBaseGC.RAWREFCOUNT_GC_AS_PYOBJ
 RAWREFCOUNT_PYOBJ_AS_GC = RawRefCountBaseGC.RAWREFCOUNT_PYOBJ_AS_GC
 RAWREFCOUNT_FINALIZER_TYPE = RawRefCountBaseGC.RAWREFCOUNT_FINALIZER_TYPE
-RAWREFCOUNT_CLEAR_WR_TYPE = RawRefCountBaseGC.RAWREFCOUNT_CLEAR_WR_TYPE
 RAWREFCOUNT_MAYBE_UNTRACK_TUPLE = \
 RawRefCountBaseGC.RAWREFCOUNT_MAYBE_UNTRACK_TUPLE
 
@@ -3103,15 +3102,13 @@
 
 def rawrefcount_init(self, dealloc_trigger_callback, tp_traverse,
  pyobj_list, tuple_list, gc_as_pyobj, pyobj_as_gc,
- finalizer_type, clear_weakref_callback,
- tuple_maybe_untrack):
+ finalizer_type, tuple_maybe_untrack):
 if not self.rrc_enabled:
 gc_flags = (GCFLAG_VISITED_RMY, GCFLAG_VISITED,
 GCFLAG_NO_HEAP_PTRS, GCFLAG_GARBAGE)
 self.rrc_gc.init(self, gc_flags, dealloc_trigger_callback,
  tp_traverse, pyobj_list, tuple_list, gc_as_pyobj,
- pyobj_as_gc, finalizer_type,
- clear_weakref_callback, tuple_maybe_untrack)
+ pyobj_as_gc, finalizer_type, tuple_maybe_untrack)
 self.rrc_enabled = True
 
 def activate_rawrefcount_cycle(self):
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -72,8 +72,6 @@
  PYOBJ_GC_HDR_PTR))
 RAWREFCOUNT_FINALIZER_TYPE = lltype.Ptr(lltype.FuncType([PYOBJ_GC_HDR_PTR],
 lltype.Signed))
-RAWREFCOUNT_CLEAR_WR_TYPE = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
-   lltype.Void))
 RAWREFCOUNT_MAYBE_UNTRACK_TUPLE = \
 lltype.Ptr(lltype.FuncType([PYOBJ_HDR_PTR], lltype.Signed))
 RAWREFCOUNT_FINALIZER_NONE = 0
@@ -90,7 +88,7 @@
 
 def init(self, gc, gc_flags, dealloc_trigger_callback, tp_traverse,
  pyobj_list, tuple_list, gc_as_pyobj, pyobj_as_gc, finalizer_type,
- clear_weakref_callback, tuple_maybe_untrack):
+ tuple_maybe_untrack):
 # see pypy/doc/discussion/rawrefcount.rst
 self.gc = gc
 (self.GCFLAG_VISITED_RMY, self.GCFLAG_VISITED,
@@ -117,7 +115,6 @@
 self.gc_as_pyobj = gc_as_pyobj
 self.pyobj_as_gc = pyobj_as_gc
 self.finalizer_type = finalizer_type
-self.clear_weakref_callback = clear_weakref_callback
 self.tuple_maybe_untrack = tuple_maybe_untrack
 self.state = self.STATE_DEFAULT
 self.cycle_enabled = True
@@ -347,7 +344,6 @@
 def _major_trace(self, pyobject, flags):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
-# TODO: add flag; if set: if 

[pypy-commit] pypy cpyext-gc-cycle: Cleaned up and moved code

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97594:aad1bca0736f
Date: 2019-09-23 14:58 +0200
http://bitbucket.org/pypy/pypy/changeset/aad1bca0736f/

Log:Cleaned up and moved code

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -719,11 +719,6 @@
 gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT,
 "refcnt", pyobj.c_ob_refcnt,
 "link", intobj)
-#if intobj: TODO fix
-#obj = llmemory.cast_int_to_adr(intobj)
-#marked = self.header(obj).tid & \
-#(GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS)
-#   debug_print("  linked obj", obj, ": marked", marked)
 
 ll_assert(gchdr.c_gc_next != lltype.nullptr(self.PYOBJ_GC_HDR),
   "gc_next is null")
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -43,6 +43,8 @@
 
 self._debug_print_snap(print_label="roots-marked")
 self._debug_check_consistency(print_label="roots-marked")
+
+self._gc_list_init(self.pyobj_old_list)
 self.state = self.STATE_MARKING
 return False
 
@@ -224,7 +226,6 @@
 # refcount > 0
 
 def _mark_rawrefcount(self):
-self._gc_list_init(self.pyobj_old_list) # TODO: move???
 # as long as new objects with cyclic a refcount > 0 or alive border
 # objects are found, increment the refcount of all referenced objects
 # of those newly found objects
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed handling of modern finalizers in rrc gc

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97595:7818db583143
Date: 2019-09-23 23:39 +0200
http://bitbucket.org/pypy/pypy/changeset/7818db583143/

Log:Fixed handling of modern finalizers in rrc gc Fixed test cases

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -109,6 +109,8 @@
 self.tuple_list = self._pygchdr(tuple_list)
 self.pyobj_old_list = self._gc_list_new()
 self.pyobj_isolate_list = self._gc_list_new()
+self.pyobj_isolate_old_list = self._gc_list_new()
+self.pyobj_isolate_dead_list = self._gc_list_new()
 self.pyobj_dead_list = self._gc_list_new()
 self.pyobj_garbage_list = self._gc_list_new()
 self.garbage_to_trace = self.gc.AddressStack()
@@ -177,7 +179,7 @@
 def next_cyclic_isolate(self):
 if not self._gc_list_is_empty(self.pyobj_isolate_list):
 gchdr = self._gc_list_pop(self.pyobj_isolate_list)
-self._gc_list_add(self.pyobj_old_list, gchdr)
+self._gc_list_add(self.pyobj_isolate_old_list, gchdr)
 return llmemory.cast_ptr_to_adr(self.gc_as_pyobj(gchdr))
 return llmemory.NULL
 
@@ -546,37 +548,9 @@
 self.gc.trace(obj, self._collect_ref_rec, None)
 return True
 
-def _check_finalizer(self):
-# Check, if the cyclic isolate from the last collection cycle
-# is reachable from outside, after the finalizers have been
-# executed (and if all finalizers have been executed). Return
-# True if some objects are reachable and thus have been resurrected.
-
-# check if the list has been fully processed since the last cycle
-# (for safety)
-found_alive = not self._gc_list_is_empty(self.pyobj_isolate_list)
-
-# check if all finalizers have actually been called (for safety)
-if not found_alive:
-found_alive = self._find_finalizer()
-
-# check if there are any objects with a reference count > 0
-if not found_alive:
-gchdr = self.pyobj_old_list.c_gc_next
-while gchdr <> self.pyobj_old_list:
-if True: # TODO: check refcount or marked (see _collect_roots)
-found_alive = True
-break
-gchdr = gchdr.c_gc_next
-
-if found_alive:
-self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+def _find_finalizer(self):
+if not self._gc_list_is_empty(self.pyobj_isolate_list):
 return True
-else:
-self._gc_list_merge(self.pyobj_old_list, self.pyobj_dead_list)
-return False
-
-def _find_finalizer(self):
 gchdr = self.pyobj_old_list.c_gc_next
 while gchdr <> self.pyobj_old_list:
 if self.finalizer_type(gchdr) == self.RAWREFCOUNT_FINALIZER_MODERN:
@@ -699,6 +673,8 @@
"pyobj_dead_list")
 self._debug_check_list(self.pyobj_isolate_list, should_print,
"pyobj_isolate_list")
+self._debug_check_list(self.pyobj_isolate_old_list, should_print,
+   "pyobj_isolate_old_list")
 # pyobj_garbage_list is not a real list, it just marks the
 # first and the last object in pyobj_list, which are garbage
 
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -6,35 +6,21 @@
 class RawRefCountIncMarkGC(RawRefCountBaseGC):
 
 def major_collection_trace_step(self):
-if not self.cycle_enabled or self.state == self.STATE_GARBAGE:
+if (not self.cycle_enabled or self.state == self.STATE_GARBAGE or
+not self._gc_list_is_empty(self.pyobj_isolate_list)):
 self._debug_check_consistency(print_label="begin-mark")
 self.p_list_old.foreach(self._major_trace, (False, False))
 self._debug_check_consistency(print_label="end-mark")
 return True
 
 if self.state == self.STATE_DEFAULT:
-# Merge all objects whose finalizer have been executed to the
-# pyobj_list (to reprocess them again in the snapshot). Finalizers
-# can only be executed once, so termination will eventually happen.
-# Objects which have not been resurrected should be freed during
-# this cycle.
-if not self._gc_list_is_empty(self.pyobj_old_list):
-self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
-# TODO: use separate list and process it after pyobj_list has been
-#   fully processed (just before modern finalizers) if 
references
-#   to separate list are encountered during take_snapshot
-#   move them to pyobj_list and include them in 

[pypy-commit] pypy cpyext-gc-cycle: Fixed bug in rrc gc tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97593:8b37cae53ec8
Date: 2019-09-23 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/8b37cae53ec8/

Log:Fixed bug in rrc gc tests

diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -5,6 +5,7 @@
 from rpython.memory.gc.rrc.mark import RawRefCountMarkGC
 from rpython.memory.gc.rrc.incmark import RawRefCountIncMarkGC
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest
+from rpython.rlib import rgc
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT
 
 PYOBJ_HDR = RawRefCountBaseGC.PYOBJ_HDR
@@ -645,7 +646,8 @@
 dest.info.ext_refcnt += 1
 if removed == "after_snap":
 remove_after_snap.append(('C', source, dest))
-elif source.info.type == "P" or dest.info.type == "P":
+elif (source.info.type == "P" or dest.info.type == "P" or
+  (source.info.type == "B" and dest.info.type == "B")):
 if (source.p is None or llmemory.cast_ptr_to_adr(source.p.next)
 == llmemory.NULL):
 if added == "after_snap":
@@ -664,6 +666,14 @@
 else:
 assert False # only 2 refs supported from pypy obj in tests
 
+if (len(add_after_snap) > 0 or len(add_border_after_snap) > 0 or
+len(add_linked_pyobj_after_snap) > 0 or
+len(add_pyobj_after_snap) > 0 or
+len(add_pypy_after_snap) > 0 or
+len(remove_after_snap) > 0):
+if self.RRCGCClass != RawRefCountIncMarkGC:
+py.test.skip('Incremental test on non-incremental gc.')
+
 # add finalizers
 for name in nodes:
 n = nodes[name]
@@ -810,87 +820,83 @@
 
 # do a collection to find cyclic isolates and clean them, if there are
 # no finalizers
-if True:
-from rpython.rlib import rgc
-state = -1
-after_snap = False
-while state <> 0:
-states = self.gc.collect_step()
-state = rgc.new_state(states)
-if (self.gc.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING and
-not after_snap):
-for obj in add_pyobj_after_snap:
-r, raddr, check_alive = self._rawrefcount_pyobj(
-tracked=obj.info.tracked, tuple=obj.info.tuple)
-r.c_ob_refcnt += obj.info.ext_refcnt
-obj.r = r
-obj.raddr = raddr
-obj.check_alive = check_alive
-for obj in add_pypy_after_snap:
-p, pref, check_alive = \
-self._rawrefcount_pypyobj(42 + i, rooted=obj.info
-  .rooted, create_old=True)
-obj.p = p
-obj.pref = pref
-obj.check_alive = check_alive
-i += 1
-for obj in add_border_after_snap:
-p, pref, r, raddr, check_alive = \
-self._rawrefcount_pair(42 + i, rooted=obj.info
-   .rooted, create_old=True,
-   tracked=obj.info.tracked,
-   tuple=obj.info.tuple,
-   is_gc=obj.info.gc)
-r.c_ob_refcnt += obj.info.ext_refcnt
-obj.r = r
-obj.raddr = raddr
-obj.p = p
-obj.pref = pref
-obj.check_alive = check_alive
-i += 1
-for obj in add_linked_pyobj_after_snap:
-r, raddr, check_alive = self._rawrefcount_pyobj(
-tracked=obj.info.tracked, tuple=obj.info.tuple,
-is_gc=obj.info.gc)
-r.c_ob_refcnt += obj.info.ext_refcnt
-obj.r = r
-obj.raddr = raddr
-old_alive = obj.check_alive
-def double_check(ext_refcnt):
-old_alive()
-check_alive(ext_refcnt)
-obj.check_alive = double_check
-self.gc.rawrefcount_create_link_pypy(obj.pref, raddr)
+state = -1
+after_snap = False
+while state <> 0:
+states = self.gc.collect_step()
+state = rgc.new_state(states)
+if 

[pypy-commit] pypy cpyext-gc-cycle: Fixed bugs in rrc incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97591:f23e180d2dfc
Date: 2019-09-23 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/f23e180d2dfc/

Log:Fixed bugs in rrc incmark Added test case and implemented missing
check for new linked proxies

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -72,24 +72,35 @@
 #set their cyclic refcount to > 0 to mark them as live
 consistent = True
 self.snapshot_consistent = True
-# simply iterate the snapshot for objects in p_list, as linked objects 
might not be freed, except by the gc
+
+# sync p_list_old (except gc-objects)
+# simply iterate the snapshot for objects in p_list, as linked objects
+# might not be freed, except by the gc
 free_p_list = self.gc.AddressStack()
 for i in range(0, self.total_objs):
 snapobj = self.snapshot_objs[i]
 if snapobj.pypy_link == 0:
-break
+break  # only look for objects in p_list
 pyobj = llmemory.cast_adr_to_ptr(snapobj.pyobj, self.PYOBJ_HDR_PTR)
 pygchdr = self.pyobj_as_gc(pyobj)
-if (pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and
+if (pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR) and
 pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
-break
+break # only look for non-gc
 if snapobj.refcnt == 0:
+# check consistency
 consistent = pyobj.c_ob_refcnt == snapobj.refcnt_original
 if not consistent:
 break
 # move to separate list
+self.p_list_old.remove(snapobj.pyobj)
 free_p_list.append(snapobj.pyobj)
 
+# look if there is a (newly) linked non-gc proxy, where the non-rc obj
+# is unmarked
+self.p_list_old_consistent = True
+self.p_list_old.foreach(self._check_consistency_p_list_old, None)
+consistent &= self.p_list_old_consistent
+
 # sync gc objects
 pygchdr = self.pyobj_list.c_gc_next
 while pygchdr <> self.pyobj_list and consistent:
@@ -116,9 +127,14 @@
 self._gc_list_add(self.pyobj_old_list, pygchdr)
 else:
 # new object, keep alive
+pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
 pyobj = self.gc_as_pyobj(pygchdr)
-pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
-# TODO: also keep reachable objects alive (in case rc proxy -> 
non-rc -> non-rc proxy -> rc obj!!!)
+if pyobj.c_ob_pypy_link != 0:
+addr = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
+if not (self.gc.header(addr).tid &
+(self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS)):
+consistent = False
+break
 pygchdr = next_old
 
 self._debug_check_consistency(print_label="end-check-consistency")
@@ -172,11 +188,19 @@
 "refcnt original", snapobj.refcnt_original,
 "link", snapobj.pypy_link)
 
+def _check_consistency_p_list_old(self, pyobject, foo):
+pyobj = llmemory.cast_adr_to_ptr(pyobject, self.PYOBJ_HDR_PTR)
+pygchdr = self.pyobj_as_gc(pyobj)
+if (pygchdr == lltype.nullptr(self.PYOBJ_GC_HDR) and
+pyobj.c_ob_pypy_link != 0):
+addr = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
+if not (self.gc.header(addr).tid &
+(self.GCFLAG_VISITED | self.GCFLAG_NO_HEAP_PTRS)):
+self.p_list_old_consistent = False
+
 def _free_p_list(self, pyobject, foo):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
-# unlink
-self.p_list_old.remove(pyobject)
 pyobj = llmemory.cast_adr_to_ptr(pyobject, self.PYOBJ_HDR_PTR)
 refcnt = pyobj.c_ob_refcnt
 if refcnt >= REFCNT_FROM_PYPY_LIGHT:
diff --git a/rpython/memory/gc/test/dot/keep_cpython_inc_3.dot 
b/rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
copy from rpython/memory/gc/test/dot/keep_cpython_inc_3.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_inc_3.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_inc_3b.dot
@@ -5,7 +5,7 @@
 "d" [type=B, alive=y];
 "e" [type=C, alive=y];
 "f" [type=C, alive=y, ext_refcnt=1, added=after_snap];
-"g" [type=B, alive=y, added=linked_after_snap];
+"g" [type=B, alive=y, added=linked_after_snap, gc=n];
 "a" -> "b" [removed=after_snap];
 "b" -> "c";
 "c" -> "d";
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 

[pypy-commit] pypy cpyext-gc-cycle: Fixed tests for non-gc proxies in incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97577:2606a6511ff0
Date: 2019-09-21 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/2606a6511ff0/

Log:Fixed tests for non-gc proxies in incmark

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -13,11 +13,6 @@
 return True
 
 if self.state == self.STATE_DEFAULT:
-# For all non-gc pyobjects which have a refcount > 0,
-# mark all reachable objects on the pypy side
-self.p_list_old.foreach(self._major_trace_nongc, False)
-# TODO: execute incrementally (own phase)
-
 # Merge all objects whose finalizer have been executed to the
 # pyobj_list (to reprocess them again in the snapshot). Finalizers
 # can only be executed once, so termination will eventually happen.
@@ -72,10 +67,29 @@
 #  * move all dead objects still in pyob_list to pyobj_old_list
 #  * for all other objects (in snapshot and new),
 #set their cyclic refcount to > 0 to mark them as live
-pygchdr = self.pyobj_list.c_gc_next
 consistent = True
 self.snapshot_consistent = True
-while pygchdr <> self.pyobj_list: # TODO: also sync p_list
+# simply iterate the snapshot for objects in p_list, as linked objects 
might not be freed, except by the gc
+free_p_list = self.gc.AddressStack()
+for i in range(0, self.total_objs):
+snapobj = self.snapshot_objs[i]
+if snapobj.pypy_link == 0:
+break
+pyobj = llmemory.cast_adr_to_ptr(snapobj.pyobj, self.PYOBJ_HDR_PTR)
+pygchdr = self.pyobj_as_gc(pyobj)
+if (pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR) and
+pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED):
+break
+if snapobj.refcnt == 0:
+consistent = pyobj.c_ob_refcnt == snapobj.refcnt_original
+if not consistent:
+break
+# move to separate list
+free_p_list.append(snapobj.pyobj)
+
+# sync gc objects
+pygchdr = self.pyobj_list.c_gc_next
+while pygchdr <> self.pyobj_list and consistent:
 next_old = pygchdr.c_gc_next
 if pygchdr.c_gc_refs > 0: # object is in snapshot
 snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
@@ -105,6 +119,8 @@
 self._debug_check_consistency(print_label="end-check-consistency")
 
 if not consistent:  # keep all objects alive
+while free_p_list.non_empty():
+self.p_list_old.append(free_p_list.pop())
 while pygchdr <> self.pyobj_list: # continue previous loop
 pygchdr.c_gc_refs = 1
 pygchdr = pygchdr.c_gc_next
@@ -114,6 +130,8 @@
 pygchdr = pygchdr.c_gc_next
 if not self._gc_list_is_empty(self.pyobj_old_list):
 self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+else:
+free_p_list.foreach(self._free_p_list, None)
 
 self._debug_check_consistency(print_label="before-snap-discard")
 
@@ -140,6 +158,19 @@
 self._debug_check_consistency(print_label="end-mark")
 return True
 
+def _free_p_list(self, pyobject, foo):
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+# unlink
+self.p_list_old.remove(pyobject)
+pyobj = llmemory.cast_adr_to_ptr(pyobject, self.PYOBJ_HDR_PTR)
+refcnt = pyobj.c_ob_refcnt
+if refcnt >= REFCNT_FROM_PYPY_LIGHT:
+refcnt -= REFCNT_FROM_PYPY_LIGHT
+elif refcnt >= REFCNT_FROM_PYPY:
+refcnt -= REFCNT_FROM_PYPY
+pyobj.c_ob_refcnt = refcnt
+
 def _collect_roots(self):
 # Subtract all internal refcounts from the cyclic refcount
 # of rawrefcounted objects
diff --git a/rpython/memory/gc/test/dot/keep_cross_nogc_1.dot 
b/rpython/memory/gc/test/dot/keep_nocycle_nogc_1.dot
copy from rpython/memory/gc/test/dot/keep_cross_nogc_1.dot
copy to rpython/memory/gc/test/dot/keep_nocycle_nogc_1.dot
--- a/rpython/memory/gc/test/dot/keep_cross_nogc_1.dot
+++ b/rpython/memory/gc/test/dot/keep_nocycle_nogc_1.dot
@@ -1,12 +1,7 @@
 digraph G {
-"a" [type=P, alive=y];
-"b" [type=B, alive=y];
-"c" [type=C, alive=y];
-"d" [type=B, alive=y, gc=n];
-"e" [type=P, alive=y, rooted=y];
+"a" [type=C, alive=y, ext_refcnt=1];
+"b" [type=B, alive=y, gc=n];
+"c" [type=P, alive=y];
 "a" -> "b";
 "b" -> "c";
-"c" -> "d";
-"d" -> "a";
-"e" -> "d";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ 

[pypy-commit] pypy cpyext-gc-cycle: Fixed bugs in rrc incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97578:9543fe367251
Date: 2019-09-21 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/9543fe367251/

Log:Fixed bugs in rrc incmark Added incremental rrc tests and debug
output of snapshot

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -35,11 +35,13 @@
 
 # Now take a snapshot
 self._take_snapshot(self.pyobj_list)
+self._debug_print_snap(print_label="after-snapshot")
 
 # collect all rawrefcounted roots
 self._collect_roots()
 # TODO: execute incrementally (own phase, save index)
 
+self._debug_print_snap(print_label="roots-marked")
 self._debug_check_consistency(print_label="roots-marked")
 self.state = self.STATE_MARKING
 return False
@@ -52,6 +54,7 @@
 if (all_rrc_marked and not self.gc.objects_to_trace.non_empty() and
 not self.gc.more_objects_to_trace.non_empty()):
 # all objects have been marked, dead objects will stay dead
+self._debug_print_snap(print_label="before-fin")
 self._debug_check_consistency(print_label="before-fin")
 self.state = self.STATE_GARBAGE_MARKING
 else:
@@ -112,7 +115,9 @@
 self._gc_list_remove(pygchdr)
 self._gc_list_add(self.pyobj_old_list, pygchdr)
 else:
-pygchdr.c_gc_refs = 1 # new object, keep alive
+# new object, keep alive
+pyobj = self.gc_as_pyobj(pygchdr)
+pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
 # TODO: also keep reachable objects alive (in case rc proxy -> 
non-rc -> non-rc proxy -> rc obj!!!)
 pygchdr = next_old
 
@@ -122,11 +127,11 @@
 while free_p_list.non_empty():
 self.p_list_old.append(free_p_list.pop())
 while pygchdr <> self.pyobj_list: # continue previous loop
-pygchdr.c_gc_refs = 1
+pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
 pygchdr = pygchdr.c_gc_next
 pygchdr = self.pyobj_old_list.c_gc_next
 while pygchdr <> self.pyobj_old_list: # resurrect "dead" objects
-pygchdr.c_gc_refs = 1
+pygchdr.c_gc_refs = 1 << self.RAWREFCOUNT_REFS_SHIFT
 pygchdr = pygchdr.c_gc_next
 if not self._gc_list_is_empty(self.pyobj_old_list):
 self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
@@ -158,6 +163,15 @@
 self._debug_check_consistency(print_label="end-mark")
 return True
 
+def _debug_print_snap(self, print_label=None):
+debug_start("snap " + print_label)
+for i in range(0, self.total_objs):
+snapobj = self.snapshot_objs[i]
+debug_print("item", snapobj.pyobj, ": snapobj", snapobj,
+"refcnt", snapobj.refcnt,
+"refcnt original", snapobj.refcnt_original,
+"link", snapobj.pypy_link)
+
 def _free_p_list(self, pyobject, foo):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
@@ -186,7 +200,7 @@
 # refcount > 0
 
 def _mark_rawrefcount(self):
-self._gc_list_init(self.pyobj_old_list)
+self._gc_list_init(self.pyobj_old_list) # TODO: move???
 # as long as new objects with cyclic a refcount > 0 or alive border
 # objects are found, increment the refcount of all referenced objects
 # of those newly found objects
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -198,8 +198,6 @@
 pyobj = self.gc_as_pyobj(gchdr)
 obj = llmemory.NULL
 if pyobj.c_ob_pypy_link <> 0:
-#intobj = pyobj.c_ob_pypy_link
-#obj = llmemory.cast_int_to_adr(intobj)
 pyobject = llmemory.cast_ptr_to_adr(pyobj)
 obj = self.refcnt_dict.get(pyobject)
 if not alive and self.gc.header(obj).tid & (
@@ -218,8 +216,6 @@
 self._traverse(pyobj, 1)
 # mark recursively, if it is a pypyobj
 if pyobj.c_ob_pypy_link <> 0:
-#intobj = pyobj.c_ob_pypy_link
-#obj = llmemory.cast_int_to_adr(intobj)
 self.gc.objects_to_trace.append(obj)
 self.gc.visit_all_objects()
 return alive
diff --git a/rpython/memory/gc/test/dot/keep_cpython_self.dot 
b/rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
copy from rpython/memory/gc/test/dot/keep_cpython_self.dot
copy to rpython/memory/gc/test/dot/keep_cpython_inc_1.dot
--- 

[pypy-commit] pypy cpyext-gc-cycle: Fixed TODOs for rrc mark and incmark for modern finalizers

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97552:9ce0c4b2a38c
Date: 2019-09-07 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/9ce0c4b2a38c/

Log:Fixed TODOs for rrc mark and incmark for modern finalizers

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -25,8 +25,13 @@
 # this cycle.
 if not self._gc_list_is_empty(self.pyobj_old_list):
 self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
-# TODO: take snapshot of pyobj_old_list and perform _collect_roots
-#   incrementally (own phase)
+# TODO: use separate list and process it after pyobj_list has been
+#   fully processed (just before modern finalizers) if 
references
+#   to separate list are encountered during take_snapshot
+#   move them to pyobj_list and include them in the snapshot.
+#   For the remaining list (before modern finalizers), check
+#   if there are external references from marked non-rc objects
+#   (rc objects were already detected during take_snapshot)
 
 # Untrack all tuples with only non-gc rrc objects and
 # promote all other tuples to the pyobj_list
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -26,6 +26,9 @@
 dead_list_empty = True
 if not self._gc_list_is_empty(self.pyobj_old_list):
 dead_list_empty = self._check_finalizer()
+# TODO: cannot work this way -> must first do full collection of
+#   new graph, bc back ref over non-rrc from new rrc graph (#1)
+# TODO: see incmark (instead of take_snapshot during collect_roots)
 
 # collect all rawrefcounted roots
 self._collect_roots()
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Include p_list in snapshot of rrc incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97553:73cc4bd3f3ca
Date: 2019-09-19 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/73cc4bd3f3ca/

Log:Include p_list in snapshot of rrc incmark Fixed some other issues
and added some TODOs

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -41,8 +41,8 @@
 PYOBJ_SNAPSHOT_OBJ = lltype.Struct('PyObject_Snapshot',
('pyobj', llmemory.Address),
('status', lltype.Signed),
+   ('refcnt_original', lltype.Signed),
('refcnt', lltype.Signed),
-   ('refcnt_external', lltype.Signed),
('refs_index', lltype.Signed),
('refs_len', lltype.Signed),
('pypy_link', lltype.Signed))
@@ -345,6 +345,7 @@
 def _major_trace(self, pyobject, flags):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+# TODO: add flag; if set: if marked, keep rc-proxy
 (use_cylicrefcnt, use_dict) = flags
 #
 pyobj = self._pyobj(pyobject)
@@ -360,7 +361,10 @@
 else:
 rc = pyobj.c_ob_refcnt
 else:
-rc = pyobj.c_ob_refcnt
+if use_dict:
+rc = pyobj.c_ob_pypy_link
+else:
+rc = pyobj.c_ob_refcnt
 
 if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0:
 pass  # the corresponding object may die
@@ -374,7 +378,7 @@
 intobj = pyobj.c_ob_pypy_link
 obj = llmemory.cast_int_to_adr(intobj)
 self.gc.objects_to_trace.append(obj)
-self.gc.visit_all_objects()
+self.gc.visit_all_objects() # TODO: execute incrementally?
 
 def _major_trace_nongc(self, pyobject, use_dict):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
@@ -416,8 +420,7 @@
 self.p_dict = new_p_dict = self.gc.AddressDict(length_estimate)
 new_p_list = self.gc.AddressStack()
 while self.p_list_old.non_empty():
-self._major_free(self.p_list_old.pop(), new_p_list,
-  new_p_dict)
+self._major_free(self.p_list_old.pop(), new_p_list, new_p_dict)
 self.p_list_old.delete()
 self.p_list_old = new_p_list
 #
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -59,8 +59,8 @@
 # all objects have been marked, dead objects will stay dead
 self._debug_check_consistency(print_label="before-fin")
 self.state = self.STATE_GARBAGE_MARKING
-
-return False
+else:
+return False
 
 # we are finished with marking, now finish things up
 ll_assert(self.state == self.STATE_GARBAGE_MARKING, "invalid state")
@@ -75,16 +75,16 @@
 pygchdr = self.pyobj_list.c_gc_next
 consistent = True
 self.snapshot_consistent = True
-while pygchdr <> self.pyobj_list:
+while pygchdr <> self.pyobj_list: # TODO: also sync p_list
 next_old = pygchdr.c_gc_next
 if pygchdr.c_gc_refs > 0: # object is in snapshot
 snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
-pygchdr.c_gc_refs = snapobj.refcnt_external
-if snapobj.refcnt_external == 0: # object considered dead
+pygchdr.c_gc_refs = snapobj.refcnt
+if snapobj.refcnt == 0: # object considered dead
 # check consistency (dead subgraphs can never change):
 pyobj = self.gc_as_pyobj(pygchdr)
 # refcount equal
-consistent = snapobj.refcnt == pyobj.c_ob_refcnt
+consistent = snapobj.refcnt_original == pyobj.c_ob_refcnt
 if not consistent:
 break
 # outgoing (internal) references equal
@@ -99,6 +99,7 @@
 self._gc_list_add(self.pyobj_old_list, pygchdr)
 else:
 pygchdr.c_gc_refs = 1 # new object, keep alive
+# TODO: also keep reachable objects alive (in case rc proxy -> 
non-rc -> non-rc proxy -> rc obj!!!)
 pygchdr = next_old
 
 self._debug_check_consistency(print_label="end-check-consistency")
@@ -148,7 +149,7 @@
 addr = self.snapshot_refs[obj.refs_index + j]
 obj_ref = llmemory.cast_adr_to_ptr(addr,

[pypy-commit] pypy cpyext-gc-cycle: Fixed tests for non-gc proxies in mark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97554:a84af3d1e5c1
Date: 2019-09-19 12:46 +0200
http://bitbucket.org/pypy/pypy/changeset/a84af3d1e5c1/

Log:Fixed tests for non-gc proxies in mark Added some TODOs and comments

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -388,11 +388,11 @@
 pygchdr = self.pyobj_as_gc(pyobj)
 if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
 if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
-rc = 0
+rc = 0 # do not mark, we will mark them later
 else:
-rc = pyobj.c_ob_refcnt
+rc = pyobj.c_ob_refcnt # take cyclic refcount
 else:
-rc = pyobj.c_ob_refcnt
+rc = pyobj.c_ob_refcnt # take cyclic refcount
 
 if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0:
 pass  # the corresponding object may die
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -239,7 +239,7 @@
 # take snapshot of p_list_old
 self.p_list_old.foreach(self._take_snapshot_pyobject, None)
 
-# take snapshot of gc objs
+# take snapshot of gc objs TODO: include finalizer_list from last cycle
 pygchdr = pygclist.c_gc_next
 while pygchdr <> pygclist:
 pyobj = self.gc_as_pyobj(pygchdr)
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -61,8 +61,8 @@
 use_cylicrc = not found_finalizer
 self._debug_check_consistency(print_label="end-mark-cyclic")
 
-# mark all pypy objects at the border which are linked to non-gc
-# pyobjs which are not directly referenced by any gc pyobj
+# mark all pypy objects at the border which are linked to live
+# non-gc pyobjs which are not directly referenced by any gc pyobj
 debug_print("use_cylicrc", use_cylicrc)
 self.p_list_old.foreach(self._major_trace, (use_cylicrc, True))  # 
TODO: set flag to keep marked, check other occurences
 self._debug_check_consistency(print_label="end-mark")
@@ -149,8 +149,6 @@
 
 def _obj_fix_refcnt(self, pyobject, ignore):
 pyobj = self._pyobj(pyobject)
-#intobj = pyobj.c_ob_pypy_link
-#obj = llmemory.cast_int_to_adr(intobj)
 obj = self.refcnt_dict.get(pyobject)
 gchdr = self.pyobj_as_gc(pyobj)
 if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
@@ -167,6 +165,11 @@
   self.GCFLAG_NO_HEAP_PTRS):
 refcnt += 1
 self._pyobj_gc_refcnt_set(gchdr, refcnt)
+else:
+debug_print("non gc obj", obj, "real-rc", pyobj.c_ob_refcnt)
+if self.gc.header(obj).tid & (self.GCFLAG_VISITED |
+  self.GCFLAG_NO_HEAP_PTRS):
+pyobj.c_ob_refcnt += 1
 
 def _mark_rawrefcount(self):
 if self._gc_list_is_empty(self.pyobj_list):
diff --git a/rpython/memory/gc/test/dot/free_cross_simple_2.dot 
b/rpython/memory/gc/test/dot/free_cross_nogc_1.dot
copy from rpython/memory/gc/test/dot/free_cross_simple_2.dot
copy to rpython/memory/gc/test/dot/free_cross_nogc_1.dot
--- a/rpython/memory/gc/test/dot/free_cross_simple_2.dot
+++ b/rpython/memory/gc/test/dot/free_cross_nogc_1.dot
@@ -2,7 +2,7 @@
 "a" [type=P, alive=n];
 "b" [type=B, alive=n];
 "c" [type=C, alive=n];
-"d" [type=B, alive=n];
+"d" [type=B, alive=n, gc=n];
 "a" -> "b";
 "b" -> "c";
 "c" -> "d";
diff --git a/rpython/memory/gc/test/dot/free_cross_simple_2.dot 
b/rpython/memory/gc/test/dot/keep_cross_nogc_1.dot
copy from rpython/memory/gc/test/dot/free_cross_simple_2.dot
copy to rpython/memory/gc/test/dot/keep_cross_nogc_1.dot
--- a/rpython/memory/gc/test/dot/free_cross_simple_2.dot
+++ b/rpython/memory/gc/test/dot/keep_cross_nogc_1.dot
@@ -1,10 +1,12 @@
 digraph G {
-"a" [type=P, alive=n];
-"b" [type=B, alive=n];
-"c" [type=C, alive=n];
-"d" [type=B, alive=n];
+"a" [type=P, alive=y];
+"b" [type=B, alive=y];
+"c" [type=C, alive=y];
+"d" [type=B, alive=y, gc=n];
+"e" [type=P, alive=y, rooted=y];
 "a" -> "b";
 "b" -> "c";
 "c" -> "d";
 "d" -> "a";
+"e" -> "d";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -26,8 +26,8 @@
 
 class TestRawRefCount(BaseDirectGCTest):
 GCClass = IncMiniMark
-RRCGCClass = RawRefCountIncMarkGC
-#RRCGCClass = RawRefCountMarkGC
+#RRCGCClass = RawRefCountIncMarkGC
+

[pypy-commit] pypy cpyext-gc-cycle: Reverted handling of modern fin for rrc mark to improve termination

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97392:4afcd17db69f
Date: 2019-09-07 10:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4afcd17db69f/

Log:Reverted handling of modern fin for rrc mark to improve termination
Added TODOs for rrc mark and incmark for modern finalizers

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -543,6 +543,36 @@
 self.gc.trace(obj, self._collect_ref_rec, None)
 return True
 
+def _check_finalizer(self):
+# Check, if the cyclic isolate from the last collection cycle
+# is reachable from outside, after the finalizers have been
+# executed (and if all finalizers have been executed). Return
+# True if some objects are reachable and thus have been resurrected.
+
+# check if the list has been fully processed since the last cycle
+# (for safety)
+found_alive = not self._gc_list_is_empty(self.pyobj_isolate_list)
+
+# check if all finalizers have actually been called (for safety)
+if not found_alive:
+found_alive = self._find_finalizer()
+
+# check if there are any objects with a reference count > 0
+if not found_alive:
+gchdr = self.pyobj_old_list.c_gc_next
+while gchdr <> self.pyobj_old_list:
+if True: # TODO: check refcount or marked (see _collect_roots)
+found_alive = True
+break
+gchdr = gchdr.c_gc_next
+
+if found_alive:
+self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+return True
+else:
+self._gc_list_merge(self.pyobj_old_list, self.pyobj_dead_list)
+return False
+
 def _find_finalizer(self):
 gchdr = self.pyobj_old_list.c_gc_next
 while gchdr <> self.pyobj_old_list:
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -13,6 +13,11 @@
 return True
 
 if self.state == self.STATE_DEFAULT:
+# For all non-gc pyobjects which have a refcount > 0,
+# mark all reachable objects on the pypy side
+self.p_list_old.foreach(self._major_trace_nongc, False)
+# TODO: execute incrementally (own phase)
+
 # Merge all objects whose finalizer have been executed to the
 # pyobj_list (to reprocess them again in the snapshot). Finalizers
 # can only be executed once, so termination will eventually happen.
@@ -20,11 +25,13 @@
 # this cycle.
 if not self._gc_list_is_empty(self.pyobj_old_list):
 self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+# TODO: take snapshot of pyobj_old_list and perform _collect_roots
+#   incrementally (own phase)
 
 # Untrack all tuples with only non-gc rrc objects and
 # promote all other tuples to the pyobj_list
 self._untrack_tuples()
-# TODO: execute incrementally? (before snapshot!)
+# TODO: execute incrementally? (before snapshot!, own phase)
 
 # Now take a snapshot
 self._take_snapshot(self.pyobj_list)
@@ -33,11 +40,6 @@
 self._collect_roots()
 # TODO: execute incrementally (own phase, save index)
 
-# For all non-gc pyobjects which have a refcount > 0,
-# mark all reachable objects on the pypy side
-self.p_list_old.foreach(self._major_trace_nongc, False)
-# TODO: execute incrementally
-
 self._debug_check_consistency(print_label="roots-marked")
 self.state = self.STATE_MARKING
 return False
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -20,18 +20,25 @@
 # Only trace and mark rawrefcounted object if we are not doing
 # something special, like building gc.garbage.
 if self.state == self.STATE_MARKING and self.cycle_enabled:
-# Merge all objects whose finalizer have been executed to the
-# pyobj_list (to reprocess them again in the snapshot). Finalizers
-# can only be executed once, so termination will eventually happen.
-# Objects which have not been resurrected should be freed during
-# this cycle.
+
+# check if objects with finalizers from last collection cycle
+# have been resurrected
+dead_list_empty = True
 if not self._gc_list_is_empty(self.pyobj_old_list):
-self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+dead_list_empty = self._check_finalizer()
 
 # collect all 

[pypy-commit] pypy cpyext-gc-cycle: Simplified handling of rrc modern finalizers

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97391:7bc15233822f
Date: 2019-09-07 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/7bc15233822f/

Log:Simplified handling of rrc modern finalizers Added TODOs to improve
pause times in rrc incmark Fixed bug in rrc mark

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -543,33 +543,10 @@
 self.gc.trace(obj, self._collect_ref_rec, None)
 return True
 
-def _check_finalizer(self):
-# Check, if the cyclic isolate from the last collection cycle
-# is reachable from outside, after the finalizers have been
-# executed (and if all finalizers have been executed).
-found_alive = self._gc_list_is_empty(self.pyobj_isolate_list)
-if not found_alive:
-found_alive = self._find_finalizer()
-if not found_alive:
-self._collect_roots(self.pyobj_old_list)
-gchdr = self.pyobj_old_list.c_gc_next
-while gchdr <> self.pyobj_old_list:
-if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0:
-found_alive = True
-break
-gchdr = gchdr.c_gc_next
-if found_alive:
-self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
-return False
-else:
-self._gc_list_merge(self.pyobj_old_list, self.pyobj_dead_list)
-return True
-
 def _find_finalizer(self):
 gchdr = self.pyobj_old_list.c_gc_next
 while gchdr <> self.pyobj_old_list:
-if self.finalizer_type(gchdr) == \
-self.RAWREFCOUNT_FINALIZER_MODERN:
+if self.finalizer_type(gchdr) == self.RAWREFCOUNT_FINALIZER_MODERN:
 return True
 gchdr = gchdr.c_gc_next
 return False
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -13,35 +13,30 @@
 return True
 
 if self.state == self.STATE_DEFAULT:
-# First, untrack all tuples with only non-gc rrc objects and
+# Merge all objects whose finalizer have been executed to the
+# pyobj_list (to reprocess them again in the snapshot). Finalizers
+# can only be executed once, so termination will eventually happen.
+# Objects which have not been resurrected should be freed during
+# this cycle.
+if not self._gc_list_is_empty(self.pyobj_old_list):
+self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+
+# Untrack all tuples with only non-gc rrc objects and
 # promote all other tuples to the pyobj_list
 self._untrack_tuples()
-
-merged_old_list = False
-# check objects with finalizers from last collection cycle
-if not self._gc_list_is_empty(self.pyobj_old_list):
-merged_old_list = self._check_finalizer()
-
-# For all non-gc pyobjects which have a refcount > 0,
-# mark all reachable objects on the pypy side
-self.p_list_old.foreach(self._major_trace_nongc, False)
+# TODO: execute incrementally? (before snapshot!)
 
 # Now take a snapshot
 self._take_snapshot(self.pyobj_list)
 
 # collect all rawrefcounted roots
-self._collect_roots(self.pyobj_list)
+self._collect_roots()
+# TODO: execute incrementally (own phase, save index)
 
-if merged_old_list:
-# set all refcounts to zero for objects in dead list
-# (might have been incremented) by fix_refcnt
-gchdr = self.pyobj_dead_list.c_gc_next
-while gchdr <> self.pyobj_dead_list:
-if (gchdr.c_gc_refs > 0 and gchdr.c_gc_refs !=
-self.RAWREFCOUNT_REFS_UNTRACKED):
-pyobj = self.snapshot_objs[gchdr.c_gc_refs - 1]
-pyobj.refcnt_external = 0
-gchdr = gchdr.c_gc_next
+# For all non-gc pyobjects which have a refcount > 0,
+# mark all reachable objects on the pypy side
+self.p_list_old.foreach(self._major_trace_nongc, False)
+# TODO: execute incrementally
 
 self._debug_check_consistency(print_label="roots-marked")
 self.state = self.STATE_MARKING
@@ -50,6 +45,7 @@
 if self.state == self.STATE_MARKING:
 # mark all objects reachable from rawrefcounted roots
 all_rrc_marked = self._mark_rawrefcount()
+# TODO: execute incrementally
 
 if (all_rrc_marked and not self.gc.objects_to_trace.non_empty() and
 not 

[pypy-commit] pypy cpyext-gc-cycle: Added consistency check in case rrc graph changed between gc-iterations

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97352:af16e13f7cbf
Date: 2019-08-31 18:24 +0200
http://bitbucket.org/pypy/pypy/changeset/af16e13f7cbf/

Log:Added consistency check in case rrc graph changed between gc-
iterations

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -41,6 +41,7 @@
 PYOBJ_SNAPSHOT_OBJ = lltype.Struct('PyObject_Snapshot',
('pyobj', llmemory.Address),
('status', lltype.Signed),
+   ('refcnt', lltype.Signed),
('refcnt_external', lltype.Signed),
('refs_index', lltype.Signed),
('refs_len', lltype.Signed),
@@ -629,6 +630,11 @@
 gchdr.c_gc_next = next
 next.c_gc_prev = gchdr
 
+def _gc_list_remove(self, gchdr):
+next = gchdr.c_gc_next
+next.c_gc_prev = gchdr.c_gc_prev
+gchdr.c_gc_prev.c_gc_next = next
+
 def _gc_list_pop(self, pygclist):
 ret = pygclist.c_gc_next
 pygclist.c_gc_next = ret.c_gc_next
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -63,26 +63,59 @@
 ll_assert(self.state == self.STATE_GARBAGE_MARKING, "invalid state")
 
 # sync snapshot with pyob_list:
-#  * move all dead objs still in pyob_list to pyobj_old_list
+#  * check the consistency of "dead" objects and keep all of them
+#alive, in case an inconsistency is found (the graph changed
+#between two pauses, so some of those objects might be alive)
+#  * move all dead objects still in pyob_list to pyobj_old_list
 #  * for all other objects (in snapshot and new),
-#set their cyclic refcount to > 0, to mark them as live
+#set their cyclic refcount to > 0 to mark them as live
 pygchdr = self.pyobj_list.c_gc_next
+consistent = True
+self.snapshot_consistent = True
 while pygchdr <> self.pyobj_list:
 next_old = pygchdr.c_gc_next
-if pygchdr.c_gc_refs > 0:
+if pygchdr.c_gc_refs > 0: # object is in snapshot
 snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
 pygchdr.c_gc_refs = snapobj.refcnt_external
-if snapobj.refcnt_external == 0:
-# remove from old list
-next = pygchdr.c_gc_next
-next.c_gc_prev = pygchdr.c_gc_prev
-pygchdr.c_gc_prev.c_gc_next = next
-# add to new list (or not, if it is a tuple)
+if snapobj.refcnt_external == 0: # object considered dead
+# check consistency (dead subgraphs can never change):
+pyobj = self.gc_as_pyobj(pygchdr)
+# refcount equal
+consistent = snapobj.refcnt == pyobj.c_ob_refcnt
+if not consistent:
+break
+# outgoing (internal) references equal
+self.snapshot_curr = snapobj
+self.snapshot_curr_index = 0
+self._check_snapshot_traverse(pyobj)
+consistent = self.snapshot_consistent
+if not consistent:
+break
+# consistent -> prepare object for collection
+self._gc_list_remove(pygchdr)
 self._gc_list_add(self.pyobj_old_list, pygchdr)
 else:
 pygchdr.c_gc_refs = 1 # new object, keep alive
 pygchdr = next_old
 
+self._debug_check_consistency(print_label="end-check-consistency")
+
+if not consistent:  # keep all objects alive
+while pygchdr <> self.pyobj_list: # continue previous loop
+pygchdr.c_gc_refs = 1
+pygchdr = pygchdr.c_gc_next
+pygchdr = self.pyobj_old_list.c_gc_next
+while pygchdr <> self.pyobj_old_list: # resurrect "dead" objects
+pygchdr.c_gc_refs = 1
+pygchdr = pygchdr.c_gc_next
+if not self._gc_list_is_empty(self.pyobj_old_list):
+self._gc_list_merge(self.pyobj_old_list, self.pyobj_list)
+
+self._debug_check_consistency(print_label="before-snap-discard")
+
+# now the snapshot is not needed any more, discard it
+self._discard_snapshot()
+
 # handle legacy finalizers (assumption: not a lot of legacy finalizers,
 # so no need to do it incrementally)
 if self._find_garbage(False):
@@ -93,8 +126,7 @@
 # handle modern finalizers
 found_finalizer = self._find_finalizer()
  

[pypy-commit] pypy cpyext-gc-cycle: Adapted rrc gc to support incremental collections (still need to reduce pauses)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97320:adc05b1fd46b
Date: 2019-08-28 14:35 +0200
http://bitbucket.org/pypy/pypy/changeset/adc05b1fd46b/

Log:Adapted rrc gc to support incremental collections (still need to
reduce pauses)

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2402,9 +2402,12 @@
 self.visit_all_objects()
 #
 # If enabled, do a major collection step for rrc objects.
+# TODO: move up before "if remaining >= estimate // 2" to
+#   improve pause times, issues:
+# - (non-inc) mark expects all objects to be marked
+# - both do not rescan nonstack-roots
 if self.rrc_enabled:
-while not rrc_finished: # TODO: remove this line to do 
incremental collection
-rrc_finished = 
self.rrc_gc.major_collection_trace_step()
+rrc_finished = self.rrc_gc.major_collection_trace_step()
 else:
 rrc_finished = True
 
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -12,9 +12,9 @@
 self._debug_check_consistency(print_label="end-mark")
 return True
 
-elif self.state == self.STATE_DEFAULT:
-# First, untrack all tuples with only non-gc rrc objects and 
promote
-# all other tuples to the pyobj_list
+if self.state == self.STATE_DEFAULT:
+# First, untrack all tuples with only non-gc rrc objects and
+# promote all other tuples to the pyobj_list
 self._untrack_tuples()
 
 merged_old_list = False
@@ -47,17 +47,25 @@
 self.state = self.STATE_MARKING
 return False
 
-elif self.state == self.STATE_MARKING:
+if self.state == self.STATE_MARKING:
 # mark all objects reachable from rawrefcounted roots
-self._mark_rawrefcount()
+all_rrc_marked = self._mark_rawrefcount()
 
-self._debug_check_consistency(print_label="before-fin")
-self.state = self.STATE_GARBAGE_MARKING
+if (all_rrc_marked and not self.gc.objects_to_trace.non_empty() and
+not self.gc.more_objects_to_trace.non_empty()):
+# all objects have been marked, dead objects will stay dead
+self._debug_check_consistency(print_label="before-fin")
+self.state = self.STATE_GARBAGE_MARKING
+
 return False
 
-# now move all dead objs still in pyob_list to garbage
-# dead -> pyobj_old_list
-# live -> set cyclic refcount to > 0
+# we are finished with marking, now finish things up
+ll_assert(self.state == self.STATE_GARBAGE_MARKING, "invalid state")
+
+# sync snapshot with pyob_list:
+#  * move all dead objs still in pyob_list to pyobj_old_list
+#  * for all other objects (in snapshot and new),
+#set their cyclic refcount to > 0, to mark them as live
 pygchdr = self.pyobj_list.c_gc_next
 while pygchdr <> self.pyobj_list:
 next_old = pygchdr.c_gc_next
@@ -75,13 +83,15 @@
 pygchdr.c_gc_refs = 1 # new object, keep alive
 pygchdr = next_old
 
-if self._find_garbage(False):  # handle legacy finalizers
+# handle legacy finalizers (assumption: not a lot of legacy finalizers,
+# so no need to do it incrementally)
+if self._find_garbage(False):
 self._mark_garbage(False)
 self._debug_check_consistency(print_label="end-legacy-fin")
 self.state = self.STATE_DEFAULT
 
-# We are finished with marking, now finish things up
-found_finalizer = self._find_finalizer()  # modern finalizers
+# handle modern finalizers
+found_finalizer = self._find_finalizer()
 if found_finalizer:
 self._gc_list_move(self.pyobj_old_list,
self.pyobj_isolate_list)
@@ -114,19 +124,22 @@
 # as long as new objects with cyclic a refcount > 0 or alive border
 # objects are found, increment the refcount of all referenced objects
 # of those newly found objects
+reached_limit = False
 found_alive = True
+simple_limit = 0
 #
-while found_alive: # TODO: working set to improve performance?
+while found_alive and not reached_limit: # TODO: working set to 
improve performance?
 found_alive = False
 for i in range(0, self.total_objs):
 obj = self.snapshot_objs[i]
 found_alive |= self._mark_rawrefcount_obj(obj)
-#
-# now all 

[pypy-commit] pypy cpyext-gc-cycle: Fixed bug when traversing rrc lists in gc

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97319:d326d15810a4
Date: 2019-08-28 13:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d326d15810a4/

Log:Fixed bug when traversing rrc lists in gc

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -192,7 +192,7 @@
 
 # special traverse for list
 if self.C._PyList_CheckExact(pyobj) != 0:
-if pyobj.c_ob_pypy_link != 0:
+if pyobj.c_ob_pypy_link != 0: # actually a refcount now
 w_obj = from_ref(space, pyobj)
 if w_obj:
 debug_print('rrc list traverse ', pyobj)
diff --git a/rpython/memory/gc/rrc/mark.py b/rpython/memory/gc/rrc/mark.py
--- a/rpython/memory/gc/rrc/mark.py
+++ b/rpython/memory/gc/rrc/mark.py
@@ -17,7 +17,7 @@
 
 # Only trace and mark rawrefcounted object if we are not doing
 # something special, like building gc.garbage.
-if (self.state == self.STATE_MARKING and self.cycle_enabled):
+if self.state == self.STATE_MARKING and self.cycle_enabled:
 merged_old_list = False
 # check objects with finalizers from last collection cycle
 if not self._gc_list_is_empty(self.pyobj_old_list):
@@ -58,10 +58,19 @@
 self.refcnt_dict.foreach(self._fix_refcnt_back, None)
 self.refcnt_dict.delete()
 self.refcnt_dict = self.gc.AddressDict()
+self.use_refcntdict = False
 
 self.state = self.STATE_DEFAULT
 return True
 
+def to_obj(self, pyobject):
+if self.use_refcntdict:
+obj = self.refcnt_dict.get(pyobject)
+else:
+obj = llmemory.cast_int_to_adr(
+self._pyobj(pyobject).c_ob_pypy_link)
+return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
 def _collect_roots(self, pygclist):
 # Initialize the cyclic refcount with the real refcount.
 self._collect_roots_init_list(pygclist)
@@ -69,6 +78,7 @@
 # Save the real refcount of objects at border
 self.p_list_old.foreach(self._obj_save_refcnt, None)
 self.o_list_old.foreach(self._obj_save_refcnt, None)
+self.use_refcntdict = True
 
 # Subtract all internal refcounts from the cyclic refcount
 # of rawrefcounted objects
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Refactored rrc to support multiple implementations

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97188:f454ba4d28f6
Date: 2019-08-14 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/f454ba4d28f6/

Log:Refactored rrc to support multiple implementations

diff too long, truncating to 2000 out of 2308 lines

diff --git a/rpython/config/translationoption.py 
b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -105,14 +105,14 @@
  "asmgcc": [("translation.gctransformer", "framework"),
 ("translation.backend", "c")],
 }),
-ChoiceOption("cpyextgc", "Garbage Collection Strategy for cpyext",
- ["boehm", "trialdeletion", "none"],
- default="trialdeletion",
+ChoiceOption("rrcgc", "Garbage Collection Strategy for raw refcounted 
objects in cpyext",
+ ["mark", "incmark", "none"],
+ default="mark",
  requires={
-"boehm": [("translation.gc", "incminimark")],
-"trialdeletion": [("translation.gc", "incminimark")],
+"mark": [("translation.gc", "incminimark")],
+"incmark": [("translation.gc", "incminimark")],
  },
- cmdline="--cpyextgc"),
+ cmdline="--rrcgc"),
 
 # other noticeable options
 BoolOption("thread", "enable use of threading primitives",
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -75,7 +75,7 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import rgc
 from rpython.memory.gc.minimarkpage import out_of_memory
-from rpython.rtyper.lltypesystem import rffi
+from rpython.memory.gc.rrc.mark import RawRefCountBaseGC, RawRefCountMarkGC
 
 #
 # Handles the objects in 2 generations:
@@ -198,6 +198,7 @@
   ('forw', llmemory.Address))
 FORWARDSTUBPTR = lltype.Ptr(FORWARDSTUB)
 NURSARRAY = lltype.Array(llmemory.Address)
+ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 
 # 
 
@@ -400,8 +401,7 @@
 # collection.
 self.probably_young_objects_with_finalizers = self.AddressDeque()
 self.old_objects_with_finalizers = self.AddressDeque()
-p = lltype.malloc(self._ADDRARRAY, 1, flavor='raw',
-  track_allocation=False)
+p = lltype.malloc(ADDRARRAY, 1, flavor='raw', track_allocation=False)
 self.singleaddr = llmemory.cast_ptr_to_adr(p)
 #
 # Two lists of all objects with destructors.
@@ -794,7 +794,8 @@
 else:
 # This does a complete minor and major collection.
 self.minor_and_major_collection()
-self.rrc_invoke_callback()
+if self.rrc_enabled:
+self.rrc_gc.invoke_callback()
 
 def collect_step(self):
 """
@@ -807,7 +808,8 @@
 old_state = self.gc_state
 self._minor_collection()
 self.major_collection_step()
-self.rrc_invoke_callback()
+if self.rrc_enabled:
+self.rrc_gc.invoke_callback()
 return rgc._encode_states(old_state, self.gc_state)
 
 def minor_collection_with_major_progress(self, extrasize=0,
@@ -848,7 +850,8 @@
 self._minor_collection()
 self.major_collection_step(extrasize)
 
-self.rrc_invoke_callback()
+if self.rrc_enabled:
+self.rrc_gc.invoke_callback()
 
 
 def collect_and_reserve(self, totalsize):
@@ -903,7 +906,7 @@
 self.minor_collection_with_major_progress()
 else:
 # Nursery too full again.  This is likely because of
-# execute_finalizers() or rrc_invoke_callback().
+# execute_finalizers() or rrc_gc.invoke_callback()().
 # we need to fix it with another call to minor_collection()
 # ---this time only the minor part so that we are sure that
 # the nursery is empty (apart from pinned objects).
@@ -1768,7 +1771,7 @@
 #
 # visit the P list from rawrefcount, if enabled.
 if self.rrc_enabled:
-self.rrc_minor_collection_trace()
+self.rrc_gc.minor_trace()
 #
 # visit the "probably young" objects with finalizers.  They
 # all survive, except if IGNORE_FINALIZER is set.
@@ -1820,7 +1823,7 @@
 #
 # visit the P and O lists from rawrefcount, if enabled.
 if self.rrc_enabled:
-self.rrc_minor_collection_free()
+self.rrc_gc.minor_collection_free()
 #
 # Walk the list of young raw-malloced objects, and either free
 # them or make them old.
@@ -2403,7 +2406,7 @@
 

[pypy-commit] pypy cpyext-gc-cycle: WIP: adapted incremental rrc to use snapshot (legacy finalizers missing)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97190:7972e94ec0ac
Date: 2019-08-15 13:44 +0200
http://bitbucket.org/pypy/pypy/changeset/7972e94ec0ac/

Log:WIP: adapted incremental rrc to use snapshot (legacy finalizers
missing)

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3161,7 +3161,7 @@
 
 def rawrefcount_end_garbage(self):
 ll_assert(self.rrc_enabled, "rawrefcount.init not called")
-self.rrc_gc.state = RawRefCountBaseGC.STATE_MARKING
+self.rrc_gc.state = RawRefCountBaseGC.STATE_DEFAULT
 
 def rawrefcount_next_garbage_pypy(self):
 ll_assert(self.rrc_enabled, "rawrefcount.init not called")
diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -61,31 +61,33 @@
 #self._debug_check_consistency(print_label="end-legacy-fin")
 self.state = self.STATE_DEFAULT
 
-# We are finished with marking, now finish things up
-#found_finalizer = self._find_finalizer()  # modern finalizers # TODO: 
from snapshot
-#if found_finalizer:
-#self._gc_list_move(self.pyobj_old_list,
-#   self.pyobj_isolate_list)
-#use_cylicrc = not found_finalizer
-use_cylicrc = True
-
 # now move all dead objs still in pyob_list to garbage
 # dead -> pyobj_old_list
 # live -> set cyclic refcount to > 0
 pygchdr = self.pyobj_list.c_gc_next
 while pygchdr <> self.pyobj_list:
 next_old = pygchdr.c_gc_next
-snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
-pygchdr.c_gc_refs = snapobj.refcnt_external
-if snapobj.refcnt_external == 0:
-# remove from old list
-next = pygchdr.c_gc_next
-next.c_gc_prev = pygchdr.c_gc_prev
-pygchdr.c_gc_prev.c_gc_next = next
-# add to new list (or not, if it is a tuple)
-self._gc_list_add(self.pyobj_old_list, pygchdr)
+if pygchdr.c_gc_refs > 0:
+snapobj = self.snapshot_objs[pygchdr.c_gc_refs - 1]
+pygchdr.c_gc_refs = snapobj.refcnt_external
+if snapobj.refcnt_external == 0:
+# remove from old list
+next = pygchdr.c_gc_next
+next.c_gc_prev = pygchdr.c_gc_prev
+pygchdr.c_gc_prev.c_gc_next = next
+# add to new list (or not, if it is a tuple)
+self._gc_list_add(self.pyobj_old_list, pygchdr)
+else:
+pygchdr.c_gc_refs = 1 # new object, keep alive
 pygchdr = next_old
 
+# We are finished with marking, now finish things up
+found_finalizer = self._find_finalizer()  # modern finalizers
+if found_finalizer:
+self._gc_list_move(self.pyobj_old_list,
+   self.pyobj_isolate_list)
+use_cylicrc = not found_finalizer
+
 # now mark all pypy objects at the border, depending on the results
 self._debug_check_consistency(print_label="end-mark-cyclic")
 debug_print("use_cylicrc", use_cylicrc)
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: WIP: adapted incremental rrc to use snapshot (finalizers still missing)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97189:615c66be0a6a
Date: 2019-08-14 17:43 +0200
http://bitbucket.org/pypy/pypy/changeset/615c66be0a6a/

Log:WIP: adapted incremental rrc to use snapshot (finalizers still
missing)

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2392,22 +2392,28 @@
 self.more_objects_to_trace = swap
 self.visit_all_objects()
 
+rrc_finished = False
+if (not self.objects_to_trace.non_empty() and
+not self.more_objects_to_trace.non_empty()):
+#
+# 'prebuilt_root_objects' might have grown since
+# we scanned it in collect_roots() (rare case).  Rescan.
+self.collect_nonstack_roots()
+self.visit_all_objects()
+#
+# If enabled, do a major collection step for rrc objects.
+if self.rrc_enabled:
+while not rrc_finished: # TODO: remove this line to do 
incremental collection
+rrc_finished = 
self.rrc_gc.major_collection_trace_step()
+else:
+rrc_finished = True
+
 # XXX A simplifying assumption that should be checked,
 # finalizers/weak references are rare and short which means that
 # they do not need a separate state and do not need to be
 # made incremental.
 # For now, the same applies to rawrefcount'ed objects.
-if (not self.objects_to_trace.non_empty() and
-not self.more_objects_to_trace.non_empty()):
-#
-# First, 'prebuilt_root_objects' might have grown since
-# we scanned it in collect_roots() (rare case).  Rescan.
-self.collect_nonstack_roots()
-self.visit_all_objects()
-#
-if self.rrc_enabled:
-self.rrc_gc.major_collection_trace()
-#
+if rrc_finished:
 ll_assert(not (self.probably_young_objects_with_finalizers
.non_empty()),
 "probably_young_objects_with_finalizers should be empty")
@@ -2723,7 +2729,7 @@
 hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
 if self.rrc_enabled and \
-self.rrc_gc.state == RawRefCountBaseGC.STATE_MARKING:
+self.rrc_gc.state == RawRefCountBaseGC.STATE_GARBAGE_MARKING:
 hdr.tid |= GCFLAG_GARBAGE
 
 if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
@@ -3155,7 +3161,7 @@
 
 def rawrefcount_end_garbage(self):
 ll_assert(self.rrc_enabled, "rawrefcount.init not called")
-self.rrc_gc.state = RawRefCountBaseGC.STATE_DEFAULT
+self.rrc_gc.state = RawRefCountBaseGC.STATE_MARKING
 
 def rawrefcount_next_garbage_pypy(self):
 ll_assert(self.rrc_enabled, "rawrefcount.init not called")
diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -21,26 +21,31 @@
 return None
 
 class RawRefCountBaseGC(object):
-# Default state, no rawrefcount specific code is executed during normal 
marking.
+# Default state.
 STATE_DEFAULT = 0
 
+# Marking state.
+STATE_MARKING = 1
+
 # Here cyclic garbage only reachable from legacy finalizers is marked.
-STATE_MARKING = 1
+STATE_GARBAGE_MARKING = 2
 
 # The state in which cyclic garbage with legacy finalizers is traced.
 # Do not mark objects during this state, because we remove the flag
 # during tracing and we do not want to trace those objects again. Also
 # during this phase no new objects can be marked, as we are only building
 # the list of cyclic garbage.
-STATE_GARBAGE = 2
+STATE_GARBAGE = 3
 
 _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 PYOBJ_SNAPSHOT_OBJ = lltype.Struct('PyObject_Snapshot',
('pyobj', llmemory.Address),
('refcnt', lltype.Signed),
-   ('refcnt_internal', lltype.Signed),
+   ('refcnt_external', lltype.Signed),
('refs_index', lltype.Signed),
-   ('refs_len', lltype.Signed))
+   ('refs_len', lltype.Signed),
+   ('pypy_link', lltype.Signed))
+PYOBJ_SNAPSHOT_OBJ_PTR = lltype.Ptr(PYOBJ_SNAPSHOT_OBJ)
 PYOBJ_SNAPSHOT = lltype.Array(PYOBJ_SNAPSHOT_OBJ,
   hints={'nolength': True})
 PYOBJ_HDR = lltype.Struct('GCHdr_PyObject',
@@ 

[pypy-commit] pypy cpyext-gc-cycle: Fixed compilation issues with rrc incmark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97192:51bfe92174e1
Date: 2019-08-16 09:07 +0200
http://bitbucket.org/pypy/pypy/changeset/51bfe92174e1/

Log:Fixed compilation issues with rrc incmark

diff --git a/rpython/memory/gc/rrc/incmark.py b/rpython/memory/gc/rrc/incmark.py
--- a/rpython/memory/gc/rrc/incmark.py
+++ b/rpython/memory/gc/rrc/incmark.py
@@ -146,6 +146,8 @@
 obj_ref.refcnt_external += 1
 # mark recursively, if it is a pypyobj
 if snapobj.pypy_link <> 0:
+intobj = snapobj.pypy_link
+obj = llmemory.cast_int_to_adr(intobj)
 self.gc.objects_to_trace.append(obj)
 self.gc.visit_all_objects()
 # mark as processed
@@ -215,7 +217,7 @@
 #
 self_adr = rffi.cast(llmemory.Address, self_ptr)
 self = cast_adr_to_nongc_instance(RawRefCountIncMarkGC, self_adr)
-self._rrc_visit_snapshot_action(pyobj, None)
+self._take_snapshot_visit_action(pyobj, None)
 return rffi.cast(rffi.INT_real, 0)
 
 def _take_snapshot_visit_action(self, pyobj, ignore):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Adapted incremental rrc to use snapshot

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r97191:13bf4458e03a
Date: 2019-08-15 14:28 +0200
http://bitbucket.org/pypy/pypy/changeset/13bf4458e03a/

Log:Adapted incremental rrc to use snapshot

diff --git a/rpython/memory/gc/rrc/base.py b/rpython/memory/gc/rrc/base.py
--- a/rpython/memory/gc/rrc/base.py
+++ b/rpython/memory/gc/rrc/base.py
@@ -40,7 +40,7 @@
 _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 PYOBJ_SNAPSHOT_OBJ = lltype.Struct('PyObject_Snapshot',
('pyobj', llmemory.Address),
-   ('refcnt', lltype.Signed),
+   ('status', lltype.Signed),
('refcnt_external', lltype.Signed),
('refs_index', lltype.Signed),
('refs_len', lltype.Signed),
@@ -442,7 +442,6 @@
 else:
 self._free(pyobject, True)
 
-
 def _untrack_tuples(self):
 gchdr = self.tuple_list.c_gc_next
 while gchdr <> self.tuple_list:
@@ -456,136 +455,7 @@
 self._gc_list_add(self.pyobj_list, gchdr)
 gchdr = gchdr_next
 
-def _collect_roots(self, pygclist):
-# Initialize the cyclic refcount with the real refcount.
-self._collect_roots_init_list(pygclist)
-
-# Save the real refcount of objects at border
-self.p_list_old.foreach(self._obj_save_refcnt, None)
-self.o_list_old.foreach(self._obj_save_refcnt, None)
-
-# Subtract all internal refcounts from the cyclic refcount
-# of rawrefcounted objects
-self._collect_roots_subtract_internal(pygclist)
-
-# For all non-gc pyobjects which have a refcount > 0,
-# mark all reachable objects on the pypy side
-self.p_list_old.foreach(self._major_trace_nongc, True)
-
-# For every object in this set, if it is marked, add 1 as a real
-# refcount (p_list => pyobj stays alive if obj stays alive).
-self.p_list_old.foreach(self._obj_fix_refcnt, None)
-self.o_list_old.foreach(self._obj_fix_refcnt, None)
-
-# now all rawrefcounted roots or live border objects have a
-# refcount > 0
-self._debug_check_consistency(print_label="rc-initialized")
-
-def _collect_roots_init_list(self, pygclist):
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
-pygchdr = pygclist.c_gc_next
-while pygchdr <> pygclist:
-refcnt = self.gc_as_pyobj(pygchdr).c_ob_refcnt
-if refcnt >= REFCNT_FROM_PYPY_LIGHT:
-refcnt -= REFCNT_FROM_PYPY_LIGHT
-elif refcnt >= REFCNT_FROM_PYPY:
-refcnt -= REFCNT_FROM_PYPY
-self._pyobj_gc_refcnt_set(pygchdr, refcnt)
-pygchdr = pygchdr.c_gc_next
-
-def _collect_roots_subtract_internal(self, pygclist):
-pygchdr = pygclist.c_gc_next
-while pygchdr <> pygclist:
-pyobj = self.gc_as_pyobj(pygchdr)
-self._traverse(pyobj, -1)
-pygchdr = pygchdr.c_gc_next
-
-def _pyobj_gc_refcnt_set(self, pygchdr, refcnt):
-pygchdr.c_gc_refs &= self.RAWREFCOUNT_REFS_MASK_FINALIZED
-pygchdr.c_gc_refs |= refcnt << self.RAWREFCOUNT_REFS_SHIFT
-
-def _obj_save_refcnt(self, pyobject, ignore):
-pyobj = self._pyobj(pyobject)
-link = llmemory.cast_int_to_adr(pyobj.c_ob_pypy_link)
-self.refcnt_dict.setitem(pyobject, link)
-pyobj.c_ob_pypy_link = pyobj.c_ob_refcnt
-
-def _obj_fix_refcnt(self, pyobject, ignore):
-pyobj = self._pyobj(pyobject)
-#intobj = pyobj.c_ob_pypy_link
-#obj = llmemory.cast_int_to_adr(intobj)
-obj = self.refcnt_dict.get(pyobject)
-gchdr = self.pyobj_as_gc(pyobj)
-if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-rc = gchdr.c_gc_refs
-refcnt = gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
-if rc == self.RAWREFCOUNT_REFS_UNTRACKED:
-debug_print("gc obj not tracked", gchdr, ": obj", obj,
-"cyclic-rc", rc)
-else:
-debug_print("gc obj tracked", gchdr, ": obj", obj, "real-rc",
-refcnt, "gc-next",
-gchdr.c_gc_next, "gc-prev", gchdr.c_gc_prev)
-if self.gc.header(obj).tid & (self.GCFLAG_VISITED |
-  self.GCFLAG_NO_HEAP_PTRS):
-refcnt += 1
-self._pyobj_gc_refcnt_set(gchdr, refcnt)
-
-def _mark_rawrefcount(self):
-if self._gc_list_is_empty(self.pyobj_list):
-self._gc_list_init(self.pyobj_old_list)
-else:
-self._gc_list_move(self.pyobj_list, self.pyobj_old_list)
-# as long as new objects with cyclic a 

[pypy-commit] pypy cpyext-gc-cycle: Fixed some issues with rrc finalizers

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96947:b048d08d615e
Date: 2019-06-30 19:41 +0200
http://bitbucket.org/pypy/pypy/changeset/b048d08d615e/

Log:Fixed some issues with rrc finalizers

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3456,7 +3456,11 @@
 self._rrc_mark_garbage()
 self._rrc_debug_check_consistency(print_label="end-legacy-fin")
 self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
-use_cylicrc = not self._rrc_find_finalizer() # modern finalizers
+found_finalizer = self._rrc_find_finalizer() # modern finalizers
+if found_finalizer:
+self._rrc_gc_list_move(self.rrc_pyobj_old_list,
+   self.rrc_pyobj_isolate_list)
+use_cylicrc = not found_finalizer
 self._rrc_debug_check_consistency(print_label="end-mark-cyclic")
 else:
 use_cylicrc = False # don't sweep any objects in cyclic isolates
@@ -3820,15 +3824,18 @@
 def _rrc_check_finalizer(self):
 # Check, if the cyclic isolate from the last collection cycle
 # is reachable from outside, after the finalizers have been
-# executed.
-self._rrc_collect_roots(self.rrc_pyobj_old_list)
-found_alive = False
-gchdr = self.rrc_pyobj_old_list.c_gc_next
-while gchdr <> self.rrc_pyobj_old_list:
-if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0:
-found_alive = True
-break
-gchdr = gchdr.c_gc_next
+# executed (and if all finalizers have been executed).
+found_alive = self._rrc_gc_list_is_empty(self.rrc_pyobj_isolate_list)
+if not found_alive:
+found_alive = self._rrc_find_finalizer()
+if not found_alive:
+self._rrc_collect_roots(self.rrc_pyobj_old_list)
+gchdr = self.rrc_pyobj_old_list.c_gc_next
+while gchdr <> self.rrc_pyobj_old_list:
+if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0:
+found_alive = True
+break
+gchdr = gchdr.c_gc_next
 if found_alive:
 self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
 self.rrc_pyobj_list)
@@ -3840,17 +3847,13 @@
 return True
 
 def _rrc_find_finalizer(self):
-found_finalizer = False
 gchdr = self.rrc_pyobj_old_list.c_gc_next
 while gchdr <> self.rrc_pyobj_old_list:
 if self.rrc_finalizer_type(gchdr) == \
 self.RAWREFCOUNT_FINALIZER_MODERN:
-found_finalizer = True
+return True
 gchdr = gchdr.c_gc_next
-if found_finalizer:
-self._rrc_gc_list_move(self.rrc_pyobj_old_list,
-   self.rrc_pyobj_isolate_list)
-return found_finalizer
+return False
 
 def _rrc_visit(pyobj, self_ptr):
 from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -62,10 +62,9 @@
 
 def rawrefcount_finalizer_type(gc):
 pyobj = self.pyobjs[self.gcobjs.index(gc)]
-if pyobj in self.pyobjs and \
-self.pyobj_finalizer.has_key(self.pyobjs.index(pyobj)):
-# TODO: improve test, so that NONE is returned, if finalizer
-#   has already been called (only for modern)
+index = self.pyobjs.index(pyobj)
+if pyobj in self.pyobjs and self.pyobj_finalizer.has_key(index) \
+and not self.pyobj_finalized.has_key(index):
 return self.pyobj_finalizer[self.pyobjs.index(pyobj)]
 else:
 return RAWREFCOUNT_FINALIZER_NONE
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed issue with linked, non-gc rrc objects (weakrefs dicts, ...)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96948:7b33b39c8a06
Date: 2019-07-04 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/7b33b39c8a06/

Log:Fixed issue with linked, non-gc rrc objects (weakrefs dicts, ...)

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3142,6 +3142,7 @@
 self.rrc_p_dict_nurs  = self.AddressDict()  # nursery keys only
 self.rrc_dealloc_trigger_callback = dealloc_trigger_callback
 self.rrc_dealloc_pending = self.AddressStack()
+self.rrc_refcnt_dict = self.AddressDict()
 self.rrc_tp_traverse = tp_traverse
 self.rrc_pyobj_list = self._pygchdr(pyobj_list)
 self.rrc_tuple_list = self._pygchdr(tuple_list)
@@ -3231,10 +3232,6 @@
 gchdr = self._rrc_gc_list_pop(self.rrc_pyobj_isolate_list)
 self._rrc_gc_list_add(self.rrc_pyobj_old_list, gchdr)
 return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr))
-#if not self._rrc_gc_list_is_empty(self.rrc_tuple_isolate_list):
-#gchdr = self._rrc_gc_list_pop(self.rrc_tuple_isolate_list)
-#self._rrc_gc_list_add(self.rrc_tuple_old_list, gchdr)
-#return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr))
 return llmemory.NULL
 
 def rawrefcount_cyclic_garbage_head(self):
@@ -3297,8 +3294,6 @@
 if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or
  not self._rrc_gc_list_is_empty(
  self.rrc_pyobj_isolate_list) or
-# not self._rrc_gc_list_is_empty(
-# self.rrc_tuple_isolate_list) or
  not self._rrc_gc_list_is_empty(
  self.rrc_pyobj_dead_list) or
  not self._rrc_gc_list_is_empty(
@@ -3470,6 +3465,17 @@
 self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc)
 self._rrc_debug_check_consistency(print_label="end-mark")
 
+# fix refcnt back
+self.rrc_refcnt_dict.foreach(self._rrc_fix_refcnt_back, None)
+self.rrc_refcnt_dict.delete()
+self.rrc_refcnt_dict = self.AddressDict()
+
+def _rrc_fix_refcnt_back(self, pyobject, link, ignore):
+pyobj = self._pyobj(pyobject)
+link_int = llmemory.cast_adr_to_int(link, "symbolic")
+pyobj.c_ob_refcnt = pyobj.c_ob_pypy_link
+pyobj.c_ob_pypy_link = link_int
+
 def _rrc_major_trace(self, pyobject, use_cylicrefcnt):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
@@ -3495,8 +3501,7 @@
 # force the corresponding object to be alive
 debug_print("pyobj stays alive", pyobj, "rc", rc, "cyclic_rc",
 cyclic_rc)
-intobj = pyobj.c_ob_pypy_link
-obj = llmemory.cast_int_to_adr(intobj)
+obj = self.rrc_refcnt_dict.get(pyobject)
 self.objects_to_trace.append(obj)
 self.visit_all_objects()
 
@@ -3519,8 +3524,9 @@
 else:
 # force the corresponding object to be alive
 debug_print("pyobj stays alive", pyobj, "rc", rc)
-intobj = pyobj.c_ob_pypy_link
-obj = llmemory.cast_int_to_adr(intobj)
+#intobj = pyobj.c_ob_pypy_link
+#obj = llmemory.cast_int_to_adr(intobj)
+obj = self.rrc_refcnt_dict.get(pyobject)
 self.objects_to_trace.append(obj)
 self.visit_all_objects()
 
@@ -3556,6 +3562,7 @@
 # Look for any weakrefs within the trash cycle and remove the callback.
 # This is only needed for weakrefs created from rawrefcounted objects
 # because weakrefs from gc-managed objects are going away anyway.
+return
 list = self.rrc_pyobj_old_list
 gchdr = list.c_gc_next
 while gchdr <> list:
@@ -3622,6 +3629,14 @@
 # Initialize the cyclic refcount with the real refcount.
 self._rrc_collect_roots_init_list(pygclist)
 
+# Save the real refcount of objects at border
+self.rrc_p_list_old.foreach(self._rrc_obj_save_refcnt, None)
+self.rrc_o_list_old.foreach(self._rrc_obj_save_refcnt, None)
+
+# Subtract all internal refcounts from the cyclic refcount
+# of rawrefcounted objects
+self._rrc_collect_roots_subtract_internal(pygclist)
+
 # For all non-gc pyobjects which have a refcount > 0,
 # mark all reachable objects on the pypy side
 self.rrc_p_list_old.foreach(self._rrc_major_trace_nongc, None)
@@ -3631,10 +3646,6 @@
 self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
 self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None)
 
-# 

[pypy-commit] pypy cpyext-gc-cycle: Implemented tuple untracking for rrc objects

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96835:a1dc8b9e7d98
Date: 2019-06-21 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/a1dc8b9e7d98/

Log:Implemented tuple untracking for rrc objects Improved stability list
traversal and removed unneccesary debug output Fixed inheritance
issues with some cypthon slots

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1216,6 +1216,15 @@
 state.C._PyPy_init_pyobj_list = rffi.llexternal(
 '_PyPy_init_pyobj_list', [], PyGC_HeadPtr,
 compilation_info=eci, _nowrapper=True)
+state.C._PyPy_init_tuple_list = rffi.llexternal(
+'_PyPy_init_tuple_list', [], PyGC_HeadPtr,
+compilation_info=eci, _nowrapper=True)
+state.C._PyTuple_MaybeUntrack = rffi.llexternal(
+'_PyTuple_MaybeUntrack', [PyObject], lltype.Signed,
+compilation_info=eci, _nowrapper=True)
+state.C._PyList_CheckExact = rffi.llexternal(
+'_PyList_CheckExact', [PyObject], lltype.Signed,
+compilation_info=eci, _nowrapper=True)
 state.C._PyPy_gc_as_pyobj = rffi.llexternal(
 '_PyPy_gc_as_pyobj', [PyGC_HeadPtr], GCHdr_PyObject,
 compilation_info=eci, _nowrapper=True)
@@ -1348,7 +1357,9 @@
 
 # initialize the pyobj_list for the gc
 pyobj_list = space.fromcache(State).C._PyPy_init_pyobj_list()
+pyobj_tuple_list = space.fromcache(State).C._PyPy_init_tuple_list()
 rawrefcount._init_pyobj_list(pyobj_list)
+rawrefcount._init_pyobj_list(pyobj_tuple_list)
 
 # we need to call this *after* the init code above, because it might
 # indirectly call some functions which are attached to pypyAPI (e.g., we
@@ -1552,6 +1563,7 @@
  source_dir / "object.c",
  source_dir / "typeobject.c",
  source_dir / "tupleobject.c",
+ source_dir / "listobject.c",
  ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/include/listobject.h 
b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,4 +1,16 @@
-/* empty */
+#ifndef Py_LISTOBJECT_H
+#define Py_LISTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #define PyList_Check(op) \
 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LIST_SUBCLASS)
 #define PyList_CheckExact(op) ((op)->ob_type == _Type)
+
+PyAPI_FUNC(Py_ssize_t) _PyList_CheckExact(PyObject *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_LISTOBJECT_H */
\ No newline at end of file
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -295,6 +295,7 @@
 ( (type *) _PyObject_GC_NewVar(typeobj, size) )
 
 extern PyGC_Head *_pypy_rawrefcount_pyobj_list;
+extern PyGC_Head *_pypy_rawrefcount_tuple_list;
 
 #define _Py_AS_GC(o) ((PyGC_Head *)(o)-1)
 #define _Py_FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1))
@@ -327,9 +328,15 @@
 #define _PyGC_REFS_REACHABLE   (-3)
 #define _PyGC_REFS_TENTATIVELY_UNREACHABLE (-4)
 
+#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
+#define PyObject_IS_GC(o) \
+(PyType_IS_GC(Py_TYPE(o)) \
+&& (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o)))
+
 #define _PyGC_IS_TRACKED(o) (_PyGC_REFS(o) != _PyGC_REFS_UNTRACKED)
-
-#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
+#define _PyObject_GC_MAY_BE_TRACKED(obj) \
+(PyObject_IS_GC(obj) && \
+(!PyTuple_CheckExact(obj) || _PyGC_IS_TRACKED(obj)))
 
 PyAPI_FUNC(void) PyObject_GC_Track(void *);
 PyAPI_FUNC(void) PyObject_GC_UnTrack(void *);
@@ -344,6 +351,16 @@
 ((PyGC_Head *)g->gc_prev)->gc_next = g; \
 _pypy_rawrefcount_pyobj_list->gc_prev = g; \
  } while(0)
+ #define _PyObject_GC_TRACK_Tuple(o) do { \
+PyGC_Head *g = _Py_AS_GC(o); \
+if (_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED) \
+Py_FatalError("GC object already tracked"); \
+_PyGCHead_SET_REFS(g, _PyGC_REFS_REACHABLE); \
+g->gc_next = _pypy_rawrefcount_tuple_list; \
+g->gc_prev = _pypy_rawrefcount_tuple_list->gc_prev; \
+((PyGC_Head *)g->gc_prev)->gc_next = g; \
+_pypy_rawrefcount_tuple_list->gc_prev = g; \
+ } while(0)
 #define _PyObject_GC_UNTRACK(o)   do { \
 PyGC_Head *g = _Py_AS_GC(o); \
 assert(_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED); \
@@ -438,6 +455,7 @@
 PyAPI_FUNC(void) _PyPy_subtype_dealloc(PyObject *);
 PyAPI_FUNC(void) _PyPy_object_dealloc(PyObject *);
 PyAPI_FUNC(PyGC_Head *) _PyPy_init_pyobj_list();
+PyAPI_FUNC(PyGC_Head *) _PyPy_init_tuple_list();
 PyAPI_FUNC(GCHdr_PyObject *) _PyPy_gc_as_pyobj(PyGC_Head *);
 PyAPI_FUNC(PyGC_Head *) _PyPy_pyobj_as_gc(GCHdr_PyObject *);
 PyAPI_FUNC(Py_ssize_t) _PyPy_finalizer_type(PyGC_Head *);

[pypy-commit] pypy cpyext-gc-cycle: Fixed issue during forking with rrc cycle detection

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96804:d4f961b90303
Date: 2019-06-14 09:24 +0200
http://bitbucket.org/pypy/pypy/changeset/d4f961b90303/

Log:Fixed issue during forking with rrc cycle detection Fixed issue with
non-rc rrc objects

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3151,6 +3151,7 @@
 self.rrc_finalizer_type = finalizer_type
 self.rrc_clear_weakref_callback = clear_weakref_callback
 self.rrc_enabled = True
+self.rrc_cycle_enabled = True
 self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
 
 def check_no_more_rawrefcount_state(self):
@@ -3164,6 +3165,12 @@
 self.rrc_p_dict.foreach(check_value_is_null, None)
 self.rrc_p_dict_nurs.foreach(check_value_is_null, None)
 
+def deactivate_rawrefcount_cycle(self):
+self.rrc_cycle_enabled = False
+
+def activate_rawrefcount_cycle(self):
+self.rrc_cycle_enabled = True
+
 def rawrefcount_create_link_pypy(self, gcobj, pyobject):
 ll_assert(self.rrc_enabled, "rawrefcount.init not called")
 obj = llmemory.cast_ptr_to_adr(gcobj)
@@ -3405,10 +3412,13 @@
 _rrc_free._always_inline_ = True
 
 def rrc_major_collection_trace(self):
-self._rrc_debug_check_consistency(print_label="begin-mark")
+if not self.rrc_cycle_enabled:
+self._rrc_debug_check_consistency(print_label="begin-mark")
+
 # Only trace and mark rawrefcounted object if we are not doing
 # something special, like building gc.garbage.
-if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+if (self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT and
+self.rrc_cycle_enabled):
 merged_old_list = False
 # check objects with finalizers from last collection cycle
 if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
@@ -3477,6 +3487,30 @@
 self.objects_to_trace.append(obj)
 self.visit_all_objects()
 
+def _rrc_major_trace_nongc(self, pyobject, ignore):
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+#
+pyobj = self._pyobj(pyobject)
+pygchdr = self.rrc_pyobj_as_gc(pyobj)
+if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
+rc = 0
+else:
+rc = pyobj.c_ob_refcnt
+else:
+rc = pyobj.c_ob_refcnt
+
+if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0:
+pass  # the corresponding object may die
+else:
+# force the corresponding object to be alive
+debug_print("pyobj stays alive", pyobj, "rc", rc)
+intobj = pyobj.c_ob_pypy_link
+obj = llmemory.cast_int_to_adr(intobj)
+self.objects_to_trace.append(obj)
+self.visit_all_objects()
+
 def rrc_major_collection_free(self):
 if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
 if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
@@ -3572,6 +3606,10 @@
 self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt)
 pygchdr = pygchdr.c_gc_next
 
+# For all non-gc pyobjects which have a refcount > 0,
+# mark all reachable objects on the pypy side
+self.rrc_p_list_old.foreach(self._rrc_major_trace_nongc, None)
+
 # For every object in this set, if it is marked, add 1 as a real
 # refcount (p_list => pyobj stays alive if obj stays alive).
 self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
diff --git a/rpython/memory/gctransform/shadowstack.py 
b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -187,6 +187,7 @@
 
 def switch_shadow_stacks(new_tid):
 # we have the wrong shadowstack right now, but it should not matter
+gcdata.gc.deactivate_rawrefcount_cycle()
 thread_stacks = gcdata.thread_stacks
 try:
 if thread_stacks is None:
@@ -210,6 +211,7 @@
 shadow_stack_pool.start_fresh_new_state()
 # done
 #
+gcdata.gc.activate_rawrefcount_cycle()
 gcdata.active_tid = new_tid
 switch_shadow_stacks._dont_inline_ = True
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed some issues with tracking/deallocating cpyext gc objects

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96723:0b73323d80cc
Date: 2019-05-30 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/0b73323d80cc/

Log:Fixed some issues with tracking/deallocating cpyext gc objects Added
rudimentary support for lists (gc issues, clear missing)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1169,6 +1169,11 @@
 [rffi.VOIDP], lltype.Void,
 compilation_info=eci,
 _nowrapper=True)
+state.C.PyObject_GC_Del = rffi.llexternal(
+mangle_name(prefix, 'PyObject_GC_Del'),
+[rffi.VOIDP], lltype.Void,
+compilation_info=eci,
+_nowrapper=True)
 state.C.PyType_GenericAlloc = rffi.llexternal(
 mangle_name(prefix, 'PyType_GenericAlloc'),
 [PyTypeObjectPtr, Py_ssize_t], PyObject,
@@ -1186,6 +1191,9 @@
 '_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
 compilation_info=eci, _nowrapper=True)
 from pypy.module.cpyext.typeobjectdefs import visitproc
+state.C.PyObject_GC_Track = rffi.llexternal(
+'PyObject_GC_Track', [rffi.VOIDP], lltype.Void,
+compilation_info=eci, _nowrapper=True)
 state.C._PyPy_tuple_traverse = rffi.llexternal(
 '_PyPy_tuple_traverse', [PyObject, visitproc, rffi.VOIDP],
 rffi.INT, compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -8,7 +8,6 @@
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
-
 PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
 
 @cpython_api([Py_ssize_t], PyObject)
@@ -30,6 +29,33 @@
 w_list.convert_to_cpy_strategy(space)
 return CPyListStrategy.unerase(w_list.lstorage)
 
+def list_traverse(space, w_list, visit, args):
+from rpython.rlib.rawrefcount import PYOBJ_HDR_PTR
+from pypy.module.cpyext.sequence import CPyListStrategy
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rlib.debug import debug_print
+assert isinstance(w_list, W_ListObject)
+obj = llmemory.cast_ptr_to_adr(w_list.lstorage)
+objint = llmemory.cast_adr_to_int(obj, "forced")
+if objint != 0:
+cpy_strategy = space.fromcache(CPyListStrategy)
+if not cpy_strategy.locked and cpy_strategy == w_list.strategy:
+debug_print('rrc do traverse ', w_list)
+index = 0
+storage = CPyListStrategy.unerase(w_list.lstorage)
+while index < storage._length:
+pyobj = storage._elems[index]
+pyobj_hdr = rffi.cast(PYOBJ_HDR_PTR, pyobj)
+if pyobj_hdr:
+vret = rffi.cast(lltype.Signed, visit(pyobj_hdr, args))
+if vret:
+return vret
+index += 1
+else:
+debug_print('rrc no traverse', w_list, 'locked',
+cpy_strategy.locked, 'strategy', w_list.strategy)
+return 0
+
 @cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, 
error=CANNOT_FAIL)
 def PyList_SET_ITEM(space, w_list, index, py_item):
 """Form of PyList_SetItem() without error checking. This is normally
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -77,9 +77,9 @@
 state = space.fromcache(State)
 return state.C._PyPy_subtype_dealloc
 
-def get_free(self, space):
+def get_free(self, space, gc):
 state = space.fromcache(State)
-return state.C.PyObject_Free
+return state.C.PyObject_GC_Del if gc else state.C.PyObject_Free
 
 def has_traverse(self, space):
 return False
@@ -156,7 +156,7 @@
 return tp_dealloc
 
 if tp_free:
-def get_free(self, space):
+def get_free(self, space, gc):
 return tp_free
 
 if tp_attach:
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -281,6 +281,7 @@
 erase, unerase = rerased.new_erasing_pair("cpylist")
 erase = staticmethod(erase)
 unerase = staticmethod(unerase)
+locked = False
 
 def _check_index(self, index, length):
 if index < 0:
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -132,10 +132,17 @@
 PyObject *
 PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems)
 {
+PyObject *obj;
+
 if (PyType_IS_GC(type))
-return (PyObject*)_PyObject_GC_NewVar(type, nitems);
+obj = (PyObject*)_PyObject_GC_NewVar(type, nitems);
 else
-return 

[pypy-commit] pypy cpyext-gc-cycle: Fixed untracked rawrefcounted objects bug and improved debug output

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96631:5621f2f5f876
Date: 2019-05-17 09:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5621f2f5f876/

Log:Fixed untracked rawrefcounted objects bug and improved debug output

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3119,7 +3119,7 @@
 RAWREFCOUNT_FINALIZER_LEGACY = 2
 RAWREFCOUNT_REFS_SHIFT = 1
 RAWREFCOUNT_REFS_MASK_FINALIZED = 1
-RAWREFCOUNT_REFS_UNTRACKED = -2
+RAWREFCOUNT_REFS_UNTRACKED = -2 << RAWREFCOUNT_REFS_SHIFT
 
 def _pyobj(self, pyobjaddr):
 return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3595,10 +3595,15 @@
 obj = llmemory.cast_int_to_adr(intobj)
 gchdr = self.rrc_pyobj_as_gc(pyobj)
 if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
-debug_print("gc obj not tracked", gchdr, ": obj", obj)
+rc = gchdr.c_gc_refs
+refcnt = pyobj.c_ob_refcnt
+if rc == self.RAWREFCOUNT_REFS_UNTRACKED:
+debug_print("gc obj not tracked", gchdr, ": obj", obj,
+"real-rc", refcnt, "cyclic-rc", rc)
 else:
-refcnt = pyobj.c_ob_refcnt
+debug_print("gc obj tracked", gchdr, ": obj", obj, "real-rc",
+refcnt, "cyclic-rc", rc, "gc-next",
+gchdr.c_gc_next, "gc-prev", gchdr.c_gc_prev)
 if refcnt >= REFCNT_FROM_PYPY_LIGHT:
 refcnt -= REFCNT_FROM_PYPY_LIGHT
 elif refcnt >= REFCNT_FROM_PYPY:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed untracked objects in rawrefcount tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96630:1c39776f6612
Date: 2019-05-16 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/1c39776f6612/

Log:Fixed untracked objects in rawrefcount tests

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3448,11 +3448,11 @@
 if use_cylicrefcnt:
 pygchdr = self.rrc_pyobj_as_gc(pyobj)
 if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
-#if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
-rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
-cyclic_rc = rc
-#else:
-#rc = pyobj.c_ob_refcnt
+if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
+rc = pygchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT
+cyclic_rc = rc
+else:
+rc = pyobj.c_ob_refcnt
 else:
 rc = pyobj.c_ob_refcnt
 else:
@@ -3595,18 +3595,18 @@
 obj = llmemory.cast_int_to_adr(intobj)
 gchdr = self.rrc_pyobj_as_gc(pyobj)
 if gchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-#if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
-#debug_print("gc obj not tracked", gchdr, ": obj", obj)
-#else:
-refcnt = pyobj.c_ob_refcnt
-if refcnt >= REFCNT_FROM_PYPY_LIGHT:
-refcnt -= REFCNT_FROM_PYPY_LIGHT
-elif refcnt >= REFCNT_FROM_PYPY:
-refcnt -= REFCNT_FROM_PYPY
-if self.header(obj).tid & (GCFLAG_VISITED |
-   GCFLAG_NO_HEAP_PTRS):
-refcnt += 1
-self._rrc_pyobj_gc_refcnt_set(gchdr, refcnt)
+if gchdr.c_gc_refs == self.RAWREFCOUNT_REFS_UNTRACKED:
+debug_print("gc obj not tracked", gchdr, ": obj", obj)
+else:
+refcnt = pyobj.c_ob_refcnt
+if refcnt >= REFCNT_FROM_PYPY_LIGHT:
+refcnt -= REFCNT_FROM_PYPY_LIGHT
+elif refcnt >= REFCNT_FROM_PYPY:
+refcnt -= REFCNT_FROM_PYPY
+if self.header(obj).tid & (GCFLAG_VISITED |
+   GCFLAG_NO_HEAP_PTRS):
+refcnt += 1
+self._rrc_pyobj_gc_refcnt_set(gchdr, refcnt)
 
 def _rrc_mark_rawrefcount(self):
 if self._rrc_gc_list_is_empty(self.rrc_pyobj_list):
@@ -3779,8 +3779,9 @@
 def _rrc_visit_action(self, pyobj, ignore):
 pygchdr = self.rrc_pyobj_as_gc(pyobj)
 if pygchdr <> lltype.nullptr(self.PYOBJ_GC_HDR):
-pygchdr.c_gc_refs += self.rrc_refcnt_add << \
- self.RAWREFCOUNT_REFS_SHIFT
+if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED:
+pygchdr.c_gc_refs += self.rrc_refcnt_add << \
+ self.RAWREFCOUNT_REFS_SHIFT
 
 def _rrc_traverse(self, pyobj, refcnt_add):
 from rpython.rlib.objectmodel import we_are_translated
diff --git a/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot 
b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
@@ -1,8 +1,8 @@
 digraph G {
 "a" [type=C, alive=n];
-"b" [type=C, alive=n, tracked=n];
-"c" [type=C, alive=n];
+"b" [type=C, alive=n];
+"c" [type=C, alive=n, tracked=n];
 "a" -> "b";
-"b" -> "c";
-"c" -> "a"
+"b" -> "a";
+"b" -> "c"
 }
diff --git a/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot 
b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
@@ -1,8 +1,8 @@
 digraph G {
 "a" [type=C, alive=y, ext_refcnt=1];
-"b" [type=C, alive=y, tracked=n];
-"c" [type=C, alive=y];
+"b" [type=C, alive=y];
+"c" [type=C, alive=y, tracked=n];
 "a" -> "b";
+"b" -> "a";
 "b" -> "c";
-"c" -> "a";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -660,9 +660,10 @@
 finalize_modern(pyobj)
 if pyobj.c_ob_refcnt == 0:
 gchdr = self.gc.rrc_pyobj_as_gc(pyobj)
-next = gchdr.c_gc_next
-next.c_gc_prev = gchdr.c_gc_prev
-gchdr.c_gc_prev.c_gc_next = next
+if gchdr.c_gc_refs != RAWREFCOUNT_REFS_UNTRACKED:
+next = gchdr.c_gc_next
+next.c_gc_prev = gchdr.c_gc_prev
+   

[pypy-commit] pypy cpyext-gc-cycle: Added support for untracked objects in rawrefcount tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96629:ec43b45f7cc4
Date: 2019-05-16 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/ec43b45f7cc4/

Log:Added support for untracked objects in rawrefcount tests Fixed a bug
in incminimark when using rawrefcounted modern finalizers

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3401,11 +3401,19 @@
 # Only trace and mark rawrefcounted object if we are not doing
 # something special, like building gc.garbage.
 if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+merged_old_list = False
 # check objects with finalizers from last collection cycle
 if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
-self._rrc_check_finalizer()
+merged_old_list = self._rrc_check_finalizer()
 # collect all rawrefcounted roots
 self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
+if merged_old_list:
+# set all refcounts to zero for objects in dead list
+# (might have been incremented) by fix_refcnt
+gchdr = self.rrc_pyobj_dead_list.c_gc_next
+while gchdr <> self.rrc_pyobj_dead_list:
+gchdr.c_gc_refs = 0
+gchdr = gchdr.c_gc_next
 self._rrc_debug_check_consistency(print_label="roots-marked")
 # mark all objects reachable from rawrefcounted roots
 self._rrc_mark_rawrefcount()
@@ -3740,10 +3748,12 @@
 if found_alive:
 self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
 self.rrc_pyobj_list)
+return False
 else:
 self._rrc_clear_weakref_callbacks()
 self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
 self.rrc_pyobj_dead_list)
+return True
 
 def _rrc_find_finalizer(self):
 found_finalizer = False
@@ -3841,10 +3851,11 @@
should_print, "rrc_pyobj_old_list")
 self._rrc_debug_check_list(self.rrc_pyobj_dead_list,
should_print, "rrc_pyobj_dead_list")
-self._rrc_debug_check_list(self.rrc_pyobj_garbage_list,
-   should_print, "rrc_pyobj_garbage_list")
 self._rrc_debug_check_list(self.rrc_pyobj_isolate_list,
should_print, "rrc_pyobj_isolate_list")
+# rrc_pyobj_garbage_list is not a real list, it just marks the
+# first and the last object in rrc_pyobj_list, which are garbage
+
 if should_print:
 debug_stop("rrc-lists " + print_label)
 
diff --git a/rpython/memory/gc/test/dot/free_cpython_simple.dot 
b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
copy from rpython/memory/gc/test/dot/free_cpython_simple.dot
copy to rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/free_cpython_simple.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_untracked_1.dot
@@ -1,6 +1,8 @@
 digraph G {
 "a" [type=C, alive=n];
-"b" [type=C, alive=n];
+"b" [type=C, alive=n, tracked=n];
+"c" [type=C, alive=n];
 "a" -> "b";
-"b" -> "a";
+"b" -> "c";
+"c" -> "a"
 }
diff --git a/rpython/memory/gc/test/dot/keep_cpython_self.dot 
b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
copy from rpython/memory/gc/test/dot/keep_cpython_self.dot
copy to rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
--- a/rpython/memory/gc/test/dot/keep_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/keep_cpython_untracked_1.dot
@@ -1,4 +1,8 @@
 digraph G {
 "a" [type=C, alive=y, ext_refcnt=1];
-"a" -> "a";
+"b" [type=C, alive=y, tracked=n];
+"c" [type=C, alive=y];
+"a" -> "b";
+"b" -> "c";
+"c" -> "a";
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -12,6 +12,7 @@
 RAWREFCOUNT_FINALIZER_MODERN = IncMiniMark.RAWREFCOUNT_FINALIZER_MODERN
 RAWREFCOUNT_FINALIZER_LEGACY = IncMiniMark.RAWREFCOUNT_FINALIZER_LEGACY
 RAWREFCOUNT_FINALIZER_NONE = IncMiniMark.RAWREFCOUNT_FINALIZER_NONE
+RAWREFCOUNT_REFS_UNTRACKED = IncMiniMark.RAWREFCOUNT_REFS_UNTRACKED
 
 S = lltype.GcForwardReference()
 S.become(lltype.GcStruct('S',
@@ -136,7 +137,8 @@
 
 return p1, p1ref, check_alive
 
-def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True):
+def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True,
+   tracked=True):
 r1 = lltype.malloc(PYOBJ_HDR, flavor='raw',
immortal=create_immortal)
 

[pypy-commit] pypy cpyext-gc-cycle: Fixed tp_clear call in cpyext (GIL handling)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96623:074fd35b8463
Date: 2019-05-16 07:56 +0200
http://bitbucket.org/pypy/pypy/changeset/074fd35b8463/

Log:Fixed tp_clear call in cpyext (GIL handling) Fixed some inheritance
issues with cpyext objects Added debug output for rrc lists WIP:
implement gc support for cpyext tuples WIP: handling of untracked
rrc objects in incminimark (keep flag, etc)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1185,6 +1185,10 @@
 state.C._PyPy_tuple_free = rffi.llexternal(
 '_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
 compilation_info=eci, _nowrapper=True)
+from pypy.module.cpyext.typeobjectdefs import visitproc
+state.C._PyPy_tuple_traverse = rffi.llexternal(
+'_PyPy_tuple_traverse', [PyObject, visitproc, rffi.VOIDP],
+rffi.INT, compilation_info=eci, _nowrapper=True)
 state.C._PyPy_tuple_dealloc = rffi.llexternal(
 '_PyPy_tuple_dealloc', [PyObject], lltype.Void,
 compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/tupleobject.h 
b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -19,6 +19,7 @@
 PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
 PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
 PyAPI_FUNC(void) _PyPy_tuple_free(void *);
+PyAPI_FUNC(int) _PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg);
 
 /* defined in varargswrapper.c */
 PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -81,6 +81,12 @@
 state = space.fromcache(State)
 return state.C.PyObject_Free
 
+def has_traverse(self, space):
+return False
+
+def get_traverse(self, space):
+return 0
+
 def allocate(self, space, w_type, itemcount=0, immortal=False):
 state = space.fromcache(State)
 ob_type = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
@@ -129,6 +135,7 @@
 tp_realize= kw.pop('realize', None)
 tp_dealloc= kw.pop('dealloc', None)
 tp_free   = kw.pop('free', None)
+tp_traverse   = kw.pop('traverse', None)
 assert not kw, "Extra arguments to make_typedescr"
 
 null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void))
@@ -159,6 +166,17 @@
 if tp_realize:
 def realize(self, space, ref):
 return tp_realize(space, ref)
+
+if tp_traverse:
+def get_traverse(self, space):
+from rpython.rtyper.lltypesystem import llmemory
+from pypy.module.cpyext.typeobjectdefs import traverseproc
+ptr = rffi.cast(traverseproc, tp_traverse)
+obj = llmemory.cast_ptr_to_adr(ptr)
+return llmemory.cast_adr_to_int(obj, "symbolic")
+
+def has_traverse(self, space):
+return True
 if typedef:
 CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/src/tupleobject.c 
b/pypy/module/cpyext/src/tupleobject.c
--- a/pypy/module/cpyext/src/tupleobject.c
+++ b/pypy/module/cpyext/src/tupleobject.c
@@ -94,4 +94,15 @@
 _PyPy_tuple_free(void *obj)
 {
 PyObject_GC_Del(obj);
+}
+
+int
+_PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg)
+{
+PyTupleObject *o = (PyTupleObject *)ob;
+Py_ssize_t i;
+
+for (i = Py_SIZE(o); --i >= 0; )
+Py_VISIT(o->ob_item[i]);
+return 0;
 }
\ No newline at end of file
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -6,6 +6,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
+from rpython.rlib.debug import debug_print
 import sys
 
 
@@ -177,6 +178,7 @@
 def _tp_traverse(pyobj_ptr, callback, args):
 from pypy.module.cpyext.api import PyObject
 from pypy.module.cpyext.typeobjectdefs import visitproc
+from pypy.module.cpyext.pyobject import cts
 # convert to pointers with correct types (PyObject)
 callback_addr = llmemory.cast_ptr_to_adr(callback)
 callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
@@ -184,9 +186,31 @@
 pyobj_addr = llmemory.cast_ptr_to_adr(pyobj_ptr)
 pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject)
 # now call tp_traverse (if possible)
-if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
-pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
-  

[pypy-commit] pypy cpyext-gc-cycle: Fixed some (de)allocation bugs in existing and derived gc cpyext-classes

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96591:0dff3316692e
Date: 2019-05-09 23:32 +0200
http://bitbucket.org/pypy/pypy/changeset/0dff3316692e/

Log:Fixed some (de)allocation bugs in existing and derived gc cpyext-
classes

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -129,7 +129,7 @@
 Py_TPFLAGS_READY Py_TPFLAGS_READYING
 METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
 METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
-Py_TPFLAGS_HEAPTYPE
+Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_GC
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
 Py_CLEANUP_SUPPORTED
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE PyBUF_WRITE
@@ -1182,6 +1182,9 @@
 [Py_ssize_t], PyObject,
 compilation_info=eci,
 _nowrapper=True)
+state.C._PyPy_tuple_free = rffi.llexternal(
+'_PyPy_tuple_free', [rffi.VOIDP], lltype.Void,
+compilation_info=eci, _nowrapper=True)
 state.C._PyPy_tuple_dealloc = rffi.llexternal(
 '_PyPy_tuple_dealloc', [PyObject], lltype.Void,
 compilation_info=eci, _nowrapper=True)
diff --git a/pypy/module/cpyext/include/tupleobject.h 
b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -18,6 +18,7 @@
 
 PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size);
 PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *);
+PyAPI_FUNC(void) _PyPy_tuple_free(void *);
 
 /* defined in varargswrapper.c */
 PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -7,7 +7,7 @@
 from pypy.module.cpyext.api import (
 cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR,
 CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject,
-PyVarObject, Py_ssize_t, init_function, cts)
+PyVarObject, Py_ssize_t, init_function, cts, Py_TPFLAGS_HAVE_GC)
 from pypy.module.cpyext.state import State
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.objspace.std.noneobject import W_NoneObject
@@ -77,37 +77,17 @@
 state = space.fromcache(State)
 return state.C._PyPy_subtype_dealloc
 
-# CCC port to C
+def get_free(self, space):
+state = space.fromcache(State)
+return state.C.PyObject_Free
+
 def allocate(self, space, w_type, itemcount=0, immortal=False):
-# typically called from PyType_GenericAlloc via typedescr.allocate
-# this returns a PyObject with ob_refcnt == 1.
-
-pytype = as_pyobj(space, w_type)
-pytype = rffi.cast(PyTypeObjectPtr, pytype)
-assert pytype
-# Don't increase refcount for non-heaptypes
-flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-if flags & Py_TPFLAGS_HEAPTYPE:
-incref(space, pytype)
-
-if pytype:
-size = pytype.c_tp_basicsize
-else:
-size = rffi.sizeof(self.basestruct)
-if pytype.c_tp_itemsize:
-size += itemcount * pytype.c_tp_itemsize
-assert size >= rffi.sizeof(PyObject.TO)
-buf = lltype.malloc(rffi.VOIDP.TO, size,
-flavor='raw', zero=True,
-add_memory_pressure=True, immortal=immortal)
-pyobj = rffi.cast(PyObject, buf)
-if pytype.c_tp_itemsize:
-pyvarobj = rffi.cast(PyVarObject, pyobj)
-pyvarobj.c_ob_size = itemcount
-pyobj.c_ob_refcnt = 1
-#pyobj.c_ob_pypy_link should get assigned very quickly
-pyobj.c_ob_type = pytype
-return pyobj
+state = space.fromcache(State)
+ob_type = rffi.cast(PyTypeObjectPtr, as_pyobj(space, w_type))
+ptup = state.ccall("PyType_GenericAlloc", ob_type, itemcount)
+if not ptup:
+state.check_and_raise_exception(always=True)
+return ptup
 
 def attach(self, space, pyobj, w_obj, w_userdata=None):
 pass
@@ -140,6 +120,7 @@
 attach: Function called to tie a raw structure to a pypy object
 realize   : Function called to create a pypy object from a raw struct
 dealloc   : a @slot_function(), similar to PyObject_dealloc
+free  : a @slot_function(), similar to PyObject_free
 """
 
 tp_basestruct = kw.pop('basestruct', PyObject.TO)
@@ -147,6 +128,7 @@
 tp_attach = kw.pop('attach', None)
 tp_realize= kw.pop('realize', None)
 tp_dealloc= kw.pop('dealloc', None)
+tp_free   = kw.pop('free', None)
 assert not kw, "Extra arguments to make_typedescr"
 
 null_dealloc = lltype.nullptr(lltype.FuncType([PyObject], lltype.Void))
@@ -166,6 +148,10 @@
 def get_dealloc(self, space):
 return tp_dealloc
 
+if tp_free:
+def get_free(self, space):
+ 

[pypy-commit] pypy cpyext-gc-cycle: Added some weakref tests to dot tests to cover important cases

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96526:e3d2ca7b1fe6
Date: 2019-04-20 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/e3d2ca7b1fe6/

Log:Added some weakref tests to dot tests to cover important cases

diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot
rename from rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
rename to rpython/memory/gc/test/dot/free_cpython_weakref_simple_1.dot
diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_2.dot
copy from rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
copy to rpython/memory/gc/test/dot/free_cpython_weakref_simple_2.dot
--- a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_2.dot
@@ -1,7 +1,8 @@
 digraph G {
 "a" [type=C, alive=n];
 "b" [type=C, alive=n];
+"c" [type=C, alive=y, ext_refcnt=1];
 "a" -> "b";
 "b" -> "a";
-"a" -> "b" [weakref=y, callback=y, clear_callback=y];
+"a" -> "c" [weakref=y, callback=y, clear_callback=y];
 }
diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_3.dot
copy from rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
copy to rpython/memory/gc/test/dot/free_cpython_weakref_simple_3.dot
--- a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_weakref_simple_3.dot
@@ -1,7 +1,8 @@
 digraph G {
 "a" [type=C, alive=n];
 "b" [type=C, alive=n];
+"c" [type=C, alive=y, ext_refcnt=1];
 "a" -> "b";
 "b" -> "a";
-"a" -> "b" [weakref=y, callback=y, clear_callback=y];
+"c" -> "a" [weakref=y, callback=y, clear_callback=n];
 }
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Improved weakref support for rawrefcount dot tests and added test

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96525:31150aa29431
Date: 2019-04-20 14:29 +0200
http://bitbucket.org/pypy/pypy/changeset/31150aa29431/

Log:Improved weakref support for rawrefcount dot tests and added test

diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_multi_1.dot
rename from rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot
rename to rpython/memory/gc/test/dot/free_cpython_weakref_multi_1.dot
diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_multi_2.dot
copy from rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot
copy to rpython/memory/gc/test/dot/free_cpython_weakref_multi_2.dot
--- a/rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_weakref_multi_2.dot
@@ -4,5 +4,5 @@
 "a" -> "b";
 "b" -> "a";
 "a" -> "b" [weakref=y, callback=y, clear_callback=y];
-"b" -> "a" [weakref=y, callback=y, clear_callback=y];
+"a" -> "b" [weakref=y, callback=y, clear_callback=y];
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -598,6 +598,7 @@
 
 # quick self check, if traverse works properly
 dests_by_source = {}
+weakrefs_added = []
 for e in g.get_edges():
 source = nodes[e.get_source()]
 dest = nodes[e.get_destination()]
@@ -607,9 +608,11 @@
 if not dests_by_source.has_key(source):
 dests_by_source[source] = []
 if weakref:
-wrs = self.pyobj_weakrefs[self.pyobjs.index(source.r)]
-# currently only one weakref supported
-dests_by_source[source].append(wrs[0].r)
+if source not in weakrefs_added: # add all weakrefs at once
+weakrefs_added.append(source)
+wrs = self.pyobj_weakrefs[self.pyobjs.index(source.r)]
+for wr in wrs:
+dests_by_source[source].append(wr.r)
 else:
 dests_by_source[source].append(dest.r)
 for source in dests_by_source:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed an error in rawrefcount dot tests and added a weakref test

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96524:3466abcd4f70
Date: 2019-04-20 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3466abcd4f70/

Log:Fixed an error in rawrefcount dot tests and added a weakref test

diff --git a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot
copy from rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
copy to rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot
--- a/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_weakref_multi.dot
@@ -4,4 +4,5 @@
 "a" -> "b";
 "b" -> "a";
 "a" -> "b" [weakref=y, callback=y, clear_callback=y];
+"b" -> "a" [weakref=y, callback=y, clear_callback=y];
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -76,7 +76,6 @@
 if gc._obj.container == weakref.p._obj:
 weakref.callback_cleared = True
 cleared = True
-assert cleared
 
 self.pyobj_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw',
 immortal=True)
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Implemented support for weakrefs in rawrefcount dot tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96523:b14c3b4ba179
Date: 2019-04-20 11:38 +0200
http://bitbucket.org/pypy/pypy/changeset/b14c3b4ba179/

Log:Implemented support for weakrefs in rawrefcount dot tests Added
first dot test with weakrefs Added simple cpython-only-cycle dot
test

diff --git a/rpython/memory/gc/test/dot/free_cpython_self.dot 
b/rpython/memory/gc/test/dot/free_cpython_simple.dot
copy from rpython/memory/gc/test/dot/free_cpython_self.dot
copy to rpython/memory/gc/test/dot/free_cpython_simple.dot
--- a/rpython/memory/gc/test/dot/free_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_simple.dot
@@ -1,4 +1,6 @@
 digraph G {
 "a" [type=C, alive=n];
-"a" -> "a";
+"b" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
 }
diff --git a/rpython/memory/gc/test/dot/free_cpython_self.dot 
b/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
copy from rpython/memory/gc/test/dot/free_cpython_self.dot
copy to rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
--- a/rpython/memory/gc/test/dot/free_cpython_self.dot
+++ b/rpython/memory/gc/test/dot/free_cpython_weakref_simple.dot
@@ -1,4 +1,7 @@
 digraph G {
 "a" [type=C, alive=n];
-"a" -> "a";
+"b" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
+"a" -> "b" [weakref=y, callback=y, clear_callback=y];
 }
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -30,21 +30,34 @@
 self.gcobjs = []
 self.pyobjs = []
 self.pyobj_refs = []
+self.pyobj_weakrefs = []
 self.pyobj_finalizer = {}
 self.pyobj_finalized = {}
 self.pyobj_resurrect = {}
 self.pyobj_delete = {}
+self.is_pygc = []
 
 def rawrefcount_tp_traverse(obj, callback, args):
 refs = self.pyobj_refs[self.pyobjs.index(obj)]
+weakrefs = self.pyobj_weakrefs[self.pyobjs.index(obj)]
 for ref in refs:
 callback(ref, args)
+for weakref in weakrefs:
+callback(weakref.r, args)
 
 def rawrefcount_gc_as_pyobj(gc):
-return self.pyobjs[self.gcobjs.index(gc)]
+index = self.gcobjs.index(gc)
+if self.is_pygc[index]:
+return self.pyobjs[index]
+else:
+assert False
 
 def rawrefcount_pyobj_as_gc(pyobj):
-return self.gcobjs[self.pyobjs.index(pyobj)]
+index = self.pyobjs.index(pyobj)
+if self.is_pygc[index]:
+return self.gcobjs[index]
+else:
+return lltype.nullptr(PYOBJ_GC_HDR)
 
 def rawrefcount_finalizer_type(gc):
 pyobj = self.pyobjs[self.gcobjs.index(gc)]
@@ -56,6 +69,15 @@
 else:
 return RAWREFCOUNT_FINALIZER_NONE
 
+def rawrefcount_clear_wr(gc):
+cleared = False
+for weakrefs in self.pyobj_weakrefs:
+for weakref in weakrefs:
+if gc._obj.container == weakref.p._obj:
+weakref.callback_cleared = True
+cleared = True
+assert cleared
+
 self.pyobj_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw',
 immortal=True)
 self.pyobj_list.c_gc_next = self.pyobj_list
@@ -65,7 +87,8 @@
  llmemory.cast_ptr_to_adr(self.pyobj_list),
  rawrefcount_gc_as_pyobj,
  rawrefcount_pyobj_as_gc,
- rawrefcount_finalizer_type)
+ rawrefcount_finalizer_type,
+ rawrefcount_clear_wr)
 
 def _collect(self, major, expected_trigger=0):
 if major:
@@ -83,6 +106,11 @@
 refs.append(pyobj_to)
 pyobj_to.c_ob_refcnt += 1
 
+def _rawrefcount_addweakref(self, pyobj_from, weakref):
+refs = self.pyobj_weakrefs[self.pyobjs.index(pyobj_from)]
+refs.append(weakref)
+weakref.r.c_ob_refcnt += 1
+
 def _rawrefcount_add_resurrect(self, pyobj_source, pyobj_target):
 refs = self.pyobj_resurrect[self.pyobjs.index(pyobj_source)] = []
 refs.append(pyobj_target)
@@ -109,23 +137,26 @@
 
 return p1, p1ref, check_alive
 
-def _rawrefcount_pyobj(self, create_immortal=False):
+def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True):
 r1 = lltype.malloc(PYOBJ_HDR, flavor='raw',
immortal=create_immortal)
 r1.c_ob_refcnt = 0
 r1.c_ob_pypy_link = 0
 r1addr = llmemory.cast_ptr_to_adr(r1)
 
-r1gc = lltype.malloc(PYOBJ_GC_HDR, flavor='raw',
- immortal=True)
-r1gc.c_gc_next = self.pyobj_list
-  

[pypy-commit] pypy cpyext-gc-cycle: Clear weakref callbacks in rawrefcounted garbage to avoid resurrection

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96453:0b7ae798b964
Date: 2019-04-12 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/0b7ae798b964/

Log:Clear weakref callbacks in rawrefcounted garbage to avoid
resurrection

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -160,6 +160,20 @@
 pyobj_dealloc_action = PyObjDeallocAction(space)
 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
+def _clear_weakref_callbacks(gcref):
+from pypy.module._weakref.interp__weakref import \
+W_Weakref, W_CallableProxy
+from pypy.module.gc.referents import \
+try_cast_gcref_to_w_root
+w_obj = try_cast_gcref_to_w_root(gcref)
+if type(w_obj) is W_Weakref:
+w_obj.w_callable = None
+elif type(w_obj) is W_CallableProxy:
+w_obj.w_callable = None
+
+self.clear_weakref_callbacks = \
+(lambda w_obj: _clear_weakref_callbacks(w_obj))
+
 def _tp_traverse(pyobj_ptr, callback, args):
 from pypy.module.cpyext.api import PyObject
 from pypy.module.cpyext.typeobjectdefs import visitproc
@@ -213,7 +227,9 @@
  self.tp_traverse),
 pypyobj_list,
 self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc,
-self.C._PyPy_finalizer_type)
+self.C._PyPy_finalizer_type,
+llhelper(rawrefcount.RAWREFCOUNT_CLEAR_WR_TYPE,
+ self.clear_weakref_callbacks))
 self.builder.attach_all(space)
 
 setup_new_method_def(space)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3112,6 +3112,8 @@
  PYOBJ_GC_HDR_PTR))
 RAWREFCOUNT_FINALIZER_TYPE = lltype.Ptr(lltype.FuncType([PYOBJ_GC_HDR_PTR],
  lltype.Signed))
+RAWREFCOUNT_CLEAR_WR_TYPE = lltype.Ptr(lltype.FuncType([llmemory.GCREF],
+lltype.Void))
 RAWREFCOUNT_FINALIZER_NONE = 0
 RAWREFCOUNT_FINALIZER_MODERN = 1
 RAWREFCOUNT_FINALIZER_LEGACY = 2
@@ -3124,7 +3126,8 @@
 return llmemory.cast_adr_to_ptr(pygchdraddr, self.PYOBJ_GC_HDR_PTR)
 
 def rawrefcount_init(self, dealloc_trigger_callback, tp_traverse,
- pyobj_list, gc_as_pyobj, pyobj_as_gc, finalizer_type):
+ pyobj_list, gc_as_pyobj, pyobj_as_gc, finalizer_type,
+ clear_weakref_callback):
 # see pypy/doc/discussion/rawrefcount.rst
 if not self.rrc_enabled:
 self.rrc_p_list_young = self.AddressStack()
@@ -3145,6 +3148,7 @@
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_finalizer_type = finalizer_type
+self.rrc_clear_weakref_callback = clear_weakref_callback
 self.rrc_enabled = True
 self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
 
@@ -3411,10 +3415,6 @@
 self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc)
 self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc)
 
-# TODO: handle weakrefs for unreachable objects and create
-# TODO: a list of callbacks, which has to be called after the
-# TODO: the GC runs
-
 def _rrc_major_trace(self, pyobject, use_cylicrefcnt):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
@@ -3459,9 +3459,48 @@
 
 if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
 if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
+self._rrc_clear_weakref_callbacks()
 self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
 self.rrc_pyobj_dead_list)
 
+def _rrc_clear_weakref_callbacks(self):
+# Look for any weakrefs within the trash cycle and remove the callback.
+# This is only needed for weakrefs created from rawrefcounted objects
+# because weakrefs from gc-managed objects are going away anyway.
+gchdr = self.rrc_pyobj_old_list.c_gc_next
+while gchdr <> self.rrc_pyobj_old_list:
+pyobj = self.rrc_gc_as_pyobj(gchdr)
+self._rrc_traverse_weakref(pyobj)
+gchdr = gchdr.c_gc_next
+
+def _rrc_visit_weakref(pyobj, self_ptr):
+from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
+

[pypy-commit] pypy cpyext-gc-cycle: Implemented garbage_pypy and improved tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96225:5d1c0b97ef5a
Date: 2019-03-08 14:23 +0100
http://bitbucket.org/pypy/pypy/changeset/5d1c0b97ef5a/

Log:Implemented garbage_pypy and improved tests

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3141,6 +3141,7 @@
 self.rrc_pyobj_isolate_list = self._rrc_gc_list_new()
 self.rrc_pyobj_dead_list = self._rrc_gc_list_new()
 self.rrc_pyobj_garbage_list = self._rrc_gc_list_new()
+self.rrc_garbage_to_trace = self.AddressStack()
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_finalizer_type = finalizer_type
@@ -3246,15 +3247,14 @@
 self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
 
 def rawrefcount_next_garbage_pypy(self):
-# We assume that next_garbage_pypy is always called before
-# next_garbage_pyobj. As pypy objects can only be in garbage, if there
-# is at least one pyobj in garbage, we can use this optimization.
-if self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
-return lltype.nullptr(llmemory.GCREF.TO)
+if self.rrc_garbage_to_trace.non_empty():
+# remove one object from the wavefront and move the wavefront
+obj = self.rrc_garbage_to_trace.pop()
+if self._rrc_garbage_visit(obj):
+return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+else:
+return lltype.nullptr(llmemory.GCREF.TO)
 else:
-# TODO: return the next pypy object which is marked with 
GCFLAG_GARBAGE and
-#   remove the flag from this object. We can safely assume 
that objects
-#   do not move, as this can only happen to old objects.
 return lltype.nullptr(llmemory.GCREF.TO)
 
 def rawrefcount_next_garbage_pyobj(self):
@@ -3620,9 +3620,28 @@
 if pyobj.c_ob_pypy_link <> 0:
 intobj = pyobj.c_ob_pypy_link
 obj = llmemory.cast_int_to_adr(intobj)
+self.rrc_garbage_to_trace.append(obj)
 self.objects_to_trace.append(obj)
 self.visit_all_objects()
 
+def _rrc_collect_obj(self, obj, ignored):
+llop.debug_nonnull_pointer(lltype.Void, obj)
+self.rrc_garbage_to_trace.append(obj)
+_rrc_collect_obj._always_inline_ = True
+
+def _rrc_collect_ref_rec(self, root, ignored):
+self._rrc_collect_obj(root.address[0], None)
+
+def _rrc_garbage_visit(self, obj):
+# If GCFLAG_GARBAGE is set, remove the flag and trace the object
+hdr = self.header(obj)
+if not (hdr.tid & GCFLAG_GARBAGE):
+return False
+hdr.tid &= ~GCFLAG_GARBAGE
+if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
+self.trace(obj, self._rrc_collect_ref_rec, None)
+return True
+
 def _rrc_check_finalizer(self):
 # Check, if the cyclic isolate from the last collection cycle
 # is reachable from outside, after the finalizers have been
diff --git a/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot 
b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot
--- a/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot
+++ b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot
@@ -3,8 +3,12 @@
 "b" [type=B, alive=y, garbage=y];
 "c" [type=P, alive=y, garbage=y];
 "d" [type=P, alive=y, rooted=y];
+"e" [type=C, alive=y, garbage=y];
+"f" [type=C, alive=y, ext_refcnt=1];
 "a" -> "b";
 "b" -> "a";
 "b" -> "c";
 "c" -> "d";
+"a" -> "e";
+"a" -> "f";
 }
\ No newline at end of file
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -672,10 +672,15 @@
 # have been added to the garbage list
 for name in nodes:
 n = nodes[name]
+garbage = n.info.garbage
 if n.info.alive:
 if n.info.type == "C":
-assert n.info.garbage != (n.raddr not in garbage_pyobj)
+assert garbage != (n.raddr not in garbage_pyobj), \
+"PyObject should " + ("" if garbage else "not ") + \
+"be in garbage"
 else:
-assert n.info.garbage != (n.pref not in garbage_pypy)
+assert garbage != (n.pref not in garbage_pypy), \
+"Object should " + ("" if garbage else "not ") + \
+"be in garbage"
 else:
-assert not n.info.garbage
\ No newline at end of file
+assert not garbage, "Object is dead, but should be in garbage"

[pypy-commit] pypy cpyext-gc-cycle: First version of rawrefcount legacy finalizer implementation

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96206:3ab559b9b893
Date: 2019-03-03 21:21 +0100
http://bitbucket.org/pypy/pypy/changeset/3ab559b9b893/

Log:First version of rawrefcount legacy finalizer implementation Test is
still failing, because garbage_pypy is not implemented yet Adapted
interface

diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -53,14 +53,15 @@
 _PyPy_finalizer_type(PyGC_Head *gc)
 {
 PyObject *op = FROM_GC(gc);
-if (!_PyGCHead_FINALIZED(gc) &&
+if (Py_TYPE(op)->tp_del != NULL) {
+return 2; // legacy (has priority over modern)
+} else if (!_PyGCHead_FINALIZED(gc) &&
 PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
 Py_TYPE(op)->tp_finalize != NULL) {
-return 1;
+return 1; // modern
 } else {
-return 0;
+return 0; // no finalizer
 }
-// TODO: legacy finalizer (tp_del)
 }
 
 void
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -120,15 +120,12 @@
 if not py_obj:
 break
 w_list.append(w_obj)
-last_py_obj = lltype.nullptr(PyObject.TO)
 while True:
-w_pyobj = rawrefcount.next_garbage_pyobj(PyObject,
- last_py_obj)
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
 if not w_pyobj:
 break
 w_obj = from_ref(space, w_pyobj)
 w_list.append(w_obj)
-last_py_obj = w_pyobj
 space.setattr(space.builtin_modules['gc'],
   space.newtext('garbage'), w_list)
 rawrefcount.end_garbage()
@@ -330,14 +327,12 @@
 if not py_obj:
 break
 w_list.append(w_obj)
-last_py_obj = lltype.nullptr(PyObject.TO)
 while True:
-w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, last_py_obj)
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
 if not w_pyobj:
 break
 w_obj = from_ref(space, w_pyobj)
 w_list.append(w_obj)
-last_py_obj = w_pyobj
 space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
   w_list)
 rawrefcount.end_garbage()
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3137,18 +3137,10 @@
 self.rrc_dealloc_pending = self.AddressStack()
 self.rrc_tp_traverse = tp_traverse
 self.rrc_pyobj_list = self._pygchdr(pyobj_list)
-self.rrc_pyobj_old_list = \
-lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
-self.rrc_pyobj_old_list.c_gc_next = self.rrc_pyobj_old_list
-self.rrc_pyobj_old_list.c_gc_prev = self.rrc_pyobj_old_list
-self.rrc_pyobj_isolate_list = \
-lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
-self.rrc_pyobj_isolate_list.c_gc_next = self.rrc_pyobj_isolate_list
-self.rrc_pyobj_isolate_list.c_gc_prev = self.rrc_pyobj_isolate_list
-self.rrc_pyobj_garbage_list = \
-lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
-self.rrc_pyobj_garbage_list.c_gc_next = self.rrc_pyobj_garbage_list
-self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list
+self.rrc_pyobj_old_list = self._rrc_gc_list_new()
+self.rrc_pyobj_isolate_list = self._rrc_gc_list_new()
+self.rrc_pyobj_dead_list = self._rrc_gc_list_new()
+self.rrc_pyobj_garbage_list = self._rrc_gc_list_new()
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_finalizer_type = finalizer_type
@@ -3224,14 +3216,14 @@
 return llmemory.NULL
 
 def rawrefcount_cyclic_garbage_head(self):
-if not self._rrc_gc_list_is_empty(self.rrc_pyobj_garbage_list):
+if not self._rrc_gc_list_is_empty(self.rrc_pyobj_dead_list):
 return llmemory.cast_ptr_to_adr(
-self.rrc_gc_as_pyobj(self.rrc_pyobj_garbage_list.c_gc_next))
+self.rrc_gc_as_pyobj(self.rrc_pyobj_dead_list.c_gc_next))
 else:
 return llmemory.NULL
 
 def rawrefcount_cyclic_garbage_remove(self):
-gchdr = self.rrc_pyobj_garbage_list.c_gc_next
+gchdr = self.rrc_pyobj_dead_list.c_gc_next
 # remove from old list
 next = gchdr.c_gc_next
 next.c_gc_prev = gchdr.c_gc_prev
@@ -3254,25 +3246,40 @@
 self.rrc_state = 

[pypy-commit] pypy cpyext-gc-cycle: Fixed issue in gc if rawrefcount is disabled

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96200:85b1a76a7d3a
Date: 2019-03-02 14:45 +0100
http://bitbucket.org/pypy/pypy/changeset/85b1a76a7d3a/

Log:Fixed issue in gc if rawrefcount is disabled Refactored rrc code to
make it easier to read

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2719,7 +2719,8 @@
 # to also set TRACK_YOUNG_PTRS here, for the write barrier.
 hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
-if self.rrc_state == self.RAWREFCOUNT_STATE_MARKING:
+if self.rrc_enabled and \
+self.rrc_state == self.RAWREFCOUNT_STATE_MARKING:
 hdr.tid |= GCFLAG_GARBAGE
 
 if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
@@ -3381,53 +3382,24 @@
 _rrc_free._always_inline_ = True
 
 def rrc_major_collection_trace(self):
+# Only trace and mark rawrefcounted object if we are not doing
+# something special, like building gc.garbage.
 if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+# check objects with finalizers from last collection cycle
 if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list):
-# Check, if the cyclic isolate from the last collection cycle
-# is reachable from outside, after the finalizers have been
-# executed.
-self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list)
-found_alive = False
-gchdr = self.rrc_pyobj_old_list.c_gc_next
-while gchdr <> self.rrc_pyobj_old_list:
-if (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0:
-found_alive = True
-break
-gchdr = gchdr.c_gc_next
-if found_alive:
-self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
-self.rrc_pyobj_list)
-else:
-self._rrc_gc_list_merge(self.rrc_pyobj_old_list,
-self.rrc_pyobj_garbage_list)
-
+self._rrc_check_finalizer()
+# collect all rawrefcounted roots
 self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list)
+# mark all objects reachable from rawrefcounted roots
 self._rrc_mark_rawrefcount()
-
-# handle legacy finalizers
-self.rrc_state = self.RAWREFCOUNT_STATE_MARKING
-# TODO: move all pyobjs to sublist of pyobj_list and reset cyclic 
refcount
-# TODO: mark all pypy-objects with special flag and mark them 
visited
-self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
-
-# now handle modern finalizers
-found_finalizer = False
-gchdr = self.rrc_pyobj_old_list.c_gc_next
-while gchdr <> self.rrc_pyobj_old_list:
-if self.rrc_finalizer_type(gchdr) == \
-self.RAWREFCOUNT_FINALIZER_MODERN:
-found_finalizer = True
-gchdr = gchdr.c_gc_next
-if found_finalizer:
-self._rrc_gc_list_move(self.rrc_pyobj_old_list,
-   self.rrc_pyobj_isolate_list)
-
-use_cylicrefcnt = not found_finalizer
+self._rrc_mark_garbage() # handle legacy finalizers
+use_cylicrc = not self._rrc_find_finalizer() # modern finalizers
 else:
-use_cylicrefcnt = False
-
-self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrefcnt)
-self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrefcnt)
+use_cylicrc = False # don't sweep any objects in cyclic isolates
+
+# now mark all pypy objects at the border, depending on the results
+self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc)
+self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc)
 
 # TODO: handle weakrefs for unreachable objects and create
 # TODO: a list of callbacks, which has to be called after the
@@ -3581,6 +3553,44 @@
 # now all rawrefcounted objects, which are alive, have a cyclic
 # refcount > 0 or are marked
 
+def _rrc_mark_garbage(self):
+self.rrc_state = self.RAWREFCOUNT_STATE_MARKING
+# TODO: move all pyobjs to sublist of pyobj_list and reset cyclic 
refcount
+# TODO: mark all pypy-objects with special flag and mark them visited
+self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
+
+def _rrc_check_finalizer(self):
+# Check, if the cyclic isolate from the last collection cycle
+# is reachable from outside, after the finalizers have been
+# executed.
+self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list)
+   

[pypy-commit] pypy cpyext-gc-cycle: Implemented additional rawrefcount states and added a gc-header flag

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96199:f0fce8ff6d08
Date: 2019-03-02 11:15 +0100
http://bitbucket.org/pypy/pypy/changeset/f0fce8ff6d08/

Log:Implemented additional rawrefcount states and added a gc-header flag
Extended interface

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -113,6 +113,7 @@
 if adr_int == llmemory.cast_adr_to_int(
 llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
+rawrefcount.begin_garbage()
 w_list = space.newlist([])
 while True:
 w_obj = rawrefcount.next_garbage_pypy(W_Root)
@@ -130,6 +131,7 @@
 last_py_obj = w_pyobj
 space.setattr(space.builtin_modules['gc'],
   space.newtext('garbage'), w_list)
+rawrefcount.end_garbage()
 print 'dealloc_trigger DONE'
 return "RETRY"
 def tp_traverse(pyobj_ptr, callback, args):
@@ -321,6 +323,7 @@
 if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
 
+rawrefcount.begin_garbage()
 w_list = space.newlist([])
 while True:
 w_obj = rawrefcount.next_garbage_pypy(W_Root)
@@ -337,6 +340,7 @@
 last_py_obj = w_pyobj
 space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
   w_list)
+rawrefcount.end_garbage()
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -163,7 +163,15 @@
 # It does not need an additional copy in trace out
 GCFLAG_SHADOW_INITIALIZED   = first_gcflag << 11
 
-_GCFLAG_FIRST_UNUSED = first_gcflag << 12# the first unused bit
+# Objects referenced only from legacy rawrefcount finalizers which have
+# not been added to gc.garbage have this flag set. It is set during the
+# rawrefcount marking phase and removed once the object is returned to
+# the caller, who is adding it to the gc.garbage list. During the next
+# collection cycle this process is repeated, as the set of objects might
+# have changed.
+GCFLAG_GARBAGE = first_gcflag << 12
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 13# the first unused bit
 
 
 # States for the incremental GC
@@ -2711,6 +2719,9 @@
 # to also set TRACK_YOUNG_PTRS here, for the write barrier.
 hdr.tid |= GCFLAG_VISITED | GCFLAG_TRACK_YOUNG_PTRS
 
+if self.rrc_state == self.RAWREFCOUNT_STATE_MARKING:
+hdr.tid |= GCFLAG_GARBAGE
+
 if self.has_gcptr(llop.extract_ushort(llgroup.HALFWORD, hdr.tid)):
 #
 # Trace the content of the object and put all objects it references
@@ -3063,15 +3074,18 @@
 
 rrc_enabled = False
 
-# The default state. Here cyclic garbage with legacy finalizers is marked.
+# Default state, no rawrefcount specific code is executed during normal 
marking.
 RAWREFCOUNT_STATE_DEFAULT = 0
 
+# Here cyclic garbage only reachable from legacy finalizers is marked.
+RAWREFCOUNT_STATE_MARKING = 1
+
 # The state in which cyclic garbage with legacy finalizers is traced.
 # Do not mark objects during this state, because we remove the flag
 # during tracing and we do not want to trace those objects again. Also
 # during this phase no new objects can be marked, as we are only building
 # the list of cyclic garbage.
-RAWREFCOUNT_STATE_GARBAGE = 1
+RAWREFCOUNT_STATE_GARBAGE = 2
 
 _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 PYOBJ_HDR = lltype.Struct('GCHdr_PyObject',
@@ -3228,19 +3242,22 @@
 gchdr.c_gc_next = next
 next.c_gc_prev = gchdr
 
+def rawrefcount_begin_garbage(self):
+# after this, no new objects should be marked with the GCFLAG_GARBAGE
+# flag
+self.rrc_state = self.RAWREFCOUNT_STATE_GARBAGE
+
+def rawrefcount_end_garbage(self):
+# now there should not be any object left with a GCFLAG_GARBAGE flag
+# set
+self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
+
 def rawrefcount_next_garbage_pypy(self):
-if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
-self.rrc_state = self.RAWREFCOUNT_STATE_GARBAGE
-
 # return the next pypy object which is only reachable from garbage
 # pyobjects, probably need two more colors for this. one for marking
 # so that they stay alive during sweep, one for marking, so they do not
 # get returned here again
-result = lltype.nullptr(llmemory.GCREF.TO)
-
-if result == 

[pypy-commit] pypy cpyext-gc-cycle: Started implementation for gc.garbage, adapted interface

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96198:77d74e85609f
Date: 2019-03-01 22:09 +0100
http://bitbucket.org/pypy/pypy/changeset/77d74e85609f/

Log:Started implementation for gc.garbage, adapted interface

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -13,7 +13,6 @@
 # context.
 ExecutionContext.cpyext_operror = None
 
-
 class State:
 def __init__(self, space):
 self.space = space
@@ -80,8 +79,6 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.pyobject import PyObject, decref, \
 incref, cts, finalize, from_ref
-w_list = space.getattr(space.builtin_modules['gc'],
-   space.newtext('garbage'))
 print 'dealloc_trigger...'
 while True:
 ob = rawrefcount.next_dead(PyObject)
@@ -116,17 +113,23 @@
 if adr_int == llmemory.cast_adr_to_int(
 llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
+w_list = space.newlist([])
 while True:
 w_obj = rawrefcount.next_garbage_pypy(W_Root)
 if not py_obj:
 break
 w_list.append(w_obj)
+last_py_obj = lltype.nullptr(PyObject.TO)
 while True:
-w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
-if not py_obj:
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject,
+ last_py_obj)
+if not w_pyobj:
 break
 w_obj = from_ref(space, w_pyobj)
 w_list.append(w_obj)
+last_py_obj = w_pyobj
+space.setattr(space.builtin_modules['gc'],
+  space.newtext('garbage'), w_list)
 print 'dealloc_trigger DONE'
 return "RETRY"
 def tp_traverse(pyobj_ptr, callback, args):
@@ -292,9 +295,6 @@
 from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
  finalize, from_ref)
 
-w_list = space.getattr(space.builtin_modules['gc'],
-   space.newtext('garbage'))
-
 while True:
 py_obj = rawrefcount.next_dead(PyObject)
 if not py_obj:
@@ -321,17 +321,22 @@
 if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
 
+w_list = space.newlist([])
 while True:
 w_obj = rawrefcount.next_garbage_pypy(W_Root)
 if not py_obj:
 break
 w_list.append(w_obj)
+last_py_obj = lltype.nullptr(PyObject.TO)
 while True:
-w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
-if not py_obj:
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject, last_py_obj)
+if not w_pyobj:
 break
 w_obj = from_ref(space, w_pyobj)
 w_list.append(w_obj)
+last_py_obj = w_pyobj
+space.setattr(space.builtin_modules['gc'], space.newtext('garbage'),
+  w_list)
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3063,6 +3063,16 @@
 
 rrc_enabled = False
 
+# The default state. Here cyclic garbage with legacy finalizers is marked.
+RAWREFCOUNT_STATE_DEFAULT = 0
+
+# The state in which cyclic garbage with legacy finalizers is traced.
+# Do not mark objects during this state, because we remove the flag
+# during tracing and we do not want to trace those objects again. Also
+# during this phase no new objects can be marked, as we are only building
+# the list of cyclic garbage.
+RAWREFCOUNT_STATE_GARBAGE = 1
+
 _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True})
 PYOBJ_HDR = lltype.Struct('GCHdr_PyObject',
   ('c_ob_refcnt', lltype.Signed),
@@ -3128,6 +3138,7 @@
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_finalizer_type = finalizer_type
 self.rrc_enabled = True
+self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT
 
 def check_no_more_rawrefcount_state(self):
 "NOT_RPYTHON: for tests"
@@ -3218,21 +3229,24 @@
 next.c_gc_prev = gchdr
 
 def rawrefcount_next_garbage_pypy(self):
+if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT:
+self.rrc_state = self.RAWREFCOUNT_STATE_GARBAGE
+
 # return the 

[pypy-commit] pypy cpyext-gc-cycle: Implemented tests for gc.garbage list and adapted interface

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96197:15bed0a0192d
Date: 2019-03-01 14:13 +0100
http://bitbucket.org/pypy/pypy/changeset/15bed0a0192d/

Log:Implemented tests for gc.garbage list and adapted interface

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -77,6 +77,7 @@
 space = self.space
 if not self.space.config.translating:
 def dealloc_trigger():
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.pyobject import PyObject, decref, \
 incref, cts, finalize, from_ref
 w_list = space.getattr(space.builtin_modules['gc'],
@@ -116,10 +117,15 @@
 llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
 while True:
-py_obj = rawrefcount.next_garbage(PyObject)
+w_obj = rawrefcount.next_garbage_pypy(W_Root)
 if not py_obj:
 break
-w_obj = from_ref(space, py_obj)
+w_list.append(w_obj)
+while True:
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
+if not py_obj:
+break
+w_obj = from_ref(space, w_pyobj)
 w_list.append(w_obj)
 print 'dealloc_trigger DONE'
 return "RETRY"
@@ -282,6 +288,7 @@
 
 
 def _rawrefcount_perform(space):
+from pypy.interpreter.baseobjspace import W_Root
 from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
  finalize, from_ref)
 
@@ -315,10 +322,15 @@
 rawrefcount.cyclic_garbage_remove()
 
 while True:
-py_obj = rawrefcount.next_garbage(PyObject)
+w_obj = rawrefcount.next_garbage_pypy(W_Root)
 if not py_obj:
 break
-w_obj = from_ref(space, py_obj)
+w_list.append(w_obj)
+while True:
+w_pyobj = rawrefcount.next_garbage_pyobj(PyObject)
+if not py_obj:
+break
+w_obj = from_ref(space, w_pyobj)
 w_list.append(w_obj)
 
 class PyObjDeallocAction(executioncontext.AsyncAction):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3217,7 +3217,22 @@
 gchdr.c_gc_next = next
 next.c_gc_prev = gchdr
 
-def rawrefcount_next_garbage(self):
+def rawrefcount_next_garbage_pypy(self):
+# return the next pypy object which is only reachable from garbage
+# pyobjects, probably need two more colors for this. one for marking
+# so that they stay alive during sweep, one for marking, so they do not
+# get returned here again
+return lltype.nullptr(llmemory.GCREF.TO)
+
+def rawrefcount_next_garbage_pyobj(self):
+# implement st objects in this list still remain in the set of
+# all pyobjs, because references could still change and cause them
+# to live again. also keep in mind, that state will create references
+# to pyobjs in this list and might increment the refcount.
+
+# use create_link_pyobj on the result to create gc objects for 
pyobjects
+#p = W_Root(42)
+#p.pyobj = ob
 return llmemory.NULL
 
 
diff --git a/rpython/memory/gc/test/dot/garbage_cpython_simple_1.dot 
b/rpython/memory/gc/test/dot/garbage_cpython_simple_1.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/garbage_cpython_simple_1.dot
@@ -0,0 +1,6 @@
+digraph G {
+"a" [type=C, alive=y, garbage=y, finalizer=legacy];
+"b" [type=C, alive=y, garbage=y];
+"a" -> "b";
+"b" -> "a";
+}
\ No newline at end of file
diff --git a/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot 
b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/garbage_cross_simple_1.dot
@@ -0,0 +1,10 @@
+digraph G {
+"a" [type=C, alive=y, garbage=y, finalizer=legacy];
+"b" [type=B, alive=y, garbage=y];
+"c" [type=P, alive=y, garbage=y];
+"d" [type=P, alive=y, rooted=y];
+"a" -> "b";
+"b" -> "a";
+"b" -> "c";
+"c" -> "d";
+}
\ No newline at end of file
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -1,17 +1,17 @@
 import os, py
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.memory.gc.incminimark import IncrementalMiniMarkGC
+from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as IncMiniMark
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest

[pypy-commit] pypy cpyext-gc-cycle: Implemented interface for gc.garbage list

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96185:38a9f6496360
Date: 2019-02-23 15:34 +0100
http://bitbucket.org/pypy/pypy/changeset/38a9f6496360/

Log:Implemented interface for gc.garbage list

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -78,7 +78,9 @@
 if not self.space.config.translating:
 def dealloc_trigger():
 from pypy.module.cpyext.pyobject import PyObject, decref, \
-incref, cts, finalize
+incref, cts, finalize, from_ref
+w_list = space.getattr(space.builtin_modules['gc'],
+   space.newtext('garbage'))
 print 'dealloc_trigger...'
 while True:
 ob = rawrefcount.next_dead(PyObject)
@@ -113,6 +115,12 @@
 if adr_int == llmemory.cast_adr_to_int(
 llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
+while True:
+py_obj = rawrefcount.next_garbage(PyObject)
+if not py_obj:
+break
+w_obj = from_ref(space, py_obj)
+w_list.append(w_obj)
 print 'dealloc_trigger DONE'
 return "RETRY"
 def tp_traverse(pyobj_ptr, callback, args):
@@ -274,7 +282,12 @@
 
 
 def _rawrefcount_perform(space):
-from pypy.module.cpyext.pyobject import PyObject, incref, decref, finalize
+from pypy.module.cpyext.pyobject import (PyObject, incref, decref,
+ finalize, from_ref)
+
+w_list = space.getattr(space.builtin_modules['gc'],
+   space.newtext('garbage'))
+
 while True:
 py_obj = rawrefcount.next_dead(PyObject)
 if not py_obj:
@@ -291,18 +304,23 @@
 py_obj = rawrefcount.cyclic_garbage_head(PyObject)
 if not py_obj:
 break
-
 pyobj = rffi.cast(PyObject, py_obj)
 adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
 if pyobj.c_ob_type.c_tp_clear:
 incref(space, py_obj)
 pyobj.c_ob_type.c_tp_clear(pyobj)
 decref(space, py_obj)
-
 head = rawrefcount.cyclic_garbage_head(PyObject)
 if adr_int == llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(head)):
 rawrefcount.cyclic_garbage_remove()
 
+while True:
+py_obj = rawrefcount.next_garbage(PyObject)
+if not py_obj:
+break
+w_obj = from_ref(space, py_obj)
+w_list.append(w_obj)
+
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
 """
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3217,6 +3217,10 @@
 gchdr.c_gc_next = next
 next.c_gc_prev = gchdr
 
+def rawrefcount_next_garbage(self):
+return llmemory.NULL
+
+
 def rrc_invoke_callback(self):
 if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or
  self.rrc_pyobj_isolate_list.c_gc_next <>
diff --git a/rpython/memory/gctransform/framework.py 
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -518,6 +518,9 @@
 self.rawrefcount_cyclic_garbage_remove_ptr = getfn(
 GCClass.rawrefcount_cyclic_garbage_remove, [s_gc],
 annmodel.s_None, inline = True)
+self.rawrefcount_next_garbage_ptr = getfn(
+GCClass.rawrefcount_next_garbage, [s_gc],
+SomeAddress(), inline = True)
 
 if GCClass.can_usually_pin_objects:
 self.pin_ptr = getfn(GCClass.pin,
@@ -1426,6 +1429,12 @@
   [self.rawrefcount_cyclic_garbage_remove_ptr,
self.c_const_gc])
 
+def gct_gc_rawrefcount_next_garbage(self, hop):
+assert hop.spaceop.result.concretetype == llmemory.Address
+hop.genop("direct_call",
+  [self.rawrefcount_next_garbage_ptr, self.c_const_gc],
+  resultvar=hop.spaceop.result)
+
 def _set_into_gc_array_part(self, op):
 if op.opname == 'setarrayitem':
 return op.args[1]
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -146,6 +146,10 @@
 return lltype.nullptr(OB_PTR_TYPE.TO)
 
 @not_rpython
+def next_garbage(OB_PTR_TYPE):
+return lltype.nullptr(OB_PTR_TYPE.TO)
+
+@not_rpython
 def _collect(track_allocation=True):
 """for tests only.  Emulates a GC collection.
 Will invoke 

[pypy-commit] pypy cpyext-gc-cycle: Added more tests for "modern" rawrefcount finalizers

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96125:a3d95a9939f4
Date: 2019-02-22 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/a3d95a9939f4/

Log:Added more tests for "modern" rawrefcount finalizers

diff --git a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot 
b/rpython/memory/gc/test/dot/free_finalizer_nocycle_1a.dot
rename from rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
rename to rpython/memory/gc/test/dot/free_finalizer_nocycle_1a.dot
diff --git a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot 
b/rpython/memory/gc/test/dot/free_finalizer_nocycle_1b.dot
copy from rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
copy to rpython/memory/gc/test/dot/free_finalizer_nocycle_1b.dot
--- a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
+++ b/rpython/memory/gc/test/dot/free_finalizer_nocycle_1b.dot
@@ -1,7 +1,7 @@
 digraph G {
 "a" [type=P, alive=n];
 "b" [type=B, alive=n];
-"c" [type=C, alive=n, finalizer=modern];
+"c" [type=C, alive=n, finalizer=modern, delete="d"];
 "d" [type=C, alive=n];
 "e" [type=B, alive=n];
 "f" [type=P, alive=n];
diff --git a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot 
b/rpython/memory/gc/test/dot/free_finalizer_nocycle_1c.dot
copy from rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
copy to rpython/memory/gc/test/dot/free_finalizer_nocycle_1c.dot
--- a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot
+++ b/rpython/memory/gc/test/dot/free_finalizer_nocycle_1c.dot
@@ -1,8 +1,8 @@
 digraph G {
 "a" [type=P, alive=n];
 "b" [type=B, alive=n];
-"c" [type=C, alive=n, finalizer=modern];
-"d" [type=C, alive=n];
+"c" [type=C, alive=n, finalizer=modern, delete="d"];
+"d" [type=C, alive=n, finalizer=modern, delete="e"];
 "e" [type=B, alive=n];
 "f" [type=P, alive=n];
 "a" -> "b";
diff --git a/rpython/memory/gc/test/dot/free_finalizer_simple.dot 
b/rpython/memory/gc/test/dot/free_finalizer_simple_1a.dot
rename from rpython/memory/gc/test/dot/free_finalizer_simple.dot
rename to rpython/memory/gc/test/dot/free_finalizer_simple_1a.dot
diff --git a/rpython/memory/gc/test/dot/free_finalizer_simple.dot 
b/rpython/memory/gc/test/dot/free_finalizer_simple_1b.dot
copy from rpython/memory/gc/test/dot/free_finalizer_simple.dot
copy to rpython/memory/gc/test/dot/free_finalizer_simple_1b.dot
--- a/rpython/memory/gc/test/dot/free_finalizer_simple.dot
+++ b/rpython/memory/gc/test/dot/free_finalizer_simple_1b.dot
@@ -1,7 +1,7 @@
 digraph G {
 "a" [type=P, alive=n];
 "b" [type=B, alive=n];
-"c" [type=C, alive=n, finalizer=modern];
+"c" [type=C, alive=n, finalizer=modern, delete="d"];
 "d" [type=C, alive=n];
 "e" [type=B, alive=n];
 "f" [type=P, alive=n];
diff --git a/rpython/memory/gc/test/dot/keep_finalizer_simple.dot 
b/rpython/memory/gc/test/dot/keep_finalizer_simple_1a.dot
rename from rpython/memory/gc/test/dot/keep_finalizer_simple.dot
rename to rpython/memory/gc/test/dot/keep_finalizer_simple_1a.dot
diff --git a/rpython/memory/gc/test/dot/keep_finalizer_simple.dot 
b/rpython/memory/gc/test/dot/keep_finalizer_simple_1b.dot
copy from rpython/memory/gc/test/dot/keep_finalizer_simple.dot
copy to rpython/memory/gc/test/dot/keep_finalizer_simple_1b.dot
--- a/rpython/memory/gc/test/dot/keep_finalizer_simple.dot
+++ b/rpython/memory/gc/test/dot/keep_finalizer_simple_1b.dot
@@ -1,7 +1,7 @@
 digraph G {
 "a" [type=P, alive=y];
 "b" [type=B, alive=y];
-"c" [type=C, alive=y, finalizer=modern, resurrect="d"];
+"c" [type=C, alive=y, finalizer=modern, resurrect="c"];
 "d" [type=C, alive=y];
 "e" [type=B, alive=y];
 "f" [type=P, alive=y];
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -31,7 +31,9 @@
 self.pyobjs = []
 self.pyobj_refs = []
 self.pyobj_finalizer = {}
+self.pyobj_finalized = {}
 self.pyobj_resurrect = {}
+self.pyobj_delete = {}
 
 def rawrefcount_tp_traverse(obj, callback, args):
 refs = self.pyobj_refs[self.pyobjs.index(obj)]
@@ -85,6 +87,10 @@
 refs = self.pyobj_resurrect[self.pyobjs.index(pyobj_source)] = []
 refs.append(pyobj_target)
 
+def _rawrefcount_add_delete(self, pyobj_source, pyobj_target):
+refs = self.pyobj_delete[self.pyobjs.index(pyobj_source)] = []
+refs.append(pyobj_target)
+
 def _rawrefcount_pypyobj(self, intval, rooted=False, create_old=True):
 p1 = self.malloc(S)
 p1.x = intval
@@ -412,8 +418,6 @@
 @py.test.mark.parametrize("file", dot_files)
 def test_dots(self, file):
 from rpython.memory.gc.test.dot import pydot
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
 
 class Node:
 def __init__(self, 

[pypy-commit] pypy cpyext-gc-cycle: Implemented wrapper for tp_finalize

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96124:78d356baf93c
Date: 2019-02-21 15:33 +0100
http://bitbucket.org/pypy/pypy/changeset/78d356baf93c/

Log:Implemented wrapper for tp_finalize

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -12,7 +12,7 @@
 getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
 ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
 cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
-getbufferproc, ssizessizeobjargproc)
+getbufferproc, ssizessizeobjargproc, destructor)
 from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj, decref
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
@@ -438,6 +438,16 @@
 fq.register_finalizer(buf)
 return buf.wrap(space)
 
+class wrap_del(W_PyCWrapperObject):
+def call(self, space, w_self, __args__):
+self.check_args(__args__, 0)
+func = self.get_func_to_call()
+func_target = rffi.cast(destructor, func)
+res = generic_cpy_call(space, func_target, w_self)
+if res == -1:
+space.fromcache(State).check_and_raise_exception(always=True)
+return space.w_None
+
 def get_richcmp_func(OP_CONST):
 class wrap_richcmp(W_PyCWrapperObject):
 def call(self, space, w_self, __args__):
@@ -832,8 +842,21 @@
 return 0
 return slot_tp_descr_set
 
+@slot_factory('tp_finalize')
+def make_tp_finalize(space, typedef, name, attr):
+w_type = space.gettypeobject(typedef)
+new_fn = w_type.lookup('__del__')
+if new_fn is None:
+return
 
-missing_wrappers = ['wrap_indexargfunc', 'wrap_del']
+@slot_function([PyObject], lltype.Void)
+@func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+def slot_tp_finalize(space, w_self):
+args = Arguments(space, [])
+return space.call_args(w_self, args)
+return slot_tp_finalize
+
+missing_wrappers = ['wrap_indexargfunc']
 for name in missing_wrappers:
 assert name not in globals()
 class missing_wrapper(W_PyCWrapperObject):
@@ -848,7 +871,6 @@
 
 missing_builtin_slots = [
 'tp_print', 'tp_compare', 'tp_getattr', 'tp_setattr', 'tp_setattro',
-'tp_finalize',
 'tp_richcompare', 'tp_del', 'tp_as_buffer.c_bf_getwritebuffer',
 'tp_as_number.c_nb_bool', 'tp_as_number.c_nb_coerce',
 'tp_as_number.c_nb_inplace_add', 'tp_as_number.c_nb_inplace_subtract',
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Added support for rawrefcount finalizers in incminimark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r96024:09b2440acb51
Date: 2019-02-16 14:35 +0100
http://bitbucket.org/pypy/pypy/changeset/09b2440acb51/

Log:Added support for rawrefcount finalizers in incminimark Added call
to tp_finalize in cpyext, if the gc found an unreachable object
(still needs some testing)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1190,7 +1190,10 @@
 state.C._PyPy_finalizer_type = rffi.llexternal(
 '_PyPy_finalizer_type', [PyGC_HeadPtr], lltype.Signed,
 compilation_info=eci, _nowrapper=True)
-
+state.C._Py_Finalize = rffi.llexternal('_Py_Finalize',
+   [PyObject], lltype.Void,
+   compilation_info=eci,
+   _nowrapper=True)
 
 def init_function(func):
 INIT_FUNCTIONS.append(func)
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -51,6 +51,7 @@
 PyAPI_FUNC(void) Py_DecRef(PyObject *);
 extern Py_ssize_t _pypy_rawrefcount_w_marker_deallocating;
 PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
+PyAPI_FUNC(void) _Py_Finalize(PyObject *);
 
 #define Py_CLEAR(op)\
 do {   \
@@ -244,6 +245,8 @@
 
 #define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL
 
+#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0)
+
 #define PyType_HasFeature(t,f)  (((t)->tp_flags & (f)) != 0)
 #define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
 
@@ -317,7 +320,7 @@
 
 #define _PyGCHead_FINALIZED(g) (((g)->gc_refs & _PyGC_REFS_MASK_FINALIZED) != 
0)
 #define _PyGCHead_SET_FINALIZED(g, v) do {  \
-(g)->gc_refs = ((g)->gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \
+   (g)->gc_refs = ((g)->gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \
 | (v != 0); \
 } while (0)
 
diff --git a/pypy/module/cpyext/parse/cpyext_object.h 
b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -311,6 +311,8 @@
/* Type attribute cache version tag. Added in version 2.6 */
unsigned int tp_version_tag;
 
+destructor tp_finalize;
+
 } PyTypeObject;
 
 typedef struct _heaptypeobject {
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -406,6 +406,13 @@
 #if w_obj is not None:
 #assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
 
+@specialize.ll()
+def finalize(space, pyobj):
+from pypy.module.cpyext.api import generic_cpy_call
+assert is_pyobj(pyobj)
+pyobj = rffi.cast(PyObject, pyobj)
+state = space.fromcache(State)
+generic_cpy_call(space, state.C._Py_Finalize, pyobj)
 
 @init_function
 def write_w_marker_deallocating(space):
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -62,9 +62,17 @@
 }
 
 Py_ssize_t
-_PyPy_finalizer_type(PyGC_Head *g)
+_PyPy_finalizer_type(PyGC_Head *gc)
 {
-return 0;
+PyObject *op = FROM_GC(gc);
+if (!_PyGCHead_FINALIZED(gc) &&
+PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
+Py_TYPE(op)->tp_finalize != NULL) {
+return 1;
+} else {
+return 0;
+}
+// TODO: legacy finalizer (tp_del)
 }
 
 void
@@ -77,6 +85,23 @@
 }
 
 void
+_Py_Finalize(PyObject *op)
+{
+PyGC_Head *gc = _Py_AS_GC(op);
+destructor finalize;
+
+if (!_PyGCHead_FINALIZED(gc) &&
+PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
+(finalize = Py_TYPE(op)->tp_finalize) != NULL) {
+_PyGCHead_SET_FINALIZED(gc, 0);
+Py_INCREF(op);
+finalize(op);
+assert(!PyErr_Occurred());
+Py_DECREF(op);
+}
+}
+
+void
 _PyPy_object_dealloc(PyObject *obj)
 {
 PyTypeObject *pto;
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -61,7 +61,7 @@
 if not self.space.config.translating:
 def dealloc_trigger():
 from pypy.module.cpyext.pyobject import PyObject, decref, \
-incref, cts
+incref, cts, finalize
 print 'dealloc_trigger...'
 while True:
 ob = rawrefcount.next_dead(PyObject)
@@ -72,6 +72,11 @@
 print 'deallocating PyObject', ob, 'of type', name
 decref(space, ob)
 while True:
+py_obj = rawrefcount.next_cyclic_isolate(PyObject)
+if not py_obj:
+

[pypy-commit] pypy cpyext-gc-cycle: Added interface for rawrefcount finalizers to the gc

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95997:a316d3f47e9f
Date: 2019-02-13 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/a316d3f47e9f/

Log:Added interface for rawrefcount finalizers to the gc Added support
for rawrefcount finalizers to dot-tests Potentially dead cross-heap
cycles are kept alive until the following collection cycle (still
missing optimization for cycles without finalizers)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1187,6 +1187,9 @@
 state.C._PyPy_pyobj_as_gc = rffi.llexternal(
 '_PyPy_pyobj_as_gc', [GCHdr_PyObject], PyGC_HeadPtr,
 compilation_info=eci, _nowrapper=True)
+state.C._PyPy_finalizer_type = rffi.llexternal(
+'_PyPy_finalizer_type', [PyGC_HeadPtr], lltype.Signed,
+compilation_info=eci, _nowrapper=True)
 
 
 def init_function(func):
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -438,6 +438,7 @@
 PyAPI_FUNC(PyGC_Head *) _PyPy_init_pyobj_list();
 PyAPI_FUNC(GCHdr_PyObject *) _PyPy_gc_as_pyobj(PyGC_Head *);
 PyAPI_FUNC(PyGC_Head *) _PyPy_pyobj_as_gc(GCHdr_PyObject *);
+PyAPI_FUNC(Py_ssize_t) _PyPy_finalizer_type(PyGC_Head *);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -61,6 +61,12 @@
 }
 }
 
+Py_ssize_t
+_PyPy_finalizer_type(PyGC_Head *g)
+{
+return 0;
+}
+
 void
 _Py_Dealloc(PyObject *obj)
 {
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -174,7 +174,8 @@
 llhelper(rawrefcount.RAWREFCOUNT_TRAVERSE,
  self.tp_traverse),
 pypyobj_list,
-self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc)
+self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc,
+self.C._PyPy_finalizer_type)
 self.builder.attach_all(space)
 
 setup_new_method_def(space)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3011,6 +3011,8 @@
  PYOBJ_HDR_PTR))
 RAWREFCOUNT_PYOBJ_AS_GC = lltype.Ptr(lltype.FuncType([PYOBJ_HDR_PTR],
  PYOBJ_GC_HDR_PTR))
+RAWREFCOUNT_FINALIZER_TYPE = lltype.Ptr(lltype.FuncType([PYOBJ_GC_HDR_PTR],
+ lltype.Signed))
 
 def _pyobj(self, pyobjaddr):
 return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR)
@@ -3018,7 +3020,7 @@
 return llmemory.cast_adr_to_ptr(pygchdraddr, self.PYOBJ_GC_HDR_PTR)
 
 def rawrefcount_init(self, dealloc_trigger_callback, tp_traverse,
- pyobj_list, gc_as_pyobj, pyobj_as_gc):
+ pyobj_list, gc_as_pyobj, pyobj_as_gc, finalizer_type):
 # see pypy/doc/discussion/rawrefcount.rst
 if not self.rrc_enabled:
 self.rrc_p_list_young = self.AddressStack()
@@ -3041,6 +3043,7 @@
 self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
+self.rrc_finalizer_type = finalizer_type
 self.rrc_enabled = True
 
 def check_no_more_rawrefcount_state(self):
@@ -3268,13 +3271,19 @@
 # TODO: pypy objects
 
 def _rrc_major_trace(self, pyobject, ignore):
-pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
-if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
-rc = pygchdr.c_gc_refs
-else:
-rc = self._pyobj(pyobject).c_ob_refcnt
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+#
+# TODO: optimization: if no finalizers are found the cyclic rc
+# TODO: can be used instead of the real rc, because the objects
+# TODO: cannot be resurrected anyway
+# pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+# if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR):
+# rc = pygchdr.c_gc_refs
+# else:
+rc = self._pyobj(pyobject).c_ob_refcnt
 
-if rc == 0:
+if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT: # or rc == 0
 pass  # the corresponding object may die
 else:
 # force the corresponding object to be alive
diff --git a/rpython/memory/gc/test/dot/free_finalizer_nocycle.dot 

[pypy-commit] pypy cpyext-gc-trialdeletion: Close branch cpyext-gc-trialdeletion.

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r95958:8ec9653041d2
Date: 2019-02-11 18:35 +
http://bitbucket.org/pypy/pypy/changeset/8ec9653041d2/

Log:Close branch cpyext-gc-trialdeletion.

___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Fixed some bugs, tests and minor issues

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95782:b59793e59182
Date: 2019-02-03 23:06 +0100
http://bitbucket.org/pypy/pypy/changeset/b59793e59182/

Log:Fixed some bugs, tests and minor issues Added new test which
currently fails and should be addressed Implemented some mocks for
the tests, still WIP Removed some debug messages

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -977,7 +977,7 @@
 # we hope that malloc removal removes the newtuple() that is
 # inserted exactly here by the varargs specializer
 
-print "start to pypy"
+# print "start to pypy"
 
 # see "Handling of the GIL" above (careful, we don't have the GIL here)
 tid = rthread.get_or_make_ident()
@@ -1091,7 +1091,7 @@
 
 _restore_gil_state(pygilstate_release, gilstate, gil_release, 
_gil_auto, tid)
 
-print "end to pypy"
+# print "end to pypy"
 
 return retval
 
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -54,7 +54,11 @@
 PyGC_Head *
 _PyPy_pyobj_as_gc(GCHdr_PyObject *obj)
 {
-return AS_GC(obj);
+if (PyType_IS_GC(((PyObject *)obj)->ob_type)) {
+return AS_GC(obj);
+} else {
+return NULL;
+}
 }
 
 void
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -60,7 +60,8 @@
 space = self.space
 if not self.space.config.translating:
 def dealloc_trigger():
-from pypy.module.cpyext.pyobject import PyObject, decref, cts
+from pypy.module.cpyext.pyobject import PyObject, decref, \
+incref, cts
 print 'dealloc_trigger...'
 while True:
 ob = rawrefcount.next_dead(PyObject)
@@ -70,11 +71,47 @@
 name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
 print 'deallocating PyObject', ob, 'of type', name
 decref(space, ob)
+while True:
+ob = rawrefcount.cyclic_garbage_head(PyObject)
+if not ob:
+break
+pto = ob.c_ob_type
+name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
+print 'clearing PyObject', ob, 'of type', name
+
+pyobj = rffi.cast(PyObject, ob)
+adr_int = llmemory.cast_adr_to_int(
+llmemory.cast_ptr_to_adr(pyobj))
+if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_clear:
+incref(space, ob)
+pyobj.c_ob_type.c_tp_clear(pyobj)
+decref(space, ob)
+
+head = rawrefcount.cyclic_garbage_head(PyObject)
+if adr_int == llmemory.cast_adr_to_int(
+llmemory.cast_ptr_to_adr(head)):
+rawrefcount.cyclic_garbage_remove()
 print 'dealloc_trigger DONE'
 return "RETRY"
-def tp_traverse(obj_addr, callback, args):
-# TODO: implement
-pass
+def tp_traverse(pyobj_ptr, callback, args):
+from pypy.module.cpyext.api import PyObject
+from pypy.module.cpyext.typeobjectdefs import visitproc
+from pypy.module.cpyext.pyobject import cts
+# convert to pointers with correct types (PyObject)
+callback_addr = llmemory.cast_ptr_to_adr(callback)
+callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
+visitproc)
+pyobj_addr = llmemory.cast_ptr_to_adr(pyobj_ptr)
+pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject)
+
+pto = pyobj.c_ob_type
+name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
+print 'traverse PyObject', pyobj, 'of type', name
+
+# now call tp_traverse (if possible)
+if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
+pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
+  args)
 rawrefcount.init(dealloc_trigger, tp_traverse)
 else:
 if space.config.translation.gc == "boehm":
@@ -227,12 +264,14 @@
 break
 
 pyobj = rffi.cast(PyObject, py_obj)
-if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_clear:
+adr_int = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(pyobj))
+if pyobj.c_ob_type.c_tp_clear:
 incref(space, py_obj)
 pyobj.c_ob_type.c_tp_clear(pyobj)
 

[pypy-commit] pypy cpyext-gc-cycle: Fixed some minor issues and added TODOs for CPython style

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95671:faf091537a23
Date: 2019-01-17 18:21 +0100
http://bitbucket.org/pypy/pypy/changeset/faf091537a23/

Log:Fixed some minor issues and added TODOs for CPython style cycle
detection

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3031,8 +3031,6 @@
 self.rrc_dealloc_pending = self.AddressStack()
 self.rrc_tp_traverse = tp_traverse
 self.rrc_pyobj_list = self._pygchdr(pyobj_list)
-self.rrc_pyobj_old_list = lltype.malloc(
-self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_enabled = True
@@ -3171,7 +3169,7 @@
 else:
 self._rrc_free(pyobject)
 
-def _rrc_free(self, pyobject):
+def _rrc_free(self, pyobject, major=False):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
 #
@@ -3179,6 +3177,11 @@
 if rc >= REFCNT_FROM_PYPY_LIGHT:
 rc -= REFCNT_FROM_PYPY_LIGHT
 if rc == 0:
+if major: # remove from old list
+pygchdr = self.rrc_pyobj_as_gc(self._pyobj(pyobject))
+next = pygchdr.c_gc_next
+next.c_gc_prev = pygchdr.c_gc_prev
+pygchdr.c_gc_prev.c_gc_next = next
 lltype.free(self._pyobj(pyobject), flavor='raw')
 else:
 # can only occur if LIGHT is used in create_link_pyobj()
@@ -3191,16 +3194,19 @@
 rc -= REFCNT_FROM_PYPY
 self._pyobj(pyobject).c_ob_pypy_link = 0
 if rc == 0:
-self.rrc_dealloc_pending.append(pyobject)
-# an object with refcnt == 0 cannot stay around waiting
-# for its deallocator to be called.  Some code (lxml)
-# expects that tp_dealloc is called immediately when
-# the refcnt drops to 0.  If it isn't, we get some
-# uncleared raw pointer that can still be used to access
-# the object; but (PyObject *)raw_pointer is then bogus
-# because after a Py_INCREF()/Py_DECREF() on it, its
-# tp_dealloc is also called!
-rc = 1
+if not major: # we do it later in major collections
+self.rrc_dealloc_pending.append(pyobject)
+# an object with refcnt == 0 cannot stay around waiting
+# for its deallocator to be called.  Some code (lxml)
+# expects that tp_dealloc is called immediately when
+# the refcnt drops to 0.  If it isn't, we get some
+# uncleared raw pointer that can still be used to access
+# the object; but (PyObject *)raw_pointer is then bogus
+# because after a Py_INCREF()/Py_DECREF() on it, its
+# tp_dealloc is also called!
+rc = 1
+else:
+rc = REFCNT_FROM_PYPY
 self._pyobj(pyobject).c_ob_refcnt = rc
 _rrc_free._always_inline_ = True
 
@@ -3209,6 +3215,28 @@
 self._rrc_mark_rawrefcount()
 self.rrc_p_list_old.foreach(self._rrc_major_trace, None)
 
+# TODO: for all unreachable objects, which are marked potentially
+# TODO: uncollectable, move them to the set of uncollectable objs
+
+# TODO: for all unreachable objects with tp_del (legacy finalizer),
+# TODO: except for border objects with a refcount of
+# TODO: REFCNT_FROM_PYPY (equals zero at this point):
+# TODO:  * mark reachable pypy objects
+# TODO:  * move reachable cpython objects back to pyobj_list
+# TODO:  * mark all reachable objects as potentially uncollectable
+
+# TODO: handle weakrefs for unreachable objects
+
+# TODO: call tp_finalize for unreachable objects
+# TODO: (could resurrect objects, so we have to do it now)
+# TODO: (set finalizer flag before calling and check if
+# TODO: finalizer was not called before)
+
+# TODO: for all objects in unreachable, check if they
+# TODO: are still unreachable. if not, abort and move all
+# TODO: unreachable back to pyobj_list and mark all reachable
+# TODO: pypy objects
+
 def _rrc_major_trace(self, pyobject, ignore):
 rc = self.rrc_pyobj_as_gc(self._pyobj(pyobject)).c_gc_refs
 if rc == 0:
@@ -3222,7 +3250,6 @@
 
 def rrc_major_collection_free(self):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
 #
 

[pypy-commit] pypy cpyext-gc-cycle: Adapted tests in gc/rawrefcount to new cycle deletion

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95672:6e15b053de37
Date: 2019-01-18 21:06 +0100
http://bitbucket.org/pypy/pypy/changeset/6e15b053de37/

Log:Adapted tests in gc/rawrefcount to new cycle deletion

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -214,13 +214,27 @@
 
 
 def _rawrefcount_perform(space):
-from pypy.module.cpyext.pyobject import PyObject, decref
+from pypy.module.cpyext.pyobject import PyObject, incref, decref
 while True:
 py_obj = rawrefcount.next_dead(PyObject)
 if not py_obj:
 break
 decref(space, py_obj)
 
+while True:
+py_obj = rawrefcount.cyclic_garbage_head(PyObject)
+if not py_obj:
+break
+
+pyobj = rffi.cast(PyObject, py_obj)
+if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_clear:
+incref(space, py_obj)
+pyobj.c_ob_type.c_tp_clear(pyobj)
+decref(space, py_obj)
+
+if py_obj == rawrefcount.cyclic_garbage_head(PyObject):
+rawrefcount.cyclic_garbage_remove()
+
 class PyObjDeallocAction(executioncontext.AsyncAction):
 """An action that invokes _Py_Dealloc() on the dying PyObjects.
 """
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3031,6 +3031,12 @@
 self.rrc_dealloc_pending = self.AddressStack()
 self.rrc_tp_traverse = tp_traverse
 self.rrc_pyobj_list = self._pygchdr(pyobj_list)
+self.rrc_pyobj_old_list = \
+lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
+self.rrc_pyobj_garbage_list = \
+lltype.malloc(self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
+self.rrc_pyobj_garbage_list.c_gc_next = self.rrc_pyobj_garbage_list
+self.rrc_pyobj_garbage_list.c_gc_prev = self.rrc_pyobj_garbage_list
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_enabled = True
@@ -3096,6 +3102,26 @@
 return self.rrc_dealloc_pending.pop()
 return llmemory.NULL
 
+def rawrefcount_cyclic_garbage_head(self):
+if self.rrc_pyobj_garbage_list.c_gc_next <> \
+self.rrc_pyobj_garbage_list:
+return llmemory.cast_ptr_to_adr(
+self.rrc_gc_as_pyobj(self.rrc_pyobj_garbage_list.c_gc_next))
+else:
+return llmemory.NULL
+
+def rawrefcount_cyclic_garbage_remove(self):
+gchdr = self.rrc_pyobj_garbage_list.c_gc_next
+# remove from old list
+next = gchdr.c_gc_next
+next.c_gc_prev = gchdr.c_gc_prev
+gchdr.c_gc_prev.c_gc_next = next
+# add to new list, may die later
+next = self.rrc_pyobj_list.c_gc_next
+self.rrc_pyobj_list.c_gc_next = gchdr
+gchdr.c_gc_prev = self.rrc_pyobj_list
+gchdr.c_gc_next = next
+next.c_gc_prev = gchdr
 
 def rrc_invoke_callback(self):
 if self.rrc_enabled and self.rrc_dealloc_pending.non_empty():
@@ -3194,19 +3220,16 @@
 rc -= REFCNT_FROM_PYPY
 self._pyobj(pyobject).c_ob_pypy_link = 0
 if rc == 0:
-if not major: # we do it later in major collections
-self.rrc_dealloc_pending.append(pyobject)
-# an object with refcnt == 0 cannot stay around waiting
-# for its deallocator to be called.  Some code (lxml)
-# expects that tp_dealloc is called immediately when
-# the refcnt drops to 0.  If it isn't, we get some
-# uncleared raw pointer that can still be used to access
-# the object; but (PyObject *)raw_pointer is then bogus
-# because after a Py_INCREF()/Py_DECREF() on it, its
-# tp_dealloc is also called!
-rc = 1
-else:
-rc = REFCNT_FROM_PYPY
+self.rrc_dealloc_pending.append(pyobject)
+# an object with refcnt == 0 cannot stay around waiting
+# for its deallocator to be called.  Some code (lxml)
+# expects that tp_dealloc is called immediately when
+# the refcnt drops to 0.  If it isn't, we get some
+# uncleared raw pointer that can still be used to access
+# the object; but (PyObject *)raw_pointer is then bogus
+# because after a Py_INCREF()/Py_DECREF() on it, its
+# tp_dealloc is also called!
+rc = 1
 self._pyobj(pyobject).c_ob_refcnt = rc
 _rrc_free._always_inline_ = True
 
@@ -3225,7 +3248,9 @@
 # TODO:  * move reachable cpython objects back to pyobj_list
 # TODO:  * 

[pypy-commit] pypy cpyext-gc-cycle: Fixed translation issues

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95663:1fdae7eeae56
Date: 2019-01-16 23:38 +0100
http://bitbucket.org/pypy/pypy/changeset/1fdae7eeae56/

Log:Fixed translation issues

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3296,17 +3296,12 @@
 pygchdr = self.rrc_pyobj_list.c_gc_next
 while pygchdr <> self.rrc_pyobj_list:
 pyobj = self.rrc_gc_as_pyobj(pygchdr)
-self._rrc_visit_pyobj = self._rrc_subtract_internal_refcnt
-self._rrc_traverse(pyobj)
+self._rrc_traverse(pyobj, -1)
 pygchdr = pygchdr.c_gc_next
 
 # now all rawrefcounted roots or live border objects have a
 # refcount > 0
 
-def _rrc_subtract_internal_refcnt(self, pyobj):
-pygchdr = self.rrc_pyobj_as_gc(pyobj)
-pygchdr.c_gc_refs -= 1
-
 def _rrc_obj_fix_refcnt(self, pyobject, ignore):
 intobj = self._pyobj(pyobject).c_ob_pypy_link
 obj = llmemory.cast_int_to_adr(intobj)
@@ -3358,10 +3353,11 @@
 gchdr.c_gc_next = next
 next.c_gc_prev = gchdr
 # increment refcounts
-self._rrc_visit_pyobj = self._rrc_increment_refcnt
-self._rrc_traverse(pyobj)
+self._rrc_traverse(pyobj, 1)
 # mark recursively, if it is a pypyobj
-if not obj is None:
+if pyobj.c_ob_pypy_link <> 0:
+intobj = pyobj.c_ob_pypy_link
+obj = llmemory.cast_int_to_adr(intobj)
 self.objects_to_trace.append(obj)
 self.visit_all_objects()
 found_alive = True
@@ -3370,31 +3366,31 @@
 # now all rawrefcounted objects, which are alive, have a cyclic
 # refcount > 0 or are marked
 
-def _rrc_increment_refcnt(self, pyobj):
-pygchdr = self.rrc_pyobj_as_gc(pyobj)
-pygchdr.c_gc_refs += 1
-
 def _rrc_visit(pyobj, self_ptr):
 from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
 #
 self_adr = rffi.cast(llmemory.Address, self_ptr)
 self = cast_adr_to_nongc_instance(IncrementalMiniMarkGC, self_adr)
-self._rrc_visit_pyobj(pyobj)
+self._rrc_visit_action(pyobj, None)
 return rffi.cast(rffi.INT_real, 0)
 
-def _rrc_traverse(self, pyobj):
+def _rrc_visit_action(self, pyobj, ignore):
+pygchdr = self.rrc_pyobj_as_gc(pyobj)
+pygchdr.c_gc_refs += self.rrc_refcnt_add
+
+def _rrc_traverse(self, pyobj, refcnt_add):
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.annlowlevel import (cast_nongc_instance_to_adr,
 llhelper)
 #
+self.rrc_refcnt_add = refcnt_add
 if we_are_translated():
 callback_ptr = llhelper(self.RAWREFCOUNT_VISIT,
 IncrementalMiniMarkGC._rrc_visit)
 self_ptr = rffi.cast(rffi.VOIDP, cast_nongc_instance_to_adr(self))
+self.rrc_tp_traverse(pyobj, callback_ptr, self_ptr)
 else:
-callback_ptr = self._rrc_visit_pyobj
-self_ptr = None
-self.rrc_tp_traverse(pyobj, callback_ptr, self_ptr)
+self.rrc_tp_traverse(pyobj, self._rrc_visit_action, None)
 
 def _rrc_gc_list_init(self, pygclist):
 pygclist.c_gc_next = pygclist
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -31,7 +31,7 @@
 def rawrefcount_tp_traverse(obj, callback, args):
 refs = self.pyobj_refs[self.pyobjs.index(obj)]
 for ref in refs:
-callback(ref)
+callback(ref, args)
 
 def rawrefcount_gc_as_pyobj(gc):
 return self.pyobjs[self.gcobjs.index(gc)]
@@ -481,10 +481,9 @@
 dests_by_source[source].append(dest.r)
 for source in dests_by_source:
 dests_target = dests_by_source[source]
-def append(pyobj):
+def append(pyobj, ignore):
 dests_target.remove(pyobj)
-self.gc._rrc_visit_pyobj = append
-self.gc._rrc_traverse(source.r)
+self.gc.rrc_tp_traverse(source.r, append, None)
 assert len(dests_target) == 0
 
 # do collection
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Implemented first version of CPython-style cycle detection

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95662:fdd107ae6257
Date: 2019-01-16 18:26 +0100
http://bitbucket.org/pypy/pypy/changeset/fdd107ae6257/

Log:Implemented first version of CPython-style cycle detection Fixed dot
tests

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3031,6 +3031,8 @@
 self.rrc_dealloc_pending = self.AddressStack()
 self.rrc_tp_traverse = tp_traverse
 self.rrc_pyobj_list = self._pygchdr(pyobj_list)
+self.rrc_pyobj_old_list = lltype.malloc(
+self.PYOBJ_GC_HDR, flavor='raw', immortal=True)
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
 self.rrc_enabled = True
@@ -3203,14 +3205,13 @@
 _rrc_free._always_inline_ = True
 
 def rrc_major_collection_trace(self):
+self._rrc_collect_rawrefcount_roots()
+self._rrc_mark_rawrefcount()
 self.rrc_p_list_old.foreach(self._rrc_major_trace, None)
 
 def _rrc_major_trace(self, pyobject, ignore):
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
-#
-rc = self._pyobj(pyobject).c_ob_refcnt
-if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT:
+rc = self.rrc_pyobj_as_gc(self._pyobj(pyobject)).c_gc_refs
+if rc == 0:
 pass # the corresponding object may die
 else:
 # force the corresponding object to be alive
@@ -3220,6 +3221,9 @@
 self.visit_all_objects()
 
 def rrc_major_collection_free(self):
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+#
 ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty 
2")
 length_estimate = self.rrc_p_dict.length()
 self.rrc_p_dict.delete()
@@ -3238,6 +3242,20 @@
 no_o_dict)
 self.rrc_o_list_old.delete()
 self.rrc_o_list_old = new_o_list
+# free all dead refcounted objects, in unreachable cycles
+pygchdr = self.rrc_pyobj_old_list.c_gc_next
+while pygchdr <> self.rrc_pyobj_old_list:
+assert pygchdr.c_gc_refs == 0
+pyobj = self.rrc_gc_as_pyobj(pygchdr)
+if pyobj.c_ob_refcnt >= REFCNT_FROM_PYPY_LIGHT:
+lltype.free(pyobj, flavor='raw')
+elif pyobj.c_ob_refcnt >= REFCNT_FROM_PYPY:
+pyobject = llmemory.cast_ptr_to_adr(pyobj)
+pyobj.c_ob_refcnt = 1
+self.rrc_dealloc_pending.append(pyobject)
+else:
+lltype.free(pyobj, flavor='raw')
+pygchdr = pygchdr.c_gc_next
 
 def _rrc_major_free(self, pyobject, surviving_list, surviving_dict):
 # The pyobject survives if the corresponding obj survives.
@@ -3251,7 +3269,110 @@
 if surviving_dict:
 surviving_dict.insertclean(obj, pyobject)
 else:
-self._rrc_free(pyobject)
+# The pyobject is freed later, if it is in old list, so
+# just unlink here.
+self._pyobj(pyobject).c_ob_pypy_link = 0
+
+def _rrc_collect_rawrefcount_roots(self):
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
+from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+#
+# Initialize the cyclic refcount with the real refcount.
+pygchdr = self.rrc_pyobj_list.c_gc_next
+while pygchdr <> self.rrc_pyobj_list:
+pygchdr.c_gc_refs = self.rrc_gc_as_pyobj(pygchdr).c_ob_refcnt
+if pygchdr.c_gc_refs >= REFCNT_FROM_PYPY_LIGHT:
+pygchdr.c_gc_refs -= REFCNT_FROM_PYPY_LIGHT
+elif pygchdr.c_gc_refs >= REFCNT_FROM_PYPY:
+pygchdr.c_gc_refs -= REFCNT_FROM_PYPY
+pygchdr = pygchdr.c_gc_next
+
+# For every object in this set, if it is marked, add 1 as a real
+# refcount
+self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None)
+
+# Subtract all internal refcounts from the cyclic refcount
+# of rawrefcounted objects
+pygchdr = self.rrc_pyobj_list.c_gc_next
+while pygchdr <> self.rrc_pyobj_list:
+pyobj = self.rrc_gc_as_pyobj(pygchdr)
+self._rrc_visit_pyobj = self._rrc_subtract_internal_refcnt
+self._rrc_traverse(pyobj)
+pygchdr = pygchdr.c_gc_next
+
+# now all rawrefcounted roots or live border objects have a
+# refcount > 0
+
+def _rrc_subtract_internal_refcnt(self, pyobj):
+pygchdr = self.rrc_pyobj_as_gc(pyobj)
+pygchdr.c_gc_refs -= 1
+
+def _rrc_obj_fix_refcnt(self, pyobject, ignore):
+intobj = 

[pypy-commit] pypy cpyext-gc-cycle: Fixed dot tests by allocating all PyPy objects old

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95616:ea95f4bc807f
Date: 2019-01-10 11:43 +0100
http://bitbucket.org/pypy/pypy/changeset/ea95f4bc807f/

Log:Fixed dot tests by allocating all PyPy objects old Add to stackroots
instead of immortal PyPy objects for dot tests Sorted dot tests

diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -64,25 +64,16 @@
 refs.append(pyobj_to)
 pyobj_to.c_ob_refcnt += 1
 
-def _rawrefcount_pypyobj(self, intval, create_old=False,
- create_immortal=False, force_external=False):
-if create_immortal:
-p1 = lltype.malloc(S, immortal=True)
-else:
-saved = self.gc.nonlarge_max
-try:
-if force_external:
-self.gc.nonlarge_max = 1
-p1 = self.malloc(S)
-finally:
-self.gc.nonlarge_max = saved
+def _rawrefcount_pypyobj(self, intval, rooted=False, create_old=True):
+p1 = self.malloc(S)
 p1.x = intval
-if create_immortal:
-self.consider_constant(p1)
-elif create_old:
+
+if create_old:
 self.stackroots.append(p1)
 self._collect(major=False)
 p1 = self.stackroots.pop()
+if rooted:
+self.stackroots.append(p1)
 p1ref = lltype.cast_opaque_ptr(llmemory.GCREF, p1)
 
 def check_alive():
@@ -116,7 +107,7 @@
 
 def _rawrefcount_pair(self, intval, is_light=False, is_pyobj=False,
   create_old=False, create_immortal=False,
-  force_external=False):
+  rooted=False, force_external=False):
 if is_light:
 rc = REFCNT_FROM_PYPY_LIGHT
 else:
@@ -139,6 +130,8 @@
 self.stackroots.append(p1)
 self._collect(major=False)
 p1 = self.stackroots.pop()
+if rooted:
+self.stackroots.append(p1)
 p1ref = lltype.cast_opaque_ptr(llmemory.GCREF, p1)
 r1 = lltype.malloc(PYOBJ_HDR, flavor='raw',
immortal=create_immortal)
@@ -389,6 +382,7 @@
 
 dot_dir = os.path.join(os.path.realpath(os.path.dirname(__file__)), "dot")
 dot_files = [file for file in os.listdir(dot_dir) if file.endswith(".dot")]
+dot_files.sort()
 
 @py.test.mark.dont_track_allocations('intentionally keep objects alive, '
  'because we do the checks ourselves')
@@ -433,7 +427,8 @@
 g = pydot.graph_from_dot_file(path)[0]
 nodes = {}
 
-# create objects from graph
+# create objects from graph (always create old to prevent moving)
+i = 0
 for n in g.get_nodes():
 name = n.get_name()
 attr = n.obj_dict['attributes']
@@ -449,15 +444,18 @@
 nodes[name] = CPythonNode(r, raddr, check_alive, info)
 elif type == "P":
 p, pref, check_alive = \
-self._rawrefcount_pypyobj(42, create_immortal=rooted)
+self._rawrefcount_pypyobj(42 + i, rooted=rooted,
+  create_old=True)
 nodes[name] = PyPyNode(p, pref, check_alive, info)
+i += 1
 elif type == "B":
 p, pref, r, raddr, check_alive =\
-self._rawrefcount_pair(42, create_immortal=rooted)
+self._rawrefcount_pair(42 + i, rooted=rooted,
+   create_old=True)
 if ext_refcnt > 0:
 r.c_ob_refcnt = ext_refcnt
 nodes[name] = BorderNode(p, pref, r, raddr, check_alive, info)
-pass
+i += 1
 
 # add references between objects from graph
 for e in g.get_edges():
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Added complex rawrefcount tests using dot files

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95614:2e7b85611e30
Date: 2018-12-21 11:55 +0100
http://bitbucket.org/pypy/pypy/changeset/2e7b85611e30/

Log:Added complex rawrefcount tests using dot files Adapted traverse
support in incminimark to support tests

diff too long, truncating to 2000 out of 9197 lines

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3253,9 +3253,6 @@
 else:
 self._rrc_free(pyobject)
 
-def _rrc_visit_pyobj(self, pyobj):
-pass
-
 def _rrc_visit(pyobj, self_ptr):
 from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
 #
@@ -3265,13 +3262,18 @@
 return rffi.cast(rffi.INT_real, 0)
 
 def _rrc_traverse(self, pyobject):
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.rtyper.annlowlevel import (cast_nongc_instance_to_adr,
 llhelper)
 #
 pyobj = self._pyobj(pyobject)
-callback_ptr = llhelper(self.RAWREFCOUNT_VISIT,
-IncrementalMiniMarkGC._rrc_visit)
-self_ptr = rffi.cast(rffi.VOIDP, cast_nongc_instance_to_adr(self))
+if we_are_translated():
+callback_ptr = llhelper(self.RAWREFCOUNT_VISIT,
+IncrementalMiniMarkGC._rrc_visit)
+self_ptr = rffi.cast(rffi.VOIDP, cast_nongc_instance_to_adr(self))
+else:
+callback_ptr = self._rrc_visit_pyobj
+self_ptr = None
 self.rrc_tp_traverse(pyobj, callback_ptr, self_ptr)
 
 def _rrc_gc_list_init(self, pygclist):
diff --git a/rpython/memory/gc/test/__init__.py 
b/rpython/memory/gc/test/dot/__init__.py
copy from rpython/memory/gc/test/__init__.py
copy to rpython/memory/gc/test/dot/__init__.py
diff --git a/rpython/memory/gc/test/dot/dot_parser.py 
b/rpython/memory/gc/test/dot/dot_parser.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/dot_parser.py
@@ -0,0 +1,555 @@
+"""Graphviz's dot language parser.
+
+The dotparser parses GraphViz files in
+dot and dot files and transforms them
+into a class representation defined by `pydot`.
+
+Author: Michael Krause 
+Fixes by: Ero Carrera 
+"""
+from __future__ import division
+from __future__ import print_function
+import sys
+
+from pyparsing import (
+nestedExpr, Literal, CaselessLiteral,
+Word, OneOrMore,
+Forward,
+Group, Optional, Combine,
+restOfLine, cStyleComment, nums, alphanums,
+printables,
+ParseException, ParseResults, CharsNotIn,
+QuotedString)
+
+import pydot
+
+__author__ = ['Michael Krause', 'Ero Carrera']
+__license__ = 'MIT'
+
+
+PY3 = sys.version_info >= (3, 0, 0)
+if PY3:
+str_type = str
+else:
+str_type = basestring
+
+
+class P_AttrList(object):
+
+def __init__(self, toks):
+
+self.attrs = {}
+i = 0
+
+while i < len(toks):
+attrname = toks[i]
+if i+2 < len(toks) and toks[i+1] == '=':
+attrvalue = toks[i+2]
+i += 3
+else:
+attrvalue = None
+i += 1
+
+self.attrs[attrname] = attrvalue
+
+
+def __repr__(self):
+
+return "%s(%r)" % (self.__class__.__name__, self.attrs)
+
+
+
+class DefaultStatement(P_AttrList):
+
+def __init__(self, default_type, attrs):
+
+self.default_type = default_type
+self.attrs = attrs
+
+def __repr__(self):
+
+return "%s(%s, %r)" % (self.__class__.__name__,
+self.default_type, self.attrs)
+
+
+top_graphs = list()
+
+def push_top_graph_stmt(str, loc, toks):
+
+attrs = {}
+g = None
+
+for element in toks:
+
+if (isinstance(element, (ParseResults, tuple, list)) and
+len(element) == 1 and
+isinstance(element[0], str_type)):
+
+element = element[0]
+
+if element == 'strict':
+attrs['strict'] = True
+
+elif element in ['graph', 'digraph']:
+
+attrs = {}
+
+g = pydot.Dot(graph_type=element, **attrs)
+attrs['type'] = element
+
+top_graphs.append( g )
+
+elif isinstance( element, str_type):
+g.set_name( element )
+
+elif isinstance(element, pydot.Subgraph):
+
+g.obj_dict['attributes'].update( element.obj_dict['attributes'] )
+g.obj_dict['edges'].update( element.obj_dict['edges'] )
+g.obj_dict['nodes'].update( element.obj_dict['nodes'] )
+g.obj_dict['subgraphs'].update( element.obj_dict['subgraphs'] )
+
+g.set_parent_graph(g)
+
+elif isinstance(element, P_AttrList):
+attrs.update(element.attrs)
+
+elif isinstance(element, (ParseResults, list)):
+add_elements(g, element)
+
+else:
+

[pypy-commit] pypy cpyext-gc-cycle: Removed extra flags in cpython refcount

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95605:878ff32d88a1
Date: 2018-07-05 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/878ff32d88a1/

Log:Removed extra flags in cpython refcount Fixed tests in
test_rawrefcount and test_cpyext Removed references from rawrefcount
to cpyext Added some comments

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1787,10 +1787,10 @@
 
 preexist_error = PyErr_Occurred(space)
 try:
-print "start cpyext_call"
+#print "start cpyext_call"
 # Call the function
 result = call_external_function(func, *boxed_args)
-print "end cpyext_call"
+#print "end cpyext_call"
 finally:
 assert cpyext_glob_tid_ptr[0] == tid
 cpyext_glob_tid_ptr[0] = tid_before
diff --git a/pypy/module/cpyext/include/boolobject.h 
b/pypy/module/cpyext/include/boolobject.h
--- a/pypy/module/cpyext/include/boolobject.h
+++ b/pypy/module/cpyext/include/boolobject.h
@@ -13,8 +13,8 @@
 #define Py_True ((PyObject *) &_Py_TrueStruct)
 
 /* Macros for returning Py_True or Py_False, respectively */
-#define Py_RETURN_TRUE do { Py_INCREF(Py_True); return Py_True; } while(0)
-#define Py_RETURN_FALSE do { Py_INCREF(Py_False); return Py_False; } while(0)
+#define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True
+#define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -2,7 +2,6 @@
 #define Py_OBJECT_H
 
 #include 
-#include 
 
 #ifdef __cplusplus
 extern "C" {
@@ -13,12 +12,7 @@
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 
-#define PY_REFCNT_FROM_PYPY (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 2)))
-#define PY_REFCNT_OVERFLOW (1L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 4) - 
1L))
-#define PY_REFCNT_MASK ((PY_REFCNT_OVERFLOW << 1L) - 1L)
-#define Py_RETURN_NONE return (PyObject *)(Py_None))->ob_refcnt & 
PY_REFCNT_OVERFLOW) == 0) ? \
-  ((PyObject *)(Py_None))->ob_refcnt++ : 
Py_IncRef((PyObject *)(Py_None))), Py_None
-
+#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 
 /*
 CPython has this for backwards compatibility with really old extensions, and 
now
@@ -40,20 +34,14 @@
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)(ob)))
 #else
 /* Fast version */
-#define Py_INCREF(ob)   do {   
  \
-if (!(((PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_OVERFLOW))   \
-((PyObject *)(ob))->ob_refcnt++;   
  \
-else   
  \
-Py_IncRef((PyObject *)(ob));   
 \
-} while (0)
-#define Py_DECREF(ob)   do {   
  \
-if PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_OVERFLOW))\
-Py_DecRef((PyObject *)(ob));   
  \
-else if (--((PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_MASK)   \
-;  
  \
-else   
  \
-_Py_Dealloc((PyObject *)(ob)); 
  \
-} while (0)
+#define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
+#define Py_DECREF(op)   \
+do {\
+if (--((PyObject *)(op))->ob_refcnt != 0)   \
+;   \
+else\
+_Py_Dealloc((PyObject *)(op));  \
+} while (0)
 
 #define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
 #define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
@@ -73,8 +61,7 @@
 }  \
 } while (0)
 
-#define Py_REFCNT(ob) PyObject *)(ob))->ob_refcnt & PY_REFCNT_OVERFLOW == 
0) ?  \
-  (((PyObject*)(ob))->ob_refcnt & PY_REFCNT_MASK) : 
_Py_RefCnt_Overflow(ob))
+#define Py_REFCNT(ob)  (((PyObject*)(ob))->ob_refcnt)
 #define Py_TYPE(ob)(((PyObject*)(ob))->ob_type)
 #define Py_SIZE(ob)(((PyVarObject*)(ob))->ob_size)
 
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- 

[pypy-commit] pypy cpyext-gc-cycle: Fixed cpyext test

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95608:ada4b64c0816
Date: 2018-08-02 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/ada4b64c0816/

Log:Fixed cpyext test

diff --git a/pypy/module/cpyext/test/test_cpyext.py 
b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -1069,6 +1069,7 @@
 (initproc)Cycle_init,  /* tp_init */
 0, /* tp_alloc */
 Cycle_new, /* tp_new */
+PyObject_GC_Del,   /* tp_free */
 };
 
 extern PyGC_Head *_pypy_rawrefcount_pyobj_list;
@@ -1078,6 +1079,8 @@
  Cycle *c = PyObject_GC_New(Cycle, );
  if (c == NULL)
  return NULL;
+ 
+ Py_INCREF(val);
  c->next = val;
 
  // TODO: check if _pypy_rawrefcount_pyobj_list contains c
@@ -1100,7 +1103,3 @@
 self.print_pyobj_list()
 c = module.create(Example(42))
 self.print_pyobj_list()
-
-# TODO: fix rawrefcount, so that the Cycle objects are properly added
-#   to the ALLOCATED list of leakfinder or alternatively not freed
-#   by collect
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Removed unnecessary code

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95603:b74906a7ac4b
Date: 2018-05-10 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/b74906a7ac4b/

Log:Removed unnecessary code

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -43,6 +43,7 @@
 from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
+from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 
 DEBUG_WRAPPER = True
 
@@ -974,6 +975,8 @@
 # we hope that malloc removal removes the newtuple() that is
 # inserted exactly here by the varargs specializer
 
+print "start to pypy"
+
 # see "Handling of the GIL" above (careful, we don't have the GIL here)
 tid = rthread.get_or_make_ident()
 _gil_auto = False
@@ -1085,6 +1088,9 @@
 rffi.stackcounter.stacks_counter -= 1
 
 _restore_gil_state(pygilstate_release, gilstate, gil_release, 
_gil_auto, tid)
+
+print "end to pypy"
+
 return retval
 
 wrapper_second_level._dont_inline_ = True
@@ -1773,8 +1779,10 @@
 
 preexist_error = PyErr_Occurred(space)
 try:
+print "start cpyext_call"
 # Call the function
 result = call_external_function(func, *boxed_args)
+print "end cpyext_call"
 finally:
 assert cpyext_glob_tid_ptr[0] == tid
 cpyext_glob_tid_ptr[0] = tid_before
diff --git a/rpython/config/translationoption.py 
b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -105,6 +105,14 @@
  "asmgcc": [("translation.gctransformer", "framework"),
 ("translation.backend", "c")],
 }),
+ChoiceOption("cpyextgc", "Garbage Collection Strategy for cpyext",
+ ["boehm", "ref", "ref_trialdel", "none"],
+ default="ref",
+ requires={
+"boehm": [("translation.gc", "incminimark")],
+"ref_trialdel": [("translation.gc", "incminimark")],
+ },
+ cmdline="--cpyextgc"),
 
 # other noticeable options
 BoolOption("thread", "enable use of threading primitives",
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -177,6 +177,8 @@
 defines = defines.copy()
 if self.config.translation.countmallocs:
 defines['COUNT_OP_MALLOCS'] = 1
+if self.config.translation.cpyextgc == "boehm":
+defines['CPYEXT_BOEHM'] = 1
 if self.config.translation.sandbox:
 defines['RPY_SANDBOXED'] = 1
 if CBuilder.have___thread is None:
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Implemented pyobj_list for rawrefcount (to be used in cpyext tests)

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95606:9e5001a6604b
Date: 2018-07-05 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/9e5001a6604b/

Log:Implemented pyobj_list for rawrefcount (to be used in cpyext tests)
Added own cpyext test file for GC-related tests

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1300,7 +1300,8 @@
 ctypes.c_void_p)
 
 # initialize the pyobj_list for the gc
-space.fromcache(State).C._PyPy_InitPyObjList()
+pyobj_list = space.fromcache(State).C._PyPy_InitPyObjList()
+rawrefcount._init_pyobj_list(pyobj_list)
 
 # we need to call this *after* the init code above, because it might
 # indirectly call some functions which are attached to pypyAPI (e.g., we
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -59,28 +59,23 @@
 def setup_rawrefcount(self):
 space = self.space
 if not self.space.config.translating:
-from pypy.module.cpyext.api import PyGC_HeadPtr
 def dealloc_trigger():
-from pypy.module.cpyext.pyobject import PyObject, decref
+from pypy.module.cpyext.pyobject import PyObject, decref, cts
 print 'dealloc_trigger...'
 while True:
 ob = rawrefcount.next_dead(PyObject)
 if not ob:
 break
-print 'deallocating PyObject', ob
+pto = ob.c_ob_type
+name = rffi.charp2str(cts.cast('char*', pto.c_tp_name))
+print 'deallocating PyObject', ob, 'of type', name
 decref(space, ob)
 print 'dealloc_trigger DONE'
 return "RETRY"
 def tp_traverse(obj_addr, callback, args):
 # TODO: implement
 pass
-# Warning: This list ist different than the list actually used
-# by the extension modules (see _PyPy_InitPyObjList).
-pyobj_list = lltype.malloc(PyGC_HeadPtr.TO,
-   flavor='raw', immortal=True, zero=True)
-pyobj_list.c_gc_next = rffi.cast(rffi.VOIDP, pyobj_list);
-pyobj_list.c_gc_next = rffi.cast(rffi.VOIDP, pyobj_list);
-rawrefcount.init(dealloc_trigger, tp_traverse, pyobj_list)
+rawrefcount.init(dealloc_trigger, tp_traverse)
 else:
 if space.config.translation.gc == "boehm":
 action = BoehmPyObjDeallocAction(space)
diff --git a/pypy/module/cpyext/test/test_cpyext_gc.py 
b/pypy/module/cpyext/test/test_cpyext_gc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_cpyext_gc.py
@@ -0,0 +1,801 @@
+import sys
+import weakref
+
+import pytest
+
+from pypy.tool.cpyext.extbuild import (
+SystemCompilationInfo, HERE, get_sys_info_app)
+from pypy.interpreter.gateway import unwrap_spec, interp2app
+from rpython.rtyper.lltypesystem import lltype, ll2ctypes
+from pypy.module.cpyext import api
+from pypy.module.cpyext.state import State
+from rpython.tool.identity_dict import identity_dict
+from rpython.tool import leakfinder
+from rpython.rlib import rawrefcount
+from rpython.tool.udir import udir
+
+only_pypy ="config.option.runappdirect and '__pypy__' not in 
sys.builtin_module_names"
+
+@api.cpython_api([], api.PyObject)
+def PyPy_Crash1(space):
+1/0
+
+@api.cpython_api([], lltype.Signed, error=-1)
+def PyPy_Crash2(space):
+1/0
+
+class SpaceCompiler(SystemCompilationInfo):
+"""Extension compiler for regular (untranslated PyPy) mode"""
+def __init__(self, space, *args, **kwargs):
+self.space = space
+SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+def load_module(self, mod, name):
+space = self.space
+api.load_extension_module(space, mod, name)
+return space.getitem(
+space.sys.get('modules'), space.wrap(name))
+
+
+def get_cpyext_info(space):
+from pypy.module.imp.importing import get_so_extension
+state = space.fromcache(State)
+api_library = state.api_lib
+if sys.platform == 'win32':
+libraries = [api_library]
+# '%s' undefined; assuming extern returning int
+compile_extra = ["/we4013"]
+# prevent linking with PythonXX.lib
+w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+(space.int_w(w_maj), space.int_w(w_min))]
+else:
+libraries = []
+if sys.platform.startswith('linux'):
+compile_extra = [
+"-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+link_extra = ["-g"]
+else:
+compile_extra = link_extra = None
+return 

[pypy-commit] pypy cpyext-gc-cycle: Fixed some formatting issues

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95613:80824650968a
Date: 2018-08-24 09:30 +0200
http://bitbucket.org/pypy/pypy/changeset/80824650968a/

Log:Fixed some formatting issues

diff --git a/pypy/module/cpyext/parse/cpyext_object.h 
b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -332,4 +332,4 @@
 typedef struct _gchdr_pyobject {
 Py_ssize_t ob_refcnt;
 Py_ssize_t ob_pypy_link;
-} GCHdr_PyObject;
\ No newline at end of file
+} GCHdr_PyObject;
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -406,6 +406,7 @@
 #if w_obj is not None:
 #assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
 
+
 @init_function
 def write_w_marker_deallocating(space):
 if we_are_translated():
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -221,4 +221,4 @@
 {
 obj->ob_size = size;
 return (PyVarObject*)PyObject_Init((PyObject*)obj, type);
-}
\ No newline at end of file
+}
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Implemented pyobj as gc and vice-versa

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95607:a189719f68e4
Date: 2018-07-06 23:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a189719f68e4/

Log:Implemented pyobj as gc and vice-versa Cleaned cpyext and
gc/rawrefcount tests Cleaned translation options

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -43,7 +43,6 @@
 from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
-from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 
 DEBUG_WRAPPER = True
 
@@ -753,6 +752,7 @@
 PyVarObject = cts.gettype('PyVarObject *')
 PyGC_Head = cts.gettype('PyGC_Head')
 PyGC_HeadPtr = cts.gettype('PyGC_Head *')
+GCHdr_PyObject = cts.gettype('GCHdr_PyObject *')
 
 Py_buffer = cts.gettype('Py_buffer')
 Py_bufferP = cts.gettype('Py_buffer *')
@@ -1175,8 +1175,17 @@
 state.C._PyPy_object_dealloc = rffi.llexternal(
 '_PyPy_object_dealloc', [PyObject], lltype.Void,
 compilation_info=eci, _nowrapper=True)
-state.C._PyPy_InitPyObjList = rffi.llexternal(
-'_PyPy_InitPyObjList', [], PyGC_HeadPtr,
+state.C._PyPy_subtype_dealloc = rffi.llexternal(
+'_PyPy_subtype_dealloc', [PyObject], lltype.Void,
+compilation_info=eci, _nowrapper=True)
+state.C._PyPy_init_pyobj_list = rffi.llexternal(
+'_PyPy_init_pyobj_list', [], PyGC_HeadPtr,
+compilation_info=eci, _nowrapper=True)
+state.C._PyPy_gc_as_pyobj = rffi.llexternal(
+'_PyPy_gc_as_pyobj', [PyGC_HeadPtr], GCHdr_PyObject,
+compilation_info=eci, _nowrapper=True)
+state.C._PyPy_pyobj_as_gc = rffi.llexternal(
+'_PyPy_pyobj_as_gc', [GCHdr_PyObject], PyGC_HeadPtr,
 compilation_info=eci, _nowrapper=True)
 
 
@@ -1300,7 +1309,7 @@
 ctypes.c_void_p)
 
 # initialize the pyobj_list for the gc
-pyobj_list = space.fromcache(State).C._PyPy_InitPyObjList()
+pyobj_list = space.fromcache(State).C._PyPy_init_pyobj_list()
 rawrefcount._init_pyobj_list(pyobj_list)
 
 # we need to call this *after* the init code above, because it might
@@ -1309,9 +1318,6 @@
 # _PyPy_Malloc)
 builder.attach_all(space)
 
-#import rpython.rlib.rawrefcount
-#rawrefcount.init_traverse(generic_cpy_call_gc)
-
 setup_init_functions(eci, prefix)
 return modulename.new(ext='')
 
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -326,8 +326,8 @@
 
 #define _PyGC_REFS(o) _PyGCHead_REFS(_Py_AS_GC(o))
 
-#define _PyGC_REFS_UNTRACKED(-2)
-#define _PyGC_REFS_REACHABLE(-3)
+#define _PyGC_REFS_UNTRACKED   (-2)
+#define _PyGC_REFS_REACHABLE   (-3)
 #define _PyGC_REFS_TENTATIVELY_UNREACHABLE (-4)
 
 #define _PyGC_IS_TRACKED(o) (_PyGC_REFS(o) != _PyGC_REFS_UNTRACKED)
@@ -435,7 +435,9 @@
 #define _PyObject_GC_Del PyObject_GC_Del
 PyAPI_FUNC(void) _PyPy_subtype_dealloc(PyObject *);
 PyAPI_FUNC(void) _PyPy_object_dealloc(PyObject *);
-PyAPI_FUNC(PyGC_Head *) _PyPy_InitPyObjList();
+PyAPI_FUNC(PyGC_Head *) _PyPy_init_pyobj_list();
+PyAPI_FUNC(GCHdr_PyObject *) _PyPy_gc_as_pyobj(PyGC_Head *);
+PyAPI_FUNC(PyGC_Head *) _PyPy_pyobj_as_gc(GCHdr_PyObject *);
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/parse/cpyext_object.h 
b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -324,7 +324,12 @@
 
 
 typedef struct _gc_head {
-void *gc_next;
-void *gc_prev;
+struct _gc_head *gc_next;
+struct _gc_head *gc_prev;
 Py_ssize_t gc_refs;
-} PyGC_Head;
\ No newline at end of file
+} PyGC_Head;
+
+typedef struct _gchdr_pyobject {
+Py_ssize_t ob_refcnt;
+Py_ssize_t ob_pypy_link;
+} GCHdr_PyObject;
\ No newline at end of file
diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c
--- a/pypy/module/cpyext/src/object.c
+++ b/pypy/module/cpyext/src/object.c
@@ -38,13 +38,25 @@
 PyGC_Head *_pypy_rawrefcount_pyobj_list = &_internal_pyobj_list;
 
 PyGC_Head *
-_PyPy_InitPyObjList()
+_PyPy_init_pyobj_list()
 {
 _pypy_rawrefcount_pyobj_list->gc_next = _pypy_rawrefcount_pyobj_list;
 _pypy_rawrefcount_pyobj_list->gc_prev = _pypy_rawrefcount_pyobj_list;
 return _pypy_rawrefcount_pyobj_list;
 }
 
+GCHdr_PyObject *
+_PyPy_gc_as_pyobj(PyGC_Head *g)
+{
+return (GCHdr_PyObject *)FROM_GC(g);
+}
+
+PyGC_Head *
+_PyPy_pyobj_as_gc(GCHdr_PyObject *obj)
+{
+return AS_GC(obj);
+}
+
 void
 _Py_Dealloc(PyObject *obj)
 {
@@ -118,7 +130,7 @@
 if (type->tp_itemsize)
 size += nitems * type->tp_itemsize;
 
-g = (PyObject*)_PyPy_Malloc(size);
+g = (PyGC_Head*)_PyPy_Malloc(size);
 if (g == NULL)
 return NULL;

[pypy-commit] pypy cpyext-gc-cycle: Added some dot tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95617:c46c894a7c06
Date: 2019-01-11 10:40 +0100
http://bitbucket.org/pypy/pypy/changeset/c46c894a7c06/

Log:Added some dot tests

diff --git a/rpython/memory/gc/test/dot/free_cpython_self.dot 
b/rpython/memory/gc/test/dot/free_cpython_self.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cpython_self.dot
@@ -0,0 +1,4 @@
+digraph G {
+"a" [type=C, alive=n];
+"a" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_1a.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_1a.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_1a.dot
@@ -0,0 +1,9 @@
+digraph G {
+"a" [type=B, alive=n];
+"b" [type=C, alive=n];
+"c" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
+"a" -> "c";
+"c" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_1b.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_1b.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_1b.dot
@@ -0,0 +1,10 @@
+digraph G {
+"a" [type=B, alive=n];
+"b" [type=C, alive=n];
+"c" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
+"a" -> "c";
+"c" -> "a";
+"b" -> "c";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_2a.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_2a.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_2a.dot
@@ -0,0 +1,9 @@
+digraph G {
+"a" [type=B, alive=n];
+"b" [type=C, alive=n];
+"c" [type=C, alive=n];
+"a" -> "b";
+"b" -> "c";
+"c" -> "a";
+"b" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_2b.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_2b.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_2b.dot
@@ -0,0 +1,11 @@
+digraph G {
+"a" [type=B, alive=n];
+"b" [type=C, alive=n];
+"c" [type=C, alive=n];
+"d" [type=B, alive=n];
+"a" -> "b";
+"b" -> "c";
+"c" -> "d";
+"b" -> "a";
+"d" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_2c.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_2c.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_2c.dot
@@ -0,0 +1,13 @@
+digraph G {
+"a" [type=B, alive=n];
+"b" [type=C, alive=n];
+"c" [type=C, alive=n];
+"d" [type=B, alive=n];
+"e" [type=P, alive=n];
+"a" -> "b";
+"b" -> "c";
+"c" -> "d";
+"b" -> "a";
+"d" -> "e";
+"e" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_3a.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_3a.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_3a.dot
@@ -0,0 +1,9 @@
+digraph G {
+"a" [type=P, alive=n];
+"b" [type=B, alive=n];
+"c" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
+"b" -> "c";
+"c" -> "b";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_3b.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_3b.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_3b.dot
@@ -0,0 +1,11 @@
+digraph G {
+"a" [type=P, alive=n];
+"b" [type=B, alive=n];
+"c" [type=C, alive=n];
+"d" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
+"b" -> "c";
+"c" -> "d";
+"d" -> "b";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_multi_3c.dot 
b/rpython/memory/gc/test/dot/free_cross_multi_3c.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_multi_3c.dot
@@ -0,0 +1,11 @@
+digraph G {
+"a" [type=P, alive=n];
+"b" [type=P, alive=n];
+"c" [type=B, alive=n];
+"d" [type=C, alive=n];
+"a" -> "b";
+"b" -> "c";
+"c" -> "d";
+"d" -> "c";
+"c" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_simple_1.dot 
b/rpython/memory/gc/test/dot/free_cross_simple_1.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_simple_1.dot
@@ -0,0 +1,6 @@
+digraph G {
+"a" [type=B, alive=n];
+"b" [type=C, alive=n];
+"a" -> "b";
+"b" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_simple_2.dot 
b/rpython/memory/gc/test/dot/free_cross_simple_2.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_simple_2.dot
@@ -0,0 +1,10 @@
+digraph G {
+"a" [type=P, alive=n];
+"b" [type=B, alive=n];
+"c" [type=C, alive=n];
+"d" [type=B, alive=n];
+"a" -> "b";
+"b" -> "c";
+"c" -> "d";
+"d" -> "a";
+}
diff --git a/rpython/memory/gc/test/dot/free_cross_simple_3.dot 
b/rpython/memory/gc/test/dot/free_cross_simple_3.dot
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/dot/free_cross_simple_3.dot
@@ -0,0 +1,12 @@
+digraph G {
+"a" [type=P, alive=n];
+"b" [type=P, alive=n];
+"c" [type=B, alive=n];
+"d" [type=C, alive=n];
+"e" [type=B, 

[pypy-commit] pypy cpyext-gc-cycle: Cleaned up code in incminimark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95612:5b3e2b3a25bf
Date: 2018-08-24 09:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5b3e2b3a25bf/

Log:Cleaned up code in incminimark

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3030,9 +3030,6 @@
 self.rrc_dealloc_trigger_callback = dealloc_trigger_callback
 self.rrc_dealloc_pending = self.AddressStack()
 self.rrc_tp_traverse = tp_traverse
-self.rrc_pyobjects_to_scan = self.AddressStack()
-self.rrc_more_pyobjects_to_scan = self.AddressStack()
-self.rrc_pyobjects_to_trace = self.AddressStack()
 self.rrc_pyobj_list = self._pygchdr(pyobj_list)
 self.rrc_gc_as_pyobj = gc_as_pyobj
 self.rrc_pyobj_as_gc = pyobj_as_gc
@@ -3205,16 +3202,8 @@
 self._pyobj(pyobject).c_ob_refcnt = rc
 _rrc_free._always_inline_ = True
 
-NO_CYCLE_DETECTION = False
-
 def rrc_major_collection_trace(self):
-debug_start("gc-rrc-trace")
-if self.NO_CYCLE_DETECTION:
-self.rrc_p_list_old.foreach(self._rrc_major_trace, None)
-else:
-self.rrc_major_collection_trace_cycle()
-self.rrc_p_list_old.foreach(self._rrc_major_trace, None) # for 
now, remove later
-debug_stop("gc-rrc-trace")
+self.rrc_p_list_old.foreach(self._rrc_major_trace, None)
 
 def _rrc_major_trace(self, pyobject, ignore):
 from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
@@ -3230,77 +3219,6 @@
 self.objects_to_trace.append(obj)
 self.visit_all_objects()
 
-def rrc_major_collection_trace_cycle(self):
-assert not self.objects_to_trace.non_empty()
-assert not self.rrc_pyobjects_to_scan.non_empty()
-assert not self.rrc_more_pyobjects_to_scan.non_empty()
-assert not self.rrc_pyobjects_to_trace.non_empty()
-
-self._rrc_gc_print_list()
-
-# initially, scan all real pyobjects (not proxies) which are linked to 
objects
-#self.rrc_p_list_old.foreach(self._rrc_major_scan_non_rc_roots, None)
-self.rrc_o_list_old.foreach(self._rrc_major_scan_non_rc_roots, None)
-
-# as long as we find new pyobjects which should be marked, recursively
-# mark them
-while self.rrc_pyobjects_to_trace.non_empty():
-while self.rrc_pyobjects_to_trace.non_empty():
-pyobject = self.rrc_pyobjects_to_trace.pop()
-self._rrc_traverse(pyobject)
-
-# see if we found new pypy objects to trace
-if self.objects_to_trace.non_empty():
-self.visit_all_objects()
-self.objects_to_trace.delete()
-self.objects_to_trace = self.AddressStack()
-
-# look if there are some pyobjects with linked objects which were
-# not marked previously, but are marked now
-swap = self.rrc_pyobjects_to_scan
-self.rrc_pyobjects_to_scan = self.rrc_more_pyobjects_to_scan
-self.rrc_more_pyobjects_to_scan = swap
-self.rrc_pyobjects_to_scan.foreach(
-self._rrc_major_scan_non_rc_roots, None)
-self.rrc_pyobjects_to_scan.delete()
-self.rrc_pyobjects_to_scan = self.AddressStack()
-
-self.rrc_more_pyobjects_to_scan.delete()
-self.rrc_more_pyobjects_to_scan = self.AddressStack()
-
-def _rrc_mark_cpyobj(self, pyobj):
-# if the pyobj is not marked, remember it and if there is a linked pypy
-# object also remember it
-visited = True # TODO: check if visited (via 'cast' to PyGC_Head)
-if not visited:
-# TODO: mark visited
-pyobject = llmemory.cast_ptr_to_adr(pyobj)
-self.rrc_more_pyobjects_to_scan.append(pyobject)
-intobj = pyobj.c_ob_pypy_link
-if intobj != 0:
-obj = llmemory.cast_int_to_adr(intobj)
-hdr = self.header(obj)
-if not (hdr.tid & GCFLAG_VISITED):
-self.objects_to_trace.append(obj)
-
-def _rrc_major_scan_non_rc_roots(self, pyobject, ignore):
-# check in the object header of the linked pypy object, if it is marked
-# or not
-pyobj = self._pyobj(pyobject)
-intobj = pyobj.c_ob_pypy_link
-obj = llmemory.cast_int_to_adr(intobj)
-hdr = self.header(obj)
-if hdr.tid & GCFLAG_VISITED:
-visited = True  # TODO: check if visited
-if not visited:
-# process the pyobject now
-# TODO: mark visited
-self.rrc_pyobjects_to_trace.append(pyobject)
-else:
-# save the pyobject for later, in case its linked object becomes
-# marked
-self.rrc_more_pyobjects_to_scan.append(pyobject)
-
 def 

[pypy-commit] pypy cpyext-gc-cycle: Fixed rawrefcount tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95610:c1219f8ea34b
Date: 2018-08-03 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/c1219f8ea34b/

Log:Fixed rawrefcount tests

diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -334,146 +334,66 @@
 self._collect(major=True)
 check_alive(0)
 
-def test_cycle_self_reference_free(self):
+def test_linked_cycle_self_reference_dies_without_external_reference(self):
+p1, p1ref, r1, r1addr, check_alive = (
+self._rawrefcount_pair(42))
+r1.c_ob_refcnt += 1
+p1.next = p1
+check_alive(+1)
+self._collect(major=True, expected_trigger=1)
+py.test.raises(RuntimeError, "p1.x")  # dead
+assert r1.c_ob_refcnt == 1  # in the pending list
+assert r1.c_ob_pypy_link == 0
+assert self.gc.rawrefcount_next_dead() == r1addr
+assert self.gc.rawrefcount_next_dead() == llmemory.NULL
+assert self.gc.rawrefcount_next_dead() == llmemory.NULL
+self.gc.check_no_more_rawrefcount_state()
+lltype.free(r1, flavor='raw')
+
+def test_linked_cycle_self_reference_survives_with_pyobj_reference(self):
 p1, p1ref, r1, r1addr, check_alive = (
 self._rawrefcount_pair(42, create_immortal=True))
+r1.c_ob_refcnt += 2  # the pyobject is kept alive
 p1.next = p1
-check_alive(0)
+check_alive(+2)
 self._collect(major=True)
-py.test.raises(RuntimeError, "r1.c_ob_refcnt")  # dead
+check_alive(+2)
+r1.c_ob_refcnt -= 1 # the external reference from pyobj is removed
+check_alive(+1)
+self._collect(major=True, expected_trigger=1)
 py.test.raises(RuntimeError, "p1.x")  # dead
+assert r1.c_ob_refcnt == 1  # in the pending list
+assert r1.c_ob_pypy_link == 0
+assert self.gc.rawrefcount_next_dead() == r1addr
+assert self.gc.rawrefcount_next_dead() == llmemory.NULL
+assert self.gc.rawrefcount_next_dead() == llmemory.NULL
+self.gc.check_no_more_rawrefcount_state()
+lltype.free(r1, flavor='raw')
 
-def test_cycle_self_reference_not_free(self):
+def test_linked_cycle_self_reference_survives_with_pypy_reference(self):
 p1, p1ref, r1, r1addr, check_alive = (
 self._rawrefcount_pair(42, create_immortal=True))
-r1.c_ob_refcnt += 1  # the pyobject is kept alive
+r1.c_ob_refcnt += 1
 p1.next = p1
+self.stackroots.append(p1)
 check_alive(+1)
 self._collect(major=True)
+assert p1.x == 42
+assert self.trigger == []
 check_alive(+1)
+p1 = self.stackroots.pop()
+check_alive(+1)
+self._collect(major=True, expected_trigger=1)
+py.test.raises(RuntimeError, "p1.x")  # dead
+assert r1.c_ob_refcnt == 1
+assert r1.c_ob_pypy_link == 0
+assert self.gc.rawrefcount_next_dead() == r1addr
+self.gc.check_no_more_rawrefcount_state()
+lltype.free(r1, flavor='raw')
 
-# def test_simple_cycle_free(self):
-# self.gc.rawrefcount_init(lambda: self.trigger.append(1))
-# r1 = self._rawrefcount_cycle_obj()
-# r2 = self._rawrefcount_cycle_obj()
-# r1.next = r2
-# r2.next = r1
-# self._rawrefcount_buffer_obj(r1)
-# self.gc.rrc_collect_cycles()
-# assert r1.base.c_ob_refcnt & REFCNT_MASK == 0
-# assert r2.base.c_ob_refcnt & REFCNT_MASK == 0
-#
-# def test_simple_cycle_not_free(self):
-# self.gc.rawrefcount_init(lambda: self.trigger.append(1))
-# r1 = self._rawrefcount_cycle_obj()
-# r2 = self._rawrefcount_cycle_obj()
-# r1.next = r2
-# r2.next = r1
-# r2.base.c_ob_refcnt += 1
-# self._rawrefcount_buffer_obj(r1)
-# self.gc.rrc_collect_cycles()
-# assert r1.base.c_ob_refcnt & REFCNT_MASK == 1
-# assert r2.base.c_ob_refcnt & REFCNT_MASK == 2
-#
-# def test_complex_cycle_free(self):
-# self.gc.rawrefcount_init(lambda: self.trigger.append(1))
-# r1 = self._rawrefcount_cycle_obj()
-# r2 = self._rawrefcount_cycle_obj()
-# r3 = self._rawrefcount_cycle_obj()
-# r1.next = r2
-# r1.prev = r2
-# r2.base.c_ob_refcnt += 1
-# r2.next = r3
-# r3.prev = r1
-# self._rawrefcount_buffer_obj(r1)
-# self.gc.rrc_collect_cycles()
-# assert r1.base.c_ob_refcnt & REFCNT_MASK == 0
-# assert r2.base.c_ob_refcnt & REFCNT_MASK == 0
-# assert r3.base.c_ob_refcnt & REFCNT_MASK == 0
-#
-# def test_complex_cycle_not_free(self):
-# self.gc.rawrefcount_init(lambda: self.trigger.append(1))
-# r1 = self._rawrefcount_cycle_obj()
-# r2 = 

[pypy-commit] pypy cpyext-gc-cycle: Fixed cpyext test

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95609:860d9f8d29b6
Date: 2018-08-02 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/860d9f8d29b6/

Log:Fixed cpyext test

diff --git a/pypy/module/cpyext/test/test_cpyext.py 
b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -937,7 +937,7 @@
 if self.runappdirect:
 skip('cannot import module with undefined functions')
 
-# TODO: remove unnecessary stuff, add tests for gc_untrack, add asserts
+# TODO: remove unnecessary stuff, add gc_(un)track asserts
 init = """
 if (Py_IsInitialized()) {
 PyObject* m;
@@ -994,7 +994,7 @@
PyObject *kwds)
 {
 Cycle *self;
-self = (Cycle *)type->tp_alloc(type, 0);
+self = PyObject_GC_New(Cycle, type);
 if (self != NULL) {
 self->next = PyString_FromString("");
 if (self->next == NULL) {
@@ -1074,23 +1074,23 @@
 
 extern PyGC_Head *_pypy_rawrefcount_pyobj_list;
 
- static PyObject * Cycle_Create(Cycle *self, PyObject *val)
- {
- Cycle *c = PyObject_GC_New(Cycle, );
- if (c == NULL)
- return NULL;
- 
- Py_INCREF(val);
- c->next = val;
+static PyObject * Cycle_Create(Cycle *self, PyObject *val)
+{
+Cycle *c = (Cycle *)Cycle_new(, NULL, NULL);
+if (c == NULL)
+   return NULL;
+
+Py_INCREF(val);
+c->next = val;
 
- // TODO: check if _pypy_rawrefcount_pyobj_list contains c
+// TODO: check if _pypy_rawrefcount_pyobj_list contains c
 
- return (PyObject *)c;
- }
- static PyMethodDef module_methods[] = {
- {"create", (PyCFunction)Cycle_Create, METH_OLDARGS, ""},
- {NULL}  /* Sentinel */
- };
+return (PyObject *)c;
+}
+static PyMethodDef module_methods[] = {
+{"create", (PyCFunction)Cycle_Create, METH_OLDARGS, ""},
+{NULL}  /* Sentinel */
+};
 """
 module = self.import_module(name='cycle', init=init, body=body)
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Added additional flags for objects

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95598:c03fe327893a
Date: 2018-03-19 10:52 +0100
http://bitbucket.org/pypy/pypy/changeset/c03fe327893a/

Log:Added additional flags for objects Implemented refcount overhead
(for non-cyclic refcount) Implemented buffer for potential roots of
cycles Fixed assert to allow for recursive cpyext calls Added some
cycle detection tests from experimental branch (disabled now)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -208,10 +208,11 @@
 # running and should not themselves release the GIL).
 #
 # **make_generic_cpy_call():** RPython to C, with the GIL held.  Before
-# the call, must assert that the global variable is 0 and set the
-# current thread identifier into the global variable.  After the call,
-# assert that the global variable still contains the current thread id,
-# and reset it to 0.
+# the call, must assert that the global variable is 0 or the current
+# thread identifier (recursive call) and set the current thread identifier
+# into the global variable.  After the call, assert that the global variable
+# still contains the current thread id, and reset it to the value it held
+# before the call.
 #
 # **make_wrapper():** C to RPython; by default assume that the GIL is
 # held, but accepts gil="acquire", "release", "around",
@@ -1763,7 +1764,8 @@
 
 # see "Handling of the GIL" above
 tid = rthread.get_ident()
-assert cpyext_glob_tid_ptr[0] == 0
+tid_before = cpyext_glob_tid_ptr[0]
+assert tid_before == 0 or tid_before == tid
 cpyext_glob_tid_ptr[0] = tid
 
 preexist_error = PyErr_Occurred(space)
@@ -1772,7 +1774,7 @@
 result = call_external_function(func, *boxed_args)
 finally:
 assert cpyext_glob_tid_ptr[0] == tid
-cpyext_glob_tid_ptr[0] = 0
+cpyext_glob_tid_ptr[0] = tid_before
 for i, ARG in unrolling_arg_types:
 # note that this loop is nicely unrolled statically by RPython
 _pyobj = to_decref[i]
diff --git a/pypy/module/cpyext/include/boolobject.h 
b/pypy/module/cpyext/include/boolobject.h
--- a/pypy/module/cpyext/include/boolobject.h
+++ b/pypy/module/cpyext/include/boolobject.h
@@ -13,8 +13,8 @@
 #define Py_True ((PyObject *) &_Py_TrueStruct)
 
 /* Macros for returning Py_True or Py_False, respectively */
-#define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True
-#define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False
+#define Py_RETURN_TRUE do { Py_INCREF(Py_True); return Py_True; } while(0)
+#define Py_RETURN_FALSE do { Py_INCREF(Py_False); return Py_False; } while(0)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -2,6 +2,7 @@
 #define Py_OBJECT_H
 
 #include 
+#include 
 
 #ifdef __cplusplus
 extern "C" {
@@ -12,7 +13,13 @@
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 
-#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
+#define PY_REFCNT_FROM_PYPY (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 2)))
+#define PY_REFCNT_GREEN (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 7)))
+#define PY_REFCNT_OVERFLOW (1L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 7) / 
2L - 1L))
+#define PY_REFCNT_MASK ((PY_REFCNT_OVERFLOW << 1L) - 1L)
+#define Py_RETURN_NONE return (PyObject *)(Py_None))->ob_refcnt & 
PY_REFCNT_OVERFLOW) == 0) ? \
+  ((PyObject *)(Py_None))->ob_refcnt++ : 
Py_IncRef((PyObject *)(Py_None))), Py_None
+
 
 /*
 CPython has this for backwards compatibility with really old extensions, and 
now
@@ -34,14 +41,21 @@
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)(ob)))
 #else
 /* Fast version */
-#define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
-#define Py_DECREF(op)   \
-do {\
-if (--((PyObject *)(op))->ob_refcnt != 0)   \
-;   \
-else\
-_Py_Dealloc((PyObject *)(op));  \
-} while (0)
+#define Py_INCREF(ob)   do {   
  \
+if (!(((PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_OVERFLOW))   \
+((PyObject *)(ob))->ob_refcnt++;   
  \
+else   
  \
+Py_IncRef((PyObject *)(ob));   
 \
+} while (0)
+#define Py_DECREF(ob)   do {   
  \
+  

[pypy-commit] pypy cpyext-gc-cycle: Removed unnecessary code

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95602:65ead3f78618
Date: 2018-04-12 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/65ead3f78618/

Log:Removed unnecessary code

diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -14,8 +14,7 @@
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 
 #define PY_REFCNT_FROM_PYPY (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 2)))
-#define PY_REFCNT_GREEN (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 7)))
-#define PY_REFCNT_OVERFLOW (1L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 7) / 
2L - 1L))
+#define PY_REFCNT_OVERFLOW (1L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 4) - 
1L))
 #define PY_REFCNT_MASK ((PY_REFCNT_OVERFLOW << 1L) - 1L)
 #define Py_RETURN_NONE return (PyObject *)(Py_None))->ob_refcnt & 
PY_REFCNT_OVERFLOW) == 0) ? \
   ((PyObject *)(Py_None))->ob_refcnt++ : 
Py_IncRef((PyObject *)(Py_None))), Py_None
@@ -48,12 +47,11 @@
 Py_IncRef((PyObject *)(ob));   
 \
 } while (0)
 #define Py_DECREF(ob)   do {   
  \
-if (!(((PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_GREEN) ||\
-(((PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_OVERFLOW))\
-Py_DecRef((PyObject *)(ob));   
 \
+if PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_OVERFLOW))\
+Py_DecRef((PyObject *)(ob));   
  \
 else if (--((PyObject *)(ob))->ob_refcnt & 
PY_REFCNT_MASK)   \
 ;  
  \
-else if ((!((PyObject *)(ob))->ob_refcnt) & 
PY_REFCNT_FROM_PYPY) \
+else   
  \
 _Py_Dealloc((PyObject *)(ob)); 
  \
 } while (0)
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -18,9 +18,8 @@
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_base_ptr
 from rpython.rlib import rawrefcount, jit
 from rpython.rlib.debug import ll_assert, fatalerror, debug_print
-from rpython.rlib.rawrefcount import (
-REFCNT_MASK, REFCNT_FROM_PYPY, REFCNT_OVERFLOW, REFCNT_CYCLE_BUFFERED,
-REFCNT_CLR_MASK, REFCNT_CLR_GREEN, REFCNT_CLR_PURPLE)
+from rpython.rlib.rawrefcount import (REFCNT_MASK, REFCNT_FROM_PYPY,
+  REFCNT_OVERFLOW)
 from pypy.module.cpyext.api import slot_function
 from pypy.module.cpyext.typeobjectdefs import visitproc
 
@@ -401,31 +400,13 @@
 rawrefcount.decref(pyobj)
 rc = pyobj.c_ob_refcnt
 if rc & REFCNT_MASK == 0:
-if rc & REFCNT_FROM_PYPY == 0 and rc & REFCNT_CLR_MASK != 
REFCNT_CLR_PURPLE:
-state = space.fromcache(State)
-generic_cpy_call(space, state.C._Py_Dealloc, pyobj)
-elif rc & REFCNT_CLR_MASK != REFCNT_CLR_GREEN:
-possible_root(space, pyobj)
+state = space.fromcache(State)
+generic_cpy_call(space, state.C._Py_Dealloc, pyobj)
 #else:
 #w_obj = rawrefcount.to_obj(W_Root, ref)
 #if w_obj is not None:
 #assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
 
-@jit.dont_look_inside
-def possible_root(space, obj):
-#debug_print("possible root", obj)
-rc = obj.c_ob_refcnt
-if not obj.c_ob_type or not obj.c_ob_type.c_tp_traverse:
-#debug_print("mark green", obj)
-rc = rc & ~REFCNT_CLR_MASK | REFCNT_CLR_GREEN
-elif rc & REFCNT_CLR_MASK != REFCNT_CLR_PURPLE:
-rc = rc & ~REFCNT_CLR_MASK | REFCNT_CLR_PURPLE
-if rc & REFCNT_CYCLE_BUFFERED == 0:
-#debug_print("mark purple", obj)
-rawrefcount.buffer_pyobj(obj)
-rc = rc | REFCNT_CYCLE_BUFFERED
-obj.c_ob_refcnt = rc
-
 @cpython_api([PyObject], lltype.Void)
 def Py_IncRef(space, obj):
 incref(space, obj)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3067,9 +3067,6 @@
 objint = llmemory.cast_adr_to_int(obj, "symbolic")
 self._pyobj(pyobject).c_ob_pypy_link = objint
 
-def rawrefcount_buffer_pyobj(self, pyobject):
-self.rrc_buffered.append(pyobject)
-
 def rawrefcount_from_obj(self, gcobj):
 obj = llmemory.cast_ptr_to_adr(gcobj)
 if self.is_in_nursery(obj):
@@ -3254,12 

[pypy-commit] pypy cpyext-gc-cycle: Fixed cpyext test

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95611:fa88e83164e0
Date: 2018-08-03 13:11 +0200
http://bitbucket.org/pypy/pypy/changeset/fa88e83164e0/

Log:Fixed cpyext test

diff --git a/pypy/module/cpyext/test/test_cpyext.py 
b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -3,7 +3,7 @@
 import pytest
 
 from pypy.tool.cpyext.extbuild import SystemCompilationInfo, HERE
-from pypy.interpreter.gateway import unwrap_spec, interp2app
+from pypy.interpreter.gateway import unwrap_spec, interp2app, ObjSpace
 from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext import api
@@ -214,8 +214,8 @@
 def debug_collect(space):
 rawrefcount._collect()
 
-def print_pyobj_list(space):
-rawrefcount._print_pyobj_list()
+def in_pygclist(space, int_addr):
+return space.wrap(rawrefcount._in_pygclist(int_addr))
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
 
@@ -227,7 +227,8 @@
 if not cls.runappdirect:
 cls.sys_info = get_cpyext_info(space)
 cls.w_debug_collect = space.wrap(interp2app(debug_collect))
-cls.w_print_pyobj_list = space.wrap(interp2app(print_pyobj_list))
+cls.w_in_pygclist = space.wrap(
+interp2app(in_pygclist, unwrap_spec=[ObjSpace, int]))
 cls.preload_builtins(space)
 else:
 def w_import_module(self, name, init=None, body='', filename=None,
@@ -929,7 +930,7 @@
  ),
 ])
 
-def test_gc_pyobj_list(self):
+def test_gc_track(self):
 """
 Test if Py_GC_Track and Py_GC_Untrack are adding and removing container
 objects from the list of all garbage-collected PyObjects.
@@ -937,17 +938,16 @@
 if self.runappdirect:
 skip('cannot import module with undefined functions')
 
-# TODO: remove unnecessary stuff, add gc_(un)track asserts
 init = """
 if (Py_IsInitialized()) {
 PyObject* m;
-if (PyType_Ready() < 0)
+if (PyType_Ready() < 0)
 return;
-m = Py_InitModule("cycle", module_methods);
+m = Py_InitModule("foo", module_methods);
 if (m == NULL)
 return;
-Py_INCREF();
-PyModule_AddObject(m, "Cycle", (PyObject *));
+Py_INCREF();
+PyModule_AddObject(m, "Foo", (PyObject *));
 }
 """
 body = """
@@ -955,85 +955,22 @@
 #include "structmember.h"
 typedef struct {
 PyObject_HEAD
-PyObject *next;
-PyObject *val;
-} Cycle;
-static PyTypeObject CycleType;
-static int Cycle_traverse(Cycle *self, visitproc visit, void *arg)
-{
-int vret;
-if (self->next) {
-vret = visit(self->next, arg);
-if (vret != 0)
-return vret;
-}
-if (self->val) {
-vret = visit(self->val, arg);
-if (vret != 0)
-return vret;
-}
-return 0;
-}
-static int Cycle_clear(Cycle *self)
-{
-PyObject *tmp;
-tmp = self->next;
-self->next = NULL;
-Py_XDECREF(tmp);
-tmp = self->val;
-self->val = NULL;
-Py_XDECREF(tmp);
-return 0;
-}
-static void Cycle_dealloc(Cycle* self)
-{
-Cycle_clear(self);
-Py_TYPE(self)->tp_free((PyObject*)self);
-}
-static PyObject* Cycle_new(PyTypeObject *type, PyObject *args,
+} Foo;
+static PyTypeObject FooType;
+static PyObject* Foo_new(PyTypeObject *type, PyObject *args,
PyObject *kwds)
 {
-Cycle *self;
-self = PyObject_GC_New(Cycle, type);
-if (self != NULL) {
-self->next = PyString_FromString("");
-if (self->next == NULL) {
-Py_DECREF(self);
-return NULL;
-}
-}
+Foo *self;
+self = PyObject_GC_New(Foo, type);
 PyObject_GC_Track(self);
 return (PyObject *)self;
 }
-static int Cycle_init(Cycle *self, PyObject *args, PyObject *kwds)
-{
-PyObject *next=NULL, *tmp;
-static char *kwlist[] = {"next", NULL};
-if (! PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist,
-  ))
-return -1;
-if (next) {
-tmp = self->next;
-Py_INCREF(next);
-self->next = next;
-Py_XDECREF(tmp);
-}
-return 0;
-}
-   

[pypy-commit] pypy cpyext-gc-cycle: Call tp_traverse from incminimark

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95599:fb1c6fe11349
Date: 2018-03-20 16:38 +0100
http://bitbucket.org/pypy/pypy/changeset/fb1c6fe11349/

Log:Call tp_traverse from incminimark Mark cpython objects reachable by
pypy objects

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1293,7 +1293,10 @@
 # if do tuple_attach of the prebuilt empty tuple, we need to call
 # _PyPy_Malloc)
 builder.attach_all(space)
-
+
+#import rpython.rlib.rawrefcount
+#rawrefcount.init_traverse(generic_cpy_call_gc)
+
 setup_init_functions(eci, prefix)
 return modulename.new(ext='')
 
@@ -1716,6 +1719,11 @@
 return make_generic_cpy_call(FT, False)(space, func, *args)
 
 @specialize.ll()
+def generic_cpy_call_gc(func, *args):
+FT = lltype.typeOf(func).TO
+return make_generic_cpy_call_gc(FT, False)(func, *args)
+
+@specialize.ll()
 def generic_cpy_call_expect_null(space, func, *args):
 FT = lltype.typeOf(func).TO
 return make_generic_cpy_call(FT, True)(space, func, *args)
@@ -1815,3 +1823,75 @@
 return result
 
 return generic_cpy_call
+
+@specialize.memo()
+def make_generic_cpy_call_gc(FT, expect_null):
+from pypy.module.cpyext.pyobject import is_pyobj, make_ref, decref
+from pypy.module.cpyext.pyobject import get_w_obj_and_decref
+from pypy.module.cpyext.pyerrors import PyErr_Occurred
+unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS))
+RESULT_TYPE = FT.RESULT
+
+# copied and modified from rffi.py
+# We need tons of care to ensure that no GC operation and no
+# exception checking occurs in call_external_function.
+argnames = ', '.join(['a%d' % i for i in range(len(FT.ARGS))])
+source = py.code.Source("""
+def cpy_call_external(funcptr, %(argnames)s):
+# NB. it is essential that no exception checking occurs here!
+res = funcptr(%(argnames)s)
+return res
+""" % locals())
+miniglobals = {'__name__':__name__, # for module name propagation
+   }
+exec source.compile() in miniglobals
+call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
+call_external_function._dont_inline_ = True
+call_external_function._gctransformer_hint_close_stack_ = True
+# don't inline, as a hack to guarantee that no GC pointer is alive
+# anywhere in call_external_function
+
+@specialize.ll()
+def generic_cpy_call(func, *args):
+boxed_args = ()
+to_decref = ()
+assert len(args) == len(FT.ARGS)
+for i, ARG in unrolling_arg_types:
+arg = args[i]
+_pyobj = None
+if is_PyObject(ARG):
+assert is_pyobj(arg)
+
+boxed_args += (arg,)
+to_decref += (_pyobj,)
+
+# see "Handling of the GIL" above
+tid = rthread.get_ident()
+tid_before = cpyext_glob_tid_ptr[0]
+assert tid_before == 0 or tid_before == tid
+cpyext_glob_tid_ptr[0] = tid
+
+try:
+# Call the function
+result = call_external_function(func, *boxed_args)
+finally:
+assert cpyext_glob_tid_ptr[0] == tid
+cpyext_glob_tid_ptr[0] = tid_before
+for i, ARG in unrolling_arg_types:
+# note that this loop is nicely unrolled statically by RPython
+_pyobj = to_decref[i]
+if _pyobj is not None:
+pyobj = rffi.cast(PyObject, _pyobj)
+rawrefcount.decref(pyobj)
+
+if is_PyObject(RESULT_TYPE):
+ret = None
+
+# Check for exception consistency
+# XXX best attempt, will miss preexisting error that is
+# overwritten with a new error of the same type
+
+return ret
+return result
+
+return generic_cpy_call
\ No newline at end of file
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -321,6 +321,8 @@
 #define _PyGC_FINALIZED(o) 1
 #define PyType_IS_GC(tp) 1
 
+/* TODO: implement like in cpython
+   (see 
https://github.com/python/cpython/blob/517da1e58f4c489d4b31579852cde5f7113da08e/Include/objimpl.h#L295)
 */
 #define PyObject_GC_Track(o)  do { } while(0)
 #define PyObject_GC_UnTrack(o)do { } while(0)
 #define _PyObject_GC_TRACK(o) do { } while(0)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -193,13 +193,27 @@
 VISIT_FUNCTYPE = rffi.CCallback([PyObject, rffi.VOIDP],
 rffi.INT_real)
 
-def traverse(obj, func_ptr):
-from pypy.module.cpyext.api import generic_cpy_call
-from pypy.module.cpyext.typeobjectdefs import 

[pypy-commit] pypy cpyext-gc-cycle: Implemented cpython-like GC list for cpyext

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95604:e5ba3fd47f96
Date: 2018-07-04 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e5ba3fd47f96/

Log:Implemented cpython-like GC list for cpyext Added some code for
cpyext-only boehm GC support

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -751,6 +751,8 @@
 PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), )
 PyVarObjectStruct = cts.gettype('PyVarObject')
 PyVarObject = cts.gettype('PyVarObject *')
+PyGC_Head = cts.gettype('PyGC_Head')
+PyGC_HeadPtr = cts.gettype('PyGC_Head *')
 
 Py_buffer = cts.gettype('Py_buffer')
 Py_bufferP = cts.gettype('Py_buffer *')
@@ -1173,6 +1175,9 @@
 state.C._PyPy_object_dealloc = rffi.llexternal(
 '_PyPy_object_dealloc', [PyObject], lltype.Void,
 compilation_info=eci, _nowrapper=True)
+state.C._PyPy_InitPyObjList = rffi.llexternal(
+'_PyPy_InitPyObjList', [], PyGC_HeadPtr,
+compilation_info=eci, _nowrapper=True)
 
 
 def init_function(func):
@@ -1294,6 +1299,9 @@
 ll2ctypes.lltype2ctypes(func.get_llhelper(space)),
 ctypes.c_void_p)
 
+# initialize the pyobj_list for the gc
+space.fromcache(State).C._PyPy_InitPyObjList()
+
 # we need to call this *after* the init code above, because it might
 # indirectly call some functions which are attached to pypyAPI (e.g., we
 # if do tuple_attach of the prebuilt empty tuple, we need to call
@@ -1826,75 +1834,3 @@
 return result
 
 return generic_cpy_call
-
-@specialize.memo()
-def make_generic_cpy_call_gc(FT, expect_null):
-from pypy.module.cpyext.pyobject import is_pyobj, make_ref, decref
-from pypy.module.cpyext.pyobject import get_w_obj_and_decref
-from pypy.module.cpyext.pyerrors import PyErr_Occurred
-unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS))
-RESULT_TYPE = FT.RESULT
-
-# copied and modified from rffi.py
-# We need tons of care to ensure that no GC operation and no
-# exception checking occurs in call_external_function.
-argnames = ', '.join(['a%d' % i for i in range(len(FT.ARGS))])
-source = py.code.Source("""
-def cpy_call_external(funcptr, %(argnames)s):
-# NB. it is essential that no exception checking occurs here!
-res = funcptr(%(argnames)s)
-return res
-""" % locals())
-miniglobals = {'__name__':__name__, # for module name propagation
-   }
-exec source.compile() in miniglobals
-call_external_function = specialize.ll()(miniglobals['cpy_call_external'])
-call_external_function._dont_inline_ = True
-call_external_function._gctransformer_hint_close_stack_ = True
-# don't inline, as a hack to guarantee that no GC pointer is alive
-# anywhere in call_external_function
-
-@specialize.ll()
-def generic_cpy_call(func, *args):
-boxed_args = ()
-to_decref = ()
-assert len(args) == len(FT.ARGS)
-for i, ARG in unrolling_arg_types:
-arg = args[i]
-_pyobj = None
-if is_PyObject(ARG):
-assert is_pyobj(arg)
-
-boxed_args += (arg,)
-to_decref += (_pyobj,)
-
-# see "Handling of the GIL" above
-tid = rthread.get_ident()
-tid_before = cpyext_glob_tid_ptr[0]
-assert tid_before == 0 or tid_before == tid
-cpyext_glob_tid_ptr[0] = tid
-
-try:
-# Call the function
-result = call_external_function(func, *boxed_args)
-finally:
-assert cpyext_glob_tid_ptr[0] == tid
-cpyext_glob_tid_ptr[0] = tid_before
-for i, ARG in unrolling_arg_types:
-# note that this loop is nicely unrolled statically by RPython
-_pyobj = to_decref[i]
-if _pyobj is not None:
-pyobj = rffi.cast(PyObject, _pyobj)
-rawrefcount.decref(pyobj)
-
-if is_PyObject(RESULT_TYPE):
-ret = None
-
-# Check for exception consistency
-# XXX best attempt, will miss preexisting error that is
-# overwritten with a new error of the same type
-
-return ret
-return result
-
-return generic_cpy_call
\ No newline at end of file
diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -279,7 +279,7 @@
  ) & ~(SIZEOF_VOID_P - 1)  \
)
 
-
+
 #define PyObject_INIT(op, typeobj) \
 ( Py_TYPE(op) = (typeobj), ((PyObject *)(op))->ob_refcnt = 1,\
   ((PyObject *)(op))->ob_pypy_link = 0, (op) )
@@ -309,22 +309,65 @@
 
 #define PyObject_GC_NewVar(type, typeobj, n) \
 ( (type *) _PyObject_GC_NewVar((typeobj), (n)) )
- 

[pypy-commit] pypy cpyext-gc-cycle: Finished implementation of dot file tests for rawrefcount

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95615:48c91f03eaa6
Date: 2018-12-27 16:22 +0100
http://bitbucket.org/pypy/pypy/changeset/48c91f03eaa6/

Log:Finished implementation of dot file tests for rawrefcount Removed
obsolete tests, that will be replaced by dot tests

diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -387,66 +387,11 @@
 self._collect(major=True)
 check_alive(0)
 
-def test_linked_cycle_self_reference_dies_without_external_reference(self):
-p1, p1ref, r1, r1addr, check_alive = (
-self._rawrefcount_pair(42))
-r1.c_ob_refcnt += 1
-p1.next = p1
-check_alive(+1)
-self._collect(major=True, expected_trigger=1)
-py.test.raises(RuntimeError, "p1.x")  # dead
-assert r1.c_ob_refcnt == 1  # in the pending list
-assert r1.c_ob_pypy_link == 0
-assert self.gc.rawrefcount_next_dead() == r1addr
-assert self.gc.rawrefcount_next_dead() == llmemory.NULL
-assert self.gc.rawrefcount_next_dead() == llmemory.NULL
-self.gc.check_no_more_rawrefcount_state()
-lltype.free(r1, flavor='raw')
-
-def test_linked_cycle_self_reference_survives_with_pyobj_reference(self):
-p1, p1ref, r1, r1addr, check_alive = (
-self._rawrefcount_pair(42, create_immortal=True))
-r1.c_ob_refcnt += 2  # the pyobject is kept alive
-p1.next = p1
-check_alive(+2)
-self._collect(major=True)
-check_alive(+2)
-r1.c_ob_refcnt -= 1 # the external reference from pyobj is removed
-check_alive(+1)
-self._collect(major=True, expected_trigger=1)
-py.test.raises(RuntimeError, "p1.x")  # dead
-assert r1.c_ob_refcnt == 1  # in the pending list
-assert r1.c_ob_pypy_link == 0
-assert self.gc.rawrefcount_next_dead() == r1addr
-assert self.gc.rawrefcount_next_dead() == llmemory.NULL
-assert self.gc.rawrefcount_next_dead() == llmemory.NULL
-self.gc.check_no_more_rawrefcount_state()
-lltype.free(r1, flavor='raw')
-
-def test_linked_cycle_self_reference_survives_with_pypy_reference(self):
-p1, p1ref, r1, r1addr, check_alive = (
-self._rawrefcount_pair(42, create_immortal=True))
-r1.c_ob_refcnt += 1
-p1.next = p1
-self.stackroots.append(p1)
-check_alive(+1)
-self._collect(major=True)
-assert p1.x == 42
-assert self.trigger == []
-check_alive(+1)
-p1 = self.stackroots.pop()
-check_alive(+1)
-self._collect(major=True, expected_trigger=1)
-py.test.raises(RuntimeError, "p1.x")  # dead
-assert r1.c_ob_refcnt == 1
-assert r1.c_ob_pypy_link == 0
-assert self.gc.rawrefcount_next_dead() == r1addr
-self.gc.check_no_more_rawrefcount_state()
-lltype.free(r1, flavor='raw')
-
 dot_dir = os.path.join(os.path.realpath(os.path.dirname(__file__)), "dot")
 dot_files = [file for file in os.listdir(dot_dir) if file.endswith(".dot")]
 
+@py.test.mark.dont_track_allocations('intentionally keep objects alive, '
+ 'because we do the checks ourselves')
 @py.test.mark.parametrize("file", dot_files)
 def test_dots(self, file):
 from rpython.memory.gc.test.dot import pydot
@@ -488,6 +433,7 @@
 g = pydot.graph_from_dot_file(path)[0]
 nodes = {}
 
+# create objects from graph
 for n in g.get_nodes():
 name = n.get_name()
 attr = n.obj_dict['attributes']
@@ -512,6 +458,8 @@
 r.c_ob_refcnt = ext_refcnt
 nodes[name] = BorderNode(p, pref, r, raddr, check_alive, info)
 pass
+
+# add references between objects from graph
 for e in g.get_edges():
 source = nodes[e.get_source()]
 dest = nodes[e.get_destination()]
@@ -526,33 +474,36 @@
 else:
 assert False  # only 2 refs supported from pypy obj
 
+# quick self check, if traverse works properly
+dests_by_source = {}
+for e in g.get_edges():
+source = nodes[e.get_source()]
+dest = nodes[e.get_destination()]
+if source.info.type == "C" or dest.info.type == "C":
+if not dests_by_source.has_key(source):
+dests_by_source[source] = []
+dests_by_source[source].append(dest.r)
+for source in dests_by_source:
+dests_target = dests_by_source[source]
+def append(self, pyobj):
+dests_target.remove(pyobj)
+self.gc._rrc_visit_pyobj = append
+self.gc._rrc_traverse(source.raddr)
+assert len(dests_target) == 0
+
+ 

[pypy-commit] pypy cpyext-gc-cycle: Directly call tp_traverse instead of via generic_cpy_call

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95601:fd6699184d11
Date: 2018-03-23 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/fd6699184d11/

Log:Directly call tp_traverse instead of via generic_cpy_call

diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -83,9 +83,8 @@
 pyobj_dealloc_action = PyObjDeallocAction(space)
 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
-def _rawrefcount_tp_traverse(space, pyobj_ptr, callback, args):
-from pypy.module.cpyext.api import (generic_cpy_call,
-PyObject)
+def _rawrefcount_tp_traverse(pyobj_ptr, callback, args):
+from pypy.module.cpyext.api import PyObject
 from pypy.module.cpyext.typeobjectdefs import visitproc
 # convert to pointers with correct types (PyObject)
 callback_addr = llmemory.cast_ptr_to_adr(callback)
@@ -95,12 +94,10 @@
 pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject)
 # now call tp_traverse (if possible)
 if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
-generic_cpy_call(space, pyobj.c_ob_type.c_tp_traverse,
- pyobj,
- callback_ptr, args)
+pyobj.c_ob_type.c_tp_traverse(pyobj, callback_ptr,
+  args)
 self.tp_traverse = (lambda o, v, a:
-_rawrefcount_tp_traverse(self.space,
- o, v, a))
+_rawrefcount_tp_traverse(o, v, a))
 
 def build_api(self):
 """NOT_RPYTHON
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-cycle: Refactored call to tp_traverse from incminimark so there are no dependencies to pypy

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-cycle
Changeset: r95600:94b062729ca4
Date: 2018-03-23 11:46 +0100
http://bitbucket.org/pypy/pypy/changeset/94b062729ca4/

Log:Refactored call to tp_traverse from incminimark so there are no
dependencies to pypy

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1719,11 +1719,6 @@
 return make_generic_cpy_call(FT, False)(space, func, *args)
 
 @specialize.ll()
-def generic_cpy_call_gc(func, *args):
-FT = lltype.typeOf(func).TO
-return make_generic_cpy_call_gc(FT, False)(func, *args)
-
-@specialize.ll()
 def generic_cpy_call_expect_null(space, func, *args):
 FT = lltype.typeOf(func).TO
 return make_generic_cpy_call(FT, True)(space, func, *args)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -20,8 +20,7 @@
 from rpython.rlib.debug import ll_assert, fatalerror, debug_print
 from rpython.rlib.rawrefcount import (
 REFCNT_MASK, REFCNT_FROM_PYPY, REFCNT_OVERFLOW, REFCNT_CYCLE_BUFFERED,
-REFCNT_CLR_MASK, REFCNT_CLR_GREEN, REFCNT_CLR_PURPLE,
-W_MARKER_DEALLOCATING)
+REFCNT_CLR_MASK, REFCNT_CLR_GREEN, REFCNT_CLR_PURPLE)
 from pypy.module.cpyext.api import slot_function
 from pypy.module.cpyext.typeobjectdefs import visitproc
 
@@ -254,6 +253,8 @@
 w_obj._cpyext_attach_pyobj(space, py_obj)
 
 
+w_marker_deallocating = W_Root()
+
 @jit.dont_look_inside
 def from_ref(space, ref):
 """
@@ -265,7 +266,7 @@
 return None
 w_obj = rawrefcount.to_obj(W_Root, ref)
 if w_obj is not None:
-if w_obj is not W_MARKER_DEALLOCATING:
+if w_obj is not w_marker_deallocating:
 return w_obj
 fatalerror(
 "*** Invalid usage of a dying CPython object ***\n"
@@ -318,7 +319,7 @@
 
 def pyobj_has_w_obj(pyobj):
 w_obj = rawrefcount.to_obj(W_Root, pyobj)
-return w_obj is not None and w_obj is not W_MARKER_DEALLOCATING
+return w_obj is not None and w_obj is not w_marker_deallocating
 
 def w_obj_has_pyobj(w_obj):
 return bool(rawrefcount.from_obj(PyObject, w_obj))
@@ -454,7 +455,7 @@
 @init_function
 def write_w_marker_deallocating(space):
 if we_are_translated():
-llptr = cast_instance_to_base_ptr(W_MARKER_DEALLOCATING)
+llptr = cast_instance_to_base_ptr(w_marker_deallocating)
 state = space.fromcache(State)
 state.C.set_marker(rffi.cast(Py_ssize_t, llptr))
 
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -1,8 +1,8 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi, lltype, llmemory
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
-from rpython.rtyper.annlowlevel import llhelper
+from rpython.rtyper.annlowlevel import llhelper, llhelper_args
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
 import sys
@@ -70,7 +70,10 @@
 decref(space, ob)
 print 'dealloc_trigger DONE'
 return "RETRY"
-rawrefcount.init(dealloc_trigger)
+def tp_traverse(obj_addr, callback, args):
+# TODO: implement
+pass
+rawrefcount.init(dealloc_trigger, tp_traverse)
 else:
 if space.config.translation.gc == "boehm":
 action = BoehmPyObjDeallocAction(space)
@@ -80,6 +83,25 @@
 pyobj_dealloc_action = PyObjDeallocAction(space)
 self.dealloc_trigger = lambda: pyobj_dealloc_action.fire()
 
+def _rawrefcount_tp_traverse(space, pyobj_ptr, callback, args):
+from pypy.module.cpyext.api import (generic_cpy_call,
+PyObject)
+from pypy.module.cpyext.typeobjectdefs import visitproc
+# convert to pointers with correct types (PyObject)
+callback_addr = llmemory.cast_ptr_to_adr(callback)
+callback_ptr = llmemory.cast_adr_to_ptr(callback_addr,
+visitproc)
+pyobj_addr = llmemory.cast_ptr_to_adr(pyobj_ptr)
+pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject)
+# now call tp_traverse (if possible)
+if pyobj.c_ob_type and pyobj.c_ob_type.c_tp_traverse:
+generic_cpy_call(space, pyobj.c_ob_type.c_tp_traverse,
+ pyobj,
+ callback_ptr, args)
+self.tp_traverse = (lambda o, v, a:
+  

[pypy-commit] extradoc extradoc: Added myself to winter sprint

[stevie_92]

Author: Stefan Beyer 
Branch: extradoc
Changeset: r5875:0e42fc53e29d
Date: 2018-03-03 15:20 +0100
http://bitbucket.org/pypy/extradoc/changeset/0e42fc53e29d/

Log:Added myself to winter sprint

diff --git a/sprintinfo/leysin-winter-2018/people.txt 
b/sprintinfo/leysin-winter-2018/people.txt
--- a/sprintinfo/leysin-winter-2018/people.txt
+++ b/sprintinfo/leysin-winter-2018/people.txt
@@ -21,6 +21,7 @@
 Alexander Schremmer  18.3/20.3  Ermina
 Ronan Lamy   17.3/23.3  Ermina
 Ren Dudfield18.3/24.3  Ermina
+Stefan Beyer 18.3/24.3  Ermina
  == ===
 
 **NOTE:** lodging is by default in Ermina.  Based on past years, there
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-trialdeletion: Added more tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r92287:74fa1f758dc8
Date: 2017-08-31 14:45 +0200
http://bitbucket.org/pypy/pypy/changeset/74fa1f758dc8/

Log:Added more tests

diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -441,3 +441,47 @@
 assert r1.base.c_ob_refcnt & REFCNT_MASK == 2
 assert r2.base.c_ob_refcnt & REFCNT_MASK == 1
 
+def test_multiple_cycles_partial_free(self):
+self.gc.rawrefcount_init(lambda: self.trigger.append(1))
+r1 = self._rawrefcount_cycle_obj()
+r2 = self._rawrefcount_cycle_obj()
+r3 = self._rawrefcount_cycle_obj()
+r4 = self._rawrefcount_cycle_obj()
+r5 = self._rawrefcount_cycle_obj()
+r1.next = r2
+r2.next = r3
+r3.next = r1
+r2.prev = r5
+r5.next = r4
+r4.next = r5
+r5.base.c_ob_refcnt += 1
+r4.base.c_ob_refcnt += 1
+self._rawrefcount_buffer_obj(r1)
+self.gc.rrc_collect_cycles()
+assert r1.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r2.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r3.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r4.base.c_ob_refcnt & REFCNT_MASK == 2
+assert r5.base.c_ob_refcnt & REFCNT_MASK == 1
+
+def test_multiple_cycles_all_free(self):
+self.gc.rawrefcount_init(lambda: self.trigger.append(1))
+r1 = self._rawrefcount_cycle_obj()
+r2 = self._rawrefcount_cycle_obj()
+r3 = self._rawrefcount_cycle_obj()
+r4 = self._rawrefcount_cycle_obj()
+r5 = self._rawrefcount_cycle_obj()
+r1.next = r2
+r2.next = r3
+r3.next = r1
+r2.prev = r5
+r5.next = r4
+r4.next = r5
+r5.base.c_ob_refcnt += 1
+self._rawrefcount_buffer_obj(r1)
+self.gc.rrc_collect_cycles()
+assert r1.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r2.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r3.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r4.base.c_ob_refcnt & REFCNT_MASK == 0
+assert r5.base.c_ob_refcnt & REFCNT_MASK == 0
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy cpyext-gc-trialdeletion: Added tests

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r92284:a478bda34d52
Date: 2017-08-31 11:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a478bda34d52/

Log:Added tests Fixed bug in generic_cpy_call if called recursively
Fixed bug in cycle detection if object is buffered twice

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -200,10 +200,11 @@
 # executes.  In non-cpyext-related code, it will thus always be 0.
 #
 # **make_generic_cpy_call():** RPython to C, with the GIL held.  Before
-# the call, must assert that the global variable is 0 and set the
-# current thread identifier into the global variable.  After the call,
-# assert that the global variable still contains the current thread id,
-# and reset it to 0.
+# the call, must assert that the global variable is 0 or the current
+# thread identifier (recursive call) and set the current thread identifier
+# into the global variable.  After the call, assert that the global variable
+# still contains the current thread id, and reset it to the value it held
+# before the call.
 #
 # **make_wrapper():** C to RPython; by default assume that the GIL is
 # held, but accepts gil="acquire", "release", "around",
@@ -1598,7 +1599,8 @@
 
 # see "Handling of the GIL" above
 tid = rthread.get_ident()
-assert cpyext_glob_tid_ptr[0] == 0
+tid_before = cpyext_glob_tid_ptr[0]
+assert tid_before == 0 or tid_before == tid
 cpyext_glob_tid_ptr[0] = tid
 
 try:
@@ -1606,7 +1608,7 @@
 result = call_external_function(func, *boxed_args)
 finally:
 assert cpyext_glob_tid_ptr[0] == tid
-cpyext_glob_tid_ptr[0] = 0
+cpyext_glob_tid_ptr[0] = tid_before
 keepalive_until_here(*keepalives)
 
 if is_PyObject(RESULT_TYPE):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -3205,9 +3205,7 @@
 from rpython.rlib.rawrefcount import (REFCNT_CYCLE_BUFFERED,
   REFCNT_CLR_MASK,
   REFCNT_CLR_PURPLE,
-  REFCNT_MASK,
-  W_MARKER_DEALLOCATING,
-  mark_deallocating)
+  REFCNT_MASK)
 obj = self._pyobj(pyobject)
 rc = obj.c_ob_refcnt
 debug_print("_rrc_cycle_mark_roots", obj)
@@ -3218,8 +3216,8 @@
 obj.c_ob_refcnt = rc & ~REFCNT_CYCLE_BUFFERED
 self.rrc_buffered.remove(pyobject)
 if rc & REFCNT_MASK == 0:
-mark_deallocating(W_MARKER_DEALLOCATING, obj)
-generic_cpy_call(True, obj.c_ob_type.c_tp_dealloc, obj)
+if obj.c_ob_type.c_tp_dealloc:
+generic_cpy_call(True, obj.c_ob_type.c_tp_dealloc, obj)
 
 def _rrc_cycle_scan_roots(self, pyobject, ignore):
 obj = self._pyobj(pyobject)
diff --git a/rpython/memory/gc/test/test_rawrefcount.py 
b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -2,11 +2,15 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.memory.gc.incminimark import IncrementalMiniMarkGC
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY
-from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT
+from rpython.rlib.rawrefcount import (REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT,
+  REFCNT_MASK)
 from pypy.module.cpyext.api import (PyObject, PyTypeObject, PyTypeObjectPtr,
 PyObjectFields, cpython_struct)
 from pypy.module.cpyext.complexobject import PyComplexObject
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.typeobjectdefs import visitproc, traverseproc
+from rpython.rtyper.annlowlevel import llhelper
+from rpython.rtyper.tool import rffi_platform
 
 PYOBJ_HDR = IncrementalMiniMarkGC.PYOBJ_HDR
 PYOBJ_HDR_PTR = IncrementalMiniMarkGC.PYOBJ_HDR_PTR
@@ -17,6 +21,17 @@
  ('prev', lltype.Ptr(S)),
  ('next', lltype.Ptr(S
 
+T = lltype.Ptr(lltype.ForwardReference())
+T.TO.become(lltype.Struct('test',
+  ('base', PyObject.TO),
+  ('next', T),
+  ('prev', T),
+  ('value', lltype.Signed)))
+
+TRAVERSE_FUNCTYPE = rffi.CCallback([PyObject, visitproc, rffi.VOIDP],
+   rffi.INT_real)
+t1 = lltype.malloc(PyTypeObject, flavor='raw', immortal=True)
+
 
 class 

[pypy-commit] pypy cpyext-gc-trialdeletion: Implemented non-incremental cycle detection, removed simple trial deletion

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r92281:2fb71fc31ef4
Date: 2017-08-06 13:48 +0200
http://bitbucket.org/pypy/pypy/changeset/2fb71fc31ef4/

Log:Implemented non-incremental cycle detection, removed simple trial
deletion

diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,6 +15,10 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
+from rpython.rlib.rawrefcount import (
+REFCNT_MASK, REFCNT_FROM_PYPY, REFCNT_OVERFLOW, REFCNT_CYCLE_BUFFERED,
+REFCNT_CLR_MASK, REFCNT_CLR_GREEN, REFCNT_CLR_PURPLE, 
+W_MARKER_DEALLOCATING)
 from rpython.rlib.debug import fatalerror, debug_print
 from pypy.module.cpyext.api import slot_function
 from pypy.module.cpyext.typeobjectdefs import visitproc
@@ -190,9 +194,6 @@
 py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY
 rawrefcount.create_link_pypy(w_obj, py_obj)
 
-
-w_marker_deallocating = W_Root()
-
 def from_ref(space, ref):
 """
 Finds the interpreter object corresponding to the given reference.  If the
@@ -203,7 +204,7 @@
 return None
 w_obj = rawrefcount.to_obj(W_Root, ref)
 if w_obj is not None:
-if w_obj is not w_marker_deallocating:
+if w_obj is not W_MARKER_DEALLOCATING:
 return w_obj
 fatalerror(
 "*** Invalid usage of a dying CPython object ***\n"
@@ -250,7 +251,7 @@
 
 def pyobj_has_w_obj(pyobj):
 w_obj = rawrefcount.to_obj(W_Root, pyobj)
-return w_obj is not None and w_obj is not w_marker_deallocating
+return w_obj is not None and w_obj is not W_MARKER_DEALLOCATING
 
 
 def is_pyobj(x):
@@ -270,27 +271,6 @@
 hop.exception_cannot_occur()
 return hop.inputconst(lltype.Bool, hop.s_result.const)
 
-def _decref(pyobj):
-if pyobj.c_ob_refcnt & rawrefcount.REFCNT_OVERFLOW == 0:
-pyobj.c_ob_refcnt -= 1
-else:
-if pyobj.c_ob_refcnt & rawrefcount.REFCNT_MASK \
-   == rawrefcount.REFCNT_OVERFLOW:
-pyobj.c_ob_refcnt -= 1
-elif rawrefcount.overflow_sub(pyobj):
-pyobj.c_ob_refcnt -= 1
-
-def _incref(pyobj):
-if pyobj.c_ob_refcnt & rawrefcount.REFCNT_OVERFLOW == 0:
-pyobj.c_ob_refcnt += 1
-else:
-if pyobj.c_ob_refcnt & rawrefcount.REFCNT_MASK \
-   == rawrefcount.REFCNT_OVERFLOW:
-pyobj.c_ob_refcnt += 1
-rawrefcount.overflow_new(pyobj)
-else:
-rawrefcount.overflow_add(pyobj)
-
 @specialize.ll()
 def make_ref(space, obj, w_userdata=None):
 """Increment the reference counter of the PyObject and return it.
@@ -301,7 +281,7 @@
 else:
 pyobj = as_pyobj(space, obj, w_userdata)
 if pyobj:
-_incref(pyobj)
+rawrefcount.incref(pyobj)
 if not is_pyobj(obj):
 keepalive_until_here(obj)
 return pyobj
@@ -321,7 +301,7 @@
 w_obj = obj
 pyobj = as_pyobj(space, w_obj)
 if pyobj:
-_decref(pyobj)
+rawrefcount.decref(pyobj)
 keepalive_until_here(w_obj)
 return w_obj
 
@@ -334,122 +314,30 @@
 if is_pyobj(obj):
 obj = rffi.cast(PyObject, obj)
 if obj:
-_decref(obj)
-
-if obj.c_ob_refcnt & rawrefcount.REFCNT_MASK == 0 and \
-   rawrefcount.get_trialdeletion_phase() != 1:
-if obj.c_ob_refcnt & rawrefcount.REFCNT_FROM_PYPY == 0:
+rawrefcount.decref(obj)
+rc = obj.c_ob_refcnt
+if (rc & REFCNT_MASK == 0):
+if (rc & REFCNT_FROM_PYPY == 0 and
+   rc & REFCNT_CLR_MASK != REFCNT_CLR_PURPLE):
 _Py_Dealloc(space, obj)
-elif obj.c_ob_refcnt & rawrefcount.REFCNT_CLR_GREEN == 0:
-if rawrefcount.get_trialdeletion_phase() == 0:
-trial_delete(space, obj)
+elif (rc & REFCNT_CLR_MASK != REFCNT_CLR_GREEN):
+possible_root(space, obj)
 else:
 get_w_obj_and_decref(space, obj)
 
-@specialize.ll()
-def refcnt_overflow(space, obj):
-if is_pyobj(obj):
-pyobj = rffi.cast(PyObject, obj)
-else:
-pyobj = as_pyobj(space, obj, None)
-if pyobj:
-if (pyobj.c_ob_refcnt & rawrefcount.REFCNT_MASK ==
-   rawrefcount.REFCNT_OVERFLOW):
-return rawrefcount.REFCNT_OVERFLOW
-else:
-return (pyobj.c_ob_refcnt & rawrefcount.REFCNT_MASK) \
-+ rawrefcount.overflow_get(pyobj)
-return 0
-
-def traverse(space, obj, visit):
-from pypy.module.cpyext.api import generic_cpy_call
-if obj.c_ob_type and obj.c_ob_type.c_tp_traverse:
-generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, visit,
- rffi.cast(rffi.VOIDP, obj))
-
-def clear(space, obj):
-from 

[pypy-commit] pypy cpyext-gc-trialdeletion: Implemented flags for concurrent cycle deletion (Bacon and Rajan 2001) with overflow handling for refcount and removed unnecessary code

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r91867:f8d7bb5f29b6
Date: 2017-07-14 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/f8d7bb5f29b6/

Log:Implemented flags for concurrent cycle deletion (Bacon and Rajan
2001) with overflow handling for refcount and removed unnecessary
code

diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -2,6 +2,7 @@
 #define Py_OBJECT_H
 
 #include 
+#include 
 
 #ifdef __cplusplus
 extern "C" {
@@ -9,10 +10,17 @@
 
 #include "cpyext_object.h"
 
+int* foo;
 #define PY_SSIZE_T_MAX ((Py_ssize_t)(((size_t)-1)>>1))
 #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
 
-#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
+#define PY_REFCNT_FROM_PYPY (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 2)))
+#define PY_REFCNT_GREEN (4L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 6)))
+#define PY_REFCNT_OVERFLOW (1L << ((long)(log(PY_SSIZE_T_MAX) / log(2) - 6) / 
2L - 1L))
+#define PY_REFCNT_MASK ((PY_REFCNT_OVERFLOW << 1L) - 1L)
+
+#define Py_RETURN_NONE return (PyObject *)(Py_None))->ob_refcnt & 
PY_REFCNT_OVERFLOW) == 0) ? \
+  ((PyObject *)(Py_None))->ob_refcnt++ : 
Py_IncRef((PyObject *)(Py_None))), Py_None
 
 /*
 CPython has this for backwards compatibility with really old extensions, and 
now
@@ -26,26 +34,34 @@
 #define PyVarObject_HEAD_INIT(type, size)  \
PyObject_HEAD_INIT(type) size,
 
-// #ifdef PYPY_DEBUG_REFCOUNT
-// /* Slow version, but useful for debugging */
+#ifdef PYPY_DEBUG_REFCOUNT
+/* Slow version, but useful for debugging */
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)(ob)))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)(ob)))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)(ob)))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)(ob)))
-// #else
-// /* Fast version */
-// #define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
-// #define Py_DECREF(op)   \
-// do {\
-// if (--((PyObject *)(op))->ob_refcnt != 0)   \
-// ;   \
-// else\
-// _Py_Dealloc((PyObject *)(op));  \
-// } while (0)
-
-// #define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while 
(0)
-// #define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while 
(0)
-// #endif
+#else
+/* Fast version */
+#define Py_INCREF(ob)  
 \
+do {   
 \
+if (!(((PyObject *)(ob))->ob_refcnt & PY_REFCNT_OVERFLOW)) 
 \
+((PyObject *)(ob))->ob_refcnt++;   
 \
+else   
 \
+Py_IncRef((PyObject *)(ob));   
 \
+} while (0)
+#define Py_DECREF(ob)  
 \
+do {   
 \
+if (!(((PyObject *)(ob))->ob_refcnt & PY_REFCNT_GREEN) ||  
 \
+(((PyObject *)(ob))->ob_refcnt & PY_REFCNT_OVERFLOW))  
 \
+Py_DecRef((PyObject *)(ob));   
 \
+else if (--((PyObject *)(ob))->ob_refcnt & PY_REFCNT_MASK) 
 \
+;  
 \
+else if (!((PyObject *)(ob))->ob_refcnt & PY_REFCNT_FROM_PYPY) 
 \
+_Py_Dealloc((PyObject *)(ob)); 
 \
+} while (0)
+#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#endif
 
 #define Py_CLEAR(op)   \
 do {   \
@@ -56,7 +72,8 @@
 }  \
 } while (0)
 
-#define Py_REFCNT(ob)  (((PyObject*)(ob))->ob_refcnt)
+#define Py_REFCNT(ob) PyObject *)(ob))->ob_refcnt & PY_REFCNT_OVERFLOW == 
0) ?  \
+  (((PyObject*)(ob))->ob_refcnt & PY_REFCNT_MASK) : 
_Py_RefCnt_Overflow(ob))
 #define Py_TYPE(ob)(((PyObject*)(ob))->ob_type)
 #define Py_SIZE(ob)(((PyVarObject*)(ob))->ob_size)
 
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -11,6 +11,7 @@
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
 from pypy.interpreter.error 

[pypy-commit] pypy cpyext-gc-trialdeletion: Implemented stubs for a trial deletion algorithm for cpyext-only reference cycles

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r91642:e65c2e7763e7
Date: 2017-06-12 20:08 +0200
http://bitbucket.org/pypy/pypy/changeset/e65c2e7763e7/

Log:Implemented stubs for a trial deletion algorithm for cpyext-only
reference cycles

diff --git a/pypy/module/cpyext/include/object.h 
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -26,26 +26,26 @@
 #define PyVarObject_HEAD_INIT(type, size)  \
PyObject_HEAD_INIT(type) size,
 
-#ifdef PYPY_DEBUG_REFCOUNT
-/* Slow version, but useful for debugging */
+// #ifdef PYPY_DEBUG_REFCOUNT
+// /* Slow version, but useful for debugging */
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)(ob)))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)(ob)))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)(ob)))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)(ob)))
-#else
-/* Fast version */
-#define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
-#define Py_DECREF(op)   \
-do {\
-if (--((PyObject *)(op))->ob_refcnt != 0)   \
-;   \
-else\
-_Py_Dealloc((PyObject *)(op));  \
-} while (0)
+// #else
+// /* Fast version */
+// #define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
+// #define Py_DECREF(op)   \
+// do {\
+// if (--((PyObject *)(op))->ob_refcnt != 0)   \
+// ;   \
+// else\
+// _Py_Dealloc((PyObject *)(op));  \
+// } while (0)
 
-#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
-#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
-#endif
+// #define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while 
(0)
+// #define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while 
(0)
+// #endif
 
 #define Py_CLEAR(op)   \
 do {   \
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -318,9 +318,30 @@
 obj.c_ob_refcnt -= 1
 if obj.c_ob_refcnt == 0:
 _Py_Dealloc(space, obj)
+else:
+trial_delete(space, obj)
 else:
-get_w_obj_and_decref(space, obj)
+get_w_obj_and_decref(space, obj)  # trial_delete?
 
+@specialize.ll()
+def trial_delete(space, obj):
+from pypy.module.cpyext.api import generic_cpy_call, slot_function
+from pypy.module.cpyext.typeobjectdefs import visitproc
+from pypy.module.cpyext.slotdefs import llslot
+
+if not obj.c_ob_type or not obj.c_ob_type.c_tp_traverse:
+return
+
+@slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def visit(space, obj, args):
+w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+print "visit", obj, w_type.name
+return 0
+
+print "trial_delete", obj, obj.c_ob_refcnt
+
+proc = rffi.cast(visitproc, llslot(space, visit))
+generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, proc, None)
 
 @cpython_api([PyObject], lltype.Void)
 def Py_IncRef(space, obj):
diff --git a/pypy/module/cpyext/test/test_cpyext_gc.py 
b/pypy/module/cpyext/test/test_cpyext_gc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_cpyext_gc.py
@@ -0,0 +1,597 @@
+import sys
+import weakref
+
+import pytest
+
+from pypy.tool.cpyext.extbuild import (
+SystemCompilationInfo, HERE, get_sys_info_app)
+from pypy.interpreter.gateway import unwrap_spec, interp2app
+from rpython.rtyper.lltypesystem import lltype, ll2ctypes
+from pypy.module.cpyext import api
+from pypy.module.cpyext.state import State
+from pypy.module.cpyext.pyobject import Py_DecRef
+from rpython.tool.identity_dict import identity_dict
+from rpython.tool import leakfinder
+from rpython.rlib import rawrefcount
+from rpython.tool.udir import udir
+
+only_pypy ="config.option.runappdirect and '__pypy__' not in 
sys.builtin_module_names"
+
+@api.cpython_api([], api.PyObject)
+def PyPy_Crash1(space):
+1/0
+
+@api.cpython_api([], lltype.Signed, error=-1)
+def PyPy_Crash2(space):
+1/0
+
+class SpaceCompiler(SystemCompilationInfo):
+"""Extension compiler for regular (untranslated PyPy) mode"""
+def __init__(self, space, *args, **kwargs):
+self.space = space
+SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+def load_module(self, mod, name):
+space = self.space
+api.load_extension_module(space, mod, name)
+return 

[pypy-commit] pypy cpyext-gc-trialdeletion: Implemented simple trial deletion for cpyext-only cycles

[stevie_92]

Author: Stefan Beyer 
Branch: cpyext-gc-trialdeletion
Changeset: r91643:ca3a43da189f
Date: 2017-06-24 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ca3a43da189f/

Log:Implemented simple trial deletion for cpyext-only cycles

diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,7 +15,9 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
-from rpython.rlib.debug import fatalerror
+from rpython.rlib.debug import fatalerror, debug_print
+from pypy.module.cpyext.api import slot_function
+from pypy.module.cpyext.typeobjectdefs import visitproc
 
 
 #
@@ -316,32 +318,106 @@
 if obj:
 assert obj.c_ob_refcnt > 0
 obj.c_ob_refcnt -= 1
-if obj.c_ob_refcnt == 0:
+if obj.c_ob_refcnt == 0 and \
+   rawrefcount.get_trialdeletion_phase() != 1:
+debug_print("dealloc", obj)
 _Py_Dealloc(space, obj)
+elif obj.c_ob_refcnt == rawrefcount.REFCNT_FROM_PYPY:
+debug_print("dead", obj)
 else:
-trial_delete(space, obj)
+if rawrefcount.get_trialdeletion_phase() == 0:
+trial_delete(space, obj)
 else:
-get_w_obj_and_decref(space, obj)  # trial_delete?
+get_w_obj_and_decref(space, obj)
+
+def traverse(space, obj, visit):
+from pypy.module.cpyext.api import generic_cpy_call
+if obj.c_ob_type and obj.c_ob_type.c_tp_traverse:
+generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, visit,
+ rffi.cast(rffi.VOIDP, obj))
+
+def clear(space, obj):
+from pypy.module.cpyext.api import generic_cpy_call
+if obj.c_ob_type:
+generic_cpy_call(space, obj.c_ob_type.c_tp_clear, obj)
+
+@slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def visit_decref(space, obj, args):
+obj.c_ob_refcnt = obj.c_ob_refcnt - 1
+debug_print("visited dec", obj, "new refcnt", obj.c_ob_refcnt)
+if (obj not in rawrefcount.get_visited()):
+rawrefcount.add_visited(obj)
+from pypy.module.cpyext.slotdefs import llslot
+traverse(space, obj, rffi.cast(visitproc, llslot(space, visit_decref)))
+return 0
+
+@slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def visit_incref(space, obj, args):
+obj.c_ob_refcnt = obj.c_ob_refcnt + 1
+debug_print("visited inc", obj, "new refcnt", obj.c_ob_refcnt)
+if (obj not in rawrefcount.get_visited()):
+rawrefcount.add_visited(obj)
+from pypy.module.cpyext.slotdefs import llslot
+traverse(space, obj, rffi.cast(visitproc, llslot(space, visit_incref)))
+return 0
 
 @specialize.ll()
 def trial_delete(space, obj):
-from pypy.module.cpyext.api import generic_cpy_call, slot_function
-from pypy.module.cpyext.typeobjectdefs import visitproc
-from pypy.module.cpyext.slotdefs import llslot
-
 if not obj.c_ob_type or not obj.c_ob_type.c_tp_traverse:
 return
 
-@slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
-def visit(space, obj, args):
-w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
-print "visit", obj, w_type.name
-return 0
+from pypy.module.cpyext.slotdefs import llslot
+visitproc_incref = rffi.cast(visitproc, llslot(space, visit_incref))
+visitproc_decref = rffi.cast(visitproc, llslot(space, visit_decref))
 
-print "trial_delete", obj, obj.c_ob_refcnt
+rawrefcount.set_trialdeletion_phase(1)
 
-proc = rffi.cast(visitproc, llslot(space, visit))
-generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, proc, None)
+debug_print("trial_delete", obj, "refct after decref", obj.c_ob_refcnt)
+
+debug_print("decref phase")
+rawrefcount.clear_visited()
+rawrefcount.add_visited(obj)
+traverse(space, obj, visitproc_decref)
+
+debug_print("checkref phase")
+visited = []
+alive = []
+for visited_obj in rawrefcount.get_visited():
+visited.append(visited_obj)
+if visited_obj.c_ob_refcnt != 0 and \
+   visited_obj.c_ob_refcnt != rawrefcount.REFCNT_FROM_PYPY:
+alive.append(visited_obj)
+debug_print("alive", visited_obj)
+
+debug_print("incref phase")
+rawrefcount.clear_visited()
+for alive_obj in alive:
+if alive_obj not in rawrefcount.get_visited():
+rawrefcount.add_visited(alive_obj)
+traverse(space, alive_obj, visitproc_incref)
+
+alive = []
+for alive_obj in rawrefcount.get_visited():
+debug_print("alive", alive_obj, alive_obj.c_ob_refcnt)
+alive.append(alive_obj)
+
+for reachable_obj in visited:
+if reachable_obj not in rawrefcount.get_visited():

[pypy-commit] pypy py3.5: Added EXT_SUFFIX for sysconfig, changed EXT_SUFFIX to Python 3 format

[stevie_92]

Author: Stefan Beyer 
Branch: py3.5
Changeset: r90478:215564273ab3
Date: 2017-03-02 12:54 +0100
http://bitbucket.org/pypy/pypy/changeset/215564273ab3/

Log:Added EXT_SUFFIX for sysconfig, changed EXT_SUFFIX to Python 3
format

diff --git a/lib-python/3/distutils/sysconfig_pypy.py 
b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -60,8 +60,8 @@
 
 def _init_posix():
 """Initialize the module as appropriate for POSIX systems."""
-so_list = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION]
-so_ext = (so_list or ['.so'])[0]
+so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+
 g = {}
 g['CC'] = "gcc -pthread"
 g['CXX'] = "g++ -pthread"
@@ -69,8 +69,9 @@
 g['CFLAGS'] = "-DNDEBUG -O2"
 g['CCSHARED'] = "-fPIC"
 g['LDSHARED'] = "gcc -pthread -shared"
-g['SO'] = so_ext
-g['SHLIB_SUFFIX'] = g['SO']
+g['EXT_SUFFIX'] = so_ext
+g['SHLIB_SUFFIX'] = so_ext
+g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
 g['AR'] = "ar"
 g['ARFLAGS'] = "rc"
 g['EXE'] = ""
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,5 +1,9 @@
 import imp
 
+so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+
 build_time_vars = {
-"SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+"EXT_SUFFIX": so_ext,
+"SHLIB_SUFFIX": so_ext,
+"SO": so_ext  # deprecated in Python 3, for backward compatibility
 }
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -2,7 +2,7 @@
 Implementation of the interpreter-level default import logic.
 """
 
-import sys, os, stat
+import sys, os, stat, platform
 
 from pypy.interpreter.module import Module, init_extra_module_attrs
 from pypy.interpreter.gateway import interp2app, unwrap_spec
@@ -22,7 +22,7 @@
 
 SO = '.pyd' if _WIN32 else '.so'
 PREFIX = 'pypy3-'
-DEFAULT_SOABI = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
+DEFAULT_SOABI_BASE = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])
 
 PYC_TAG = '%s%d%d' % ((PREFIX,) + PYPY_VERSION[:2])   # 'pypy3-XY'
 
@@ -35,7 +35,7 @@
 if space.config.objspace.soabi is not None:
 soabi = space.config.objspace.soabi
 else:
-soabi = DEFAULT_SOABI
+soabi = DEFAULT_SOABI_BASE
 
 if not soabi:
 return SO
@@ -43,6 +43,17 @@
 if not space.config.translating:
 soabi += 'i'
 
+platform_name = sys.platform
+if platform_name == 'linux2':
+platform_name = 'linux'
+
+soabi += '-' + platform.machine() + '-' + platform_name
+
+if platform_name == 'linux':
+soabi += '-gnu'
+if sys.maxsize == (2**31 - 1) and platform.machine() == 'x86_64':
+soabi += 'x32'
+
 return '.' + soabi + SO
 
 def log_pyverbose(space, level, message):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: (mjacob, stevie) added SO variable to sysconfig

[stevie_92]

Author: Stefan Beyer 
Branch: 
Changeset: r90475:bbcb5f789c19
Date: 2017-03-02 12:39 +0100
http://bitbucket.org/pypy/pypy/changeset/bbcb5f789c19/

Log:(mjacob, stevie) added SO variable to sysconfig

diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,2 +1,5 @@
+import imp
+
 build_time_vars = {
+"SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 }
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy fix-global: Added tests for bugfix

[stevie_92]

Author: Stefan Beyer 
Branch: fix-global
Changeset: r90435:b8d66cc71819
Date: 2017-02-28 21:59 +0100
http://bitbucket.org/pypy/pypy/changeset/b8d66cc71819/

Log:Added tests for bugfix

diff --git a/pypy/interpreter/astcompiler/test/test_misc.py 
b/pypy/interpreter/astcompiler/test/test_misc.py
--- a/pypy/interpreter/astcompiler/test/test_misc.py
+++ b/pypy/interpreter/astcompiler/test/test_misc.py
@@ -12,3 +12,23 @@
 assert mangle("__foo", "__Bar") == "_Bar__foo"
 assert mangle("__foo", "___") == "__foo"
 assert mangle("___foo", "__Bar") == "_Bar___foo"
+
+def app_test_warning_to_error_translation():
+import warnings
+
+with warnings.catch_warnings():
+warnings.filterwarnings("error", module="")
+statement = """\
+def wrong1():
+a = 1
+b = 2
+global a
+global b
+"""
+try:
+   compile(statement, '', 'exec')
+except SyntaxError as err:
+   assert err.lineno is not None
+   assert err.filename is not None
+   assert err.offset is not None
+   assert err.message is not None
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy fix-global: Fixed bug translating SyntaxWarning to SyntaxError

[stevie_92]

Author: Stefan Beyer 
Branch: fix-global
Changeset: r90434:2e54fda87e45
Date: 2017-02-28 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/2e54fda87e45/

Log:Fixed bug translating SyntaxWarning to SyntaxError Tests in lib-
python/3/test/test_global.py are now sucessful

diff --git a/pypy/interpreter/astcompiler/misc.py 
b/pypy/interpreter/astcompiler/misc.py
--- a/pypy/interpreter/astcompiler/misc.py
+++ b/pypy/interpreter/astcompiler/misc.py
@@ -9,7 +9,7 @@
 try:
 warnings.warn_explicit(msg, SyntaxWarning, fn, lineno)
 except SyntaxWarning:
-raise SyntaxError(msg, fn, lineno, offset)
+raise SyntaxError(msg, (fn, lineno, offset, msg))
 """, filename=__file__)
 _emit_syntax_warning = app.interphook("syntax_warning")
 del app
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit