[pypy-commit] pypy gc-del-3: in-progress: "test_transformed_gc -k Inc" seems happy

[arigo]

Author: Armin Rigo 
Branch: gc-del-3
Changeset: r84184:70f42e6f2872
Date: 2016-05-04 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/70f42e6f2872/

Log:in-progress: "test_transformed_gc -k Inc" seems happy

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -7,6 +7,7 @@
 from rpython.memory.support import get_address_stack, get_address_deque
 from rpython.memory.support import AddressDict, null_address_dict
 from rpython.rtyper.lltypesystem.llmemory import NULL, raw_malloc_usage
+from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
 
 TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed),
  ('size', lltype.Signed),
@@ -40,8 +41,8 @@
 self.finalizer_lock = False
 
 def mark_finalizer_to_run(self, fq_index, obj):
-fdeque = self.get_run_finalizer_queue(self.AddressDeque, fq_index)
-fdeque.append(obj)
+handlers = self.finalizer_handlers()
+self._adr2deque(handlers[fq_index].deque).append(obj)
 
 def post_setup(self):
 # More stuff that needs to be initialized when the GC is already
@@ -64,8 +65,7 @@
 
 def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
 is_gcarrayofgcptr,
-finalizer_trigger,
-get_run_finalizer_queue,
+finalizer_handlers,
 destructor_or_custom_trace,
 offsets_to_gc_pointers,
 fixed_size, varsize_item_sizes,
@@ -79,8 +79,7 @@
 fast_path_tracing,
 has_gcptr,
 cannot_pin):
-self.finalizer_trigger = finalizer_trigger
-self.get_run_finalizer_queue = get_run_finalizer_queue
+self.finalizer_handlers = finalizer_handlers
 self.destructor_or_custom_trace = destructor_or_custom_trace
 self.is_varsize = is_varsize
 self.has_gcptr_in_varsize = has_gcptr_in_varsize
@@ -332,12 +331,10 @@
 enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
 def enum_pending_finalizers(self, callback, arg):
+handlers = self.finalizer_handlers()
 i = 0
-while True:
-fdeque = self.get_run_finalizer_queue(self.AddressDeque, i)
-if fdeque is None:
-break
-fdeque.foreach(callback, arg)
+while i < len(handlers):
+self._adr2deque(handlers[i].deque).foreach(callback, arg)
 i += 1
 enum_pending_finalizers._annspecialcase_ = 'specialize:arg(1)'
 
@@ -379,23 +376,25 @@
 def debug_check_object(self, obj):
 pass
 
+def _adr2deque(self, adr):
+return cast_adr_to_nongc_instance(self.AddressDeque, adr)
+
 def execute_finalizers(self):
 if self.finalizer_lock:
 return  # the outer invocation of execute_finalizers() will do it
 self.finalizer_lock = True
 try:
+handlers = self.finalizer_handlers()
 i = 0
-while True:
-fdeque = self.get_run_finalizer_queue(self.AddressDeque, i)
-if fdeque is None:
-break
-if fdeque.non_empty():
-self.finalizer_trigger(i)
+while i < len(handlers):
+if self._adr2deque(handlers[i].deque).non_empty():
+handlers[i].trigger()
 i += 1
 finally:
 self.finalizer_lock = False
 
 def finalizer_next_dead(self, fq_index):
+
 fdeque = self.get_run_finalizer_queue(self.AddressDeque, fq_index)
 if fdeque.non_empty():
 obj = fdeque.popleft()
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1568,8 +1568,8 @@
 def register_finalizer(self, fq_index, gcobj):
 from rpython.rtyper.lltypesystem import rffi
 obj = llmemory.cast_ptr_to_adr(gcobj)
+fq_index = rffi.cast(llmemory.Address, fq_index)
 self.probably_young_objects_with_finalizers.append(obj)
-fq_index = rffi.cast(llmemory.Address, fq_index)
 self.probably_young_objects_with_finalizers.append(fq_index)
 
 # --
diff --git a/rpython/memory/gctransform/framework.py 
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -9,8 +9,10 @@
 from rpython.memory import gctypelayout
 from rpython.memory.gctransform.log import log
 from rpython.memory.gctransform.support import get_rtti, ll_call_destructor
+from rpython.memory.gctransform.support import ll_report_finalizer_error
 from rpython.memory.gctransform.transform import 

[pypy-commit] pypy gc-del-3: in-progress

[arigo]

Author: Armin Rigo 
Branch: gc-del-3
Changeset: r84179:979bc16d2cc9
Date: 2016-05-03 22:48 +0200
http://bitbucket.org/pypy/pypy/changeset/979bc16d2cc9/

Log:in-progress

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -6,6 +6,9 @@
 from rpython.memory.support import DEFAULT_CHUNK_SIZE
 from rpython.memory.support import get_address_stack, get_address_deque
 from rpython.memory.support import AddressDict, null_address_dict
+from rpython.memory.support import make_list_of_nongc_instances
+from rpython.memory.support import list_set_nongc_instance
+from rpython.memory.support import list_get_nongc_instance
 from rpython.rtyper.lltypesystem.llmemory import NULL, raw_malloc_usage
 
 TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed),
@@ -33,7 +36,7 @@
 self.config = config
 assert isinstance(translated_to_c, bool)
 self.translated_to_c = translated_to_c
-self._finalizer_queue_objects = []
+self.run_finalizer_queues = make_list_of_nongc_instances(0)
 
 def setup(self):
 # all runtime mutable values' setup should happen here
@@ -42,17 +45,23 @@
 
 def register_finalizer_index(self, fq, index):
 "NOT_RPYTHON"
-while len(self._finalizer_queue_objects) <= index:
-self._finalizer_queue_objects.append(None)
-if self._finalizer_queue_objects[index] is None:
-fq._reset()
-fq._gc_deque = self.AddressDeque()
-self._finalizer_queue_objects[index] = fq
-else:
-assert self._finalizer_queue_objects[index] is fq
+if len(self.run_finalizer_queues) <= index:
+array = make_list_of_nongc_instances(index + 1)
+for i in range(len(self.run_finalizer_queues)):
+array[i] = self.run_finalizer_queues[i]
+self.run_finalizer_queues = array
+#
+fdold = list_get_nongc_instance(self.AddressDeque,
+   self.run_finalizer_queues, index)
+list_set_nongc_instance(self.run_finalizer_queues, index,
+self.AddressDeque())
+if fdold is not None:
+fdold.delete()
 
 def mark_finalizer_to_run(self, fq_index, obj):
-self._finalizer_queue_objects[fq_index]._gc_deque.append(obj)
+fdeque = list_get_nongc_instance(self.AddressDeque,
+ self.run_finalizer_queues, fq_index)
+fdeque.append(obj)
 
 def post_setup(self):
 # More stuff that needs to be initialized when the GC is already
@@ -61,7 +70,7 @@
 self.DEBUG = env.read_from_env('PYPY_GC_DEBUG')
 
 def _teardown(self):
-self._finalizer_queue_objects = [] # for tests
+pass
 
 def can_optimize_clean_setarrayitems(self):
 return True # False in case of card marking
@@ -342,10 +351,11 @@
 
 def enum_pending_finalizers(self, callback, arg):
 i = 0
-while i < len(self._finalizer_queue_objects):
-fq = self._finalizer_queue_objects[i]
-if fq is not None:
-fq._gc_deque.foreach(callback, arg)
+while i < len(self.run_finalizer_queues):
+fdeque = list_get_nongc_instance(self.AddressDeque,
+ self.run_finalizer_queues, i)
+if fdeque is not None:
+fdeque.foreach(callback, arg)
 i += 1
 enum_pending_finalizers._annspecialcase_ = 'specialize:arg(1)'
 
@@ -393,14 +403,24 @@
 self.finalizer_lock = True
 try:
 i = 0
-while i < len(self._finalizer_queue_objects):
-fq = self._finalizer_queue_objects[i]
-if fq is not None and fq._gc_deque.non_empty():
+while i < len(self.run_finalizer_queues):
+fdeque = list_get_nongc_instance(self.AddressDeque,
+ self.run_finalizer_queues, i)
+if fdeque is not None and fdeque.non_empty():
 self.finalizer_trigger(i)
 i += 1
 finally:
 self.finalizer_lock = False
 
+def finalizer_next_dead(self, fq_index):
+fdeque = list_get_nongc_instance(self.AddressDeque,
+ self.run_finalizer_queues, fq_index)
+if fdeque.non_empty():
+obj = fdeque.popleft()
+else:
+obj = llmemory.NULL
+return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
 
 class MovingGCBase(GCBase):
 moving_gc = True
diff --git a/rpython/memory/gctransform/framework.py 
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -255,6 +255,7 @@
 
 self.layoutbuilder.encode_type_shapes_now()
 

[pypy-commit] pypy gc-del-3: in-progress

[arigo]

Author: Armin Rigo 
Branch: gc-del-3
Changeset: r84137:1d403288f1ac
Date: 2016-05-02 19:48 +0200
http://bitbucket.org/pypy/pypy/changeset/1d403288f1ac/

Log:in-progress

diff --git a/rpython/translator/backendopt/finalizer.py 
b/rpython/translator/backendopt/finalizer.py
--- a/rpython/translator/backendopt/finalizer.py
+++ b/rpython/translator/backendopt/finalizer.py
@@ -5,25 +5,30 @@
 class FinalizerError(Exception):
 """__del__() is used for lightweight RPython destructors,
 but the FinalizerAnalyzer found that it is not lightweight.
+
+The set of allowed operations is restrictive for a good reason
+- it's better to be safe. Specifically disallowed operations:
+
+* anything that escapes self
+* anything that can allocate
 """
 
 class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
 """ Analyzer that determines whether a finalizer is lightweight enough
 so it can be called without all the complicated logic in the garbage
-collector. The set of operations here is restrictive for a good reason
-- it's better to be safe. Specifically disallowed operations:
-
-* anything that escapes self
-* anything that can allocate
+collector.
 """
 ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
  'direct_ptradd', 'force_cast', 'track_alloc_stop',
  'raw_free', 'adr_eq', 'adr_ne']
 
 def check_light_finalizer(self, graph):
+self._origin = graph
 result = self.analyze_direct_call(graph)
+del self._origin
 if result is self.top_result():
-raise FinalizerError(FinalizerError.__doc__, graph)
+msg = '%s\nIn %r' % (FinalizerError.__doc__, graph)
+raise FinalizerError(msg)
 
 def analyze_simple_operation(self, op, graphinfo):
 if op.opname in self.ok_operations:
@@ -41,4 +46,10 @@
 if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
 # primitive type
 return self.bottom_result()
-return self.top_result()
+
+if not hasattr(self, '_origin'):# for tests
+return self.top_result()
+msg = '%s\nFound this forbidden operation:\n%r\nin %r\nfrom %r' % (
+FinalizerError.__doc__, op, graphinfo,
+getattr(self, '_origin', '?'))
+raise FinalizerError(msg)
diff --git a/rpython/translator/backendopt/test/test_finalizer.py 
b/rpython/translator/backendopt/test/test_finalizer.py
--- a/rpython/translator/backendopt/test/test_finalizer.py
+++ b/rpython/translator/backendopt/test/test_finalizer.py
@@ -26,8 +26,12 @@
 t.view()
 a = FinalizerAnalyzer(t)
 fgraph = graphof(t, func_to_analyze)
-result = a.analyze_light_finalizer(fgraph)
-return result
+try:
+a.check_light_finalizer(fgraph)
+except FinalizerError as e:
+print e
+return a.top_result()   # True
+return a.bottom_result()# False
 
 def test_nothing(self):
 def f():
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy gc-del-3: in-progress

[arigo]

Author: Armin Rigo 
Branch: gc-del-3
Changeset: r84132:585ea21d4f7f
Date: 2016-05-02 19:31 +0200
http://bitbucket.org/pypy/pypy/changeset/585ea21d4f7f/

Log:in-progress

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1565,6 +1565,13 @@
 self.header(shadow).tid |= GCFLAG_VISITED
 new_shadow_object_dict.setitem(obj, shadow)
 
+def register_finalizer(self, fq_index, gcobj):
+from rpython.rtyper.lltypesystem import rffi
+obj = llmemory.cast_ptr_to_adr(gcobj)
+self.probably_young_objects_with_finalizers.append(obj)
+fq_index = rffi.cast(llmemory.Address, fq_index)
+self.probably_young_objects_with_finalizers.append(fq_index)
+
 # --
 # Nursery collection
 
@@ -2617,12 +2624,12 @@
 def deal_with_young_objects_with_finalizers(self):
 while self.probably_young_objects_with_finalizers.non_empty():
 obj = self.probably_young_objects_with_finalizers.popleft()
-fin_nr = self.probably_young_objects_with_finalizers.popleft()
-singleaddr.address[0] = obj
-self._trace_drag_out1(singleaddr)
-obj = singleaddr.address[0]
-self.old_objects_with_light_finalizers.append(obj)
-self.old_objects_with_light_finalizers.append(fin_nr)
+fq_nr = self.probably_young_objects_with_finalizers.popleft()
+self.singleaddr.address[0] = obj
+self._trace_drag_out1(self.singleaddr)
+obj = self.singleaddr.address[0]
+self.old_objects_with_finalizers.append(obj)
+self.old_objects_with_finalizers.append(fq_nr)
 
 def deal_with_objects_with_finalizers(self):
 # Walk over list of objects with finalizers.
@@ -2635,14 +2642,17 @@
 marked = self.AddressDeque()
 pending = self.AddressStack()
 self.tmpstack = self.AddressStack()
-while self.objects_with_finalizers.non_empty():
-x = self.objects_with_finalizers.popleft()
+while self.old_objects_with_finalizers.non_empty():
+x = self.old_objects_with_finalizers.popleft()
+fq_nr = self.old_objects_with_finalizers.popleft()
 ll_assert(self._finalization_state(x) != 1,
   "bad finalization state 1")
 if self.header(x).tid & GCFLAG_VISITED:
 new_with_finalizer.append(x)
+new_with_finalizer.append(fq_nr)
 continue
 marked.append(x)
+marked.append(fq_nr)
 pending.append(x)
 while pending.non_empty():
 y = pending.pop()
@@ -2662,9 +2672,11 @@
 
 while marked.non_empty():
 x = marked.popleft()
+fq_nr = marked.popleft()
 state = self._finalization_state(x)
 ll_assert(state >= 2, "unexpected finalization state < 2")
 if state == 2:
+# XXX use fq_nr here
 self.run_finalizers.append(x)
 # we must also fix the state from 2 to 3 here, otherwise
 # we leave the GCFLAG_FINALIZATION_ORDERING bit behind
@@ -2672,12 +2684,13 @@
 self._recursively_bump_finalization_state_from_2_to_3(x)
 else:
 new_with_finalizer.append(x)
+new_with_finalizer.append(fq_nr)
 
 self.tmpstack.delete()
 pending.delete()
 marked.delete()
-self.objects_with_finalizers.delete()
-self.objects_with_finalizers = new_with_finalizer
+self.old_objects_with_finalizers.delete()
+self.old_objects_with_finalizers = new_with_finalizer
 
 def _append_if_nonnull(pointer, stack):
 stack.append(pointer.address[0])
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.memory import gctypelayout
 from rpython.flowspace.model import Constant
+from rpython.rlib import rgc
 
 
 class GCManagedHeap(object):
@@ -20,6 +21,7 @@
 self.llinterp = llinterp
 self.prepare_graphs(flowgraphs)
 self.gc.setup()
+self.finalizer_queues = {}
 self.has_write_barrier_from_array = hasattr(self.gc,
 'write_barrier_from_array')
 
@@ -187,6 +189,20 @@
 def thread_run(self):
 pass
 
+def get_finalizer_queue_index(self, fq_tag):
+assert fq_tag.expr == 'FinalizerQueue TAG'
+fq = fq_tag.default
+return self.finalizer_queues.setdefault(fq, len(self.finalizer_queues))
+
+def gc_fq_next_dead(self, fq_tag):
+index = self.get_finalizer_queue_index(fq_tag)
+xxx
+
+def gc_fq_register(self, fq_tag, ptr):
+

[pypy-commit] pypy gc-del-3: in-progress

[arigo]

Author: Armin Rigo 
Branch: gc-del-3
Changeset: r84129:7edef1bf570b
Date: 2016-05-02 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/7edef1bf570b/

Log:in-progress

diff --git a/pypy/doc/discussion/finalizer-order.rst 
b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -90,10 +90,10 @@
 To find the queued items, call ``fin.next_dead()`` repeatedly.  It
 returns the next queued item, or ``None`` when the queue is empty.
 
-It is not allowed to cumulate several ``FinalizerQueue`` instances for
-objects of the same class.  Calling ``fin.register_finalizer(obj)``
-several times with the same arguments is fine (and will only register
-``obj`` once).
+It is allowed in theory to cumulate several different
+``FinalizerQueue`` instances for objects of the same class, and
+(always in theory) the same ``obj`` could be registered several times
+in the same queue, or in several queues.  This is not tested though.
 
 
 Ordering of finalizers
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -60,8 +60,7 @@
 
 def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
 is_gcarrayofgcptr,
-getfinalizer,
-getlightfinalizer,
+destructor_or_custom_trace,
 offsets_to_gc_pointers,
 fixed_size, varsize_item_sizes,
 varsize_offset_to_variable_part,
@@ -74,8 +73,7 @@
 fast_path_tracing,
 has_gcptr,
 cannot_pin):
-self.getfinalizer = getfinalizer
-self.getlightfinalizer = getlightfinalizer
+self.destructor_or_custom_trace = destructor_or_custom_trace
 self.is_varsize = is_varsize
 self.has_gcptr_in_varsize = has_gcptr_in_varsize
 self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -136,13 +134,13 @@
 the four malloc_[fixed,var]size[_clear]() functions.
 """
 size = self.fixed_size(typeid)
-needs_finalizer = bool(self.getfinalizer(typeid))
-finalizer_is_light = bool(self.getlightfinalizer(typeid))
+needs_destructor = (bool(self.destructor_or_custom_trace(typeid))
+and not self.has_custom_trace(typeid))
 contains_weakptr = self.weakpointer_offset(typeid) >= 0
-assert not (needs_finalizer and contains_weakptr)
+assert not (needs_destructor and contains_weakptr)
 if self.is_varsize(typeid):
 assert not contains_weakptr
-assert not needs_finalizer
+assert not needs_destructor
 itemsize = self.varsize_item_sizes(typeid)
 offset_to_length = self.varsize_offset_to_length(typeid)
 if self.malloc_zero_filled:
@@ -157,8 +155,7 @@
 malloc_fixedsize = self.malloc_fixedsize_clear
 else:
 malloc_fixedsize = self.malloc_fixedsize
-ref = malloc_fixedsize(typeid, size, needs_finalizer,
-   finalizer_is_light,
+ref = malloc_fixedsize(typeid, size, needs_destructor,
contains_weakptr)
 # lots of cast and reverse-cast around...
 ref = llmemory.cast_ptr_to_adr(ref)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -372,10 +372,19 @@
 
 self.gc_state = STATE_SCANNING
 #
-# A list of all objects with finalizers (these are never young).
-self.objects_with_finalizers = self.AddressDeque()
-self.young_objects_with_light_finalizers = self.AddressStack()
-self.old_objects_with_light_finalizers = self.AddressStack()
+# Two lists of all objects with finalizers.  Actually they are lists
+# of pairs (finalization_queue_nr, object).  "probably young objects"
+# are all traced and moved to the "old" list by the next minor
+# collection.
+self.probably_young_objects_with_finalizers = self.AddressDeque()
+self.old_objects_with_finalizers = self.AddressDeque()
+p = lltype.malloc(self._ADDRARRAY, 1, flavor='raw',
+  track_allocation=False)
+self.singleaddr = llmemory.cast_ptr_to_adr(p)
+#
+# Two lists of all objects with destructors.
+self.young_objects_with_destructors = self.AddressStack()
+self.old_objects_with_destructors = self.AddressStack()
 #
 # Two lists of the objects with weakrefs.  No weakref can be an
 # old object weakly pointing to a young object: indeed, weakrefs
@@ -599,25 +608,16 @@
 
 
 def