Author: Antonio Cuni <anto.c...@gmail.com>
Branch:
Changeset: r94361:5bd740a5496c
Date: 2018-04-17 12:09 +0200
http://bitbucket.org/pypy/pypy/changeset/5bd740a5496c/
Log: merge the gc-hooks branch: it is now possible to install app-level
hooks which are triggered whenever a specific GC activity occurs
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -121,6 +121,160 @@
alive by GC objects, but not accounted in the GC
+GC Hooks
+--------
+
+GC hooks are user-defined functions which are called whenever a specific GC
+event occur, and can be used to monitor GC activity and pauses. You can
+install the hooks by setting the following attributes:
+
+``gc.hook.on_gc_minor``
+ Called whenever a minor collection occurs. It corresponds to
+ ``gc-minor`` sections inside ``PYPYLOG``.
+
+``gc.hook.on_gc_collect_step``
+ Called whenever an incremental step of a major collection occurs. It
+ corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``.
+
+``gc.hook.on_gc_collect``
+ Called after the last incremental step, when a major collection is fully
+ done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``.
+
+To uninstall a hook, simply set the corresponding attribute to ``None``. To
+install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look
+for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you
+can call ``gc.hooks.reset()``.
+
+The functions called by the hooks receive a single ``stats`` argument, which
+contains various statistics about the event.
+
+Note that PyPy cannot call the hooks immediately after a GC event, but it has
+to wait until it reaches a point in which the interpreter is in a known state
+and calling user-defined code is harmless. It might happen that multiple
+events occur before the hook is invoked: in this case, you can inspect the
+value ``stats.count`` to know how many times the event occured since the last
+time the hook was called. Similarly, ``stats.duration`` contains the
+**total** time spent by the GC for this specific event since the last time the
+hook was called.
+
+On the other hand, all the other fields of the ``stats`` object are relative
+only to the **last** event of the series.
+
+The attributes for ``GcMinorStats`` are:
+
+``count``
+ The number of minor collections occured since the last hook call.
+
+``duration``
+ The total time spent inside minor collections since the last hook
+ call. See below for more information on the unit.
+
+ ``total_memory_used``
+ The amount of memory used at the end of the minor collection, in
+ bytes. This include the memory used in arenas (for GC-managed memory) and
+ raw-malloced memory (e.g., the content of numpy arrays).
+
+``pinned_objects``
+ the number of pinned objects.
+
+
+The attributes for ``GcCollectStepStats`` are:
+
+``count``, ``duration``
+ See above.
+
+``oldstate``, ``newstate``
+ Integers which indicate the state of the GC before and after the step.
+
+The value of ``oldstate`` and ``newstate`` is one of these constants, defined
+inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``,
+``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string
+representation of it by indexing the ``GC_STATS`` tuple.
+
+
+The attributes for ``GcCollectStats`` are:
+
+``count``
+ See above.
+
+``num_major_collects``
+ The total number of major collections which have been done since the
+ start. Contrarily to ``count``, this is an always-growing counter and it's
+ not reset between invocations.
+
+``arenas_count_before``, ``arenas_count_after``
+ Number of arenas used before and after the major collection.
+
+``arenas_bytes``
+ Total number of bytes used by GC-managed objects.
+
+``rawmalloc_bytes_before``, ``rawmalloc_bytes_after``
+ Total number of bytes used by raw-malloced objects, before and after the
+ major collection.
+
+Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is
+because all the GC work is done inside ``gc-collect-step``:
+``gc-collect-done`` is used only to give additional stats, but doesn't do any
+actual work.
+
+A note about the ``duration`` field: depending on the architecture and
+operating system, PyPy uses different ways to read timestamps, so ``duration``
+is expressed in varying units. It is possible to know which by calling
+``__pypy__.debug_get_timestamp_unit()``, which can be one of the following
+values:
+
+``tsc``
+ The default on ``x86`` machines: timestamps are expressed in CPU ticks, as
+ read by the `Time Stamp Counter`_.
+
+``ns``
+ Timestamps are expressed in nanoseconds.
+
+``QueryPerformanceCounter``
+ On Windows, in case for some reason ``tsc`` is not available: timestamps
+ are read using the win API ``QueryPerformanceCounter()``.
+
+
+Unfortunately, there does not seem to be a reliable standard way for
+converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs
+it is enough to divide the ticks by the maximum nominal frequency of the CPU.
+For this reason, PyPy gives the raw value, and leaves the job of doing the
+conversion to external libraries.
+
+Here is an example of GC hooks in use::
+
+ import sys
+ import gc
+
+ class MyHooks(object):
+ done = False
+
+ def on_gc_minor(self, stats):
+ print 'gc-minor: count = %02d, duration = %d' %
(stats.count,
+
stats.duration)
+
+ def on_gc_collect_step(self, stats):
+ old = gc.GcCollectStepStats.GC_STATES[stats.oldstate]
+ new = gc.GcCollectStepStats.GC_STATES[stats.newstate]
+ print 'gc-collect-step: %s --> %s' % (old, new)
+ print ' count = %02d, duration = %d' %
(stats.count,
+
stats.duration)
+
+ def on_gc_collect(self, stats):
+ print 'gc-collect-done: count = %02d' % stats.count
+ self.done = True
+
+ hooks = MyHooks()
+ gc.hooks.set(hooks)
+
+ # simulate some GC activity
+ lst = []
+ while not hooks.done:
+ lst = [lst, 1, 2, 3]
+
+
+.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter
+
.. _minimark-environment-variables:
Environment variables
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -10,3 +10,7 @@
Fix a rare GC bug that was introduced more than one year ago, but was
not diagnosed before issue #2752.
+
+.. branch: gc-hooks
+
+Introduce GC hooks, as documented in doc/gc_info.rst
diff --git a/pypy/goal/targetpypystandalone.py
b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -215,6 +215,7 @@
usage = SUPPRESS_USAGE
take_options = True
+ space = None
def opt_parser(self, config):
parser = to_optparse(config, useoptions=["objspace.*"],
@@ -364,15 +365,21 @@
from pypy.module.pypyjit.hooks import pypy_hooks
return PyPyJitPolicy(pypy_hooks)
+ def get_gchooks(self):
+ from pypy.module.gc.hook import LowLevelGcHooks
+ if self.space is None:
+ raise Exception("get_gchooks must be called afeter
get_entry_point")
+ return self.space.fromcache(LowLevelGcHooks)
+
def get_entry_point(self, config):
- space = make_objspace(config)
+ self.space = make_objspace(config)
# manually imports app_main.py
filename = os.path.join(pypydir, 'interpreter', 'app_main.py')
app = gateway.applevel(open(filename).read(), 'app_main.py',
'app_main')
app.hidden_applevel = False
- w_dict = app.getwdict(space)
- entry_point, _ = create_entry_point(space, w_dict)
+ w_dict = app.getwdict(self.space)
+ entry_point, _ = create_entry_point(self.space, w_dict)
return entry_point, None, PyPyAnnotatorPolicy()
@@ -381,7 +388,7 @@
'jitpolicy', 'get_entry_point',
'get_additional_config_options']:
ns[name] = getattr(self, name)
-
+ ns['get_gchooks'] = self.get_gchooks
PyPyTarget().interface(globals())
diff --git a/pypy/interpreter/executioncontext.py
b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -404,7 +404,7 @@
self._periodic_actions = []
self._nonperiodic_actions = []
self.has_bytecode_counter = False
- self.fired_actions = None
+ self._fired_actions_reset()
# the default value is not 100, unlike CPython 2.7, but a much
# larger value, because we use a technique that not only allows
# but actually *forces* another thread to run whenever the counter
@@ -416,13 +416,28 @@
"""Request for the action to be run before the next opcode."""
if not action._fired:
action._fired = True
- if self.fired_actions is None:
- self.fired_actions = []
- self.fired_actions.append(action)
+ self._fired_actions_append(action)
# set the ticker to -1 in order to force action_dispatcher()
# to run at the next possible bytecode
self.reset_ticker(-1)
+ def _fired_actions_reset(self):
+ # linked list of actions. We cannot use a normal RPython list because
+ # we want AsyncAction.fire() to be marked as @rgc.collect: this way,
+ # we can call it from e.g. GcHooks or cpyext's dealloc_trigger.
+ self._fired_actions_first = None
+ self._fired_actions_last = None
+
+ @rgc.no_collect
+ def _fired_actions_append(self, action):
+ assert action._next is None
+ if self._fired_actions_first is None:
+ self._fired_actions_first = action
+ self._fired_actions_last = action
+ else:
+ self._fired_actions_last._next = action
+ self._fired_actions_last = action
+
@not_rpython
def register_periodic_action(self, action, use_bytecode_counter):
"""
@@ -467,9 +482,9 @@
action.perform(ec, frame)
# nonperiodic actions
- list = self.fired_actions
- if list is not None:
- self.fired_actions = None
+ action = self._fired_actions_first
+ if action:
+ self._fired_actions_reset()
# NB. in case there are several actions, we reset each
# 'action._fired' to false only when we're about to call
# 'action.perform()'. This means that if
@@ -477,9 +492,10 @@
# the corresponding perform(), the fire() has no
# effect---which is the effect we want, because
# perform() will be called anyway.
- for action in list:
+ while action is not None:
action._fired = False
action.perform(ec, frame)
+ action._next, action = None, action._next
self.action_dispatcher = action_dispatcher
@@ -512,10 +528,12 @@
to occur between two opcodes, not at a completely random time.
"""
_fired = False
+ _next = None
def __init__(self, space):
self.space = space
+ @rgc.no_collect
def fire(self):
"""Request for the action to be run before the next opcode.
The action must have been registered at space initalization time."""
diff --git a/pypy/interpreter/test/test_executioncontext.py
b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -37,6 +37,37 @@
pass
assert i == 9
+ def test_action_queue(self):
+ events = []
+
+ class Action1(executioncontext.AsyncAction):
+ def perform(self, ec, frame):
+ events.append('one')
+
+ class Action2(executioncontext.AsyncAction):
+ def perform(self, ec, frame):
+ events.append('two')
+
+ space = self.space
+ a1 = Action1(space)
+ a2 = Action2(space)
+ a1.fire()
+ a2.fire()
+ space.appexec([], """():
+ n = 5
+ return n + 2
+ """)
+ assert events == ['one', 'two']
+ #
+ events[:] = []
+ a1.fire()
+ space.appexec([], """():
+ n = 5
+ return n + 2
+ """)
+ assert events == ['one']
+
+
def test_periodic_action(self):
from pypy.interpreter.executioncontext import ActionFlag
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -82,6 +82,8 @@
'debug_stop' : 'interp_debug.debug_stop',
'debug_print_once' : 'interp_debug.debug_print_once',
'debug_flush' : 'interp_debug.debug_flush',
+ 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp',
+ 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit',
'builtinify' : 'interp_magic.builtinify',
'hidden_applevel' : 'interp_magic.hidden_applevel',
'get_hidden_tb' : 'interp_magic.get_hidden_tb',
diff --git a/pypy/module/__pypy__/interp_debug.py
b/pypy/module/__pypy__/interp_debug.py
--- a/pypy/module/__pypy__/interp_debug.py
+++ b/pypy/module/__pypy__/interp_debug.py
@@ -1,6 +1,6 @@
from pypy.interpreter.gateway import unwrap_spec
from rpython.rlib import debug, jit
-
+from rpython.rlib import rtimer
@jit.dont_look_inside
@unwrap_spec(category='text')
@@ -28,3 +28,18 @@
@jit.dont_look_inside
def debug_flush(space):
debug.debug_flush()
+
+def debug_read_timestamp(space):
+ return space.newint(rtimer.read_timestamp())
+
+def debug_get_timestamp_unit(space):
+ unit = rtimer.get_timestamp_unit()
+ if unit == rtimer.UNIT_TSC:
+ unit_str = 'tsc'
+ elif unit == rtimer.UNIT_NS:
+ unit_str = 'ns'
+ elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER:
+ unit_str = 'QueryPerformanceCounter'
+ else:
+ unit_str = 'UNKNOWN(%d)' % unit
+ return space.newtext(unit_str)
diff --git a/pypy/module/__pypy__/test/test_debug.py
b/pypy/module/__pypy__/test/test_debug.py
--- a/pypy/module/__pypy__/test/test_debug.py
+++ b/pypy/module/__pypy__/test/test_debug.py
@@ -48,3 +48,14 @@
from __pypy__ import debug_flush
debug_flush()
# assert did not crash
+
+ def test_debug_read_timestamp(self):
+ from __pypy__ import debug_read_timestamp
+ a = debug_read_timestamp()
+ b = debug_read_timestamp()
+ assert b > a
+
+ def test_debug_get_timestamp_unit(self):
+ from __pypy__ import debug_get_timestamp_unit
+ unit = debug_get_timestamp_unit()
+ assert unit in ('tsc', 'ns', 'QueryPerformanceCounter')
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -34,5 +34,7 @@
'get_typeids_z': 'referents.get_typeids_z',
'get_typeids_list': 'referents.get_typeids_list',
'GcRef': 'referents.W_GcRef',
+ 'hooks': 'space.fromcache(hook.W_AppLevelHooks)',
+ 'GcCollectStepStats': 'hook.W_GcCollectStepStats',
})
MixedModule.__init__(self, space, w_name)
diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/gc/hook.py
@@ -0,0 +1,317 @@
+from rpython.memory.gc.hook import GcHooks
+from rpython.memory.gc import incminimark
+from rpython.rlib.nonconst import NonConstant
+from rpython.rlib.rarithmetic import r_uint, r_longlong
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty,
GetSetProperty
+from pypy.interpreter.executioncontext import AsyncAction
+
+class LowLevelGcHooks(GcHooks):
+ """
+ These are the low-level hooks which are called directly from the GC.
+
+ They can't do much, because the base class marks the methods as
+ @rgc.no_collect.
+
+ This is expected to be a singleton, created by space.fromcache, and it is
+ integrated with the translation by targetpypystandalone.get_gchooks
+ """
+
+ def __init__(self, space):
+ self.space = space
+ self.w_hooks = space.fromcache(W_AppLevelHooks)
+
+ def is_gc_minor_enabled(self):
+ return self.w_hooks.gc_minor_enabled
+
+ def is_gc_collect_step_enabled(self):
+ return self.w_hooks.gc_collect_step_enabled
+
+ def is_gc_collect_enabled(self):
+ return self.w_hooks.gc_collect_enabled
+
+ def on_gc_minor(self, duration, total_memory_used, pinned_objects):
+ action = self.w_hooks.gc_minor
+ action.count += 1
+ action.duration += duration
+ action.total_memory_used = total_memory_used
+ action.pinned_objects = pinned_objects
+ action.fire()
+
+ def on_gc_collect_step(self, duration, oldstate, newstate):
+ action = self.w_hooks.gc_collect_step
+ action.count += 1
+ action.duration += duration
+ action.oldstate = oldstate
+ action.newstate = newstate
+ action.fire()
+
+ def on_gc_collect(self, num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after):
+ action = self.w_hooks.gc_collect
+ action.count += 1
+ action.num_major_collects = num_major_collects
+ action.arenas_count_before = arenas_count_before
+ action.arenas_count_after = arenas_count_after
+ action.arenas_bytes = arenas_bytes
+ action.rawmalloc_bytes_before = rawmalloc_bytes_before
+ action.rawmalloc_bytes_after = rawmalloc_bytes_after
+ action.fire()
+
+
+class W_AppLevelHooks(W_Root):
+
+ def __init__(self, space):
+ self.space = space
+ self.gc_minor_enabled = False
+ self.gc_collect_step_enabled = False
+ self.gc_collect_enabled = False
+ self.gc_minor = GcMinorHookAction(space)
+ self.gc_collect_step = GcCollectStepHookAction(space)
+ self.gc_collect = GcCollectHookAction(space)
+
+ def descr_get_on_gc_minor(self, space):
+ return self.gc_minor.w_callable
+
+ def descr_set_on_gc_minor(self, space, w_obj):
+ self.gc_minor_enabled = not space.is_none(w_obj)
+ self.gc_minor.w_callable = w_obj
+ self.gc_minor.fix_annotation()
+
+ def descr_get_on_gc_collect_step(self, space):
+ return self.gc_collect_step.w_callable
+
+ def descr_set_on_gc_collect_step(self, space, w_obj):
+ self.gc_collect_step_enabled = not space.is_none(w_obj)
+ self.gc_collect_step.w_callable = w_obj
+ self.gc_collect_step.fix_annotation()
+
+ def descr_get_on_gc_collect(self, space):
+ return self.gc_collect.w_callable
+
+ def descr_set_on_gc_collect(self, space, w_obj):
+ self.gc_collect_enabled = not space.is_none(w_obj)
+ self.gc_collect.w_callable = w_obj
+ self.gc_collect.fix_annotation()
+
+ def descr_set(self, space, w_obj):
+ w_a = space.getattr(w_obj, space.newtext('on_gc_minor'))
+ w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step'))
+ w_c = space.getattr(w_obj, space.newtext('on_gc_collect'))
+ self.descr_set_on_gc_minor(space, w_a)
+ self.descr_set_on_gc_collect_step(space, w_b)
+ self.descr_set_on_gc_collect(space, w_c)
+
+ def descr_reset(self, space):
+ self.descr_set_on_gc_minor(space, space.w_None)
+ self.descr_set_on_gc_collect_step(space, space.w_None)
+ self.descr_set_on_gc_collect(space, space.w_None)
+
+
+class GcMinorHookAction(AsyncAction):
+ count = 0
+ duration = r_longlong(0)
+ total_memory_used = 0
+ pinned_objects = 0
+
+ def __init__(self, space):
+ AsyncAction.__init__(self, space)
+ self.w_callable = space.w_None
+
+ def reset(self):
+ self.count = 0
+ self.duration = r_longlong(0)
+
+ def fix_annotation(self):
+ # the annotation of the class and its attributes must be completed
+ # BEFORE we do the gc transform; this makes sure that everything is
+ # annotated with the correct types
+ if NonConstant(False):
+ self.count = NonConstant(-42)
+ self.duration = NonConstant(r_longlong(-42))
+ self.total_memory_used = NonConstant(r_uint(42))
+ self.pinned_objects = NonConstant(-42)
+ self.fire()
+
+ def perform(self, ec, frame):
+ w_stats = W_GcMinorStats(
+ self.count,
+ self.duration,
+ self.total_memory_used,
+ self.pinned_objects)
+ self.reset()
+ self.space.call_function(self.w_callable, w_stats)
+
+
+class GcCollectStepHookAction(AsyncAction):
+ count = 0
+ duration = r_longlong(0)
+ oldstate = 0
+ newstate = 0
+
+ def __init__(self, space):
+ AsyncAction.__init__(self, space)
+ self.w_callable = space.w_None
+
+ def reset(self):
+ self.count = 0
+ self.duration = r_longlong(0)
+
+ def fix_annotation(self):
+ # the annotation of the class and its attributes must be completed
+ # BEFORE we do the gc transform; this makes sure that everything is
+ # annotated with the correct types
+ if NonConstant(False):
+ self.count = NonConstant(-42)
+ self.duration = NonConstant(r_longlong(-42))
+ self.oldstate = NonConstant(-42)
+ self.newstate = NonConstant(-42)
+ self.fire()
+
+ def perform(self, ec, frame):
+ w_stats = W_GcCollectStepStats(
+ self.count,
+ self.duration,
+ self.oldstate,
+ self.newstate)
+ self.reset()
+ self.space.call_function(self.w_callable, w_stats)
+
+
+class GcCollectHookAction(AsyncAction):
+ count = 0
+ num_major_collects = 0
+ arenas_count_before = 0
+ arenas_count_after = 0
+ arenas_bytes = 0
+ rawmalloc_bytes_before = 0
+ rawmalloc_bytes_after = 0
+
+ def __init__(self, space):
+ AsyncAction.__init__(self, space)
+ self.w_callable = space.w_None
+
+ def reset(self):
+ self.count = 0
+
+ def fix_annotation(self):
+ # the annotation of the class and its attributes must be completed
+ # BEFORE we do the gc transform; this makes sure that everything is
+ # annotated with the correct types
+ if NonConstant(False):
+ self.count = NonConstant(-42)
+ self.num_major_collects = NonConstant(-42)
+ self.arenas_count_before = NonConstant(-42)
+ self.arenas_count_after = NonConstant(-42)
+ self.arenas_bytes = NonConstant(r_uint(42))
+ self.rawmalloc_bytes_before = NonConstant(r_uint(42))
+ self.rawmalloc_bytes_after = NonConstant(r_uint(42))
+ self.fire()
+
+ def perform(self, ec, frame):
+ w_stats = W_GcCollectStats(self.count,
+ self.num_major_collects,
+ self.arenas_count_before,
+ self.arenas_count_after,
+ self.arenas_bytes,
+ self.rawmalloc_bytes_before,
+ self.rawmalloc_bytes_after)
+ self.reset()
+ self.space.call_function(self.w_callable, w_stats)
+
+
+class W_GcMinorStats(W_Root):
+
+ def __init__(self, count, duration, total_memory_used, pinned_objects):
+ self.count = count
+ self.duration = duration
+ self.total_memory_used = total_memory_used
+ self.pinned_objects = pinned_objects
+
+
+class W_GcCollectStepStats(W_Root):
+
+ def __init__(self, count, duration, oldstate, newstate):
+ self.count = count
+ self.duration = duration
+ self.oldstate = oldstate
+ self.newstate = newstate
+
+
+class W_GcCollectStats(W_Root):
+ def __init__(self, count, num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after):
+ self.count = count
+ self.num_major_collects = num_major_collects
+ self.arenas_count_before = arenas_count_before
+ self.arenas_count_after = arenas_count_after
+ self.arenas_bytes = arenas_bytes
+ self.rawmalloc_bytes_before = rawmalloc_bytes_before
+ self.rawmalloc_bytes_after = rawmalloc_bytes_after
+
+
+# just a shortcut to make the typedefs shorter
+def wrap_many_ints(cls, names):
+ d = {}
+ for name in names:
+ d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint")
+ return d
+
+
+W_AppLevelHooks.typedef = TypeDef(
+ "GcHooks",
+ on_gc_minor = GetSetProperty(
+ W_AppLevelHooks.descr_get_on_gc_minor,
+ W_AppLevelHooks.descr_set_on_gc_minor),
+
+ on_gc_collect_step = GetSetProperty(
+ W_AppLevelHooks.descr_get_on_gc_collect_step,
+ W_AppLevelHooks.descr_set_on_gc_collect_step),
+
+ on_gc_collect = GetSetProperty(
+ W_AppLevelHooks.descr_get_on_gc_collect,
+ W_AppLevelHooks.descr_set_on_gc_collect),
+
+ set = interp2app(W_AppLevelHooks.descr_set),
+ reset = interp2app(W_AppLevelHooks.descr_reset),
+ )
+
+W_GcMinorStats.typedef = TypeDef(
+ "GcMinorStats",
+ **wrap_many_ints(W_GcMinorStats, (
+ "count",
+ "duration",
+ "total_memory_used",
+ "pinned_objects"))
+ )
+
+W_GcCollectStepStats.typedef = TypeDef(
+ "GcCollectStepStats",
+ STATE_SCANNING = incminimark.STATE_SCANNING,
+ STATE_MARKING = incminimark.STATE_MARKING,
+ STATE_SWEEPING = incminimark.STATE_SWEEPING,
+ STATE_FINALIZING = incminimark.STATE_FINALIZING,
+ GC_STATES = tuple(incminimark.GC_STATES),
+ **wrap_many_ints(W_GcCollectStepStats, (
+ "count",
+ "duration",
+ "oldstate",
+ "newstate"))
+ )
+
+W_GcCollectStats.typedef = TypeDef(
+ "GcCollectStats",
+ **wrap_many_ints(W_GcCollectStats, (
+ "count",
+ "num_major_collects",
+ "arenas_count_before",
+ "arenas_count_after",
+ "arenas_bytes",
+ "rawmalloc_bytes_before",
+ "rawmalloc_bytes_after"))
+ )
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/gc/test/test_hook.py
@@ -0,0 +1,176 @@
+import pytest
+from rpython.rlib.rarithmetic import r_uint
+from pypy.module.gc.hook import LowLevelGcHooks
+from pypy.interpreter.baseobjspace import ObjSpace
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+
+class AppTestGcHooks(object):
+
+ def setup_class(cls):
+ if cls.runappdirect:
+ pytest.skip("these tests cannot work with -A")
+ space = cls.space
+ gchooks = space.fromcache(LowLevelGcHooks)
+
+ @unwrap_spec(ObjSpace, int, r_uint, int)
+ def fire_gc_minor(space, duration, total_memory_used, pinned_objects):
+ gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects)
+
+ @unwrap_spec(ObjSpace, int, int, int)
+ def fire_gc_collect_step(space, duration, oldstate, newstate):
+ gchooks.fire_gc_collect_step(duration, oldstate, newstate)
+
+ @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint)
+ def fire_gc_collect(space, a, b, c, d, e, f):
+ gchooks.fire_gc_collect(a, b, c, d, e, f)
+
+ @unwrap_spec(ObjSpace)
+ def fire_many(space):
+ gchooks.fire_gc_minor(5, 0, 0)
+ gchooks.fire_gc_minor(7, 0, 0)
+ gchooks.fire_gc_collect_step(5, 0, 0)
+ gchooks.fire_gc_collect_step(15, 0, 0)
+ gchooks.fire_gc_collect_step(22, 0, 0)
+ gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6)
+
+ cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor))
+ cls.w_fire_gc_collect_step =
space.wrap(interp2app(fire_gc_collect_step))
+ cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect))
+ cls.w_fire_many = space.wrap(interp2app(fire_many))
+
+ def test_default(self):
+ import gc
+ assert gc.hooks.on_gc_minor is None
+ assert gc.hooks.on_gc_collect_step is None
+ assert gc.hooks.on_gc_collect is None
+
+ def test_on_gc_minor(self):
+ import gc
+ lst = []
+ def on_gc_minor(stats):
+ lst.append((stats.count,
+ stats.duration,
+ stats.total_memory_used,
+ stats.pinned_objects))
+ gc.hooks.on_gc_minor = on_gc_minor
+ self.fire_gc_minor(10, 20, 30)
+ self.fire_gc_minor(40, 50, 60)
+ assert lst == [
+ (1, 10, 20, 30),
+ (1, 40, 50, 60),
+ ]
+ #
+ gc.hooks.on_gc_minor = None
+ self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is
disabled
+ assert lst == [
+ (1, 10, 20, 30),
+ (1, 40, 50, 60),
+ ]
+
+ def test_on_gc_collect_step(self):
+ import gc
+ lst = []
+ def on_gc_collect_step(stats):
+ lst.append((stats.count,
+ stats.duration,
+ stats.oldstate,
+ stats.newstate))
+ gc.hooks.on_gc_collect_step = on_gc_collect_step
+ self.fire_gc_collect_step(10, 20, 30)
+ self.fire_gc_collect_step(40, 50, 60)
+ assert lst == [
+ (1, 10, 20, 30),
+ (1, 40, 50, 60),
+ ]
+ #
+ gc.hooks.on_gc_collect_step = None
+ self.fire_gc_collect_step(70, 80, 90) # won't fire
+ assert lst == [
+ (1, 10, 20, 30),
+ (1, 40, 50, 60),
+ ]
+
+ def test_on_gc_collect(self):
+ import gc
+ lst = []
+ def on_gc_collect(stats):
+ lst.append((stats.count,
+ stats.num_major_collects,
+ stats.arenas_count_before,
+ stats.arenas_count_after,
+ stats.arenas_bytes,
+ stats.rawmalloc_bytes_before,
+ stats.rawmalloc_bytes_after))
+ gc.hooks.on_gc_collect = on_gc_collect
+ self.fire_gc_collect(1, 2, 3, 4, 5, 6)
+ self.fire_gc_collect(7, 8, 9, 10, 11, 12)
+ assert lst == [
+ (1, 1, 2, 3, 4, 5, 6),
+ (1, 7, 8, 9, 10, 11, 12),
+ ]
+ #
+ gc.hooks.on_gc_collect = None
+ self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire
+ assert lst == [
+ (1, 1, 2, 3, 4, 5, 6),
+ (1, 7, 8, 9, 10, 11, 12),
+ ]
+
+ def test_consts(self):
+ import gc
+ S = gc.GcCollectStepStats
+ assert S.STATE_SCANNING == 0
+ assert S.STATE_MARKING == 1
+ assert S.STATE_SWEEPING == 2
+ assert S.STATE_FINALIZING == 3
+ assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING')
+
+ def test_cumulative(self):
+ import gc
+ class MyHooks(object):
+
+ def __init__(self):
+ self.minors = []
+ self.steps = []
+
+ def on_gc_minor(self, stats):
+ self.minors.append((stats.count, stats.duration))
+
+ def on_gc_collect_step(self, stats):
+ self.steps.append((stats.count, stats.duration))
+
+ on_gc_collect = None
+
+ myhooks = MyHooks()
+ gc.hooks.set(myhooks)
+ self.fire_many()
+ assert myhooks.minors == [(2, 12)]
+ assert myhooks.steps == [(3, 42)]
+
+ def test_clear_queue(self):
+ import gc
+ class MyHooks(object):
+
+ def __init__(self):
+ self.lst = []
+
+ def on_gc_minor(self, stats):
+ self.lst.append('minor')
+
+ def on_gc_collect_step(self, stats):
+ self.lst.append('step')
+
+ def on_gc_collect(self, stats):
+ self.lst.append('collect')
+
+ myhooks = MyHooks()
+ gc.hooks.set(myhooks)
+ self.fire_many()
+ assert myhooks.lst == ['minor', 'step', 'collect']
+ myhooks.lst[:] = []
+ self.fire_gc_minor(0, 0, 0)
+ assert myhooks.lst == ['minor']
+ gc.hooks.reset()
+ assert gc.hooks.on_gc_minor is None
+ assert gc.hooks.on_gc_collect_step is None
+ assert gc.hooks.on_gc_collect is None
diff --git a/pypy/module/gc/test/test_ztranslation.py
b/pypy/module/gc/test/test_ztranslation.py
--- a/pypy/module/gc/test/test_ztranslation.py
+++ b/pypy/module/gc/test/test_ztranslation.py
@@ -1,4 +1,9 @@
from pypy.objspace.fake.checkmodule import checkmodule
def test_checkmodule():
- checkmodule('gc')
+ # we need to ignore GcCollectStepStats, else checkmodule fails. I think
+ # this happens because W_GcCollectStepStats.__init__ is only called from
+ # GcCollectStepHookAction.perform() and the fake objspace doesn't know
+ # about those: so, perform() is never annotated and the annotator thinks
+ # W_GcCollectStepStats has no attributes
+ checkmodule('gc', ignore=['GcCollectStepStats'])
diff --git a/pypy/objspace/fake/checkmodule.py
b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -4,6 +4,7 @@
def checkmodule(*modnames, **kwds):
translate_startup = kwds.pop('translate_startup', True)
+ ignore = set(kwds.pop('ignore', ()))
assert not kwds
config = get_pypy_config(translating=True)
space = FakeObjSpace(config)
@@ -17,6 +18,8 @@
module.init(space)
modules.append(module)
for name in module.loaders:
+ if name in ignore:
+ continue
seeobj_w.append(module._load_lazily(space, name))
if hasattr(module, 'submodules'):
for cls in module.submodules.itervalues():
diff --git a/rpython/jit/codewriter/jtransform.py
b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -2164,6 +2164,11 @@
oopspecindex=EffectInfo.OS_MATH_READ_TIMESTAMP,
extraeffect=EffectInfo.EF_CANNOT_RAISE)
+ def rewrite_op_ll_get_timestamp_unit(self, op):
+ op1 = self.prepare_builtin_call(op, "ll_get_timestamp_unit", [])
+ return self.handle_residual_call(op1,
+ extraeffect=EffectInfo.EF_CANNOT_RAISE)
+
def rewrite_op_jit_force_quasi_immutable(self, op):
v_inst, c_fieldname = op.args
descr1 = self.cpu.fielddescrof(v_inst.concretetype.TO,
diff --git a/rpython/jit/codewriter/support.py
b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -285,6 +285,9 @@
from rpython.rlib import rtimer
return rtimer.read_timestamp()
+def _ll_0_ll_get_timestamp_unit():
+ from rpython.rlib import rtimer
+ return rtimer.get_timestamp_unit()
# math support
# ------------
diff --git a/rpython/jit/metainterp/test/test_ajit.py
b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2577,6 +2577,14 @@
res = self.interp_operations(f, [])
assert res
+ def test_get_timestamp_unit(self):
+ import time
+ from rpython.rlib import rtimer
+ def f():
+ return rtimer.get_timestamp_unit()
+ unit = self.interp_operations(f, [])
+ assert unit == rtimer.UNIT_NS
+
def test_bug688_multiple_immutable_fields(self):
myjitdriver = JitDriver(greens=[], reds=['counter','context'])
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -5,6 +5,7 @@
from rpython.memory.support import DEFAULT_CHUNK_SIZE
from rpython.memory.support import get_address_stack, get_address_deque
from rpython.memory.support import AddressDict, null_address_dict
+from rpython.memory.gc.hook import GcHooks
from rpython.rtyper.lltypesystem.llmemory import NULL, raw_malloc_usage
from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
@@ -25,7 +26,7 @@
_totalroots_rpy = 0 # for inspector.py
def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
- translated_to_c=True):
+ translated_to_c=True, hooks=None):
self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
self.AddressStack = get_address_stack(chunk_size)
self.AddressDeque = get_address_deque(chunk_size)
@@ -34,6 +35,9 @@
self.config = config
assert isinstance(translated_to_c, bool)
self.translated_to_c = translated_to_c
+ if hooks is None:
+ hooks = GcHooks() # the default hooks are empty
+ self.hooks = hooks
def setup(self):
# all runtime mutable values' setup should happen here
diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/hook.py
@@ -0,0 +1,70 @@
+from rpython.rlib import rgc
+
+# WARNING: at the moment of writing, gc hooks are implemented only for
+# incminimark. Please add calls to hooks to the other GCs if you need it.
+class GcHooks(object):
+ """
+ Base class to write your own GC hooks.
+
+ Subclasses are expected to override the on_* methods. Note that such
+ methods can do only simple stuff such as updating statistics and/or
+ setting a flag: in particular, they cannot do anything which can possibly
+ trigger a GC collection.
+ """
+
+ def is_gc_minor_enabled(self):
+ return False
+
+ def is_gc_collect_step_enabled(self):
+ return False
+
+ def is_gc_collect_enabled(self):
+ return False
+
+ def on_gc_minor(self, duration, total_memory_used, pinned_objects):
+ """
+ Called after a minor collection
+ """
+
+ def on_gc_collect_step(self, duration, oldstate, newstate):
+ """
+ Called after each individual step of a major collection, in case the
GC is
+ incremental.
+
+ ``oldstate`` and ``newstate`` are integers which indicate the GC
+ state; for incminimark, see incminimark.STATE_* and
+ incminimark.GC_STATES.
+ """
+
+
+ def on_gc_collect(self, num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after):
+ """
+ Called after a major collection is fully done
+ """
+
+ # the fire_* methods are meant to be called from the GC are should NOT be
+ # overridden
+
+ @rgc.no_collect
+ def fire_gc_minor(self, duration, total_memory_used, pinned_objects):
+ if self.is_gc_minor_enabled():
+ self.on_gc_minor(duration, total_memory_used, pinned_objects)
+
+ @rgc.no_collect
+ def fire_gc_collect_step(self, duration, oldstate, newstate):
+ if self.is_gc_collect_step_enabled():
+ self.on_gc_collect_step(duration, oldstate, newstate)
+
+ @rgc.no_collect
+ def fire_gc_collect(self, num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after):
+ if self.is_gc_collect_enabled():
+ self.on_gc_collect(num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -73,6 +73,7 @@
from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
from rpython.rlib.objectmodel import specialize
from rpython.rlib import rgc
+from rpython.rlib.rtimer import read_timestamp
from rpython.memory.gc.minimarkpage import out_of_memory
#
@@ -1643,6 +1644,7 @@
"""Perform a minor collection: find the objects from the nursery
that remain alive and move them out."""
#
+ start = read_timestamp()
debug_start("gc-minor")
#
# All nursery barriers are invalid from this point on. They
@@ -1830,8 +1832,8 @@
# from the nursery that we just moved out.
self.size_objects_made_old += r_uint(self.nursery_surviving_size)
#
- debug_print("minor collect, total memory used:",
- self.get_total_memory_used())
+ total_memory_used = self.get_total_memory_used()
+ debug_print("minor collect, total memory used:", total_memory_used)
debug_print("number of pinned objects:",
self.pinned_objects_in_nursery)
if self.DEBUG >= 2:
@@ -1840,6 +1842,11 @@
self.root_walker.finished_minor_collection()
#
debug_stop("gc-minor")
+ duration = read_timestamp() - start
+ self.hooks.fire_gc_minor(
+ duration=duration,
+ total_memory_used=total_memory_used,
+ pinned_objects=self.pinned_objects_in_nursery)
def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN !=
0,
@@ -2241,7 +2248,9 @@
# Note - minor collections seem fast enough so that one
# is done before every major collection step
def major_collection_step(self, reserving_size=0):
+ start = read_timestamp()
debug_start("gc-collect-step")
+ oldstate = self.gc_state
debug_print("starting gc state: ", GC_STATES[self.gc_state])
# Debugging checks
if self.pinned_objects_in_nursery == 0:
@@ -2421,6 +2430,13 @@
self.stat_rawmalloced_total_size, " => ",
self.rawmalloced_total_size)
debug_stop("gc-collect-done")
+ self.hooks.fire_gc_collect(
+ num_major_collects=self.num_major_collects,
+ arenas_count_before=self.stat_ac_arenas_count,
+ arenas_count_after=self.ac.arenas_count,
+ arenas_bytes=self.ac.total_memory_used,
+ rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
+ rawmalloc_bytes_after=self.rawmalloced_total_size)
#
# Set the threshold for the next major collection to be when we
# have allocated 'major_collection_threshold' times more than
@@ -2472,6 +2488,11 @@
debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
debug_stop("gc-collect-step")
+ duration = read_timestamp() - start
+ self.hooks.fire_gc_collect_step(
+ duration=duration,
+ oldstate=oldstate,
+ newstate=self.gc_state)
def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):
if self.header(obj).tid & GCFLAG_VISITED:
diff --git a/rpython/memory/gc/test/test_direct.py
b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -70,6 +70,9 @@
class BaseDirectGCTest(object):
GC_PARAMS = {}
+ def get_extra_gc_params(self):
+ return {}
+
def setup_method(self, meth):
from rpython.config.translationoption import
get_combined_translation_config
config = get_combined_translation_config(translating=True).translation
@@ -78,6 +81,7 @@
if hasattr(meth, 'GC_PARAMS'):
GC_PARAMS.update(meth.GC_PARAMS)
GC_PARAMS['translated_to_c'] = False
+ GC_PARAMS.update(self.get_extra_gc_params())
self.gc = self.GCClass(config, **GC_PARAMS)
self.gc.DEBUG = True
self.rootwalker = DirectRootWalker(self)
diff --git a/rpython/memory/gc/test/test_hook.py
b/rpython/memory/gc/test/test_hook.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/test_hook.py
@@ -0,0 +1,125 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.memory.gc.hook import GcHooks
+from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
+
+
+class MyGcHooks(GcHooks):
+
+ def __init__(self):
+ GcHooks.__init__(self)
+ self._gc_minor_enabled = False
+ self._gc_collect_step_enabled = False
+ self._gc_collect_enabled = False
+ self.reset()
+
+ def is_gc_minor_enabled(self):
+ return self._gc_minor_enabled
+
+ def is_gc_collect_step_enabled(self):
+ return self._gc_collect_step_enabled
+
+ def is_gc_collect_enabled(self):
+ return self._gc_collect_enabled
+
+ def reset(self):
+ self.minors = []
+ self.steps = []
+ self.collects = []
+ self.durations = []
+
+ def on_gc_minor(self, duration, total_memory_used, pinned_objects):
+ self.durations.append(duration)
+ self.minors.append({
+ 'total_memory_used': total_memory_used,
+ 'pinned_objects': pinned_objects})
+
+ def on_gc_collect_step(self, duration, oldstate, newstate):
+ self.durations.append(duration)
+ self.steps.append({
+ 'oldstate': oldstate,
+ 'newstate': newstate})
+
+ def on_gc_collect(self, num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after):
+ self.collects.append({
+ 'num_major_collects': num_major_collects,
+ 'arenas_count_before': arenas_count_before,
+ 'arenas_count_after': arenas_count_after,
+ 'arenas_bytes': arenas_bytes,
+ 'rawmalloc_bytes_before': rawmalloc_bytes_before,
+ 'rawmalloc_bytes_after': rawmalloc_bytes_after})
+
+
+class TestIncMiniMarkHooks(BaseDirectGCTest):
+ from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
+
+ def get_extra_gc_params(self):
+ return {'hooks': MyGcHooks()}
+
+ def setup_method(self, m):
+ BaseDirectGCTest.setup_method(self, m)
+ size = llmemory.sizeof(S) + self.gc.gcheaderbuilder.size_gc_header
+ self.size_of_S = llmemory.raw_malloc_usage(size)
+
+ def test_on_gc_minor(self):
+ self.gc.hooks._gc_minor_enabled = True
+ self.malloc(S)
+ self.gc._minor_collection()
+ assert self.gc.hooks.minors == [
+ {'total_memory_used': 0, 'pinned_objects': 0}
+ ]
+ assert self.gc.hooks.durations[0] > 0
+ self.gc.hooks.reset()
+ #
+ # these objects survive, so the total_memory_used is > 0
+ self.stackroots.append(self.malloc(S))
+ self.stackroots.append(self.malloc(S))
+ self.gc._minor_collection()
+ assert self.gc.hooks.minors == [
+ {'total_memory_used': self.size_of_S*2, 'pinned_objects': 0}
+ ]
+
+ def test_on_gc_collect(self):
+ from rpython.memory.gc import incminimark as m
+ self.gc.hooks._gc_collect_step_enabled = True
+ self.gc.hooks._gc_collect_enabled = True
+ self.malloc(S)
+ self.gc.collect()
+ assert self.gc.hooks.steps == [
+ {'oldstate': m.STATE_SCANNING, 'newstate': m.STATE_MARKING},
+ {'oldstate': m.STATE_MARKING, 'newstate': m.STATE_SWEEPING},
+ {'oldstate': m.STATE_SWEEPING, 'newstate': m.STATE_FINALIZING},
+ {'oldstate': m.STATE_FINALIZING, 'newstate': m.STATE_SCANNING}
+ ]
+ assert self.gc.hooks.collects == [
+ {'num_major_collects': 1,
+ 'arenas_count_before': 0,
+ 'arenas_count_after': 0,
+ 'arenas_bytes': 0,
+ 'rawmalloc_bytes_after': 0,
+ 'rawmalloc_bytes_before': 0}
+ ]
+ assert len(self.gc.hooks.durations) == 4 # 4 steps
+ for d in self.gc.hooks.durations:
+ assert d > 0
+ self.gc.hooks.reset()
+ #
+ self.stackroots.append(self.malloc(S))
+ self.gc.collect()
+ assert self.gc.hooks.collects == [
+ {'num_major_collects': 2,
+ 'arenas_count_before': 1,
+ 'arenas_count_after': 1,
+ 'arenas_bytes': self.size_of_S,
+ 'rawmalloc_bytes_after': 0,
+ 'rawmalloc_bytes_before': 0}
+ ]
+
+ def test_hook_disabled(self):
+ self.gc._minor_collection()
+ self.gc.collect()
+ assert self.gc.hooks.minors == []
+ assert self.gc.hooks.steps == []
+ assert self.gc.hooks.collects == []
diff --git a/rpython/memory/gctransform/framework.py
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -116,7 +116,7 @@
class BaseFrameworkGCTransformer(GCTransformer):
root_stack_depth = None # for tests to override
- def __init__(self, translator):
+ def __init__(self, translator, gchooks=None):
from rpython.memory.gc.base import choose_gc_from_config
super(BaseFrameworkGCTransformer, self).__init__(translator,
@@ -162,7 +162,8 @@
self.finalizer_queue_indexes = {}
self.finalizer_handlers = []
- gcdata.gc = GCClass(translator.config.translation, **GC_PARAMS)
+ gcdata.gc = GCClass(translator.config.translation, hooks=gchooks,
+ **GC_PARAMS)
root_walker = self.build_root_walker()
root_walker.finished_minor_collection_func = finished_minor_collection
self.root_walker = root_walker
diff --git a/rpython/memory/test/test_transformed_gc.py
b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -14,7 +14,9 @@
from rpython.conftest import option
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rarithmetic import LONG_BIT
+from rpython.rlib.nonconst import NonConstant
from rpython.rtyper.rtyper import llinterp_backend
+from rpython.memory.gc.hook import GcHooks
WORD = LONG_BIT // 8
@@ -48,6 +50,7 @@
gcpolicy = None
GC_CAN_MOVE = False
taggedpointers = False
+ gchooks = None
def setup_class(cls):
cls.marker = lltype.malloc(rffi.CArray(lltype.Signed), 1,
@@ -112,7 +115,8 @@
fixup(t)
cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
- gcpolicy=cls.gcpolicy)
+ gcpolicy=cls.gcpolicy,
+ gchooks=cls.gchooks)
cbuild.make_entrypoint_wrapper = False
db = cbuild.build_database()
entrypointptr = cbuild.getentrypointptr()
@@ -1388,6 +1392,48 @@
assert res([]) == 0
+class GcHooksStats(object):
+ minors = 0
+ steps = 0
+ collects = 0
+
+ def reset(self):
+ # the NonConstant are needed so that the annotator annotates the
+ # fields as a generic SomeInteger(), instead of a constant 0. A call
+ # to this method MUST be seen during normal annotation, else the class
+ # is annotated only during GC transform, when it's too late
+ self.minors = NonConstant(0)
+ self.steps = NonConstant(0)
+ self.collects = NonConstant(0)
+
+
+class MyGcHooks(GcHooks):
+
+ def __init__(self, stats=None):
+ self.stats = stats or GcHooksStats()
+
+ def is_gc_minor_enabled(self):
+ return True
+
+ def is_gc_collect_step_enabled(self):
+ return True
+
+ def is_gc_collect_enabled(self):
+ return True
+
+ def on_gc_minor(self, duration, total_memory_used, pinned_objects):
+ self.stats.minors += 1
+
+ def on_gc_collect_step(self, duration, oldstate, newstate):
+ self.stats.steps += 1
+
+ def on_gc_collect(self, num_major_collects,
+ arenas_count_before, arenas_count_after,
+ arenas_bytes, rawmalloc_bytes_before,
+ rawmalloc_bytes_after):
+ self.stats.collects += 1
+
+
class TestIncrementalMiniMarkGC(TestMiniMarkGC):
gcname = "incminimark"
@@ -1405,6 +1451,8 @@
}
root_stack_depth = 200
+ gchooks = MyGcHooks()
+
def define_malloc_array_of_gcptr(self):
S = lltype.GcStruct('S', ('x', lltype.Signed))
A = lltype.GcArray(lltype.Ptr(S))
@@ -1438,6 +1486,34 @@
res = run([])
assert res
+ def define_gc_hooks(cls):
+ gchooks = cls.gchooks
+ # it is important that we fish .stats OUTSIDE f(); we cannot see
+ # gchooks from within RPython code
+ stats = gchooks.stats
+ def f():
+ stats.reset()
+ # trigger two major collections
+ llop.gc__collect(lltype.Void)
+ llop.gc__collect(lltype.Void)
+ return (10000 * stats.collects +
+ 100 * stats.steps +
+ 1 * stats.minors)
+ return f
+
+ def test_gc_hooks(self):
+ run = self.runner("gc_hooks")
+ count = run([])
+ collects, count = divmod(count, 10000)
+ steps, minors = divmod(count, 100)
+ #
+ # note: the following asserts are slightly fragile, as they assume
+ # that we do NOT run any minor collection apart the ones triggered by
+ # major_collection_step
+ assert collects == 2 # 2 collections, manually triggered
+ assert steps == 4 * collects # 4 steps for each major collection
+ assert minors == steps # one minor collection for each step
+
# ________________________________________________________________
# tagged pointers
diff --git a/rpython/rlib/rtimer.py b/rpython/rlib/rtimer.py
--- a/rpython/rlib/rtimer.py
+++ b/rpython/rlib/rtimer.py
@@ -7,6 +7,11 @@
_is_64_bit = r_uint.BITS > 32
+# unit of values returned by read_timestamp. Should be in sync with the ones
+# defined in translator/c/debug_print.h
+UNIT_TSC = 0
+UNIT_NS = 1 # nanoseconds
+UNIT_QUERY_PERFORMANCE_COUNTER = 2
def read_timestamp():
# Returns a longlong on 32-bit, and a regular int on 64-bit.
@@ -17,6 +22,11 @@
else:
return longlongmask(x)
+def get_timestamp_unit():
+ # an unit which is as arbitrary as the way we build the result of
+ # read_timestamp :)
+ return UNIT_NS
+
class ReadTimestampEntry(ExtRegistryEntry):
_about_ = read_timestamp
@@ -35,3 +45,15 @@
else:
resulttype = rffi.LONGLONG
return hop.genop("ll_read_timestamp", [], resulttype=resulttype)
+
+
+class ReadTimestampEntry(ExtRegistryEntry):
+ _about_ = get_timestamp_unit
+
+ def compute_result_annotation(self):
+ from rpython.annotator.model import SomeInteger
+ return SomeInteger(nonneg=True)
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.genop("ll_get_timestamp_unit", [], resulttype=lltype.Signed)
diff --git a/rpython/rlib/test/test_rtimer.py b/rpython/rlib/test/test_rtimer.py
--- a/rpython/rlib/test/test_rtimer.py
+++ b/rpython/rlib/test/test_rtimer.py
@@ -1,28 +1,56 @@
import time
-
-from rpython.rlib.rtimer import read_timestamp
+import platform
+from rpython.rlib import rtimer
from rpython.rtyper.test.test_llinterp import interpret
from rpython.translator.c.test.test_genc import compile
-def timer():
- t1 = read_timestamp()
- start = time.time()
- while time.time() - start < 0.1:
- # busy wait
- pass
- t2 = read_timestamp()
- return t2 - t1
+class TestTimer(object):
-def test_timer():
- diff = timer()
- # We're counting ticks, verify they look correct
- assert diff > 1000
+ @staticmethod
+ def timer():
+ t1 = rtimer.read_timestamp()
+ start = time.time()
+ while time.time() - start < 0.1:
+ # busy wait
+ pass
+ t2 = rtimer.read_timestamp()
+ return t2 - t1
-def test_annotation():
- diff = interpret(timer, [])
- assert diff > 1000
+ def test_direct(self):
+ diff = self.timer()
+ # We're counting ticks, verify they look correct
+ assert diff > 1000
-def test_compile_c():
- function = compile(timer, [])
- diff = function()
- assert diff > 1000
\ No newline at end of file
+ def test_annotation(self):
+ diff = interpret(self.timer, [])
+ assert diff > 1000
+
+ def test_compile_c(self):
+ function = compile(self.timer, [])
+ diff = function()
+ assert diff > 1000
+
+
+class TestGetUnit(object):
+
+ @staticmethod
+ def get_unit():
+ return rtimer.get_timestamp_unit()
+
+ def test_direct(self):
+ unit = self.get_unit()
+ assert unit == rtimer.UNIT_NS
+
+ def test_annotation(self):
+ unit = interpret(self.get_unit, [])
+ assert unit == rtimer.UNIT_NS
+
+ def test_compile_c(self):
+ function = compile(self.get_unit, [])
+ unit = function()
+ if platform.processor() in ('x86', 'x86_64'):
+ assert unit == rtimer.UNIT_TSC
+ else:
+ assert unit in (rtimer.UNIT_TSC,
+ rtimer.UNIT_NS,
+ rtimer.UNIT_QUERY_PERFORMANCE_COUNTER)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py
b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -445,6 +445,7 @@
'get_write_barrier_from_array_failing_case': LLOp(sideeffects=False),
'gc_get_type_info_group': LLOp(sideeffects=False),
'll_read_timestamp': LLOp(canrun=True),
+ 'll_get_timestamp_unit': LLOp(canrun=True),
# __________ GC operations __________
diff --git a/rpython/rtyper/lltypesystem/opimpl.py
b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -696,6 +696,10 @@
from rpython.rlib.rtimer import read_timestamp
return read_timestamp()
+def op_ll_get_timestamp_unit():
+ from rpython.rlib.rtimer import get_timestamp_unit
+ return get_timestamp_unit()
+
def op_debug_fatalerror(ll_msg):
from rpython.rtyper.lltypesystem import lltype, rstr
from rpython.rtyper.llinterp import LLFatalError
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -29,6 +29,7 @@
def __init__(self, translator=None, standalone=False,
gcpolicyclass=None,
+ gchooks=None,
exctransformer=None,
thread_enabled=False,
sandbox=False):
@@ -56,7 +57,7 @@
self.namespace = CNameManager()
if translator is not None:
- self.gctransformer = self.gcpolicy.gettransformer(translator)
+ self.gctransformer = self.gcpolicy.gettransformer(translator,
gchooks)
self.completed = False
self.instrument_ncounter = 0
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -94,7 +94,7 @@
class RefcountingGcPolicy(BasicGcPolicy):
- def gettransformer(self, translator):
+ def gettransformer(self, translator, gchooks):
from rpython.memory.gctransform import refcounting
return refcounting.RefcountingGCTransformer(translator)
@@ -175,7 +175,7 @@
class BoehmGcPolicy(BasicGcPolicy):
- def gettransformer(self, translator):
+ def gettransformer(self, translator, gchooks):
from rpython.memory.gctransform import boehm
return boehm.BoehmGCTransformer(translator)
@@ -302,9 +302,9 @@
class BasicFrameworkGcPolicy(BasicGcPolicy):
- def gettransformer(self, translator):
+ def gettransformer(self, translator, gchooks):
if hasattr(self, 'transformerclass'): # for rpython/memory tests
- return self.transformerclass(translator)
+ return self.transformerclass(translator, gchooks=gchooks)
raise NotImplementedError
def struct_setup(self, structdefnode, rtti):
@@ -439,9 +439,9 @@
class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy):
- def gettransformer(self, translator):
+ def gettransformer(self, translator, gchooks):
from rpython.memory.gctransform import shadowstack
- return shadowstack.ShadowStackFrameworkGCTransformer(translator)
+ return shadowstack.ShadowStackFrameworkGCTransformer(translator,
gchooks)
def enter_roots_frame(self, funcgen, (c_gcdata, c_numcolors)):
numcolors = c_numcolors.value
@@ -484,9 +484,9 @@
class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):
- def gettransformer(self, translator):
+ def gettransformer(self, translator, gchooks):
from rpython.memory.gctransform import asmgcroot
- return asmgcroot.AsmGcRootFrameworkGCTransformer(translator)
+ return asmgcroot.AsmGcRootFrameworkGCTransformer(translator, gchooks)
def GC_KEEPALIVE(self, funcgen, v):
return 'pypy_asm_keepalive(%s);' % funcgen.expr(v)
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -64,13 +64,14 @@
split = False
def __init__(self, translator, entrypoint, config, gcpolicy=None,
- secondary_entrypoints=()):
+ gchooks=None, secondary_entrypoints=()):
self.translator = translator
self.entrypoint = entrypoint
self.entrypoint_name = getattr(self.entrypoint, 'func_name', None)
self.originalentrypoint = entrypoint
self.config = config
self.gcpolicy = gcpolicy # for tests only, e.g. rpython/memory/
+ self.gchooks = gchooks
self.eci = self.get_eci()
self.secondary_entrypoints = secondary_entrypoints
@@ -91,6 +92,7 @@
exctransformer = translator.getexceptiontransformer()
db = LowLevelDatabase(translator, standalone=self.standalone,
gcpolicyclass=gcpolicyclass,
+ gchooks=self.gchooks,
exctransformer=exctransformer,
thread_enabled=self.config.translation.thread,
sandbox=self.config.translation.sandbox)
diff --git a/rpython/translator/c/src/asm_gcc_x86.h
b/rpython/translator/c/src/asm_gcc_x86.h
--- a/rpython/translator/c/src/asm_gcc_x86.h
+++ b/rpython/translator/c/src/asm_gcc_x86.h
@@ -70,6 +70,7 @@
// lfence
// I don't know how important it is, comment talks about time warps
+#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC
#ifndef PYPY_CPU_HAS_STANDARD_PRECISION
/* On x86-32, we have to use the following hacks to set and restore
diff --git a/rpython/translator/c/src/asm_gcc_x86_64.h
b/rpython/translator/c/src/asm_gcc_x86_64.h
--- a/rpython/translator/c/src/asm_gcc_x86_64.h
+++ b/rpython/translator/c/src/asm_gcc_x86_64.h
@@ -7,5 +7,6 @@
val = (_rdx << 32) | _rax; \
} while (0)
+#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC
#define RPy_YieldProcessor() asm("pause")
diff --git a/rpython/translator/c/src/asm_msvc.h
b/rpython/translator/c/src/asm_msvc.h
--- a/rpython/translator/c/src/asm_msvc.h
+++ b/rpython/translator/c/src/asm_msvc.h
@@ -13,3 +13,4 @@
#include <intrin.h>
#pragma intrinsic(__rdtsc)
#define READ_TIMESTAMP(val) do { val = (long long)__rdtsc(); } while (0)
+#define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_TSC
diff --git a/rpython/translator/c/src/debug_print.h
b/rpython/translator/c/src/debug_print.h
--- a/rpython/translator/c/src/debug_print.h
+++ b/rpython/translator/c/src/debug_print.h
@@ -51,6 +51,11 @@
RPY_EXTERN long pypy_have_debug_prints;
RPY_EXPORTED FILE *pypy_debug_file;
+/* these should be in sync with the values defined in rlib/rtimer.py */
+#define TIMESTAMP_UNIT_TSC 0
+#define TIMESTAMP_UNIT_NS 1
+#define TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER 2
+
#define OP_LL_READ_TIMESTAMP(val) READ_TIMESTAMP(val)
#include "src/asm.h"
@@ -62,11 +67,15 @@
# ifdef _WIN32
# define READ_TIMESTAMP(val) QueryPerformanceCounter((LARGE_INTEGER*)&(val))
+# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_QUERY_PERFORMANCE_COUNTER
# else
RPY_EXTERN long long pypy_read_timestamp(void);
# define READ_TIMESTAMP(val) (val) = pypy_read_timestamp()
+# define READ_TIMESTAMP_UNIT TIMESTAMP_UNIT_NS
# endif
#endif
+
+#define OP_LL_GET_TIMESTAMP_UNIT(res) res = READ_TIMESTAMP_UNIT
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -413,11 +413,13 @@
translator.frozen = True
standalone = self.standalone
+ get_gchooks = self.extra.get('get_gchooks', lambda: None)
+ gchooks = get_gchooks()
if standalone:
from rpython.translator.c.genc import CStandaloneBuilder
cbuilder = CStandaloneBuilder(self.translator, self.entry_point,
- config=self.config,
+ config=self.config, gchooks=gchooks,
secondary_entrypoints=
self.secondary_entrypoints + annotated_jit_entrypoints)
else:
@@ -426,7 +428,8 @@
cbuilder = CLibraryBuilder(self.translator, self.entry_point,
functions=functions,
name='libtesting',
- config=self.config)
+ config=self.config,
+ gchooks=gchooks)
if not standalone: # xxx more messy
cbuilder.modulename = self.extmod_name
database = cbuilder.build_database()
diff --git a/rpython/translator/goal/targetgcbench.py
b/rpython/translator/goal/targetgcbench.py
--- a/rpython/translator/goal/targetgcbench.py
+++ b/rpython/translator/goal/targetgcbench.py
@@ -1,16 +1,25 @@
from rpython.translator.goal import gcbench
+from rpython.memory.test.test_transformed_gc import MyGcHooks, GcHooksStats
# _____ Define and setup target ___
+GC_HOOKS_STATS = GcHooksStats()
+
+def entry_point(argv):
+ GC_HOOKS_STATS.reset()
+ ret = gcbench.entry_point(argv)
+ minors = GC_HOOKS_STATS.minors
+ steps = GC_HOOKS_STATS.steps
+ collects = GC_HOOKS_STATS.collects
+ print 'GC hooks statistics'
+ print ' gc-minor: ', minors
+ print ' gc-collect-step: ', steps
+ print ' gc-collect: ', collects
+ return ret
+
+def get_gchooks():
+ return MyGcHooks(GC_HOOKS_STATS)
+
def target(*args):
gcbench.ENABLE_THREADS = False # not RPython
- return gcbench.entry_point, None
-
-"""
-Why is this a stand-alone target?
-
-The above target specifies None as the argument types list.
-This is a case treated specially in the driver.py . If the list
-of input types is empty, it is meant to be a list of strings,
-actually implementing argv of the executable.
-"""
+ return entry_point, None
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
