Author: Matti Picus <matti.pi...@gmail.com> Branch: release-pypy3.5-6.x Changeset: r94395:aa0e948e1bc4 Date: 2018-04-20 14:25 +0300 http://bitbucket.org/pypy/pypy/changeset/aa0e948e1bc4/
Log: merge py3.5 into release diff too long, truncating to 2000 out of 4118 lines diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py --- a/lib-python/3/test/test_exceptions.py +++ b/lib-python/3/test/test_exceptions.py @@ -164,10 +164,10 @@ is_pypy = check_impl_detail(pypy=True) check('def fact(x):\n\treturn x!\n', 2, 10) - check('1 +\n', 1, 4 - is_pypy) - check('def spam():\n print(1)\n print(2)', 3, 0 if is_pypy else 10) - check('Python = "Python" +', 1, 20 - is_pypy) - check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20 - is_pypy) + check('1 +\n', 1, 4) + check('def spam():\n print(1)\n print(2)', 3, 2 if is_pypy else 10) + check('Python = "Python" +', 1, 20) + check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) @cpython_only def testSettingException(self): diff --git a/lib-python/3/test/test_fstring.py b/lib-python/3/test/test_fstring.py --- a/lib-python/3/test/test_fstring.py +++ b/lib-python/3/test/test_fstring.py @@ -319,7 +319,7 @@ ["f'{3)+(4}'", ]) - self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + self.assertAllRaise(SyntaxError, r'end of line \(EOL\) while scanning string literal', ["f'{\n}'", ]) @@ -741,7 +741,7 @@ self.assertEqual('{d[0]}'.format(d=d), 'integer') def test_invalid_expressions(self): - self.assertAllRaise(SyntaxError, 'invalid syntax', + self.assertAllRaise(SyntaxError, "closing parenthesis '.' does not match opening parenthesis '.'", [r"f'{a[4)}'", r"f'{a(4]}'", ]) diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -121,6 +121,166 @@ alive by GC objects, but not accounted in the GC +GC Hooks +-------- + +GC hooks are user-defined functions which are called whenever a specific GC +event occur, and can be used to monitor GC activity and pauses. You can +install the hooks by setting the following attributes: + +``gc.hook.on_gc_minor`` + Called whenever a minor collection occurs. It corresponds to + ``gc-minor`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect_step`` + Called whenever an incremental step of a major collection occurs. It + corresponds to ``gc-collect-step`` sections inside ``PYPYLOG``. + +``gc.hook.on_gc_collect`` + Called after the last incremental step, when a major collection is fully + done. It corresponds to ``gc-collect-done`` sections inside ``PYPYLOG``. + +To uninstall a hook, simply set the corresponding attribute to ``None``. To +install all hooks at once, you can call ``gc.hooks.set(obj)``, which will look +for methods ``on_gc_*`` on ``obj``. To uninstall all the hooks at once, you +can call ``gc.hooks.reset()``. + +The functions called by the hooks receive a single ``stats`` argument, which +contains various statistics about the event. + +Note that PyPy cannot call the hooks immediately after a GC event, but it has +to wait until it reaches a point in which the interpreter is in a known state +and calling user-defined code is harmless. It might happen that multiple +events occur before the hook is invoked: in this case, you can inspect the +value ``stats.count`` to know how many times the event occured since the last +time the hook was called. Similarly, ``stats.duration`` contains the +**total** time spent by the GC for this specific event since the last time the +hook was called. + +On the other hand, all the other fields of the ``stats`` object are relative +only to the **last** event of the series. + +The attributes for ``GcMinorStats`` are: + +``count`` + The number of minor collections occured since the last hook call. + +``duration`` + The total time spent inside minor collections since the last hook + call. See below for more information on the unit. + +``duration_min`` + The duration of the fastest minor collection since the last hook call. + +``duration_max`` + The duration of the slowest minor collection since the last hook call. + + ``total_memory_used`` + The amount of memory used at the end of the minor collection, in + bytes. This include the memory used in arenas (for GC-managed memory) and + raw-malloced memory (e.g., the content of numpy arrays). + +``pinned_objects`` + the number of pinned objects. + + +The attributes for ``GcCollectStepStats`` are: + +``count``, ``duration``, ``duration_min``, ``duration_max`` + See above. + +``oldstate``, ``newstate`` + Integers which indicate the state of the GC before and after the step. + +The value of ``oldstate`` and ``newstate`` is one of these constants, defined +inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``, +``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string +representation of it by indexing the ``GC_STATS`` tuple. + + +The attributes for ``GcCollectStats`` are: + +``count`` + See above. + +``num_major_collects`` + The total number of major collections which have been done since the + start. Contrarily to ``count``, this is an always-growing counter and it's + not reset between invocations. + +``arenas_count_before``, ``arenas_count_after`` + Number of arenas used before and after the major collection. + +``arenas_bytes`` + Total number of bytes used by GC-managed objects. + +``rawmalloc_bytes_before``, ``rawmalloc_bytes_after`` + Total number of bytes used by raw-malloced objects, before and after the + major collection. + +Note that ``GcCollectStats`` has **not** got a ``duration`` field. This is +because all the GC work is done inside ``gc-collect-step``: +``gc-collect-done`` is used only to give additional stats, but doesn't do any +actual work. + +A note about the ``duration`` field: depending on the architecture and +operating system, PyPy uses different ways to read timestamps, so ``duration`` +is expressed in varying units. It is possible to know which by calling +``__pypy__.debug_get_timestamp_unit()``, which can be one of the following +values: + +``tsc`` + The default on ``x86`` machines: timestamps are expressed in CPU ticks, as + read by the `Time Stamp Counter`_. + +``ns`` + Timestamps are expressed in nanoseconds. + +``QueryPerformanceCounter`` + On Windows, in case for some reason ``tsc`` is not available: timestamps + are read using the win API ``QueryPerformanceCounter()``. + + +Unfortunately, there does not seem to be a reliable standard way for +converting ``tsc`` ticks into nanoseconds, although in practice on modern CPUs +it is enough to divide the ticks by the maximum nominal frequency of the CPU. +For this reason, PyPy gives the raw value, and leaves the job of doing the +conversion to external libraries. + +Here is an example of GC hooks in use:: + + import sys + import gc + + class MyHooks(object): + done = False + + def on_gc_minor(self, stats): + print 'gc-minor: count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect_step(self, stats): + old = gc.GcCollectStepStats.GC_STATES[stats.oldstate] + new = gc.GcCollectStepStats.GC_STATES[stats.newstate] + print 'gc-collect-step: %s --> %s' % (old, new) + print ' count = %02d, duration = %d' % (stats.count, + stats.duration) + + def on_gc_collect(self, stats): + print 'gc-collect-done: count = %02d' % stats.count + self.done = True + + hooks = MyHooks() + gc.hooks.set(hooks) + + # simulate some GC activity + lst = [] + while not hooks.done: + lst = [lst, 1, 2, 3] + + +.. _`Time Stamp Counter`: https://en.wikipedia.org/wiki/Time_Stamp_Counter + .. _minimark-environment-variables: Environment variables diff --git a/pypy/doc/release-v6.0.0.rst b/pypy/doc/release-v6.0.0.rst --- a/pypy/doc/release-v6.0.0.rst +++ b/pypy/doc/release-v6.0.0.rst @@ -18,6 +18,8 @@ getting started writing code. We have improved our parser to emit more friendly `syntax errors`_, making PyPy not only faster but more friendly. +The GC now has `hooks`_ to gain more insights into its performance + The Windows PyPy3.5 release is still considered beta-quality. There are open issues with unicode handling especially around system calls and c-extensions. @@ -53,6 +55,7 @@ .. _`blog post`: https://morepypy.blogspot.it/2017/10/cape-of-good-hope-for-pypy-hello-from.html .. _pygobject: https://lazka.github.io/posts/2018-04_pypy-pygobject/index.html .. _`syntax errors`: https://morepypy.blogspot.com/2018/04/improving-syntaxerror-in-pypy.html +.. _`hooks`: gc_info.html#gc-hooks What is PyPy? ============= @@ -101,8 +104,9 @@ * Added missing attributes to C-API ``instancemethod`` on pypy3 * Store error state in thread-local storage for C-API. * Fix JIT bugs exposed in the sre module -* Improve speed of Python parser, improve ParseError messages slightly +* Improve speed of Python parser, improve ParseError messages and SyntaxError * Handle JIT hooks more efficiently +* Fix a rare GC bug exposed by intensive use of cpyext `Buffer` s We also refactored many parts of the JIT bridge optimizations, as well as cpyext internals, and together with new contributors fixed issues, added new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,6 +3,7 @@ ========================== .. this is a revision shortly after release-pypy-6.0.0 -.. startrev: f22145c34985 +.. startrev: ad79cc0ce9a8 + diff --git a/pypy/doc/whatsnew-pypy2-6.0.0.rst b/pypy/doc/whatsnew-pypy2-6.0.0.rst --- a/pypy/doc/whatsnew-pypy2-6.0.0.rst +++ b/pypy/doc/whatsnew-pypy2-6.0.0.rst @@ -113,3 +113,16 @@ Improve line offsets that are reported by SyntaxError. Improve error messages for a few situations, including mismatched parenthesis. + +.. branch: issue2752 + +Fix a rare GC bug that was introduced more than one year ago, but was +not diagnosed before issue #2752. + +.. branch: gc-hooks + +Introduce GC hooks, as documented in doc/gc_info.rst + +.. branch: gc-hook-better-timestamp + +Improve GC hooksd diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -3,5 +3,5 @@ ======================== .. this is the revision after release-pypy3.5-v6.0 -.. startrev: bf74662ee4fa +.. startrev: 580e3e26cd32 diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -223,6 +223,7 @@ usage = SUPPRESS_USAGE take_options = True + space = None def opt_parser(self, config): parser = to_optparse(config, useoptions=["objspace.*"], @@ -382,15 +383,21 @@ from pypy.module.pypyjit.hooks import pypy_hooks return PyPyJitPolicy(pypy_hooks) + def get_gchooks(self): + from pypy.module.gc.hook import LowLevelGcHooks + if self.space is None: + raise Exception("get_gchooks must be called afeter get_entry_point") + return self.space.fromcache(LowLevelGcHooks) + def get_entry_point(self, config): - space = make_objspace(config) + self.space = make_objspace(config) # manually imports app_main.py filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False - w_dict = app.getwdict(space) - entry_point, _ = create_entry_point(space, w_dict) + w_dict = app.getwdict(self.space) + entry_point, _ = create_entry_point(self.space, w_dict) return entry_point, None, PyPyAnnotatorPolicy() @@ -399,7 +406,7 @@ 'jitpolicy', 'get_entry_point', 'get_additional_config_options']: ns[name] = getattr(self, name) - + ns['get_gchooks'] = self.get_gchooks PyPyTarget().interface(globals()) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -392,7 +392,7 @@ self._periodic_actions = [] self._nonperiodic_actions = [] self.has_bytecode_counter = False - self.fired_actions = None + self._fired_actions_reset() # the default value is not 100, unlike CPython 2.7, but a much # larger value, because we use a technique that not only allows # but actually *forces* another thread to run whenever the counter @@ -404,13 +404,28 @@ """Request for the action to be run before the next opcode.""" if not action._fired: action._fired = True - if self.fired_actions is None: - self.fired_actions = [] - self.fired_actions.append(action) + self._fired_actions_append(action) # set the ticker to -1 in order to force action_dispatcher() # to run at the next possible bytecode self.reset_ticker(-1) + def _fired_actions_reset(self): + # linked list of actions. We cannot use a normal RPython list because + # we want AsyncAction.fire() to be marked as @rgc.collect: this way, + # we can call it from e.g. GcHooks or cpyext's dealloc_trigger. + self._fired_actions_first = None + self._fired_actions_last = None + + @rgc.no_collect + def _fired_actions_append(self, action): + assert action._next is None + if self._fired_actions_first is None: + self._fired_actions_first = action + self._fired_actions_last = action + else: + self._fired_actions_last._next = action + self._fired_actions_last = action + @not_rpython def register_periodic_action(self, action, use_bytecode_counter): """ @@ -455,9 +470,9 @@ action.perform(ec, frame) # nonperiodic actions - list = self.fired_actions - if list is not None: - self.fired_actions = None + action = self._fired_actions_first + if action: + self._fired_actions_reset() # NB. in case there are several actions, we reset each # 'action._fired' to false only when we're about to call # 'action.perform()'. This means that if @@ -465,9 +480,10 @@ # the corresponding perform(), the fire() has no # effect---which is the effect we want, because # perform() will be called anyway. - for action in list: + while action is not None: action._fired = False action.perform(ec, frame) + action._next, action = None, action._next self.action_dispatcher = action_dispatcher @@ -500,10 +516,12 @@ to occur between two opcodes, not at a completely random time. """ _fired = False + _next = None def __init__(self, space): self.space = space + @rgc.no_collect def fire(self): """Request for the action to be run before the next opcode. The action must have been registered at space initalization time.""" diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -36,6 +36,37 @@ pass assert i == 9 + def test_action_queue(self): + events = [] + + class Action1(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('one') + + class Action2(executioncontext.AsyncAction): + def perform(self, ec, frame): + events.append('two') + + space = self.space + a1 = Action1(space) + a2 = Action2(space) + a1.fire() + a2.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one', 'two'] + # + events[:] = [] + a1.fire() + space.appexec([], """(): + n = 5 + return n + 2 + """) + assert events == ['one'] + + def test_periodic_action(self): from pypy.interpreter.executioncontext import ActionFlag diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -83,6 +83,8 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'debug_read_timestamp' : 'interp_debug.debug_read_timestamp', + 'debug_get_timestamp_unit' : 'interp_debug.debug_get_timestamp_unit', 'builtinify' : 'interp_magic.builtinify', 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,11 +1,14 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib import debug, jit - +from rpython.rlib import rtimer @jit.dont_look_inside -@unwrap_spec(category='text') -def debug_start(space, category): - debug.debug_start(category) +@unwrap_spec(category='text', timestamp=bool) +def debug_start(space, category, timestamp=False): + res = debug.debug_start(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @jit.dont_look_inside def debug_print(space, args_w): @@ -13,10 +16,12 @@ debug.debug_print(' '.join(parts)) @jit.dont_look_inside -@unwrap_spec(category='text') -def debug_stop(space, category): - debug.debug_stop(category) - +@unwrap_spec(category='text', timestamp=bool) +def debug_stop(space, category, timestamp=False): + res = debug.debug_stop(category, timestamp=timestamp) + if timestamp: + return space.newint(res) + return space.w_None @unwrap_spec(category='text') def debug_print_once(space, category, args_w): @@ -28,3 +33,18 @@ @jit.dont_look_inside def debug_flush(space): debug.debug_flush() + +def debug_read_timestamp(space): + return space.newint(rtimer.read_timestamp()) + +def debug_get_timestamp_unit(space): + unit = rtimer.get_timestamp_unit() + if unit == rtimer.UNIT_TSC: + unit_str = 'tsc' + elif unit == rtimer.UNIT_NS: + unit_str = 'ns' + elif unit == rtimer.UNIT_QUERY_PERFORMANCE_COUNTER: + unit_str = 'QueryPerformanceCounter' + else: + unit_str = 'UNKNOWN(%d)' % unit + return space.newtext(unit_str) diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -48,3 +48,26 @@ from __pypy__ import debug_flush debug_flush() # assert did not crash + + def test_debug_read_timestamp(self): + from __pypy__ import debug_read_timestamp + a = debug_read_timestamp() + b = debug_read_timestamp() + assert b > a + + def test_debug_get_timestamp_unit(self): + from __pypy__ import debug_get_timestamp_unit + unit = debug_get_timestamp_unit() + assert unit in ('tsc', 'ns', 'QueryPerformanceCounter') + + def test_debug_start_stop_timestamp(self): + import time + from __pypy__ import debug_start, debug_stop, debug_read_timestamp + assert debug_start('foo') is None + assert debug_stop('foo') is None + ts1 = debug_start('foo', timestamp=True) + t = time.time() + while time.time() - t < 0.02: + pass + ts2 = debug_stop('foo', timestamp=True) + assert ts2 > ts1 diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -70,7 +70,15 @@ length = wchar_helper.unicode_size_as_char32(u) return (w_value, length + 1) else: - explicitlength = space.getindex_w(w_value, space.w_OverflowError) + try: + explicitlength = space.getindex_w(w_value, + space.w_OverflowError) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, + "expected new array length or list/tuple/str, " + "not %T", w_value) + raise if explicitlength < 0: raise oefmt(space.w_ValueError, "negative array length") return (space.w_None, explicitlength) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -278,6 +278,9 @@ assert repr(q).startswith("<cdata 'int *' 0x") assert p == q assert hash(p) == hash(q) + e = py.test.raises(TypeError, newp, new_array_type(BPtr, None), None) + assert str(e.value) == ( + "expected new array length or list/tuple/str, not NoneType") def test_pointer_bool(): BInt = new_primitive_type("int") @@ -359,6 +362,9 @@ assert int(c) == ord(b'A') py.test.raises(TypeError, cast, BChar, b'foo') py.test.raises(TypeError, cast, BChar, u+'foo') + e = py.test.raises(TypeError, newp, new_array_type(BPtr, None), 12.3) + assert str(e.value) == ( + "expected new array length or list/tuple/str, not float") def test_reading_pointer_to_pointer(): BVoidP = new_pointer_type(new_void_type()) diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -149,7 +149,7 @@ # No tzinfo return py_datetime = rffi.cast(PyDateTime_Time, py_obj) - w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) + w_tzinfo = space.getattr(w_obj, space.newtext('_tzinfo')) if space.is_none(w_tzinfo): py_datetime.c_hastzinfo = cts.cast('unsigned char', 0) py_datetime.c_tzinfo = lltype.nullptr(PyObject.TO) diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -187,7 +187,4 @@ self.attrib = True import gc module.subclass_with_attribute(Sub, "addattrib", "attrib", gc.collect) - if self.runappdirect: - assert Sub.__module__ == 'pypy.module.cpyext.test.test_arraymodule' - assert str(Sub) == "<class 'pypy.module.cpyext.test.test_arraymodule.Sub'>" - + assert Sub.__module__ == __name__ diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -27,11 +27,8 @@ if(PyUnicode_GetSize(s) != 11) { result = -PyUnicode_GetSize(s); } -#ifdef PYPY_VERSION - // Slightly silly test that tp_basicsize is reasonable. - if(s->ob_type->tp_basicsize != sizeof(void*)*12) + if(s->ob_type->tp_basicsize != sizeof(PyUnicodeObject)) result = s->ob_type->tp_basicsize; -#endif // PYPY_VERSION Py_DECREF(s); return PyLong_FromLong(result); """), diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py --- a/pypy/module/gc/__init__.py +++ b/pypy/module/gc/__init__.py @@ -34,5 +34,7 @@ 'get_typeids_z': 'referents.get_typeids_z', 'get_typeids_list': 'referents.get_typeids_list', 'GcRef': 'referents.W_GcRef', + 'hooks': 'space.fromcache(hook.W_AppLevelHooks)', + 'GcCollectStepStats': 'hook.W_GcCollectStepStats', }) MixedModule.__init__(self, space, w_name) diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/hook.py @@ -0,0 +1,341 @@ +from rpython.memory.gc.hook import GcHooks +from rpython.memory.gc import incminimark +from rpython.rlib.nonconst import NonConstant +from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty +from pypy.interpreter.executioncontext import AsyncAction + +class LowLevelGcHooks(GcHooks): + """ + These are the low-level hooks which are called directly from the GC. + + They can't do much, because the base class marks the methods as + @rgc.no_collect. + + This is expected to be a singleton, created by space.fromcache, and it is + integrated with the translation by targetpypystandalone.get_gchooks + """ + + def __init__(self, space): + self.space = space + self.w_hooks = space.fromcache(W_AppLevelHooks) + + def is_gc_minor_enabled(self): + return self.w_hooks.gc_minor_enabled + + def is_gc_collect_step_enabled(self): + return self.w_hooks.gc_collect_step_enabled + + def is_gc_collect_enabled(self): + return self.w_hooks.gc_collect_enabled + + def on_gc_minor(self, duration, total_memory_used, pinned_objects): + action = self.w_hooks.gc_minor + action.count += 1 + action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) + action.total_memory_used = total_memory_used + action.pinned_objects = pinned_objects + action.fire() + + def on_gc_collect_step(self, duration, oldstate, newstate): + action = self.w_hooks.gc_collect_step + action.count += 1 + action.duration += duration + action.duration_min = min(action.duration_min, duration) + action.duration_max = max(action.duration_max, duration) + action.oldstate = oldstate + action.newstate = newstate + action.fire() + + def on_gc_collect(self, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + action = self.w_hooks.gc_collect + action.count += 1 + action.num_major_collects = num_major_collects + action.arenas_count_before = arenas_count_before + action.arenas_count_after = arenas_count_after + action.arenas_bytes = arenas_bytes + action.rawmalloc_bytes_before = rawmalloc_bytes_before + action.rawmalloc_bytes_after = rawmalloc_bytes_after + action.fire() + + +class W_AppLevelHooks(W_Root): + + def __init__(self, space): + self.space = space + self.gc_minor_enabled = False + self.gc_collect_step_enabled = False + self.gc_collect_enabled = False + self.gc_minor = GcMinorHookAction(space) + self.gc_collect_step = GcCollectStepHookAction(space) + self.gc_collect = GcCollectHookAction(space) + + def descr_get_on_gc_minor(self, space): + return self.gc_minor.w_callable + + def descr_set_on_gc_minor(self, space, w_obj): + self.gc_minor_enabled = not space.is_none(w_obj) + self.gc_minor.w_callable = w_obj + self.gc_minor.fix_annotation() + + def descr_get_on_gc_collect_step(self, space): + return self.gc_collect_step.w_callable + + def descr_set_on_gc_collect_step(self, space, w_obj): + self.gc_collect_step_enabled = not space.is_none(w_obj) + self.gc_collect_step.w_callable = w_obj + self.gc_collect_step.fix_annotation() + + def descr_get_on_gc_collect(self, space): + return self.gc_collect.w_callable + + def descr_set_on_gc_collect(self, space, w_obj): + self.gc_collect_enabled = not space.is_none(w_obj) + self.gc_collect.w_callable = w_obj + self.gc_collect.fix_annotation() + + def descr_set(self, space, w_obj): + w_a = space.getattr(w_obj, space.newtext('on_gc_minor')) + w_b = space.getattr(w_obj, space.newtext('on_gc_collect_step')) + w_c = space.getattr(w_obj, space.newtext('on_gc_collect')) + self.descr_set_on_gc_minor(space, w_a) + self.descr_set_on_gc_collect_step(space, w_b) + self.descr_set_on_gc_collect(space, w_c) + + def descr_reset(self, space): + self.descr_set_on_gc_minor(space, space.w_None) + self.descr_set_on_gc_collect_step(space, space.w_None) + self.descr_set_on_gc_collect(space, space.w_None) + + +class GcMinorHookAction(AsyncAction): + total_memory_used = 0 + pinned_objects = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) + self.total_memory_used = NonConstant(r_uint(42)) + self.pinned_objects = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcMinorStats( + self.count, + self.duration, + self.duration_min, + self.duration_max, + self.total_memory_used, + self.pinned_objects) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectStepHookAction(AsyncAction): + oldstate = 0 + newstate = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + self.duration = r_longlong(0) + self.duration_min = r_longlong(longlongmax) + self.duration_max = r_longlong(0) + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.duration = NonConstant(r_longlong(-42)) + self.duration_min = NonConstant(r_longlong(-42)) + self.duration_max = NonConstant(r_longlong(-42)) + self.oldstate = NonConstant(-42) + self.newstate = NonConstant(-42) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStepStats( + self.count, + self.duration, + self.duration_min, + self.duration_max, + self.oldstate, + self.newstate) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class GcCollectHookAction(AsyncAction): + num_major_collects = 0 + arenas_count_before = 0 + arenas_count_after = 0 + arenas_bytes = 0 + rawmalloc_bytes_before = 0 + rawmalloc_bytes_after = 0 + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.w_callable = space.w_None + self.reset() + + def reset(self): + self.count = 0 + + def fix_annotation(self): + # the annotation of the class and its attributes must be completed + # BEFORE we do the gc transform; this makes sure that everything is + # annotated with the correct types + if NonConstant(False): + self.count = NonConstant(-42) + self.num_major_collects = NonConstant(-42) + self.arenas_count_before = NonConstant(-42) + self.arenas_count_after = NonConstant(-42) + self.arenas_bytes = NonConstant(r_uint(42)) + self.rawmalloc_bytes_before = NonConstant(r_uint(42)) + self.rawmalloc_bytes_after = NonConstant(r_uint(42)) + self.fire() + + def perform(self, ec, frame): + w_stats = W_GcCollectStats(self.count, + self.num_major_collects, + self.arenas_count_before, + self.arenas_count_after, + self.arenas_bytes, + self.rawmalloc_bytes_before, + self.rawmalloc_bytes_after) + self.reset() + self.space.call_function(self.w_callable, w_stats) + + +class W_GcMinorStats(W_Root): + + def __init__(self, count, duration, duration_min, duration_max, + total_memory_used, pinned_objects): + self.count = count + self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max + self.total_memory_used = total_memory_used + self.pinned_objects = pinned_objects + + +class W_GcCollectStepStats(W_Root): + + def __init__(self, count, duration, duration_min, duration_max, + oldstate, newstate): + self.count = count + self.duration = duration + self.duration_min = duration_min + self.duration_max = duration_max + self.oldstate = oldstate + self.newstate = newstate + + +class W_GcCollectStats(W_Root): + def __init__(self, count, num_major_collects, + arenas_count_before, arenas_count_after, + arenas_bytes, rawmalloc_bytes_before, + rawmalloc_bytes_after): + self.count = count + self.num_major_collects = num_major_collects + self.arenas_count_before = arenas_count_before + self.arenas_count_after = arenas_count_after + self.arenas_bytes = arenas_bytes + self.rawmalloc_bytes_before = rawmalloc_bytes_before + self.rawmalloc_bytes_after = rawmalloc_bytes_after + + +# just a shortcut to make the typedefs shorter +def wrap_many_ints(cls, names): + d = {} + for name in names: + d[name] = interp_attrproperty(name, cls=cls, wrapfn="newint") + return d + + +W_AppLevelHooks.typedef = TypeDef( + "GcHooks", + on_gc_minor = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_minor, + W_AppLevelHooks.descr_set_on_gc_minor), + + on_gc_collect_step = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect_step, + W_AppLevelHooks.descr_set_on_gc_collect_step), + + on_gc_collect = GetSetProperty( + W_AppLevelHooks.descr_get_on_gc_collect, + W_AppLevelHooks.descr_set_on_gc_collect), + + set = interp2app(W_AppLevelHooks.descr_set), + reset = interp2app(W_AppLevelHooks.descr_reset), + ) + +W_GcMinorStats.typedef = TypeDef( + "GcMinorStats", + **wrap_many_ints(W_GcMinorStats, ( + "count", + "duration", + "duration_min", + "duration_max", + "total_memory_used", + "pinned_objects")) + ) + +W_GcCollectStepStats.typedef = TypeDef( + "GcCollectStepStats", + STATE_SCANNING = incminimark.STATE_SCANNING, + STATE_MARKING = incminimark.STATE_MARKING, + STATE_SWEEPING = incminimark.STATE_SWEEPING, + STATE_FINALIZING = incminimark.STATE_FINALIZING, + GC_STATES = tuple(incminimark.GC_STATES), + **wrap_many_ints(W_GcCollectStepStats, ( + "count", + "duration", + "duration_min", + "duration_max", + "oldstate", + "newstate")) + ) + +W_GcCollectStats.typedef = TypeDef( + "GcCollectStats", + **wrap_many_ints(W_GcCollectStats, ( + "count", + "num_major_collects", + "arenas_count_before", + "arenas_count_after", + "arenas_bytes", + "rawmalloc_bytes_before", + "rawmalloc_bytes_after")) + ) diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py new file mode 100644 --- /dev/null +++ b/pypy/module/gc/test/test_hook.py @@ -0,0 +1,178 @@ +import pytest +from rpython.rlib.rarithmetic import r_uint +from pypy.module.gc.hook import LowLevelGcHooks +from pypy.interpreter.baseobjspace import ObjSpace +from pypy.interpreter.gateway import interp2app, unwrap_spec + +class AppTestGcHooks(object): + + def setup_class(cls): + if cls.runappdirect: + pytest.skip("these tests cannot work with -A") + space = cls.space + gchooks = space.fromcache(LowLevelGcHooks) + + @unwrap_spec(ObjSpace, int, r_uint, int) + def fire_gc_minor(space, duration, total_memory_used, pinned_objects): + gchooks.fire_gc_minor(duration, total_memory_used, pinned_objects) + + @unwrap_spec(ObjSpace, int, int, int) + def fire_gc_collect_step(space, duration, oldstate, newstate): + gchooks.fire_gc_collect_step(duration, oldstate, newstate) + + @unwrap_spec(ObjSpace, int, int, int, r_uint, r_uint, r_uint) + def fire_gc_collect(space, a, b, c, d, e, f): + gchooks.fire_gc_collect(a, b, c, d, e, f) + + @unwrap_spec(ObjSpace) + def fire_many(space): + gchooks.fire_gc_minor(5, 0, 0) + gchooks.fire_gc_minor(7, 0, 0) + gchooks.fire_gc_collect_step(5, 0, 0) + gchooks.fire_gc_collect_step(15, 0, 0) + gchooks.fire_gc_collect_step(22, 0, 0) + gchooks.fire_gc_collect(1, 2, 3, 4, 5, 6) + + cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor)) + cls.w_fire_gc_collect_step = space.wrap(interp2app(fire_gc_collect_step)) + cls.w_fire_gc_collect = space.wrap(interp2app(fire_gc_collect)) + cls.w_fire_many = space.wrap(interp2app(fire_many)) + + def test_default(self): + import gc + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None + + def test_on_gc_minor(self): + import gc + lst = [] + def on_gc_minor(stats): + lst.append((stats.count, + stats.duration, + stats.total_memory_used, + stats.pinned_objects)) + gc.hooks.on_gc_minor = on_gc_minor + self.fire_gc_minor(10, 20, 30) + self.fire_gc_minor(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_minor = None + self.fire_gc_minor(70, 80, 90) # won't fire because the hooks is disabled + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect_step(self): + import gc + lst = [] + def on_gc_collect_step(stats): + lst.append((stats.count, + stats.duration, + stats.oldstate, + stats.newstate)) + gc.hooks.on_gc_collect_step = on_gc_collect_step + self.fire_gc_collect_step(10, 20, 30) + self.fire_gc_collect_step(40, 50, 60) + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + # + gc.hooks.on_gc_collect_step = None + self.fire_gc_collect_step(70, 80, 90) # won't fire + assert lst == [ + (1, 10, 20, 30), + (1, 40, 50, 60), + ] + + def test_on_gc_collect(self): + import gc + lst = [] + def on_gc_collect(stats): + lst.append((stats.count, + stats.num_major_collects, + stats.arenas_count_before, + stats.arenas_count_after, + stats.arenas_bytes, + stats.rawmalloc_bytes_before, + stats.rawmalloc_bytes_after)) + gc.hooks.on_gc_collect = on_gc_collect + self.fire_gc_collect(1, 2, 3, 4, 5, 6) + self.fire_gc_collect(7, 8, 9, 10, 11, 12) + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + # + gc.hooks.on_gc_collect = None + self.fire_gc_collect(42, 42, 42, 42, 42, 42) # won't fire + assert lst == [ + (1, 1, 2, 3, 4, 5, 6), + (1, 7, 8, 9, 10, 11, 12), + ] + + def test_consts(self): + import gc + S = gc.GcCollectStepStats + assert S.STATE_SCANNING == 0 + assert S.STATE_MARKING == 1 + assert S.STATE_SWEEPING == 2 + assert S.STATE_FINALIZING == 3 + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + + def test_cumulative(self): + import gc + class MyHooks(object): + + def __init__(self): + self.minors = [] + self.steps = [] + + def on_gc_minor(self, stats): + self.minors.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) + + def on_gc_collect_step(self, stats): + self.steps.append((stats.count, stats.duration, + stats.duration_min, stats.duration_max)) + + on_gc_collect = None + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.minors == [(2, 12, 5, 7)] + assert myhooks.steps == [(3, 42, 5, 22)] + + def test_clear_queue(self): + import gc + class MyHooks(object): + + def __init__(self): + self.lst = [] + + def on_gc_minor(self, stats): + self.lst.append('minor') + + def on_gc_collect_step(self, stats): + self.lst.append('step') + + def on_gc_collect(self, stats): + self.lst.append('collect') + + myhooks = MyHooks() + gc.hooks.set(myhooks) + self.fire_many() + assert myhooks.lst == ['minor', 'step', 'collect'] + myhooks.lst[:] = [] + self.fire_gc_minor(0, 0, 0) + assert myhooks.lst == ['minor'] + gc.hooks.reset() + assert gc.hooks.on_gc_minor is None + assert gc.hooks.on_gc_collect_step is None + assert gc.hooks.on_gc_collect is None diff --git a/pypy/module/gc/test/test_ztranslation.py b/pypy/module/gc/test/test_ztranslation.py --- a/pypy/module/gc/test/test_ztranslation.py +++ b/pypy/module/gc/test/test_ztranslation.py @@ -1,4 +1,9 @@ from pypy.objspace.fake.checkmodule import checkmodule def test_checkmodule(): - checkmodule('gc') + # we need to ignore GcCollectStepStats, else checkmodule fails. I think + # this happens because W_GcCollectStepStats.__init__ is only called from + # GcCollectStepHookAction.perform() and the fake objspace doesn't know + # about those: so, perform() is never annotated and the annotator thinks + # W_GcCollectStepStats has no attributes + checkmodule('gc', ignore=['GcCollectStepStats']) diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py --- a/pypy/objspace/fake/checkmodule.py +++ b/pypy/objspace/fake/checkmodule.py @@ -4,6 +4,7 @@ def checkmodule(*modnames, **kwds): translate_startup = kwds.pop('translate_startup', True) + ignore = set(kwds.pop('ignore', ())) assert not kwds config = get_pypy_config(translating=True) space = FakeObjSpace(config) @@ -17,6 +18,8 @@ module.init(space) modules.append(module) for name in module.loaders: + if name in ignore: + continue seeobj_w.append(module._load_lazily(space, name)) if hasattr(module, 'submodules'): for cls in module.submodules.itervalues(): diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py deleted file mode 100755 --- a/pypy/tool/pytest/genreportdata.py +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env python -import py -import sys - -mydir = py.path.local(__file__).dirpath().realpath() -from pypy.tool.pytest import htmlreport -from pypy.tool.pytest import confpath - -if __name__ == '__main__': - if len(sys.argv) > 1: - testresultdir = py.path.local(sys.argv[1]) - assert testresultdir.check(dir=1) - else: - testresultdir = confpath.testresultdir - assert testresultdir.check(dir=1) - try: - resultwc = py.path.svnwc(testresultdir) - print "updating", resultwc - resultwc.update() - except (KeyboardInterrupt, RuntimeError): - raise - except Exception as e: #py.process.ExecutionFailed,e: - print >> sys.stderr, "Warning: ",e #Subversion update failed" - - print "traversing", mydir - rep = htmlreport.HtmlReport(testresultdir) - rep.parselatest() - - print "making html files" - rep.makeindex(testresultdir.join('index.html')) diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py deleted file mode 100644 --- a/pypy/tool/pytest/htmlreport.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python - -""" -the html test reporter - -""" -import sys, os, re -import pprint -import py -from pypy.tool.pytest import result -from pypy.tool.pytest.overview import ResultCache - -# -# various interesting path objects -# - -html = py.xml.html -NBSP = py.xml.raw(" ") - -class HtmlReport(object): - def __init__(self, resultdir): - self.resultcache = ResultCache(resultdir) - - def parselatest(self): - self.resultcache.parselatest() - - # - # rendering - # - - def render_latest_table(self, results): - table = html.table( - [html.th(x, align='left') - for x in ("failure", "filename", "revision", - "user", "platform", "elapsed", - "options", "last error line" - )], - ) - r = results[:] - def f(x, y): - xnum = x.isok() and 1 or (x.istimeout() and 2 or 3) - ynum = y.isok() and 1 or (y.istimeout() and 2 or 3) - res = -cmp(xnum, ynum) - if res == 0: - return cmp(x['execution-time'], y['execution-time']) - return res - r.sort(f) - for result in r: - table.append(self.render_result_row(result)) - return table - - def render_result_row(self, result): - dp = py.path.local(result['fspath']) - - options = " ".join([x for x in result.get('options', []) if x!= 'core']) - if not options: - options = NBSP - - failureratio = 100 * (1.0 - result.ratio_of_passed()) - self.data[result.testname] = failureratio - return html.tr( - html.td("%.2f%%" % failureratio, - style = "background-color: %s" % (getresultcolor(result),)), - html.td(self.render_test_references(result)), - html.td(result['pypy-revision']), - html.td(result['userhost'][:15]), - html.td(result['platform']), - html.td("%.2fs" % result['execution-time']), - html.td(options), - html.td(result.repr_short_error() or NBSP) - ) - - def getrelpath(self, p): - return p.relto(self.indexpath.dirpath()) - - def render_test_references(self, result): - dest = self.make_single_test_result(result) - #XXX: ask hg for differences between test and vendor branch - modified = result.ismodifiedtest() and " [mod]" or "" - return html.div(html.a(result.path.purebasename + modified, - href=self.getrelpath(dest)), - style="background-color: transparent") - - def make_single_test_result(self, result): - cache = self.indexpath.dirpath('.cache', result['userhost'][:15]) - cache.ensure(dir=1) - dest = cache.join(result.path.basename).new(ext='.html') - doc = ViewResult(result) - doc.writetopath(dest) - return dest - - def getcorelists(self): - def iscore(result): - return 'core' in result.get('options', []) - coretests = [] - noncoretests = [] - for name in self.resultcache.getnames(): - result = self.resultcache.getlatestrelevant(name) - if iscore(result): - coretests.append(result) - else: - noncoretests.append(result) - return coretests, noncoretests - - # generate html files - # - def makeindex(self, indexpath, detail="PyPy - latest"): - self.indexpath = indexpath - self.data = {} - doc = Document(title='pypy test results') - body = doc.body - coretests, noncoretests = self.getcorelists() - body.append(html.h2("%s compliance test results - " - "core tests" % detail)) - - body.append(self.render_test_summary('core', coretests)) - body.append(self.render_latest_table(coretests)) - body.append( - html.h2("%s compliance test results - non-core tests" % detail)) - body.append(self.render_test_summary('noncore', noncoretests)) - body.append(self.render_latest_table(noncoretests)) - doc.writetopath(indexpath) - datapath = indexpath.dirpath().join('data') - d = datapath.open('w') - print >>d, "data = ", - pprint.pprint(self.data, stream=d) - d.close() - self.data = None - - def render_test_summary(self, tag, tests): - ok = len([x for x in tests if x.isok()]) - err = len([x for x in tests if x.iserror()]) - to = len([x for x in tests if x.istimeout()]) - numtests = ok + err + to - assert numtests == len(tests) - assert numtests - - t = html.table() - sum100 = numtests / 100.0 - def row(*args): - return html.tr(*[html.td(arg) for arg in args]) - - sum_passed = sum([x.ratio_of_passed() for x in tests]) - compliancy = sum_passed/sum100 - self.data['%s-compliancy' % tag] = compliancy - t.append(row(html.b("tests compliancy"), - html.b("%.2f%%" % (compliancy,)))) - - passed = ok/sum100 - self.data['%s-passed' % tag] = passed - t.append(row("testmodules passed completely", "%.2f%%" % passed)) - failed = err/sum100 - self.data['%s-failed' % tag] = failed - t.append(row("testmodules (partially) failed", "%.2f%%" % failed)) - timedout = to/sum100 - self.data['%s-timedout' % tag] = timedout - t.append(row("testmodules timeout", "%.2f%%" % timedout)) - return t - -class Document(object): - def __init__(self, title=None): - self.body = html.body() - self.head = html.head() - self.doc = html.html(self.head, self.body) - if title is not None: - self.head.append( - html.meta(name="title", content=title)) - self.head.append( - html.link(rel="Stylesheet", type="text/css", href="/pypy/default.css")) - - def writetopath(self, p): - assert p.ext == '.html' - self.head.append( - html.meta(name="Content-Type", content="text/html;charset=UTF-8") - ) - s = self.doc.unicode().encode('utf-8') - p.write(s) - -def getresultcolor(result): - if result.isok(): - color = "#00ee00" - elif result.iserror(): - color = "#ee0000" - elif result.istimeout: - color = "#0000ee" - else: - color = "#444444" - return color - -class ViewResult(Document): - def __init__(self, result): - title = "%s testresult" % (result.path.purebasename,) - super(ViewResult, self).__init__(title=title) - color = getresultcolor(result) - self.body.append(html.h2(title, - style="background-color: %s" % color)) - self.body.append(self.render_meta_info(result)) - - for name in ('reportdiff', 'stdout', 'stderr'): - try: - text = result.getnamedtext(name) - except KeyError: - continue - self.body.append(html.h3(name)) - self.body.append(html.pre(text)) - - def render_meta_info(self, result): - t = html.table() - items = result.items() - items.sort() - for name, value in items: - if name.lower() == name: - t.append(html.tr( - html.td(name), html.td(value))) - return t - -class TestOfHtmlReportClass: - def setup_class(cls): - py.test.skip('needs move to own test file') - cls.testresultdir = confpath.testresultdir - cls.rep = rep = HtmlReport() - rep.parse_all(cls.testresultdir) - - def test_pickling(self): - # test pickling of report - tempdir = py.test.ensuretemp('reportpickle') - picklepath = tempdir.join('report.pickle') - picklepath.dump(self.rep) - x = picklepath.load() - assert len(x.results) == len(self.rep.results) - - def test_render_latest(self): - t = self.rep.render_latest_table(self.rep.results) - assert unicode(t) - -mydir = py.path.local(__file__).dirpath() - -def getpicklepath(): - return mydir.join('.htmlreport.pickle') diff --git a/pypy/tool/pytest/overview.py b/pypy/tool/pytest/overview.py deleted file mode 100644 --- a/pypy/tool/pytest/overview.py +++ /dev/null @@ -1,56 +0,0 @@ -from pypy.tool.pytest import result -import sys - -class ResultCache: - def __init__(self, resultdir): - self.resultdir = resultdir - self.name2result = {} - - def parselatest(self): - def filefilter(p): - return p.check(fnmatch='test_*.txt', file=1) - def rec(p): - return p.check(dotfile=0) - for x in self.resultdir.visit(filefilter, rec): - self.parse_one(x) - - def parse_one(self, resultpath): - try: - res = result.ResultFromMime(resultpath) - ver = res['testreport-version'] - if ver != "1.1" and ver != "1.1.1": - raise TypeError - except TypeError: # xxx - print >>sys.stderr, "could not parse %s" % resultpath - return - name = res.testname - print name - self.name2result.setdefault(name, []).append(res) - return res - - def getnames(self): - return self.name2result.keys() - - def getlatest(self, name, timeout=0, error=0, ok=0): - l = [] - resultlist = self.name2result[name] - maxrev = 0 - maxresult = None - for res in resultlist: - resrev = res['pypy-revision'] - if resrev == 'unknown': - continue - if resrev <= maxrev: - continue - if timeout or error or ok: - if not (timeout and res.istimeout() or - error and res.iserror() or - ok and res.isok()): - continue - maxrev = resrev - maxresult = res - return maxresult - - def getlatestrelevant(self, name): - # get the latest revision that did not time out. - return self.getlatest(name, error=1, ok=1) or self.getlatest(name) diff --git a/pypy/tool/pytest/result.py b/pypy/tool/pytest/result.py deleted file mode 100644 --- a/pypy/tool/pytest/result.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -import py -import re - -class Result(object): - def __init__(self, init=True): - self._headers = {} - self._blocks = {} - self._blocknames = [] - if init: - stdinit(self) - - def __setitem__(self, name, value): - self._headers[name.lower()] = value - - def __getitem__(self, name): - return self._headers[name.lower()] - - def get(self, name, default): - return self._headers.get(name, default) - - def __delitem__(self, name): - del self._headers[name.lower()] - - def items(self): - return self._headers.items() - - def addnamedtext(self, name, text): - assert isinstance(text, basestring) - assert isinstance(name, str) - self._blocknames.append(name) - self._blocks[name] = text - - def getnamedtext(self, name): - return self._blocks[name] - - def repr_short_error(self): - if not self.isok(): - if 'reportdiff' in self._blocks: - return "output comparison failed, see reportdiff" - else: - text = self.getnamedtext('stderr') - lines = text.strip().split('\n') - if lines: - return lines[-1] - - def repr_mimemessage(self): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - - outer = MIMEMultipart() - items = self._headers.items() - items.sort() - reprs = {} - for name, value in items: - assert ':' not in name - chars = map(ord, name) - assert min(chars) >= 33 and max(chars) <= 126 - outer[name] = str(value) - if not isinstance(value, str): - typename = type(value).__name__ - assert typename in vars(py.std.__builtin__) - reprs[name] = typename - - outer['_reprs'] = repr(reprs) - - for name in self._blocknames: - text = self._blocks[name] - m = MIMEText(text) - m.add_header('Content-Disposition', 'attachment', filename=name) - outer.attach(m) - return outer - - def grep_nr(self,text,section='stdout'): - stdout = self._blocks[section] - find = re.search('%s(?P<nr>\d+)'%text,stdout) - if find: - return float(find.group('nr')) - return 0. - - def ratio_of_passed(self): - if self.isok(): - return 1. - elif self.istimeout(): - return 0. - else: - nr = self.grep_nr('Ran ') - if nr > 0: - return (nr - (self.grep_nr('errors=') + self.grep_nr('failures=')))/nr - else: - passed = self.grep_nr('TestFailed: ',section='stderr') - run = self.grep_nr('TestFailed: \d+/',section='stderr') - if run > 0: - return passed/run - else: - run = self.grep_nr('TestFailed: \d+ of ',section='stderr') - if run > 0 : - return (run-passed)/run - else: - return 0.0 - - def isok(self): - return self['outcome'].lower() == 'ok' - - def iserror(self): - return self['outcome'].lower()[:3] == 'err' or self['outcome'].lower() == 'fail' - - def istimeout(self): - return self['outcome'].lower() == 't/o' - -# XXX backward compatibility -def sanitize(msg, path): - if 'exit-status' in msg.keys(): - return msg - f = open(str(path), 'r') - msg = f.read() - f.close() - for broken in ('exit status', 'cpu model', 'cpu mhz'): - valid = broken.replace(' ','-') - invalid = msg.find(broken+':') - msg = (msg[:invalid] + valid + - msg[invalid+len(valid):]) - from email import message_from_string - msg = message_from_string(msg) - return msg - -def sanitize_reprs(reprs): - if 'exit status' in reprs: - reprs['exit-status'] = reprs.pop('exit status') - -class ResultFromMime(Result): - def __init__(self, path): - super(ResultFromMime, self).__init__(init=False) - f = open(str(path), 'r') - from email import message_from_file - msg = message_from_file(f) - f.close() - msg = sanitize(msg, path) - # XXX security wise evil (keep in mind once we accept reporsts - # from anonymous - #print msg['_reprs'] - self._reprs = eval(msg['_reprs']) - del msg['_reprs'] - sanitize_reprs(self._reprs) - for name, value in msg.items(): - if name in self._reprs: - value = eval(value) # XXX security - self._headers[name] = value - self.fspath = self['fspath'] - if self['platform'] == 'win32' and '\\' in self.fspath: - self.testname = self.fspath.split('\\')[-1] - else: - self.testname = self.fspath.split('/')[-1] - #if sys.platform != 'win32' and '\\' in self.fspath: - # self.fspath = py.path.local(self['fspath'].replace('\\' - self.path = path - - payload = msg.get_payload() - if payload: - for submsg in payload: - assert submsg.get_content_type() == 'text/plain' - fn = submsg.get_filename() - assert fn - # XXX we need to deal better with encodings to - # begin with - content = submsg.get_payload() - for candidate in 'utf8', 'latin1': - try: - text = unicode(content, candidate) - except UnicodeDecodeError: - continue - else: - unicode(content, candidate) - self.addnamedtext(fn, text) - - def ismodifiedtest(self): - # XXX we need proper cross-platform paths! - return 'modified' in self.fspath - - def __repr__(self): - return '<%s (%s) %r rev=%s>' %(self.__class__.__name__, - self['outcome'], - self.fspath, - self['pypy-revision']) - -def stdinit(result): - import getpass - import socket - try: - username = getpass.getuser() - except: - username = 'unknown' - userhost = '%s@%s' % (username, socket.gethostname()) - result['testreport-version'] = "1.1.1" - result['userhost'] = userhost - result['platform'] = sys.platform - result['python-version-info'] = sys.version_info - info = try_getcpuinfo() - if info is not None: - result['cpu-model'] = info.get('model name', "unknown") - result['cpu-mhz'] = info.get('cpu mhz', 'unknown') -# -# -# -def try_getcpuinfo(): - if sys.platform.startswith('linux'): - cpuinfopath = py.path.local('/proc/cpuinfo') - if cpuinfopath.check(file=1): - d = {} - for line in cpuinfopath.readlines(): - if line.strip(): - name, value = line.split(':', 1) - name = name.strip().lower() - d[name] = value.strip() - return d diff --git a/pypy/tool/pytest/test/data/test___all__.txt b/pypy/tool/pytest/test/data/test___all__.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test___all__.txt +++ /dev/null @@ -1,94 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============0790678169==" -MIME-Version: 1.0 -execution-time: 1445.14346004 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py -options: ['core', '_sre'] -outcome: T/O -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 17 23:51:59 2005 -testreport-version: 1.1 -timeout: 1369.0 -userhost: anderslehmann@anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_all (__main__.AllTest) ... ERROR - -====================================================================== -ERROR: test_all (__main__.AllTest) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 163, in test_all - self.check_all("tty") - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 26, in check_all - "%s has no __all__ attribute" % modname) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 208, in verify - raise TestFailed(reason) -TestFailed: tty has no __all__ attribute - ----------------------------------------------------------------------- - ---===============0790678169== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking <type 'module'> -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking <type 'file'> -faking <type 'posix.stat_result'> -faking <type 'posix.statvfs_result'> -fake-wrapping interp file <open file '<stdout>', mode 'w' at 0x12068> -fake-wrapping interp file <open file '<stderr>', mode 'w' at 0x120b0> -fake-wrapping interp file <open file '<stdin>', mode 'r' at 0x12020> -faking <type '_socket.socket'> -faking <type 'classobj'> -faking <type 'PyCObject'> -faking <type 'time.struct_time'> -==========================timedout========================== -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 192 in <module> - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test___all__.py", line 189 in test_main - test_support.run_unittest(AllTest) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 262 in run_suite - result = runner.run(suite) -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/atexit.py", line 29 in _run_exitfuncs - print >> sys.stderr, "Error in atexit._run_exitfuncs:" -KeyboardInterrupt -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/pypy/tool/alarm.py", line 43, in ? - execfile(_main_with_alarm(finished)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 206, in ? - sys.exit(main_(sys.argv)) - File "/Users/anderslehmann/pypy/pypy/bin/py.py", line 115, in main_ - if not main.run_toplevel(space, doit, verbose=Options.verbose): - File "/Users/anderslehmann/pypy/pypy/interpreter/main.py", line 150, in run_toplevel - operationerr.print_application_traceback(space) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 83, in print_application_traceback - self.print_app_tb_only(file) - File "/Users/anderslehmann/pypy/pypy/interpreter/error.py", line 104, in print_app_tb_only - l = linecache.getline(fname, lineno) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 14, in getline - lines = getlines(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 40, in getlines - return updatecache(filename) - File "/Library/Frameworks/Python.framework/Versions/2.4/lib/python2.4/linecache.py", line 101, in updatecache - lines = fp.readlines() -KeyboardInterrupt - ---===============0790678169==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_compile.txt b/pypy/tool/pytest/test/data/test_compile.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_compile.txt +++ /dev/null @@ -1,111 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============2137793924==" -MIME-Version: 1.0 -execution-time: 34.8464071751 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py -options: ['core', '_sre'] -outcome: ERR -platform: darwin -pypy-revision: 16114 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Thu Aug 18 03:08:18 2005 -testreport-version: 1.1 -timeout: 1521.0 -userhost: anderslehmann@anders-lehmanns-15-powerbook-g4.local -_reprs: {'execution-time': 'float', 'python-version-info': 'tuple', - 'options': 'list', 'timeout': 'float', 'pypy-revision': 'int', - 'exit status': 'int'} - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stdout" - -test_argument_handling (__main__.TestSpecifics) ... FAIL -test_argument_order (__main__.TestSpecifics) ... FAIL -test_complex_args (__main__.TestSpecifics) ... ok -test_debug_assignment (__main__.TestSpecifics) ... FAIL -test_duplicate_global_local (__main__.TestSpecifics) ... ok -test_exec_with_general_mapping_for_locals (__main__.TestSpecifics) ... ok -test_float_literals (__main__.TestSpecifics) ... ok -test_for_distinct_code_objects (__main__.TestSpecifics) ... ok -test_import (__main__.TestSpecifics) ... FAIL -test_indentation (__main__.TestSpecifics) ... ok -test_literals_with_leading_zeroes (__main__.TestSpecifics) ... ok -test_none_assignment (__main__.TestSpecifics) ... FAIL -test_sequence_unpacking_error (__main__.TestSpecifics) ... ok -test_syntax_error (__main__.TestSpecifics) ... ok -test_unary_minus (__main__.TestSpecifics) ... ok - -====================================================================== -FAIL: test_argument_handling (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 18, in test_argument_handling - self.assertRaises(SyntaxError, eval, 'lambda a,a:0') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_argument_order (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 127, in test_argument_order - self.fail("non-default args after default") -AssertionError: non-default args after default - -====================================================================== -FAIL: test_debug_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 10, in test_debug_assignment - self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_import (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 253, in test_import - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') -AssertionError: SyntaxError not raised - -====================================================================== -FAIL: test_none_assignment (__main__.TestSpecifics) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 211, in test_none_assignment - self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') -AssertionError: SyntaxError not raised - ----------------------------------------------------------------------- -Ran 15 tests in 14.363s - -FAILED (failures=5) - ---===============2137793924== -Content-Type: text/plain; charset="us-ascii" -MIME-Version: 1.0 -Content-Transfer-Encoding: 7bit -Content-Disposition: attachment; filename="stderr" - -faking <type 'module'> -Loading grammar /Users/anderslehmann/pypy/pypy/interpreter/pyparser/data/Grammar2.4 -faking <type 'file'> -faking <type 'posix.stat_result'> -faking <type 'posix.statvfs_result'> -fake-wrapping interp file <open file '<stdout>', mode 'w' at 0x12068> -fake-wrapping interp file <open file '<stderr>', mode 'w' at 0x120b0> -fake-wrapping interp file <open file '<stdin>', mode 'r' at 0x12020> -Traceback (application-level): - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 268 in <module> - test_main() - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_compile.py", line 265 in test_main - test_support.run_unittest(TestSpecifics) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 290 in run_unittest - run_suite(suite, testclass) - File "/Users/anderslehmann/pypy/lib-python/2.4.1/test/test_support.py", line 274 in run_suite - raise TestFailed(msg) -TestFailed: errors occurred in __main__.TestSpecifics - ---===============2137793924==-- \ No newline at end of file diff --git a/pypy/tool/pytest/test/data/test_descr.txt b/pypy/tool/pytest/test/data/test_descr.txt deleted file mode 100644 --- a/pypy/tool/pytest/test/data/test_descr.txt +++ /dev/null @@ -1,233 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1265023865==" -MIME-Version: 1.0 -execution-time: 4098.8407588 -exit status: 1 -fspath: /Users/anderslehmann/pypy/lib-python/modified-2.4.1/test/test_descr.py -options: ['oldstyle', 'core'] -outcome: ERR -platform: darwin -pypy-revision: 16388 -python-version-info: (2, 4, 1, 'final', 0) -startdate: Wed Aug 24 16:54:12 2005 _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit