Author: Ronan Lamy <ronan.l...@gmail.com> Branch: py3tests Changeset: r94340:8cfa4b755c8e Date: 2018-04-14 20:12 +0100 http://bitbucket.org/pypy/pypy/changeset/8cfa4b755c8e/
Log: hg merge apptest-file diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -52,3 +52,13 @@ .. branch: refactor-slots Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures diff --git a/pypy/interpreter/test/apptest_pyframe.py b/pypy/interpreter/test/apptest_pyframe.py --- a/pypy/interpreter/test/apptest_pyframe.py +++ b/pypy/interpreter/test/apptest_pyframe.py @@ -13,7 +13,7 @@ import sys f = sys._getframe() assert f.f_globals is globals() - raises(AttributeError, "f.f_globals = globals()") + pytest.raises(TypeError, "f.f_globals = globals()") def test_f_builtins(): import sys, builtins diff --git a/pypy/interpreter/test/fixtures.py b/pypy/interpreter/test/fixtures.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/fixtures.py @@ -0,0 +1,5 @@ +from _pytest.tmpdir import TempdirFactory + +def tempfile(space, config): + tmpdir = TempdirFactory(config).getbasetemp() + return space.newtext(str(tmpdir / 'tempfile1')) diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,4 +1,3 @@ -from rpython.tool import udir from pypy.conftest import option from pypy.interpreter.gateway import interp2app diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -82,10 +82,10 @@ def PyTraceBack_Here(space, w_frame): from pypy.interpreter.pytraceback import record_application_traceback state = space.fromcache(State) - if state.operror is None: + if state.get_exception() is None: return -1 frame = space.interp_w(PyFrame, w_frame) - record_application_traceback(space, state.operror, frame, 0) + record_application_traceback(space, state.get_exception(), frame, 0) return 0 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -67,9 +67,10 @@ @cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) - if state.operror is None: + operror = state.get_exception() + if operror is None: return None - return state.operror.w_type # borrowed ref + return operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -2,11 +2,18 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter import executioncontext +from pypy.interpreter.executioncontext import ExecutionContext from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rdynload import DLLHANDLE from rpython.rlib import rawrefcount import sys + +# Keep track of exceptions raised in cpyext for a particular execution +# context. +ExecutionContext.cpyext_operror = None + + class State: def __init__(self, space): self.space = space @@ -18,7 +25,8 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.operror = None + ec = self.space.getexecutioncontext() + ec.cpyext_operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track @@ -37,17 +45,24 @@ def set_exception(self, operror): self.clear_exception() - self.operror = operror + ec = self.space.getexecutioncontext() + ec.cpyext_operror = operror def clear_exception(self): """Clear the current exception state, and return the operror.""" - operror = self.operror - self.operror = None + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror + ec.cpyext_operror = None return operror + def get_exception(self): + ec = self.space.getexecutioncontext() + return ec.cpyext_operror + @specialize.arg(1) def check_and_raise_exception(self, always=False): - operror = self.operror + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror if operror: self.clear_exception() raise operror diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -39,7 +39,7 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - operror = state.operror + operror = state.get_exception() if not operror: raise Exception("DID NOT RAISE") if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py --- a/pypy/module/cpyext/test/test_pyerrors.py +++ b/pypy/module/cpyext/test/test_pyerrors.py @@ -43,7 +43,8 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.operror.get_w_value(space), + operror = state.get_exception() + assert space.eq_w(operror.get_w_value(space), space.wrap("a value")) api.PyErr_Clear() @@ -51,12 +52,14 @@ def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.operror.w_type, space.w_KeyError) - assert space.eq_w(state.operror.get_w_value(space), space.w_None) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_KeyError) + assert space.eq_w(operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.operror.w_type, space.w_MemoryError) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_Warning(self, space, api, capfd): @@ -475,3 +478,59 @@ '''), ]) raises(SystemError, module.oops) + + def test_error_thread_race(self): + # Check race condition: thread 0 returns from cpyext with error set, + # after thread 1 has set an error but before it returns. + module = self.import_extension('foo', [ + ("emit_error", "METH_VARARGS", + ''' + PyThreadState *save = NULL; + PyGILState_STATE gilsave; + + /* NB. synchronization due to GIL */ + static volatile int flag = 0; + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + return NULL; + + /* Proceed in thread 1 first */ + save = PyEval_SaveThread(); + while (id == 0 && flag == 0); + gilsave = PyGILState_Ensure(); + + PyErr_Format(PyExc_ValueError, "%d", id); + + /* Proceed in thread 0 first */ + if (id == 1) flag = 1; + PyGILState_Release(gilsave); + while (id == 1 && flag == 1); + PyEval_RestoreThread(save); + + if (id == 0) flag = 0; + return NULL; + ''' + ), + ]) + + import threading + + failures = [] + + def worker(arg): + try: + module.emit_error(arg) + failures.append(True) + except Exception as exc: + if str(exc) != str(arg): + failures.append(exc) + + threads = [threading.Thread(target=worker, args=(j,)) + for j in (0, 1)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not failures diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1403,6 +1403,7 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + @py.test.mark.skipif("sys.platform != 'win32'") def test_rename(self): os = self.posix fname = self.path2 + 'rename.txt' diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,6 +1,7 @@ +import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -40,9 +41,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 -@example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -15,6 +15,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.objectmodel import ( import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.rlib.rarithmetic import ovfcheck from rpython.rlib import longlong2float from rpython.tool.sourcetools import func_with_new_name @@ -848,7 +849,12 @@ """Extend w_list from a generic iterable""" length_hint = self.space.length_hint(w_iterable, 0) if length_hint: - w_list._resize_hint(w_list.length() + length_hint) + try: + newsize_hint = ovfcheck(w_list.length() + length_hint) + except OverflowError: + pass + else: + w_list._resize_hint(newsize_hint) extended = _do_extend_from_iterable(self.space, w_list, w_iterable) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -618,6 +618,18 @@ assert l == [1.2, 2.3, 3.4, 4.5] assert l is l0 + def test_extend_iterable_length_hint_overflow(self): + import sys + class CustomIterable(object): + def __iter__(self): + if False: + yield + def __length_hint__(self): + return sys.maxsize + a = [1, 2, 3, 4] + a.extend(CustomIterable()) + assert a == [1, 2, 3, 4] + def test_sort(self): l = l0 = [1, 5, 3, 0] l.sort() diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -198,7 +198,7 @@ w_BuiltinAssertionError = space.getitem(space.builtin.w_dict, space.wrap('AssertionError')) w_metaclass = space.type(w_BuiltinAssertionError) - w_init = space.wrap(gateway.interp2app_temp(my_init)) + w_init = space.wrap(gateway.interp2app(my_init)) w_dict = space.getattr(w_BuiltinAssertionError, space.wrap('__dict__')) w_dict = space.call_method(w_dict, 'copy') # fixup __module__, since the new type will be is_heaptype() == True diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -23,20 +23,21 @@ os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) w_source = space.newtext(source) fname = str(self.fspath) + w_name = space.newtext(str(self.fspath.purebasename)) w_fname = space.newtext(fname) if self.rewrite_asserts: - w_mod = space.appexec([w_rootdir, w_source, w_fname], - """(rootdir, source, fname): + w_mod = space.appexec([w_rootdir, w_source, w_fname, w_name], + """(rootdir, source, fname, name): import sys sys.path.insert(0, rootdir) from ast_rewrite import rewrite_asserts, create_module co = rewrite_asserts(source, fname) - mod = create_module(fname, co) + mod = create_module(name, co) return mod """) else: - w_mod = create_module(space, w_fname, fname, source) + w_mod = create_module(space, w_name, fname, source) mod_dict = w_mod.getdict(space).unwrap(space) items = [] for name, w_obj in mod_dict.items(): diff --git a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py --- a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py +++ b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py @@ -3,14 +3,21 @@ import ast import itertools import marshal -import struct import sys -from ast_util import assertrepr_compare, format_explanation as _format_explanation +from ast_util import callbinrepr, format_explanation as _format_explanation # pytest caches rewritten pycs in __pycache__. -PYTEST_TAG = sys.implementation.cache_tag + "-PYTEST" +if hasattr(sys, "pypy_version_info"): + impl = "pypy" +elif sys.platform == "java": + impl = "jython" +else: + impl = "cpython" +ver = sys.version_info +PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1]) +del ver, impl PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT @@ -29,6 +36,7 @@ # import. However, there's little reason deviate, and I hope # sometime to be able to use imp.load_compiled to load them. (See # the comment in load_module above.) + import struct try: fp = open(pyc, "wb") except IOError: @@ -91,6 +99,7 @@ Return rewritten code if successful or None if not. """ + import struct try: fp = open(pyc, "rb") except IOError: @@ -161,7 +170,7 @@ done = True if done: break - custom = assertrepr_compare(ops[i], each_obj[i], each_obj[i + 1]) + custom = callbinrepr(ops[i], each_obj[i], each_obj[i + 1]) if custom is not None: return custom return expl diff --git a/pypy/tool/pytest/ast-rewriter/ast_util.py b/pypy/tool/pytest/ast-rewriter/ast_util.py --- a/pypy/tool/pytest/ast-rewriter/ast_util.py +++ b/pypy/tool/pytest/ast-rewriter/ast_util.py @@ -90,6 +90,13 @@ s = s[:maxsize] return s +def callbinrepr(op, left, right): + new_expl = assertrepr_compare(op, left, right) + new_expl = [line.replace("\n", "\\n") for line in new_expl] + res = "\n~".join(new_expl) + res = res.replace("%", "%%") + return res + def assertrepr_compare(op, left, right, verbose=False): """Return specialised explanations for some operators/operands""" diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,15 +96,14 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: - raise AssertionError("annotationoftype(%r)" % (t,)) + raise TypeError("Annotation of type %r not supported" % (t,)) class Sig(object): diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -259,6 +259,26 @@ intmask(). +Type Enforcing and Checking +--------------------------- + +RPython provides a helper decorator to force RPython-level types on function +arguments. The decorator, called ``enforceargs()``, accepts as parameters the +types expected to match the arguments of the function. + +Functions decorated with ``enforceargs()`` have their function signature +analyzed and their RPython-level type inferred at import time (for further +details about the flavor of translation performed in RPython, see the +`Annotation pass documentation`_). Encountering types not supported by RPython +will raise a ``TypeError``. + +``enforceargs()`` by default also performs type checking of parameter types +each time the function is invoked. To disable this behavior, it's possible to +pass the ``typecheck=False`` parameter to the decorator. + +.. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator + + Exception rules --------------- diff --git a/rpython/doc/translation.rst b/rpython/doc/translation.rst --- a/rpython/doc/translation.rst +++ b/rpython/doc/translation.rst @@ -48,7 +48,7 @@ be present in memory as a form that is "static enough" in the sense of :doc:`RPython <rpython>`. -2. The Annotator_ performs a global analysis starting from an specified +2. The Annotator_ performs a global analysis starting from a specified entry point to deduce type and other information about what each variable can contain at run-time, :ref:`building flow graphs <flow-graphs>` as it encounters them. diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # <length> # (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # <length> # (<box1> <index> <descr> <box2>) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# <length> +# (<const> <box2>) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -27,6 +30,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +65,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +105,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) dct = {box: cls for box, known_class in boxes_known_classes @@ -321,3 +326,74 @@ self.check_trace_count(3) self.check_resops(guard_value=1) self.check_resops(getarrayitem_gc_i=5) + + def test_bridge_call_loopinvariant(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + a = get() + a = get() + res += a.x + if y > n: + res += 1 + res += get().x + a.x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) + + @pytest.mark.xfail() + def test_bridge_call_loopinvariant_2(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + if y > n: + res += get().x + res += 1 + res += get().x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -40,7 +40,7 @@ class FakeOptimizer(object): metainterp_sd = None - optheap = None + optheap = optrewrite = None def __init__(self, trace=None): self.trace = trace diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1592,8 +1592,7 @@ index = self.get_finalizer_queue_index(hop) c_index = rmodel.inputconst(lltype.Signed, index) v_ptr = hop.spaceop.args[1] - v_ptr = hop.genop("cast_opaque_ptr", [v_ptr], - resulttype=llmemory.GCREF) + assert v_ptr.concretetype == llmemory.GCREF hop.genop("direct_call", [self.register_finalizer_ptr, self.c_const_gc, c_index, v_ptr]) diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -235,11 +235,11 @@ obj = deque.popleft() else: obj = llmemory.NULL - return llmemory.cast_adr_to_ptr(obj, rclass.OBJECTPTR) + return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) def gc_fq_register(self, fq_tag, ptr): index = self.get_finalizer_queue_index(fq_tag) - ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) + assert lltype.typeOf(ptr) == llmemory.GCREF self.gc.register_finalizer(index, ptr) # ____________________________________________________________ diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -120,7 +120,7 @@ """ Decorate a function with forcing of RPython-level types on arguments. None means no enforcing. - When not translated, the type of the actual arguments are checked against + When not translated, the type of the actual arguments is checked against the enforced types every time the function is called. You can disable the typechecking by passing ``typecheck=False`` to @enforceargs. """ @@ -147,8 +147,7 @@ # they are already homogeneous, so we only check the first # item. The case of empty list/dict is handled inside typecheck() if isinstance(arg, list): - item = arg[0] - return [get_type_descr_of_argument(item)] + return [get_type_descr_of_argument(arg[0])] elif isinstance(arg, dict): key, value = next(arg.iteritems()) return {get_type_descr_of_argument(key): get_type_descr_of_argument(value)} diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -378,13 +378,12 @@ class FinalizerQueue(object): """A finalizer queue. See pypy/doc/discussion/finalizer-order.rst. - Note: only works with the framework GCs (like minimark). It is - ignored with Boehm or with refcounting (used by tests). """ # Must be subclassed, and the subclass needs these attributes: # # Class: # the class (or base class) of finalized objects + # --or-- None to handle low-level GCREFs directly # # def finalizer_trigger(self): # called to notify that new items have been put in the queue @@ -397,11 +396,13 @@ def next_dead(self): if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance + from rpython.rtyper.lltypesystem.llmemory import GCREF + from rpython.rtyper.annlowlevel import cast_gcref_to_instance tag = FinalizerQueue._get_tag(self) - ptr = llop.gc_fq_next_dead(OBJECTPTR, tag) - return cast_base_ptr_to_instance(self.Class, ptr) + ptr = llop.gc_fq_next_dead(GCREF, tag) + if self.Class is not None: + ptr = cast_gcref_to_instance(self.Class, ptr) + return ptr try: return self._queue.popleft() except (AttributeError, IndexError): @@ -410,14 +411,18 @@ @specialize.arg(0) @jit.dont_look_inside def register_finalizer(self, obj): - assert isinstance(obj, self.Class) + from rpython.rtyper.lltypesystem.llmemory import GCREF + if self.Class is None: + assert lltype.typeOf(obj) == GCREF + else: + assert isinstance(obj, self.Class) if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr + from rpython.rtyper.annlowlevel import cast_instance_to_gcref tag = FinalizerQueue._get_tag(self) - ptr = cast_instance_to_base_ptr(obj) - llop.gc_fq_register(lltype.Void, tag, ptr) + if self.Class is not None: + obj = cast_instance_to_gcref(obj) + llop.gc_fq_register(lltype.Void, tag, obj) return else: self._untranslated_register_finalizer(obj) diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py --- a/rpython/rlib/rstruct/nativefmttable.py +++ b/rpython/rlib/rstruct/nativefmttable.py @@ -130,6 +130,13 @@ sizeof_double = native_fmttable['d']['size'] sizeof_float = native_fmttable['f']['size'] +# Copy CPython's behavior of using short's size and alignment for half-floats. +native_fmttable['e'] = {'size': native_fmttable['h']['size'], + 'alignment': native_fmttable['h']['alignment'], + 'pack': std.pack_halffloat, + 'unpack': std.unpack_halffloat, + } + # ____________________________________________________________ # # A PyPy extension: accepts the 'u' format character in native mode, diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py --- a/rpython/rlib/rstruct/test/test_pack.py +++ b/rpython/rlib/rstruct/test/test_pack.py @@ -139,9 +139,6 @@ self.check('d', 123.456789) def test_pack_halffloat(self): - if self.fmttable is nativefmttable.native_fmttable: - # Host Python cannot handle half floats. - return size = 2 wbuf = MutableStringBuffer(size) self.mypack_into('e', wbuf, 6.5e+04) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -599,3 +599,94 @@ e = py.test.raises(TyperError, gengraph, f, []) assert str(e.value).startswith('the RPython-level __del__() method in') + + def test_translated_boehm(self): + self._test_translated(use_gc="boehm", llcase=False) + + def test_translated_boehm_ll(self): + self._test_translated(use_gc="boehm", llcase=True) + + def test_translated_incminimark(self): + self._test_translated(use_gc="incminimark", llcase=False) + + def test_translated_incminimark_ll(self): + self._test_translated(use_gc="incminimark", llcase=True) + + def _test_translated(self, use_gc, llcase): + import subprocess + from rpython.rlib import objectmodel + from rpython.translator.interactive import Translation + # + class Seen: + count = 0 + class MySimpleFQ(rgc.FinalizerQueue): + if not llcase: + Class = T_Root + else: + Class = None + def finalizer_trigger(self): + seen.count += 1 + seen = Seen() + fq = MySimpleFQ() + if not llcase: + EMPTY = None + llbuilder = T_Int + else: + from rpython.rtyper.annlowlevel import llstr + EMPTY = lltype.nullptr(llmemory.GCREF.TO) + def llbuilder(n): + return lltype.cast_opaque_ptr(llmemory.GCREF, llstr(str(n))) + + def subfunc(): + w0 = llbuilder(40); fq.register_finalizer(w0) + w1 = llbuilder(41); fq.register_finalizer(w1) + w2 = llbuilder(42); fq.register_finalizer(w2) + w3 = llbuilder(43); fq.register_finalizer(w3) + w4 = llbuilder(44); fq.register_finalizer(w4) + w5 = llbuilder(45); fq.register_finalizer(w5) + w6 = llbuilder(46); fq.register_finalizer(w6) + w7 = llbuilder(47); fq.register_finalizer(w7) + w8 = llbuilder(48); fq.register_finalizer(w8) + w9 = llbuilder(49); fq.register_finalizer(w9) + gc.collect() + assert seen.count == 0 + assert fq.next_dead() is EMPTY + objectmodel.keepalive_until_here(w0) + objectmodel.keepalive_until_here(w1) + objectmodel.keepalive_until_here(w2) + objectmodel.keepalive_until_here(w3) + objectmodel.keepalive_until_here(w4) + objectmodel.keepalive_until_here(w5) + objectmodel.keepalive_until_here(w6) + objectmodel.keepalive_until_here(w7) + objectmodel.keepalive_until_here(w8) + objectmodel.keepalive_until_here(w9) + + def main(argv): + assert fq.next_dead() is EMPTY + subfunc() + gc.collect(); gc.collect(); gc.collect() + assert seen.count > 0 + n = fq.next_dead() + while True: + if not llcase: + assert type(n) is T_Int and 40 <= n.x <= 49 + else: + from rpython.rtyper.lltypesystem.rstr import STR + assert lltype.typeOf(n) is llmemory.GCREF + p = lltype.cast_opaque_ptr(lltype.Ptr(STR), n) + assert len(p.chars) == 2 + assert p.chars[0] == "4" + assert "0" <= p.chars[1] <= "9" + n = fq.next_dead() + if n is EMPTY: + break + print "OK!" + return 0 + # + t = Translation(main, gc=use_gc) + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit