Author: Richard Plangger <planri...@gmail.com> Branch: ppc-vsx-support Changeset: r85753:6d7932738c3d Date: 2016-07-18 15:53 +0200 http://bitbucket.org/pypy/pypy/changeset/6d7932738c3d/
Log: merge default diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -199,10 +199,13 @@ return tp._alignmentofinstances() @builtinify -def byref(cdata): +def byref(cdata, offset=0): # "pointer" is imported at the end of this module to avoid circular # imports - return pointer(cdata) + ptr = pointer(cdata) + if offset != 0: + ptr._buffer[0] += offset + return ptr def cdata_from_address(self, address): # fix the address: turn it into as unsigned, in case it's a negative number diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -335,3 +335,60 @@ This will disable SELinux's protection and allow PyPy to configure correctly. Be sure to enable it again if you need it! + + +How should I report a bug? +-------------------------- + +Our bug tracker is here: https://bitbucket.org/pypy/pypy/issues/ + +Missing features or incompatibilities with CPython are considered +bugs, and they are welcome. (See also our list of `known +incompatibilities`__.) + +.. __: http://pypy.org/compat.html + +For bugs of the kind "I'm getting a PyPy crash or a strange +exception", please note that: **We can't do anything without +reproducing the bug ourselves**. We cannot do anything with +tracebacks from gdb, or core dumps. This is not only because the +standard PyPy is compiled without debug symbols. The real reason is +that a C-level traceback is usually of no help at all in PyPy. +Debugging PyPy can be annoying. + +In more details: + +* First, please give the exact PyPy version, and the OS. + +* It might help focus our search if we know if the bug can be + reproduced on a "``pypy --jit off``" or not. If "``pypy --jit + off``" always works, then the problem might be in the JIT. + Otherwise, we know we can ignore that part. + +* If you got the bug using only Open Source components, please give a + step-by-step guide that we can follow to reproduce the problem + ourselves. Don't assume we know anything about any program other + than PyPy. We would like a guide that we can follow point by point + (without guessing or having to figure things out) + on a machine similar to yours, starting from a bare PyPy, until we + see the same problem. (If you can, you can try to reduce the number + of steps and the time it needs to run, but that is not mandatory.) + +* If the bug involves Closed Source components, or just too many Open + Source components to install them all ourselves, then maybe you can + give us some temporary ssh access to a machine where the bug can be + reproduced. Or, maybe we can download a VirtualBox or VMWare + virtual machine where the problem occurs. + +* If giving us access would require us to use tools other than ssh, + make appointments, or sign a NDA, then we can consider a commerical + support contract for a small sum of money. + +* If even that is not possible for you, then sorry, we can't help. + +Of course, you can try to debug the problem yourself, and we can help +you get started if you ask on the #pypy IRC channel, but be prepared: +debugging an annoying PyPy problem usually involves quite a lot of gdb +in auto-generated C code, and at least some knowledge about the +various components involved, from PyPy's own RPython source code to +the GC and possibly the JIT. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -53,3 +53,36 @@ Refactor PyTupleObject to look like cpython's and allow subclassing PyTuple_Type + +.. branch: call-via-pyobj + +Use offsets from PyTypeObject to find actual c function to call rather than +fixed functions, allows function override after PyType_Ready is called + +.. branch: issue2335 + +Avoid exhausting the stack in the JIT due to successive guard +failures in the same Python function ending up as successive levels of +RPython functions, while at app-level the traceback is very short + +.. branch: use-madv-free + +Try harder to memory to the OS. See e.g. issue #2336. Note that it does +not show up as a reduction of the VIRT column in ``top``, and the RES +column might also not show the reduction, particularly on Linux >= 4.5 or +on OS/X: it uses MADV_FREE, which only marks the pages as returnable to +the OS if the memory is low. + +.. branch: cpyext-slotdefs2 + +Fill in more slots when creating a PyTypeObject from a W_TypeObject +More slots are still TBD, like tp_print and richcmp + +.. branch: json-surrogates + +Align json module decode with the cpython's impl, fixes issue 2345 + +.. branch: issue2343 + +Copy CPython's logic more closely for handling of ``__instancecheck__()`` +and ``__subclasscheck__()``. Fixes issue 2343. diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py --- a/pypy/module/__builtin__/abstractinst.py +++ b/pypy/module/__builtin__/abstractinst.py @@ -46,9 +46,65 @@ raise # propagate other errors return space.type(w_obj) + +# ---------- isinstance ---------- + + +def p_recursive_isinstance_w(space, w_inst, w_cls): + # Copied straight from CPython 2.7. Does not handle 'cls' being a tuple. + if (isinstance(w_cls, W_ClassObject) and + isinstance(w_inst, W_InstanceObject)): + return w_inst.w_class.is_subclass_of(w_cls) + + if space.isinstance_w(w_cls, space.w_type): + return p_recursive_isinstance_type_w(space, w_inst, w_cls) + + check_class(space, w_cls, "isinstance() arg 2 must be a class, type," + " or tuple of classes and types") + try: + w_abstractclass = space.getattr(w_inst, space.wrap('__class__')) + except OperationError as e: + if e.async(space): # ignore most exceptions + raise + return False + else: + return p_abstract_issubclass_w(space, w_abstractclass, w_cls) + + +def p_recursive_isinstance_type_w(space, w_inst, w_type): + # subfunctionality of p_recursive_isinstance_w(): assumes that w_type is + # a type object. Copied straight from CPython 2.7. + if space.isinstance_w(w_inst, w_type): + return True + try: + w_abstractclass = space.getattr(w_inst, space.wrap('__class__')) + except OperationError as e: + if e.async(space): # ignore most exceptions + raise + else: + if w_abstractclass is not space.type(w_inst): + if space.isinstance_w(w_abstractclass, space.w_type): + return space.issubtype_w(w_abstractclass, w_type) + return False + + @jit.unroll_safe def abstract_isinstance_w(space, w_obj, w_klass_or_tuple, allow_override=False): """Implementation for the full 'isinstance(obj, klass_or_tuple)'.""" + # Copied from CPython 2.7's PyObject_Isinstance(). Additionally, + # if 'allow_override' is False (the default), then don't try to + # use a custom __instancecheck__ method. + + # WARNING: backward compatibility function name here. CPython + # uses the name "abstract" to refer to the logic of handling + # class-like objects, with a "__bases__" attribute. This function + # here is not related to that and implements the full + # PyObject_IsInstance() logic. + + # Quick test for an exact match + if space.type(w_obj) is w_klass_or_tuple: + return True + # -- case (anything, tuple) # XXX it might be risky that the JIT sees this if space.isinstance_w(w_klass_or_tuple, space.w_tuple): @@ -58,68 +114,59 @@ return False # -- case (anything, type) - try: - if allow_override: - w_result = space.isinstance_allow_override(w_obj, w_klass_or_tuple) - else: - w_result = space.isinstance(w_obj, w_klass_or_tuple) - except OperationError as e: # if w_klass_or_tuple was not a type, ignore it - if not e.match(space, space.w_TypeError): - raise # propagate other errors - else: - if space.is_true(w_result): - return True - # From now on we know that w_klass_or_tuple is indeed a type. - # Try also to compare it with obj.__class__, if this is not - # the same as type(obj). - try: - w_pretendtype = space.getattr(w_obj, space.wrap('__class__')) - if space.is_w(w_pretendtype, space.type(w_obj)): - return False # common case: obj.__class__ is type(obj) - if not allow_override: - return space.issubtype_w(w_pretendtype, w_klass_or_tuple) - w_result = space.issubtype_allow_override(w_pretendtype, - w_klass_or_tuple) - except OperationError as e: - if e.async(space): - raise - return False # ignore most exceptions - else: - return space.is_true(w_result) + if allow_override: + w_check = space.lookup(w_klass_or_tuple, "__instancecheck__") + if w_check is not None: + # this is the common case: all type objects have a method + # __instancecheck__. The one in the base 'type' type calls + # back p_recursive_isinstance_type_w() from the present module. + return space.is_true(space.get_and_call_function( + w_check, w_klass_or_tuple, w_obj)) - # -- case (old-style instance, old-style class) - if isinstance(w_klass_or_tuple, W_ClassObject): - if isinstance(w_obj, W_InstanceObject): - return w_obj.w_class.is_subclass_of(w_klass_or_tuple) - return _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple) + return p_recursive_isinstance_w(space, w_obj, w_klass_or_tuple) -def _abstract_isinstance_w_helper(space, w_obj, w_klass_or_tuple): - # -- case (anything, abstract-class) - check_class(space, w_klass_or_tuple, - "isinstance() arg 2 must be a class, type," - " or tuple of classes and types") - try: - w_abstractclass = space.getattr(w_obj, space.wrap('__class__')) - except OperationError as e: - if e.async(space): # ignore most exceptions - raise - return False - else: - return _issubclass_recurse(space, w_abstractclass, w_klass_or_tuple) +# ---------- issubclass ---------- @jit.unroll_safe -def _issubclass_recurse(space, w_derived, w_top): - """Internal helper for abstract cases. Here, w_top cannot be a tuple.""" - if space.is_w(w_derived, w_top): - return True - w_bases = _get_bases(space, w_derived) - if w_bases is not None: - for w_base in space.fixedview(w_bases): - if _issubclass_recurse(space, w_base, w_top): +def p_abstract_issubclass_w(space, w_derived, w_cls): + # Copied straight from CPython 2.7, function abstract_issubclass(). + # Don't confuse this with the function abstract_issubclass_w() below. + # Here, w_cls cannot be a tuple. + while True: + if space.is_w(w_derived, w_cls): + return True + w_bases = _get_bases(space, w_derived) + if w_bases is None: + return False + bases_w = space.fixedview(w_bases) + last_index = len(bases_w) - 1 + if last_index < 0: + return False + # Avoid recursivity in the single inheritance case; in general, + # don't recurse on the last item in the tuple (loop instead). + for i in range(last_index): + if p_abstract_issubclass_w(space, bases_w[i], w_cls): return True - return False + w_derived = bases_w[last_index] + + +def p_recursive_issubclass_w(space, w_derived, w_cls): + # From CPython's function of the same name (which as far as I can tell + # is not recursive). Copied straight from CPython 2.7. + if (space.isinstance_w(w_cls, space.w_type) and + space.isinstance_w(w_derived, space.w_type)): + return space.issubtype_w(w_derived, w_cls) + # + if (isinstance(w_derived, W_ClassObject) and + isinstance(w_cls, W_ClassObject)): + return w_derived.is_subclass_of(w_cls) + # + check_class(space, w_derived, "issubclass() arg 1 must be a class") + check_class(space, w_cls, "issubclass() arg 2 must be a class" + " or tuple of classes") + return p_abstract_issubclass_w(space, w_derived, w_cls) @jit.unroll_safe @@ -127,37 +174,31 @@ allow_override=False): """Implementation for the full 'issubclass(derived, klass_or_tuple)'.""" - # -- case (class-like-object, tuple-of-classes) + # WARNING: backward compatibility function name here. CPython + # uses the name "abstract" to refer to the logic of handling + # class-like objects, with a "__bases__" attribute. This function + # here is not related to that and implements the full + # PyObject_IsSubclass() logic. There is also p_abstract_issubclass_w(). + + # -- case (anything, tuple-of-classes) if space.isinstance_w(w_klass_or_tuple, space.w_tuple): for w_klass in space.fixedview(w_klass_or_tuple): if abstract_issubclass_w(space, w_derived, w_klass, allow_override): return True return False - # -- case (type, type) - try: - if not allow_override: - return space.issubtype_w(w_derived, w_klass_or_tuple) - w_result = space.issubtype_allow_override(w_derived, w_klass_or_tuple) - except OperationError as e: # if one of the args was not a type, ignore it - if not e.match(space, space.w_TypeError): - raise # propagate other errors - else: - return space.is_true(w_result) + # -- case (anything, type) + if allow_override: + w_check = space.lookup(w_klass_or_tuple, "__subclasscheck__") + if w_check is not None: + # this is the common case: all type objects have a method + # __subclasscheck__. The one in the base 'type' type calls + # back p_recursive_issubclass_w() from the present module. + return space.is_true(space.get_and_call_function( + w_check, w_klass_or_tuple, w_derived)) - # -- case (old-style class, old-style class) - if isinstance(w_derived, W_ClassObject): - if isinstance(w_klass_or_tuple, W_ClassObject): - return w_derived.is_subclass_of(w_klass_or_tuple) - else: - check_class(space, w_derived, "issubclass() arg 1 must be a class") - # from here on, we are sure that w_derived is a class-like object + return p_recursive_issubclass_w(space, w_derived, w_klass_or_tuple) - # -- case (class-like-object, abstract-class) - check_class(space, w_klass_or_tuple, - "issubclass() arg 2 must be a class, type," - " or tuple of classes and types") - return _issubclass_recurse(space, w_derived, w_klass_or_tuple) # ------------------------------------------------------------ # Exception helpers diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py --- a/pypy/module/__builtin__/test/test_abstractinst.py +++ b/pypy/module/__builtin__/test/test_abstractinst.py @@ -216,3 +216,26 @@ c = C() assert isinstance(c, C) assert not called + + def test_instancecheck_exception_not_eaten(self): + class M(object): + def __instancecheck__(self, obj): + raise TypeError("foobar") + + e = raises(TypeError, isinstance, 42, M()) + assert str(e.value) == "foobar" + + def test_issubclass_exception_not_eaten(self): + class M(object): + def __subclasscheck__(self, subcls): + raise TypeError("foobar") + + e = raises(TypeError, issubclass, 42, M()) + assert str(e.value) == "foobar" + + def test_issubclass_no_fallback(self): + class M(object): + def __subclasscheck__(self, subcls): + return False + + assert issubclass(42, M()) is False diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -360,10 +360,11 @@ hexdigits = self.getslice(start, i) try: val = int(hexdigits, 16) - if val & 0xfc00 == 0xd800: + if sys.maxunicode > 65535 and 0xd800 <= val <= 0xdfff: # surrogate pair - val = self.decode_surrogate_pair(i, val) - i += 6 + if self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u': + val = self.decode_surrogate_pair(i, val) + i += 6 except ValueError: self._raise("Invalid \uXXXX escape (char %d)", i-1) return # help the annotator to know that we'll never go beyond @@ -375,8 +376,9 @@ return i def decode_surrogate_pair(self, i, highsurr): - if self.ll_chars[i] != '\\' or self.ll_chars[i+1] != 'u': - self._raise("Unpaired high surrogate at char %d", i) + """ uppon enter the following must hold: + chars[i] == "\\" and chars[i+1] == "u" + """ i += 2 hexdigits = self.getslice(i, i+4) lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -184,6 +184,12 @@ res = _pypyjson.loads('"z\\ud834\\udd20x"') assert res == expected + def test_surrogate_pair(self): + import _pypyjson + json = '{"a":"\\uD83D"}' + res = _pypyjson.loads(json) + assert res == {u'a': u'\ud83d'} + def test_tab_in_string_should_fail(self): import _pypyjson # http://json.org/JSON_checker/test/fail25.json diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -561,8 +561,10 @@ #define PyObject_TypeCheck(ob, tp) \ ((ob)->ob_type == (tp) || PyType_IsSubtype((ob)->ob_type, (tp))) -#define Py_TRASHCAN_SAFE_BEGIN(pyObj) -#define Py_TRASHCAN_SAFE_END(pyObj) +#define Py_TRASHCAN_SAFE_BEGIN(pyObj) do { +#define Py_TRASHCAN_SAFE_END(pyObj) ; } while(0); +/* note: the ";" at the start of Py_TRASHCAN_SAFE_END is needed + if the code has a label in front of the macro call */ /* Copied from CPython ----------------------------- */ PyAPI_FUNC(int) PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *); diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -1,4 +1,4 @@ -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError, oefmt @@ -10,9 +10,10 @@ from pypy.module.cpyext.api import ( CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O, METH_STATIC, METH_VARARGS, PyObject, PyObjectFields, bootstrap_function, - build_type_checkers, cpython_api, cpython_struct, generic_cpy_call) + build_type_checkers, cpython_api, cpython_struct, generic_cpy_call, + PyTypeObjectPtr) from pypy.module.cpyext.pyobject import ( - Py_DecRef, from_ref, make_ref, make_typedescr) + Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr) PyCFunction_typedef = rffi.COpaquePtr(typedef='PyCFunction') PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) @@ -151,28 +152,45 @@ class W_PyCWrapperObject(W_Root): def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func): + wrapper_func_kwds, doc, func, offset=None): self.space = space self.method_name = method_name self.wrapper_func = wrapper_func self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func + self.offset = offset pyo = rffi.cast(PyObject, pto) w_type = from_ref(space, pyo) assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type def call(self, space, w_self, w_args, w_kw): + func_to_call = self.func + if self.offset: + pto = as_pyobj(space, self.w_objclass) + # make ptr the equivalent of this, using the offsets + #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) + if pto: + cptr = rffi.cast(rffi.CCHARP, pto) + for o in self.offset: + ptr = rffi.cast(rffi.VOIDPP, rffi.ptradd(cptr, o))[0] + cptr = rffi.cast(rffi.CCHARP, ptr) + func_to_call = rffi.cast(rffi.VOIDP, cptr) + else: + # Should never happen, assert to get a traceback + assert False, "failed to convert w_type %s to PyObject" % str( + self.w_objclass) + assert func_to_call if self.wrapper_func is None: assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, self.func, + return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, w_kw) if space.is_true(w_kw): raise oefmt(space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - return self.wrapper_func(space, w_self, w_args, self.func) + return self.wrapper_func(space, w_self, w_args, func_to_call) def descr_method_repr(self): return self.space.wrap("<slot wrapper '%s' of '%s' objects>" % @@ -301,12 +319,6 @@ def PyDescr_NewClassMethod(space, w_type, method): return space.wrap(W_PyCClassMethodObject(space, method, w_type)) -def PyDescr_NewWrapper(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func): - # not exactly the API sig - return space.wrap(W_PyCWrapperObject(space, pto, method_name, - wrapper_func, wrapper_func_kwds, doc, func)) - @cpython_api([lltype.Ptr(PyMethodDef), PyObject, CONST_STRING], PyObject) def Py_FindMethod(space, table, w_obj, name_ptr): """Return a bound method object for an extension type implemented in diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -380,6 +380,7 @@ ('tp_as_number.c_nb_absolute', '__abs__'), ('tp_as_number.c_nb_invert', '__invert__'), ('tp_as_number.c_nb_index', '__index__'), + ('tp_as_number.c_nb_hex', '__hex__'), ('tp_str', '__str__'), ('tp_repr', '__repr__'), ('tp_iter', '__iter__'), @@ -398,7 +399,7 @@ # binary functions for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__subtract__'), + ('tp_as_number.c_nb_subtract', '__sub__'), ('tp_as_number.c_nb_multiply', '__mul__'), ('tp_as_number.c_nb_divide', '__div__'), ('tp_as_number.c_nb_remainder', '__mod__'), @@ -408,6 +409,8 @@ ('tp_as_number.c_nb_and', '__and__'), ('tp_as_number.c_nb_xor', '__xor__'), ('tp_as_number.c_nb_or', '__or__'), + ('tp_as_sequence.c_sq_concat', '__add__'), + ('tp_as_sequence.c_sq_inplace_concat', '__iadd__') ]: if name == tp_name: slot_fn = w_type.getdictvalue(space, attr) @@ -421,8 +424,26 @@ api_func = slot_func.api_func handled = True + # binary-with-Py_ssize_t-type + for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'), + ('tp_as_sequence.c_sq_repeat', '__mul__'), + ('tp_as_sequence.c_sq_repeat', '__mul__'), + ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), + ]: + if name == tp_name: + slot_fn = w_type.getdictvalue(space, attr) + if slot_fn is None: + return + + @cpython_api([PyObject, Py_ssize_t], PyObject, header=header) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.wrap(arg)) + api_func = slot_func.api_func + handled = True + # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', ''), + for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), ]: if name == tp_name: slot_fn = w_type.getdictvalue(space, attr) @@ -522,6 +543,8 @@ api_func = slot_tp_new.api_func else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce + # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length + # richcmpfunc(s) return return lambda: llhelper(api_func.functype, api_func.get_wrapper(space)) diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2144,6 +2144,13 @@ return array_new(type, args, NULL); } +static PyObject * +switch_multiply(void) +{ + Arraytype.tp_as_number->nb_multiply = array_base_multiply; + Py_RETURN_NONE; +}; + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ an array of basic values: characters, integers, floating point\n\ @@ -2394,6 +2401,7 @@ /* No functions in array module. */ static PyMethodDef a_methods[] = { {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, + {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -747,6 +747,7 @@ if (PyType_Ready(&UnicodeSubtype3) < 0) INITERROR; + TupleLike.tp_flags = Py_TPFLAGS_DEFAULT; TupleLike.tp_base = &PyTuple_Type; if (PyType_Ready(&TupleLike) < 0) INITERROR; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -84,3 +84,7 @@ arr = module.array('i', [2]) res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] + module.switch_multiply() + res = [1, 2, 3] * arr + assert res == [2, 4, 6] + diff --git a/pypy/module/cpyext/test/test_bytearrayobject.py b/pypy/module/cpyext/test/test_bytearrayobject.py --- a/pypy/module/cpyext/test/test_bytearrayobject.py +++ b/pypy/module/cpyext/test/test_bytearrayobject.py @@ -87,11 +87,12 @@ module = self.import_extension('foo', [ ("getbytearray", "METH_NOARGS", """ - PyObject* s1 = PyByteArray_FromStringAndSize("test", 4); + const char *c; + PyObject *s2, *s1 = PyByteArray_FromStringAndSize("test", 4); if (s1 == NULL) return NULL; - const char* c = PyByteArray_AsString(s1); - PyObject* s2 = PyByteArray_FromStringAndSize(c, 4); + c = PyByteArray_AsString(s1); + s2 = PyByteArray_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; """), diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -113,9 +113,10 @@ module = self.import_extension('foo', [ ("getbytes", "METH_NOARGS", """ - PyObject* s1 = PyBytes_FromStringAndSize("test", 4); - char* c = PyBytes_AsString(s1); - PyObject* s2 = PyBytes_FromStringAndSize(c, 4); + char *c; + PyObject* s2, *s1 = PyBytes_FromStringAndSize("test", 4); + c = PyBytes_AsString(s1); + s2 = PyBytes_FromStringAndSize(c, 4); Py_DECREF(s1); return s2; """), diff --git a/pypy/module/cpyext/test/test_datetime.py b/pypy/module/cpyext/test/test_datetime.py --- a/pypy/module/cpyext/test/test_datetime.py +++ b/pypy/module/cpyext/test/test_datetime.py @@ -176,13 +176,15 @@ """), ("test_datetime_macros", "METH_NOARGS", """ + PyObject* obj; + PyDateTime_DateTime *dt; PyDateTime_IMPORT; if (!PyDateTimeAPI) { PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI"); return NULL; } - PyObject* obj = PyDateTime_FromDateAndTime(2000, 6, 6, 6, 6, 6, 6); - PyDateTime_DateTime* dt = (PyDateTime_DateTime*)obj; + obj = PyDateTime_FromDateAndTime(2000, 6, 6, 6, 6, 6, 6); + dt = (PyDateTime_DateTime*)obj; PyDateTime_GET_YEAR(obj); PyDateTime_GET_YEAR(dt); @@ -209,13 +211,15 @@ """), ("test_time_macros", "METH_NOARGS", """ + PyObject* obj; + PyDateTime_Time* t; PyDateTime_IMPORT; if (!PyDateTimeAPI) { PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI"); return NULL; } - PyObject* obj = PyTime_FromTime(6, 6, 6, 6); - PyDateTime_Time* t = (PyDateTime_Time*)obj; + obj = PyTime_FromTime(6, 6, 6, 6); + t = (PyDateTime_Time*)obj; PyDateTime_TIME_GET_HOUR(obj); PyDateTime_TIME_GET_HOUR(t); @@ -233,13 +237,15 @@ """), ("test_delta_macros", "METH_NOARGS", """ + PyObject* obj; + PyDateTime_Delta* delta; PyDateTime_IMPORT; if (!PyDateTimeAPI) { PyErr_SetString(PyExc_RuntimeError, "No PyDateTimeAPI"); return NULL; } - PyObject* obj = PyDelta_FromDSU(6, 6, 6); - PyDateTime_Delta* delta = (PyDateTime_Delta*)obj; + obj = PyDelta_FromDSU(6, 6, 6); + delta = (PyDateTime_Delta*)obj; #if defined(PYPY_VERSION) || PY_VERSION_HEX >= 0x03030000 // These macros are only defined in CPython 3.x and PyPy. diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py --- a/pypy/module/cpyext/test/test_listobject.py +++ b/pypy/module/cpyext/test/test_listobject.py @@ -64,7 +64,7 @@ assert space.unwrap(w_s) == [2, 1] class AppTestListObject(AppTestCpythonExtensionBase): - def test_listobject(self): + def test_basic_listobject(self): import sys module = self.import_extension('foo', [ ("newlist", "METH_NOARGS", @@ -104,6 +104,15 @@ Py_RETURN_NONE; """ ), + ('test_tp_as_', "METH_NOARGS", + ''' + PyObject *l = PyList_New(3); + int ok = l->ob_type->tp_as_sequence != NULL; /* 1 */ + ok += 2 * (l->ob_type->tp_as_number == NULL); /* 2 */ + Py_DECREF(l); + return PyLong_FromLong(ok); /* should be 3 */ + ''' + ), ]) l = module.newlist() assert l == [3, -5, 1000] @@ -137,6 +146,9 @@ module.setlistitem(l,0) assert l == [None, 2, 3] + # tp_as_sequence should be filled, but tp_as_number should be NULL + assert module.test_tp_as_() == 3 + def test_list_macros(self): """The PyList_* macros cast, and calls expecting that build.""" module = self.import_extension('foo', [ diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -227,4 +227,40 @@ """)]) assert module.from_str() == 0 - + def test_slots(self): + module = self.import_extension('foo', [ + ("has_sub", "METH_NOARGS", + """ + PyObject *ret, *obj = PyLong_FromLong(42); + if (obj->ob_type->tp_as_number->nb_subtract) + ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj); + else + ret = PyLong_FromLong(-1); + Py_DECREF(obj); + return ret; + """), + ("has_pow", "METH_NOARGS", + """ + PyObject *ret, *obj = PyLong_FromLong(42); + PyObject *one = PyLong_FromLong(1); + if (obj->ob_type->tp_as_number->nb_power) + ret = obj->ob_type->tp_as_number->nb_power(obj, one, one); + else + ret = PyLong_FromLong(-1); + Py_DECREF(obj); + return ret; + """), + ("has_hex", "METH_NOARGS", + """ + PyObject *ret, *obj = PyLong_FromLong(42); + if (obj->ob_type->tp_as_number->nb_hex) + ret = obj->ob_type->tp_as_number->nb_hex(obj); + else + ret = PyLong_FromLong(-1); + Py_DECREF(obj); + return ret; + """)]) + assert module.has_sub() == 0 + assert module.has_pow() == 0 + assert module.has_hex() == '0x2aL' + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -17,9 +17,9 @@ generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder, - PyObjectFields, Py_TPFLAGS_BASETYPE) + PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, + W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, W_PyCMethodObject, W_PyCFunctionObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( @@ -30,7 +30,7 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( - PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, + PyGetSetDef, PyMemberDef, newfunc, PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs) from pypy.objspace.std.typeobject import W_TypeObject, find_best_base @@ -238,7 +238,7 @@ i += 1 def update_all_slots(space, w_type, pto): - # XXX fill slots in pto + # fill slots in pto # Not very sure about it, but according to # test_call_tp_dealloc_when_created_from_python, we should not # overwrite slots that are already set: these ones are probably @@ -272,6 +272,15 @@ if len(slot_names) == 1: if not getattr(pto, slot_names[0]): setattr(pto, slot_names[0], slot_func_helper) + elif (w_type.getname(space) in ('list', 'tuple') and + slot_names[0] == 'c_tp_as_number'): + # XXX hack - hwo can we generalize this? The problem is method + # names like __mul__ map to more than one slot, and we have no + # convenient way to indicate which slots CPython have filled + # + # We need at least this special case since Numpy checks that + # (list, tuple) do __not__ fill tp_as_number + pass else: assert len(slot_names) == 2 struct = getattr(pto, slot_names[0]) @@ -296,6 +305,7 @@ for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: if method_name in dict_w: continue + offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] if len(slot_names) == 1: func = getattr(pto, slot_names[0]) else: @@ -303,14 +313,16 @@ struct = getattr(pto, slot_names[0]) if not struct: continue + offset.append(rffi.offsetof(lltype.typeOf(struct).TO, slot_names[1])) func = getattr(struct, slot_names[1]) func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue if wrapper_func is None and wrapper_func_kwds is None: continue - dict_w[method_name] = PyDescr_NewWrapper(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func_voidp) + w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, + wrapper_func_kwds, doc, func_voidp, offset=offset) + dict_w[method_name] = space.wrap(w_obj) if pto.c_tp_new: add_tp_new_wrapper(space, dict_w, pto) @@ -730,6 +742,7 @@ try: w_obj = _type_realize(space, py_obj) finally: + name = rffi.charp2str(pto.c_tp_name) pto.c_tp_flags &= ~Py_TPFLAGS_READYING pto.c_tp_flags |= Py_TPFLAGS_READY return w_obj @@ -811,7 +824,8 @@ base = pto.c_tp_base base_pyo = rffi.cast(PyObject, pto.c_tp_base) if base and not base.c_tp_flags & Py_TPFLAGS_READY: - type_realize(space, rffi.cast(PyObject, base_pyo)) + name = rffi.charp2str(base.c_tp_name) + type_realize(space, base_pyo) if base and not pto.c_ob_type: # will be filled later pto.c_ob_type = base.c_ob_type if not pto.c_tp_bases: diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -254,7 +254,7 @@ idx = space.str_w(w_idx) return self.getfield(space, idx) if space.is_w(w_idx, space.w_Ellipsis): - return self + return self.descr_view(space, space.type(self)) elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool(): if w_idx.ndims() > 0: w_ret = self.getitem_filter(space, w_idx) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2614,17 +2614,11 @@ import numpy as np import sys a = np.array(1.5) - if '__pypy__' in sys.builtin_module_names: - assert a[...] is a - else: - assert a[...].base is a + assert a[...].base is a a[...] = 2.5 assert a == 2.5 a = np.array([1, 2, 3]) - if '__pypy__' in sys.builtin_module_names: - assert a[...] is a - else: - assert a[...].base is a + assert a[...].base is a a[...] = 4 assert (a == [4, 4, 4]).all() assert a[..., 0] == 4 diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_pointers.py @@ -265,3 +265,10 @@ class A(object): _byref = byref A._byref(c_int(5)) + + def test_byref_with_offset(self): + c = c_int() + d = byref(c) + base = cast(d, c_void_p).value + for i in [0, 1, 4, 1444, -10293]: + assert cast(byref(c, i), c_void_p).value == base + i diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -509,21 +509,6 @@ def isinstance(space, w_inst, w_type): return space.wrap(space.isinstance_w(w_inst, w_type)) - def issubtype_allow_override(space, w_sub, w_type): - w_check = space.lookup(w_type, "__subclasscheck__") - if w_check is None: - raise oefmt(space.w_TypeError, "issubclass not supported here") - return space.get_and_call_function(w_check, w_type, w_sub) - - def isinstance_allow_override(space, w_inst, w_type): - if space.type(w_inst) is w_type: - return space.w_True # fast path copied from cpython - w_check = space.lookup(w_type, "__instancecheck__") - if w_check is not None: - return space.get_and_call_function(w_check, w_type, w_inst) - else: - return space.isinstance(w_inst, w_type) - # helpers diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -1091,6 +1091,29 @@ C() # the lookup of '__new__' succeeds in 'int', # but the lookup of '__init__' fails + def test_instancecheck(self): + assert int.__instancecheck__(42) is True + assert int.__instancecheck__(42.0) is False + class Foo: + __class__ = int + assert int.__instancecheck__(Foo()) is False + class Bar(object): + __class__ = int + assert int.__instancecheck__(Bar()) is True + + def test_subclasscheck(self): + assert int.__subclasscheck__(bool) is True + assert int.__subclasscheck__(float) is False + class Foo: + __class__ = int + assert int.__subclasscheck__(Foo) is False + class Bar(object): + __class__ = int + assert int.__subclasscheck__(Bar) is False + class AbstractClass(object): + __bases__ = (int,) + assert int.__subclasscheck__(AbstractClass()) is True + class AppTestWithMethodCacheCounter: spaceconfig = {"objspace.std.withmethodcachecounter": True} @@ -1290,5 +1313,3 @@ assert not self.compares_by_identity(X) del X.__eq__ assert self.compares_by_identity(X) - - diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -6,6 +6,7 @@ from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\ descr_get_dict, dict_descr, Member, TypeDef from pypy.interpreter.astcompiler.misc import mangle +from pypy.module.__builtin__ import abstractinst from rpython.rlib.jit import (promote, elidable_promote, we_are_jitted, elidable, dont_look_inside, unroll_safe) @@ -899,13 +900,15 @@ # ____________________________________________________________ -@gateway.unwrap_spec(w_obj=W_TypeObject, w_sub=W_TypeObject) +@gateway.unwrap_spec(w_obj=W_TypeObject) def type_issubtype(w_obj, space, w_sub): - return space.newbool(w_sub.issubtype(w_obj)) + return space.newbool( + abstractinst.p_recursive_issubclass_w(space, w_sub, w_obj)) @gateway.unwrap_spec(w_obj=W_TypeObject) def type_isinstance(w_obj, space, w_inst): - return space.newbool(space.type(w_inst).issubtype(w_obj)) + return space.newbool( + abstractinst.p_recursive_isinstance_type_w(space, w_inst, w_obj)) W_TypeObject.typedef = TypeDef("type", __new__ = gateway.interp2app(descr__new__), diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1143,45 +1143,35 @@ @arguments("cpu", "i", "R", "d", returns="i") def bhimpl_residual_call_r_i(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d", returns="r") def bhimpl_residual_call_r_r(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, None, args_r, None, calldescr) @arguments("cpu", "i", "R", "d") def bhimpl_residual_call_r_v(cpu, func, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, None, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="i") def bhimpl_residual_call_ir_i(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d", returns="r") def bhimpl_residual_call_ir_r(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "d") def bhimpl_residual_call_ir_v(cpu, func, args_i, args_r, calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, None, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="i") def bhimpl_residual_call_irf_i(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_i(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="r") def bhimpl_residual_call_irf_r(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_r(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d", returns="f") def bhimpl_residual_call_irf_f(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_f(func, args_i, args_r, args_f, calldescr) @arguments("cpu", "i", "I", "R", "F", "d") def bhimpl_residual_call_irf_v(cpu, func, args_i,args_r,args_f,calldescr): - workaround2200.active = True return cpu.bh_call_v(func, args_i, args_r, args_f, calldescr) # conditional calls - note that they cannot return stuff @@ -1209,54 +1199,44 @@ @arguments("cpu", "j", "R", returns="i") def bhimpl_inline_call_r_i(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R", returns="r") def bhimpl_inline_call_r_r(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "R") def bhimpl_inline_call_r_v(cpu, jitcode, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), None, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="i") def bhimpl_inline_call_ir_i(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", returns="r") def bhimpl_inline_call_ir_r(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R") def bhimpl_inline_call_ir_v(cpu, jitcode, args_i, args_r): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, None, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="i") def bhimpl_inline_call_irf_i(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_i(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="r") def bhimpl_inline_call_irf_r(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_r(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F", returns="f") def bhimpl_inline_call_irf_f(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_f(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @arguments("cpu", "j", "I", "R", "F") def bhimpl_inline_call_irf_v(cpu, jitcode, args_i, args_r, args_f): - workaround2200.active = True return cpu.bh_call_v(jitcode.get_fnaddr_as_int(), args_i, args_r, args_f, jitcode.calldescr) @@ -1543,8 +1523,6 @@ if not self.nextblackholeinterp: self._exit_frame_with_exception(current_exc) return current_exc - finally: - workaround2200.active = False # # pass the frame's return value to the caller caller = self.nextblackholeinterp @@ -1717,10 +1695,3 @@ # _run_forever(firstbh, current_exc) convert_and_run_from_pyjitpl._dont_inline_ = True - -# ____________________________________________________________ - -class WorkaroundIssue2200(object): - pass -workaround2200 = WorkaroundIssue2200() -workaround2200.active = False diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -620,23 +620,28 @@ raise jitexc.DoneWithThisFrameVoid() class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_int_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.INT - result = metainterp_sd.cpu.get_int_value(deadframe, 0) - raise jitexc.DoneWithThisFrameInt(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameInt(self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_ref_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu - result = cpu.get_ref_value(deadframe, 0) - raise jitexc.DoneWithThisFrameRef(cpu, result) + raise jitexc.DoneWithThisFrameRef(cpu, self.get_result(cpu, deadframe)) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): + def get_result(self, cpu, deadframe): + return cpu.get_float_value(deadframe, 0) def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.FLOAT - result = metainterp_sd.cpu.get_float_value(deadframe, 0) - raise jitexc.DoneWithThisFrameFloat(result) + cpu = metainterp_sd.cpu + raise jitexc.DoneWithThisFrameFloat(self.get_result(cpu, deadframe)) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -24,15 +24,15 @@ # this is the initial size of the trace - note that we probably # want something that would fit the inital "max_trace_length" INIT_SIZE = 30000 - MIN_SHORT = 0 - MAX_SHORT = 2**16 - 1 - check_range = True + MIN_VALUE = 0 + MAX_VALUE = 2**16 - 1 class BigModel: INIT_SIZE = 30000 - STORAGE_TP = lltype.Signed - check_range = False - # we can move SMALL ints here, if necessary + STORAGE_TP = rffi.UINT + MIN_VALUE = 0 + MAX_VALUE = int(2**31 - 1) # we could go to 2**32-1 on 64-bit, but + # that seems already far too huge def get_model(self): return _get_model(self.metainterp_sd) @@ -295,9 +295,8 @@ if self._pos >= len(self._ops): # grow by 2X self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops) - if model.check_range: - if not model.MIN_SHORT <= v <= model.MAX_SHORT: - raise FrontendTagOverflow + if not model.MIN_VALUE <= v <= model.MAX_VALUE: + raise FrontendTagOverflow self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v) self._pos += 1 diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2543,7 +2543,13 @@ elif box.type == history.REF: args.append(box.getref_base()) elif box.type == history.FLOAT: args.append(box.getfloatstorage()) else: assert 0 - self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + res = self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args) + kind = history.getkind(lltype.typeOf(res)) + if kind == 'void': raise jitexc.DoneWithThisFrameVoid() + if kind == 'int': raise jitexc.DoneWithThisFrameInt(res) + if kind == 'ref': raise jitexc.DoneWithThisFrameRef(self.cpu, res) + if kind == 'float': raise jitexc.DoneWithThisFrameFloat(res) + raise AssertionError(kind) def prepare_resume_from_failure(self, deadframe, inputargs, resumedescr): exception = self.cpu.grab_exc_value(deadframe) diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4470,6 +4470,28 @@ self.meta_interp(f, []) + def test_issue2335_recursion(self): + # Reproduces issue #2335: same as issue #2200, but the workaround + # in c4c54cb69aba was not enough. + driver = JitDriver(greens=["level"], reds=["i"]) + def enter(level, i): + if level == 0: + f(1) # recursive call + driver.can_enter_jit(level=level, i=i) + def f(level): + i = 0 if level == 0 else 298 + while True: + driver.jit_merge_point(level=level, i=i) + i += 1 + if i >= 300: + return i + promote(i + 1) # a failing guard + enter(level, i) + def main(): + set_param(None, 'trace_eagerness', 999999) + f(0) + self.meta_interp(main, []) + def test_pending_setarrayitem_with_indirect_constant_index(self): driver = JitDriver(greens=[], reds='auto') class X: diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -529,7 +529,7 @@ def make_enter_function(self, jd): from rpython.jit.metainterp.warmstate import WarmEnterState state = WarmEnterState(self, jd) - maybe_compile_and_run = state.make_entry_point() + maybe_compile_and_run, EnterJitAssembler = state.make_entry_point() jd.warmstate = state def crash_in_jit(e): @@ -560,6 +560,7 @@ maybe_enter_jit._always_inline_ = True jd._maybe_enter_jit_fn = maybe_enter_jit jd._maybe_compile_and_run_fn = maybe_compile_and_run + jd._EnterJitAssembler = EnterJitAssembler def make_driverhook_graphs(self): s_Str = annmodel.SomeString() @@ -842,12 +843,8 @@ # while 1: # try: # return portal(*args) - # except ContinueRunningNormally, e: - # *args = *e.new_args - # except DoneWithThisFrame, e: - # return e.return - # except ExitFrameWithException, e: - # raise Exception, e.value + # except JitException, e: + # return handle_jitexception(e) # # def portal(*args): # while 1: @@ -884,90 +881,79 @@ rtyper = self.translator.rtyper RESULT = PORTALFUNC.RESULT result_kind = history.getkind(RESULT) + assert result_kind.startswith(jd.result_type) ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn + EnterJitAssembler = jd._EnterJitAssembler def ll_portal_runner(*args): - start = True - while 1: - try: - # maybe enter from the function's start. Note that the - # 'start' variable is constant-folded away because it's - # the first statement in the loop. - if start: - maybe_compile_and_run( - state.increment_function_threshold, *args) - # - # then run the normal portal function, i.e. the - # interpreter's main loop. It might enter the jit - # via maybe_enter_jit(), which typically ends with - # handle_fail() being called, which raises on the - # following exceptions --- catched here, because we - # want to interrupt the whole interpreter loop. - return support.maybe_on_top_of_llinterp(rtyper, - portal_ptr)(*args) - except jitexc.ContinueRunningNormally as e: + try: + # maybe enter from the function's start. + maybe_compile_and_run( + state.increment_function_threshold, *args) + # + # then run the normal portal function, i.e. the + # interpreter's main loop. It might enter the jit + # via maybe_enter_jit(), which typically ends with + # handle_fail() being called, which raises on the + # following exceptions --- catched here, because we + # want to interrupt the whole interpreter loop. + return support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except jitexc.JitException as e: + result = handle_jitexception(e) + if result_kind != 'void': + result = specialize_value(RESULT, result) + return result + + def handle_jitexception(e): + # XXX there are too many exceptions all around... + while True: + if isinstance(e, EnterJitAssembler): + try: + return e.execute() + except jitexc.JitException as e: + continue + # + if isinstance(e, jitexc.ContinueRunningNormally): args = () for ARGTYPE, attrname, count in portalfunc_ARGS: x = getattr(e, attrname)[count] x = specialize_value(ARGTYPE, x) args = args + (x,) - start = False - continue - except jitexc.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except jitexc.DoneWithThisFrameInt as e: - assert result_kind == 'int' - return specialize_value(RESULT, e.result) - except jitexc.DoneWithThisFrameRef as e: - assert result_kind == 'ref' - return specialize_value(RESULT, e.result) - except jitexc.DoneWithThisFrameFloat as e: - assert result_kind == 'float' - return specialize_value(RESULT, e.result) - except jitexc.ExitFrameWithExceptionRef as e: + try: + result = support.maybe_on_top_of_llinterp(rtyper, + portal_ptr)(*args) + except jitexc.JitException as e: + continue + if result_kind != 'void': + result = unspecialize_value(result) + return result + # + if result_kind == 'void': + if isinstance(e, jitexc.DoneWithThisFrameVoid): + return None + if result_kind == 'int': + if isinstance(e, jitexc.DoneWithThisFrameInt): + return e.result + if result_kind == 'ref': + if isinstance(e, jitexc.DoneWithThisFrameRef): + return e.result + if result_kind == 'float': + if isinstance(e, jitexc.DoneWithThisFrameFloat): + return e.result + # + if isinstance(e, jitexc.ExitFrameWithExceptionRef): value = ts.cast_to_baseclass(e.value) if not we_are_translated(): raise LLException(ts.get_typeptr(value), value) else: value = cast_base_ptr_to_instance(Exception, value) + assert value is not None raise value - - def handle_jitexception(e): - # XXX the bulk of this function is mostly a copy-paste from above - try: - raise e - except jitexc.ContinueRunningNormally as e: - args = () - for ARGTYPE, attrname, count in portalfunc_ARGS: - x = getattr(e, attrname)[count] - x = specialize_value(ARGTYPE, x) - args = args + (x,) - result = ll_portal_runner(*args) - if result_kind != 'void': - result = unspecialize_value(result) - return result - except jitexc.DoneWithThisFrameVoid: - assert result_kind == 'void' - return - except jitexc.DoneWithThisFrameInt as e: - assert result_kind == 'int' - return e.result - except jitexc.DoneWithThisFrameRef as e: - assert result_kind == 'ref' - return e.result - except jitexc.DoneWithThisFrameFloat as e: - assert result_kind == 'float' - return e.result - except jitexc.ExitFrameWithExceptionRef as e: - value = ts.cast_to_baseclass(e.value) - if not we_are_translated(): - raise LLException(ts.get_typeptr(value), value) - else: - value = cast_base_ptr_to_instance(Exception, value) - raise value + # + raise AssertionError("all cases should have been handled") jd._ll_portal_runner = ll_portal_runner # for debugging jd.portal_runner_ptr = self.helper_func(jd._PTR_PORTAL_FUNCTYPE, diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -2,7 +2,7 @@ import weakref from rpython.jit.codewriter import support, heaptracker, longlong -from rpython.jit.metainterp import resoperation, history +from rpython.jit.metainterp import resoperation, history, jitexc from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib.debug import have_debug_prints_for from rpython.rlib.jit import PARAMETERS @@ -348,8 +348,9 @@ def make_entry_point(self): "NOT_RPYTHON" - if hasattr(self, 'maybe_compile_and_run'): - return self.maybe_compile_and_run + from rpython.jit.metainterp import compile + if hasattr(self, 'entry_point_fns'): + return self.entry_point_fns warmrunnerdesc = self.warmrunnerdesc metainterp_sd = warmrunnerdesc.metainterp_sd @@ -362,6 +363,8 @@ confirm_enter_jit = self.confirm_enter_jit range_red_args = unrolling_iterable( range(num_green_args, num_green_args + jitdriver_sd.num_red_args)) + name_red_args = unrolling_iterable( + [(i, 'arg%d' % i) for i in range(jitdriver_sd.num_red_args)]) # get a new specialized copy of the method ARGS = [] for kind in jitdriver_sd.red_args_types: @@ -376,6 +379,7 @@ func_execute_token = self.cpu.make_execute_token(*ARGS) cpu = self.cpu jitcounter = self.warmrunnerdesc.jitcounter + result_type = jitdriver_sd.result_type def execute_assembler(loop_token, *args): # Call the backend to run the 'looptoken' with the given @@ -396,8 +400,23 @@ # # Handle the failure fail_descr = cpu.get_latest_descr(deadframe) + # First, a fast path to avoid raising and immediately catching + # a DoneWithThisFrame exception + if result_type == history.VOID: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrVoid): + return None + if result_type == history.INT: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrInt): + return fail_descr.get_result(cpu, deadframe) + if result_type == history.REF: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrRef): + return fail_descr.get_result(cpu, deadframe) + if result_type == history.FLOAT: + if isinstance(fail_descr, compile.DoneWithThisFrameDescrFloat): + return fail_descr.get_result(cpu, deadframe) + # + # General case fail_descr.handle_fail(deadframe, metainterp_sd, jitdriver_sd) - # assert 0, "should have raised" def bound_reached(hash, cell, *args): @@ -419,7 +438,8 @@ def maybe_compile_and_run(increment_threshold, *args): """Entry point to the JIT. Called at the point with the - can_enter_jit() hint. + can_enter_jit() hint, and at the start of a function + with a different threshold. """ # Look for the cell corresponding to the current greenargs. # Search for the JitCell that is of the correct subclass of @@ -439,14 +459,6 @@ bound_reached(hash, None, *args) return - # Workaround for issue #2200, maybe temporary. This is not - # a proper fix, but only a hack that should work well enough - # for PyPy's main jitdriver... See test_issue2200_recursion - from rpython.jit.metainterp.blackhole import workaround2200 - if workaround2200.active: - workaround2200.active = False - return - # Here, we have found 'cell'. # if cell.flags & (JC_TRACING | JC_TEMPORARY): @@ -484,14 +496,27 @@ execute_args = () for i in range_red_args: execute_args += (unspecialize_value(args[i]), ) - # run it! this executes until interrupted by an exception - execute_assembler(procedure_token, *execute_args) - assert 0, "should not reach this point" + # run it, but from outside in ll_portal_runner, not from here + # (this avoids RPython-level recursion with no corresponding + # app-level recursion, as shown by issues 2200 and 2335) + raise EnterJitAssembler(procedure_token, *execute_args) + + class EnterJitAssembler(jitexc.JitException): + def __init__(self, procedure_token, *args): + self.procedure_token = procedure_token + for i, argname in name_red_args: + setattr(self, argname, args[i]) + def execute(self): + args = () + for i, argname in name_red_args: + args += (getattr(self, argname), ) + return execute_assembler(self.procedure_token, *args) maybe_compile_and_run._dont_inline_ = True - self.maybe_compile_and_run = maybe_compile_and_run self.execute_assembler = execute_assembler - return maybe_compile_and_run + self.entry_point_fns = (maybe_compile_and_run, + EnterJitAssembler) + return self.entry_point_fns # ---------- diff --git a/rpython/memory/gc/minimarkpage.py b/rpython/memory/gc/minimarkpage.py --- a/rpython/memory/gc/minimarkpage.py +++ b/rpython/memory/gc/minimarkpage.py @@ -395,6 +395,7 @@ if arena.nfreepages == arena.totalpages: # # The whole arena is empty. Free it. + llarena.arena_reset(arena.base, self.arena_size, 4) llarena.arena_free(arena.base) lltype.free(arena, flavor='raw', track_allocation=False) # diff --git a/rpython/rlib/nonconst.py b/rpython/rlib/nonconst.py --- a/rpython/rlib/nonconst.py +++ b/rpython/rlib/nonconst.py @@ -39,7 +39,4 @@ def specialize_call(self, hop): hop.exception_cannot_occur() - retval = Constant(hop.r_result.convert_const(hop.args_v[0].value)) - retval.concretetype = hop.r_result.lowleveltype - return retval - + return hop.inputarg(hop.r_result, arg=0) diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -70,13 +70,20 @@ CConfig.MREMAP_MAYMOVE = ( rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE")) CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)') - # a dirty hack, this is probably a macro + CConfig.has_madvise = rffi_platform.Has('madvise(NULL, 0, 0)') + # ^^ both are a dirty hack, this is probably a macro + + CConfig.MADV_DONTNEED = ( + rffi_platform.DefinedConstantInteger('MADV_DONTNEED')) + CConfig.MADV_FREE = ( + rffi_platform.DefinedConstantInteger('MADV_FREE')) elif _MS_WINDOWS: constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS', + 'MEM_RESET'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) @@ -144,6 +151,7 @@ if _POSIX: has_mremap = cConfig['has_mremap'] + has_madvise = cConfig['has_madvise'] c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR, macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO) @@ -154,6 +162,9 @@ if has_mremap: c_mremap, _ = external('mremap', [PTR, size_t, size_t, rffi.ULONG], PTR) + if has_madvise: + _, c_madvise_safe = external('madvise', [PTR, size_t, rffi.INT], + rffi.INT, _nowrapper=True) # this one is always safe _pagesize = rffi_platform.getintegerfunctionresult('getpagesize', @@ -222,6 +233,9 @@ VirtualAlloc, VirtualAlloc_safe = winexternal('VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP) + _, _VirtualAlloc_safe_no_wrapper = winexternal('VirtualAlloc', + [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], + rffi.VOIDP, _nowrapper=True) _, _VirtualProtect_safe = winexternal('VirtualProtect', [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], BOOL) @@ -755,6 +769,39 @@ else: free = c_munmap_safe + if sys.platform.startswith('linux'): + assert has_madvise + assert MADV_DONTNEED is not None + if MADV_FREE is None: + MADV_FREE = 8 # from the kernel sources of Linux >= 4.5 + class CanUseMadvFree: + ok = -1 + can_use_madv_free = CanUseMadvFree() + def madvise_free(addr, map_size): + # We don't know if we are running on a recent enough kernel + # that supports MADV_FREE. Check that at runtime: if the + # first call to madvise(MADV_FREE) fails, we assume it's + # because of EINVAL and we fall back to MADV_DONTNEED. + if can_use_madv_free.ok != 0: + res = c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, MADV_FREE)) + if can_use_madv_free.ok == -1: + can_use_madv_free.ok = (rffi.cast(lltype.Signed, res) == 0) + if can_use_madv_free.ok == 0: + c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, MADV_DONTNEED)) + elif has_madvise and not (MADV_FREE is MADV_DONTNEED is None): + use_flag = MADV_FREE if MADV_FREE is not None else MADV_DONTNEED + def madvise_free(addr, map_size): + c_madvise_safe(rffi.cast(PTR, addr), + rffi.cast(size_t, map_size), + rffi.cast(rffi.INT, use_flag)) + else: + def madvice_free(addr, map_size): + "No madvice() on this platform" + elif _MS_WINDOWS: def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0): # XXX flags is or-ed into access by now. @@ -907,4 +954,11 @@ def free(ptr, map_size): VirtualFree_safe(ptr, 0, MEM_RELEASE) -# register_external here? + def madvise_free(addr, map_size): + r = _VirtualAlloc_safe_no_wrapper( + rffi.cast(rffi.VOIDP, addr), + rffi.cast(rffi.SIZE_T, map_size), + rffi.cast(DWORD, MEM_RESET), + rffi.cast(DWORD, PAGE_READWRITE)) + from rpython.rlib import debug + debug.debug_print("madvise_free:", r) diff --git a/rpython/rlib/test/test_nonconst.py b/rpython/rlib/test/test_nonconst.py --- a/rpython/rlib/test/test_nonconst.py +++ b/rpython/rlib/test/test_nonconst.py @@ -7,6 +7,7 @@ from rpython.annotator.annrpython import RPythonAnnotator from rpython.conftest import option from rpython.annotator.model import SomeInstance +from rpython.rtyper.test.test_llinterp import interpret def test_nonconst(): def nonconst_f(): @@ -18,7 +19,6 @@ assert s.knowntype is int assert not hasattr(s, 'const') - def test_nonconst_list(): def nonconst_l(): a = NonConstant([1, 2, 3]) @@ -56,3 +56,8 @@ if option.view: a.translator.view() + +def test_already_not_const(): + def fn(x): + return NonConstant(x) + assert interpret(fn, [5]) == 5 diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -5,6 +5,8 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib import rmmap as mmap from rpython.rlib.rmmap import RTypeError, RValueError, alloc, free +from rpython.rlib.rmmap import madvise_free + class TestMMap: def setup_class(cls): @@ -490,6 +492,7 @@ data[i] = chr(i & 0xff) for i in range(0, map_size, 171): assert data[i] == chr(i & 0xff) + madvise_free(data, map_size) free(data, map_size) def test_compile_alloc_free(): diff --git a/rpython/rtyper/lltypesystem/llarena.py b/rpython/rtyper/lltypesystem/llarena.py --- a/rpython/rtyper/lltypesystem/llarena.py +++ b/rpython/rtyper/lltypesystem/llarena.py @@ -52,7 +52,7 @@ del self.objectptrs[offset] del self.objectsizes[offset] obj._free() - if zero and zero != 3: + if zero in (1, 2): initialbyte = "0" else: initialbyte = "#" @@ -335,6 +335,8 @@ * 1: clear, optimized for a very large area of memory * 2: clear, optimized for a small or medium area of memory * 3: fill with garbage + * 4: large area of memory that can benefit from MADV_FREE + (i.e. contains garbage, may be zero-filled or not) """ arena_addr = getfakearenaaddress(arena_addr) arena_addr.arena.reset(zero, arena_addr.offset, size) @@ -410,16 +412,19 @@ self.pagesize = 0 def _cleanup_(self): self.pagesize = 0 + def get(self): + pagesize = self.pagesize + if pagesize == 0: + pagesize = rffi.cast(lltype.Signed, legacy_getpagesize()) + self.pagesize = pagesize + return pagesize + posixpagesize = PosixPageSize() def clear_large_memory_chunk(baseaddr, size): from rpython.rlib import rmmap - pagesize = posixpagesize.pagesize - if pagesize == 0: - pagesize = rffi.cast(lltype.Signed, legacy_getpagesize()) - posixpagesize.pagesize = pagesize - + pagesize = posixpagesize.get() if size > 2 * pagesize: lowbits = rffi.cast(lltype.Signed, baseaddr) & (pagesize - 1) if lowbits: # clear the initial misaligned part, if any @@ -443,6 +448,24 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear + class PosixPageSize: + def get(self): + from rpython.rlib import rmmap + return rmmap.PAGESIZE + posixpagesize = PosixPageSize() + +def madvise_arena_free(baseaddr, size): + from rpython.rlib import rmmap + + pagesize = posixpagesize.get() + baseaddr = rffi.cast(lltype.Signed, baseaddr) + aligned_addr = (baseaddr + pagesize - 1) & ~(pagesize - 1) + size -= (aligned_addr - baseaddr) + if size >= pagesize: + rmmap.madvise_free(rffi.cast(rmmap.PTR, aligned_addr), + size & ~(pagesize - 1)) + + if os.name == "posix": from rpython.translator.tool.cbuild import ExternalCompilationInfo _eci = ExternalCompilationInfo(includes=['sys/mman.h']) @@ -509,6 +532,8 @@ clear_large_memory_chunk(arena_addr, size) elif zero == 3: llop.raw_memset(lltype.Void, arena_addr, ord('#'), size) + elif zero == 4: + madvise_arena_free(arena_addr, size) else: llmemory.raw_memclear(arena_addr, size) llimpl_arena_reset._always_inline_ = True diff --git a/rpython/rtyper/lltypesystem/test/test_llarena.py b/rpython/rtyper/lltypesystem/test/test_llarena.py --- a/rpython/rtyper/lltypesystem/test/test_llarena.py +++ b/rpython/rtyper/lltypesystem/test/test_llarena.py @@ -1,6 +1,6 @@ -import py +import py, os -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llarena from rpython.rtyper.lltypesystem.llarena import (arena_malloc, arena_reset, arena_reserve, arena_free, round_up_for_allocation, ArenaError, arena_new_view, arena_shrink_obj, arena_protect, has_protect) @@ -299,6 +299,29 @@ p.x = 125 assert p.x == 125 +def test_madvise_arena_free(): + from rpython.rlib import rmmap + + if os.name != 'posix': + py.test.skip("posix only") + pagesize = llarena.posixpagesize.get() + prev = rmmap.madvise_free + try: + seen = [] + def my_madvise_free(addr, size): + assert lltype.typeOf(addr) == rmmap.PTR + seen.append((addr, size)) + rmmap.madvise_free = my_madvise_free + llarena.madvise_arena_free( + rffi.cast(llmemory.Address, 123 * pagesize + 1), + pagesize * 7 - 2) + finally: + rmmap.madvise_free = prev + assert len(seen) == 1 + addr, size = seen[0] + assert rffi.cast(lltype.Signed, addr) == 124 * pagesize + assert size == pagesize * 5 + class TestStandalone(test_standalone.StandaloneTests): def test_compiled_arena_protect(self): _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit