Author: Richard Plangger <planri...@gmail.com> Branch: py3.5 Changeset: r86460:304a36465049 Date: 2016-08-24 10:29 +0200 http://bitbucket.org/pypy/pypy/changeset/304a36465049/
Log: merge py3k diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -166,8 +166,8 @@ else: return self.value - def __buffer__(self): - return memoryview(self._buffer) + def __buffer__(self, flags): + return buffer(self._buffer) def _get_b_base(self): try: @@ -208,7 +208,7 @@ def cdata_from_address(self, address): # fix the address: turn it into as unsigned, in case it's a negative number - address = address & (sys.maxsize * 2 + 1) + address = address & (sys.maxint * 2 + 1) instance = self.__new__(self) lgt = getattr(self, '_length_', 1) instance._buffer = self._ffiarray.fromaddress(address, lgt) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -515,7 +515,7 @@ tovar, errcode) return # - elif isinstance(tp, (model.StructOrUnion, model.EnumType)): + elif isinstance(tp, model.StructOrUnionOrEnum): # a struct (not a struct pointer) as a function argument self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' % (tovar, self._gettypenum(tp), fromvar)) @@ -572,7 +572,7 @@ elif isinstance(tp, model.ArrayType): return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(model.PointerType(tp.item))) - elif isinstance(tp, model.StructType): + elif isinstance(tp, model.StructOrUnion): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( tp._get_c_name(), context)) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -308,7 +308,7 @@ elif isinstance(tp, model.ArrayType): return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( var, self._gettypenum(model.PointerType(tp.item))) - elif isinstance(tp, model.StructType): + elif isinstance(tp, model.StructOrUnion): if tp.fldnames is None: raise TypeError("'%s' is used as %s, but is opaque" % ( tp._get_c_name(), context)) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -144,3 +144,14 @@ ``type.__dict__`` now returns a ``dict_proxy`` object, like on CPython. Previously it returned what looked like a regular dict object (but it was already read-only). + + +.. branch: const-fold-we-are-jitted + +Reduce the size of the generated C code by constant-folding ``we_are_jitted`` +in non-jitcode. + +.. branch: memoryview-attributes + +Support for memoryview attributes (format, itemsize, ...). +Extends the cpyext emulation layer. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -585,6 +585,11 @@ self.sys = Module(self, w_name) self.sys.install() + from pypy.module.imp import Module + w_name = self.wrap('imp') + mod = Module(self, w_name) + mod.install() + from pypy.module.__builtin__ import Module w_name = self.wrap('builtins') self.builtin = Module(self, w_name) @@ -1996,7 +2001,7 @@ ObjSpace.IrregularOpTable = [ 'wrap', - 'bytes_w', + 'str_w', 'int_w', 'float_w', 'uint_w', diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -79,7 +79,7 @@ class AppTestRecompiler: - spaceconfig = dict(usemodules=['_cffi_backend', 'imp', 'cpyext', 'struct']) + spaceconfig = dict(usemodules=['_cffi_backend', 'imp']) def setup_class(cls): if cls.runappdirect: diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -120,7 +120,7 @@ Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O -Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS +Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_CLEANUP_SUPPORTED """.split() @@ -651,6 +651,7 @@ #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)), ('internal', rffi.VOIDP) )) +Py_bufferP = lltype.Ptr(Py_buffer) @specialize.memo() def is_PyObject(TYPE): diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,8 +1,37 @@ +from pypy.interpreter.error import oefmt from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib import buffer from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, Py_buffer) -from pypy.module.cpyext.pyobject import PyObject, Py_DecRef +from pypy.module.cpyext.pyobject import PyObject + +@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) +def PyObject_CheckBuffer(space, w_obj): + """Return 1 if obj supports the buffer interface otherwise 0.""" + return 0 # the bf_getbuffer field is never filled by cpyext + +@cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], + rffi.INT_real, error=-1) +def PyObject_GetBuffer(space, w_obj, view, flags): + """Export obj into a Py_buffer, view. These arguments must + never be NULL. The flags argument is a bit field indicating what + kind of buffer the caller is prepared to deal with and therefore what + kind of buffer the exporter is allowed to return. The buffer interface + allows for complicated memory sharing possibilities, but some caller may + not be able to handle all the complexity but may want to see if the + exporter will let them take a simpler view to its memory. + + Some exporters may not be able to share memory in every possible way and + may need to raise errors to signal to some consumers that something is + just not possible. These errors should be a BufferError unless + there is another error that is actually causing the problem. The + exporter can use flags information to simplify how much of the + Py_buffer structure is filled in with non-default values and/or + raise an error if the object can't support a simpler view of its memory. + + 0 is returned on success and -1 on error.""" + raise oefmt(space.w_TypeError, + "PyPy does not yet implement the new buffer interface") @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL) def PyBuffer_IsContiguous(space, view, fortran): diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -16,7 +16,7 @@ @cpython_api([PyObject], PyObject) def PyMemoryView_GET_BASE(space, w_obj): # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER - raise NotImplementedError + raise NotImplementedError('PyMemoryView_GET_BUFFER') @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL) def PyMemoryView_GET_BUFFER(space, w_obj): diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -7,11 +7,11 @@ cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES, Py_buffer, mangle_name, pypy_decl) from pypy.module.cpyext.typeobjectdefs import ( - unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, + unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, ternaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, - getbufferproc, ssizessizeobjargproc) + getbufferproc, readbufferproc, ssizessizeobjargproc) from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -298,11 +298,23 @@ # Similar to Py_buffer _immutable_ = True - def __init__(self, ptr, size, w_obj): + def __init__(self, ptr, size, w_obj, format='B', shape=None, + strides=None, ndim=1, itemsize=1, readonly=True): self.ptr = ptr self.size = size self.w_obj = w_obj # kept alive - self.readonly = True + self.format = format + if not shape: + self.shape = [size] + else: + self.shape = shape + if not strides: + self.strides = [1] + else: + self.strides = strides + self.ndim = ndim + self.itemsize = itemsize + self.readonly = readonly def getlength(self): return self.size @@ -313,14 +325,38 @@ def get_raw_address(self): return rffi.cast(rffi.CCHARP, self.ptr) + def getformat(self): + return self.format + + def getshape(self): + return self.shape + + def getitemsize(self): + return self.itemsize + def wrap_getbuffer(space, w_self, w_args, func): func_target = rffi.cast(getbufferproc, func) - with lltype.scoped_alloc(Py_buffer) as view: - flags = rffi.cast(rffi.INT_real, 0) - ret = generic_cpy_call(space, func_target, w_self, view, flags) - if rffi.cast(lltype.Signed, ret) == -1: + with lltype.scoped_alloc(Py_buffer) as pybuf: + _flags = 0 + if space.len_w(w_args) > 0: + _flags = space.int_w(space.listview(w_args)[0]) + flags = rffi.cast(rffi.INT_real,_flags) + size = generic_cpy_call(space, func_target, w_self, pybuf, flags) + if widen(size) < 0: space.fromcache(State).check_and_raise_exception(always=True) - return space.newbuffer(CPyBuffer(view.c_buf, view.c_len, w_self)) + ptr = pybuf.c_buf + size = pybuf.c_len + ndim = widen(pybuf.c_ndim) + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = [pybuf.c_strides[i] for i in range(ndim)] + if pybuf.c_format: + format = rffi.charp2str(pybuf.c_format) + else: + format = 'B' + return space.newbuffer(CPyBuffer(ptr, size, w_self, format=format, + ndim=ndim, shape=shape, strides=strides, + itemsize=pybuf.c_itemsize, + readonly=widen(pybuf.c_readonly))) def get_richcmp_func(OP_CONST): def inner(space, w_self, w_args, func): @@ -542,6 +578,21 @@ w_stararg=w_args, w_starstararg=w_kwds) return space.call_args(space.get(new_fn, w_self), args) api_func = slot_tp_new.api_func + elif name == 'tp_as_buffer.c_bf_getbuffer': + buff_fn = w_type.getdictvalue(space, '__buffer__') + if buff_fn is None: + return + @cpython_api([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, header=None, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def buff_w(space, w_self, pybuf, flags): + # XXX this is wrong, needs a test + raise oefmt(space.w_NotImplemented, + "calling bf_getbuffer on a builtin type not supported yet") + #args = Arguments(space, [w_self], + # w_stararg=w_args, w_starstararg=w_kwds) + #return space.call_args(space.get(buff_fn, w_self), args) + api_func = buff_w.api_func else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length @@ -673,23 +724,23 @@ "x.__delitem__(y) <==> del x[y]"), BINSLOT("__add__", nb_add, slot_nb_add, - "+"), + "+"), RBINSLOT("__radd__", nb_add, slot_nb_add, "+"), BINSLOT("__sub__", nb_subtract, slot_nb_subtract, - "-"), + "-"), RBINSLOT("__rsub__", nb_subtract, slot_nb_subtract, "-"), BINSLOT("__mul__", nb_multiply, slot_nb_multiply, - "*"), + "*"), RBINSLOT("__rmul__", nb_multiply, slot_nb_multiply, "*"), BINSLOT("__mod__", nb_remainder, slot_nb_remainder, - "%"), + "%"), RBINSLOT("__rmod__", nb_remainder, slot_nb_remainder, "%"), BINSLOTNOTINFIX("__divmod__", nb_divmod, slot_nb_divmod, - "divmod(x, y)"), + "divmod(x, y)"), RBINSLOTNOTINFIX("__rdivmod__", nb_divmod, slot_nb_divmod, "divmod(y, x)"), NBSLOT("__pow__", nb_power, slot_nb_power, wrap_ternaryfunc, @@ -819,11 +870,19 @@ slotdefs = eval(slotdefs_str) # PyPy addition slotdefs += ( + # XXX that might not be what we want! TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""), ) +if not PY3: + slotdefs += ( + TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""), + ) + + # partial sort to solve some slot conflicts: # Number slots before Mapping slots before Sequence slots. +# also prefer the new buffer interface # These are the only conflicts between __name__ methods def slotdef_sort_key(slotdef): if slotdef.slot_name.startswith('tp_as_number'): @@ -832,6 +891,10 @@ return 2 if slotdef.slot_name.startswith('tp_as_sequence'): return 3 + if slotdef.slot_name == 'tp_as_buffer.c_bf_getbuffer': + return 100 + if slotdef.slot_name == 'tp_as_buffer.c_bf_getreadbuffer': + return 101 return 0 slotdefs = sorted(slotdefs, key=slotdef_sort_key) diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/buffer_test.c @@ -0,0 +1,243 @@ +#include <Python.h> +#include <stdlib.h> +#include <stdio.h> + +/* + * Adapted from https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol, + * which is copyright Jake Vanderplas and released under the BSD license + */ + +/* Structure defines a 1-dimensional strided array */ +typedef struct{ + int* arr; + long length; +} MyArray; + +/* initialize the array with integers 0...length */ +void initialize_MyArray(MyArray* a, long length){ + int i; + a->length = length; + a->arr = (int*)malloc(length * sizeof(int)); + for(i=0; i<length; i++){ + a->arr[i] = i; + } +} + +/* free the memory when finished */ +void deallocate_MyArray(MyArray* a){ + free(a->arr); + a->arr = NULL; +} + +/* tools to print the array */ +char* stringify(MyArray* a, int nmax){ + char* output = (char*) malloc(nmax * 20); + int k, pos = sprintf(&output[0], "["); + + for (k=0; k < a->length && k < nmax; k++){ + pos += sprintf(&output[pos], " %d", a->arr[k]); + } + if(a->length > nmax) + pos += sprintf(&output[pos], "..."); + sprintf(&output[pos], " ]"); + return output; +} + +void print_MyArray(MyArray* a, int nmax){ + char* s = stringify(a, nmax); + printf("%s", s); + free(s); +} + +/* This is where we define the PyMyArray object structure */ +typedef struct { + PyObject_HEAD + /* Type-specific fields go below. */ + MyArray arr; +} PyMyArray; + + +/* This is the __init__ function, implemented in C */ +static int +PyMyArray_init(PyMyArray *self, PyObject *args, PyObject *kwds) +{ + // init may have already been called + if (self->arr.arr != NULL) { + deallocate_MyArray(&self->arr); + } + + int length = 0; + static char *kwlist[] = {"length", NULL}; + if (! PyArg_ParseTupleAndKeywords(args, kwds, "|i", kwlist, &length)) + return -1; + + if (length < 0) + length = 0; + + initialize_MyArray(&self->arr, length); + + return 0; +} + + +/* this function is called when the object is deallocated */ +static void +PyMyArray_dealloc(PyMyArray* self) +{ + deallocate_MyArray(&self->arr); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* This function returns the string representation of our object */ +static PyObject * +PyMyArray_str(PyMyArray * self) +{ + char* s = stringify(&self->arr, 10); + PyObject* ret = PyUnicode_FromString(s); + free(s); + return ret; +} + +/* Here is the buffer interface function */ +static int +PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (view == NULL) { + PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer"); + return -1; + } + if (flags == 0) { + PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer"); + return -1; + } + + PyMyArray* self = (PyMyArray*)obj; + view->obj = (PyObject*)self; + view->buf = (void*)self->arr.arr; + view->len = self->arr.length * sizeof(int); + view->readonly = 0; + view->itemsize = sizeof(int); + view->format = "i"; // integer + view->ndim = 1; + view->shape = &self->arr.length; // length-1 sequence of dimensions + view->strides = &view->itemsize; // for the simple case we can do this + view->suboffsets = NULL; + view->internal = NULL; + + Py_INCREF(self); // need to increase the reference count + return 0; +} + +static PyBufferProcs PyMyArray_as_buffer = { +#if PY_MAJOR_VERSION < 3 + (readbufferproc)0, + (writebufferproc)0, + (segcountproc)0, + (charbufferproc)0, +#endif + (getbufferproc)PyMyArray_getbuffer, + (releasebufferproc)0, // we do not require any special release function +}; + + +/* Here is the type structure: we put the above functions in the appropriate place + in order to actually define the Python object type */ +static PyTypeObject PyMyArrayType = { + PyVarObject_HEAD_INIT(NULL, 0) + "pymyarray.PyMyArray", /* tp_name */ + sizeof(PyMyArray), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyMyArray_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)PyMyArray_str, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)PyMyArray_str, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + &PyMyArray_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ + "PyMyArray object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyMyArray_init, /* tp_init */ +}; + +static PyMethodDef buffer_functions[] = { + {NULL, NULL} /* Sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "buffer_test", + "Module Doc", + -1, + buffer_functions; + NULL, + NULL, + NULL, + NULL, +}; +#define INITERROR return NULL + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +PyInit_buffer_test(void) + +#else + +#define INITERROR return + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +initbuffer_test(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *m= PyModule_Create(&moduledef); +#else + PyObject *m= Py_InitModule("buffer_test", buffer_functions); +#endif + if (m == NULL) + INITERROR; + PyMyArrayType.tp_new = PyType_GenericNew; + if (PyType_Ready(&PyMyArrayType) < 0) + INITERROR; + Py_INCREF(&PyMyArrayType); + PyModule_AddObject(m, "PyMyArray", (PyObject *)&PyMyArrayType); +#if PY_MAJOR_VERSION >=3 + return m; +#endif +} diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -9,7 +9,10 @@ py.test.skip("unsupported before Python 2.7") w_hello = space.newbytes("hello") + assert api.PyObject_CheckBuffer(w_hello) w_view = api.PyMemoryView_FromObject(w_hello) + w_char = space.call_method(w_view, '__getitem__', space.wrap(0)) + assert space.eq_w(w_char, space.wrap('h')) w_bytes = space.call_method(w_view, "tobytes") assert space.unwrap(w_bytes) == "hello" @@ -55,3 +58,15 @@ @pytest.mark.skipif(True, reason='write a test for this') def test_get_base_and_get_buffer(self, space, api): assert False # XXX test PyMemoryView_GET_BASE, PyMemoryView_GET_BUFFER + +class AppTestBufferProtocol(AppTestCpythonExtensionBase): + def test_buffer_protocol(self): + import struct + module = self.import_module(name='buffer_test') + arr = module.PyMyArray(10) + y = memoryview(arr) + assert y.format == 'i' + assert y.shape == (10,) + s = y[3] + assert len(s) == struct.calcsize('i') + assert s == struct.pack('i', 3) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -17,7 +17,8 @@ generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder, - PyObjectFields, Py_TPFLAGS_BASETYPE, Py_buffer, PyTypeObject, PyTypeObjectPtr) + PyObjectFields, Py_TPFLAGS_BASETYPE, PyTypeObject, PyTypeObjectPtr, + Py_TPFLAGS_HAVE_NEWBUFFER) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction_typedef, PyMethodDef, W_PyCMethodObject, W_PyCFunctionObject) @@ -514,6 +515,7 @@ bytes_getbuffer.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf pto.c_tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER + pto.c_tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER @cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): @@ -681,6 +683,8 @@ pto.c_tp_setattro = base.c_tp_setattro if not pto.c_tp_getattro: pto.c_tp_getattro = base.c_tp_getattro + if not pto.c_tp_as_buffer: + pto.c_tp_as_buffer = base.c_tp_as_buffer finally: Py_DecRef(space, base_pyo) diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py --- a/pypy/module/cpyext/typeobjectdefs.py +++ b/pypy/module/cpyext/typeobjectdefs.py @@ -1,10 +1,11 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.lltypesystem.lltype import Ptr, FuncType, Void from pypy.module.cpyext.api import (cpython_struct, Py_ssize_t, Py_ssize_tP, - PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_buffer, + PyVarObjectFields, PyTypeObject, PyTypeObjectPtr, FILEP, Py_TPFLAGS_READYING, Py_TPFLAGS_READY, Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.modsupport import PyMethodDef +from pypy.module.cpyext.api import Py_bufferP P, FT, PyO = Ptr, FuncType, PyObject @@ -53,7 +54,11 @@ wrapperfunc = P(FT([PyO, PyO, rffi.VOIDP], PyO)) wrapperfunc_kwds = P(FT([PyO, PyO, rffi.VOIDP, PyO], PyO)) -getbufferproc = P(FT([PyO, Ptr(Py_buffer), rffi.INT_real], rffi.INT_real)) +readbufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t)) +writebufferproc = P(FT([PyO, Py_ssize_t, rffi.VOIDPP], Py_ssize_t)) +segcountproc = P(FT([PyO, Py_ssize_tP], Py_ssize_t)) +charbufferproc = P(FT([PyO, Py_ssize_t, rffi.CCHARPP], Py_ssize_t)) +getbufferproc = P(FT([PyO, Py_bufferP, rffi.INT_real], rffi.INT_real)) releasebufferproc = P(FT([PyO, Ptr(Py_buffer)], Void)) @@ -126,6 +131,10 @@ )) PyBufferProcs = cpython_struct("PyBufferProcs", ( + ("bf_getreadbuffer", readbufferproc), + ("bf_getwritebuffer", writebufferproc), + ("bf_getsegcount", segcountproc), + ("bf_getcharbuffer", charbufferproc), ("bf_getbuffer", getbufferproc), ("bf_releasebuffer", releasebufferproc), )) diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -19,7 +19,6 @@ UserDelAction) from pypy.interpreter.pyframe import PyFrame - class BogusBytecode(Exception): pass @@ -383,6 +382,9 @@ # XXX even the hacks have hacks if s == 'size': # used in _array() but never called by tests return IntObject(0) + if s == '__buffer__': + # descr___buffer__ does not exist on W_Root + return self.w_None return getattr(w_obj, 'descr_' + s)(self, *args) @specialize.arg(1) diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -704,3 +704,20 @@ def get_raw_address(self): from rpython.rtyper.lltypesystem import rffi return rffi.ptradd(self.impl.storage, self.impl.start) + + def getformat(self): + return self.impl.dtype.char + + def getitemsize(self): + return self.impl.dtype.elsize + + def getndim(self): + return len(self.impl.shape) + + def getshape(self): + return self.impl.shape + + def getstrides(self): + return self.impl.strides + + diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -468,7 +468,8 @@ except OperationError as e: if not e.match(space, space.w_TypeError): raise - w_buffer = space.getattr(w_buffer, space.wrap('__buffer__')) + w_buffer = space.call_method(w_buffer, '__buffer__', + space.newint(space.BUF_FULL_RO)) buf = _getbuffer(space, w_buffer) ts = buf.getlength() diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -3617,13 +3617,35 @@ assert str(exc.value) == "assignment destination is read-only" class A(object): - __buffer__ = 'abc' + def __buffer__(self, flags): + return 'abc' data = A() a = np.frombuffer(data, 'c') #assert a.base is data.__buffer__ assert a.tostring() == 'abc' + def test_memoryview(self): + import numpy as np + import sys + if sys.version_info[:2] > (3, 2): + # In Python 3.3 the representation of empty shape, strides and sub-offsets + # is an empty tuple instead of None. + # http://docs.python.org/dev/whatsnew/3.3.html#api-changes + EMPTY = () + else: + EMPTY = None + x = np.array([1, 2, 3, 4, 5], dtype='i') + y = memoryview('abc') + assert y.format == 'B' + y = memoryview(x) + assert y.format == 'i' + assert y.shape == (5,) + assert y.ndim == 1 + assert y.strides == (4,) + assert y.suboffsets == EMPTY + assert y.itemsize == 4 + def test_fromstring(self): import sys from numpy import fromstring, dtype diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -423,7 +423,7 @@ guard_false(i114, descr=...) --TICK-- i123 = arraylen_gc(p67, descr=<ArrayP .>) - i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize__Signed), 6, descr=<Calli . i EF=5 OS=110>) + i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=<Calli . i EF=5 OS=110>) check_memory_error(i119) raw_store(i119, 0, i160, descr=<ArrayS 2>) raw_store(i119, 2, i160, descr=<ArrayS 2>) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -1963,3 +1963,21 @@ ffi, "test_function_returns_opaque", "?") assert str(e.value) == ("function foo: 'struct a' is used as result type," " but is opaque") + +def test_function_returns_union(): + ffi = FFI() + ffi.cdef("union u1 { int a, b; }; union u1 f1(int);") + lib = verify(ffi, "test_function_returns_union", """ + union u1 { int a, b; }; + static union u1 f1(int x) { union u1 u; u.b = x; return u; } + """) + assert lib.f1(51).a == 51 + +def test_function_returns_partial_struct(): + ffi = FFI() + ffi.cdef("struct a { int a; ...; }; struct a f1(int);") + lib = verify(ffi, "test_function_returns_partial_struct", """ + struct a { int b, a, c; }; + static struct a f1(int x) { struct a s = {0}; s.a = x; return s; } + """) + assert lib.f1(52).a == 52 diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py --- a/pypy/objspace/std/bytesobject.py +++ b/pypy/objspace/std/bytesobject.py @@ -423,6 +423,18 @@ space.check_buf_flags(flags, True) return StringBuffer(self._value) + def readbuf_w(self, space): + return StringBuffer(self._value) + + def writebuf_w(self, space): + raise oefmt(space.w_TypeError, + "Cannot use string as modifiable buffer") + + def descr_getbuffer(self, space, w_flags): + #from pypy.objspace.std.bufferobject import W_Buffer + #return W_Buffer(StringBuffer(self._value)) + return self + def listview_int(self): return _create_list_from_bytes(self._value) @@ -654,11 +666,6 @@ def descr_upper(self, space): return W_BytesObject(self._value.upper()) - def descr_hex(self, space): - from pypy.objspace.std.bytearrayobject import _array_to_hexstring - return _array_to_hexstring(space, self.buffer_w(space, space.BUF_SIMPLE)) - - def _create_list_from_bytes(value): # need this helper function to allow the jit to look inside and inline @@ -838,7 +845,6 @@ fromhex = interp2app(W_BytesObject.descr_fromhex, as_classmethod=True), maketrans = interp2app(W_BytesObject.descr_maketrans, as_classmethod=True), - hex = interp2app(W_BytesObject.descr_hex) ) W_BytesObject.typedef.flag_sequence_bug_compat = True diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -1030,4 +1030,5 @@ if x >= HASH_MODULUS: x -= HASH_MODULUS - return intmask(intmask(x) * sign) + h = intmask(intmask(x) * sign) + return h - (h == -1) diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -254,12 +254,12 @@ def _make_generic_descr_binop(opname): if opname not in COMMUTATIVE_OPS: raise Exception("Not supported") - + methname = opname + '_' if opname in ('and', 'or') else opname descr_rname = 'descr_r' + opname op = getattr(rbigint, methname) intop = getattr(rbigint, "int_" + methname) - + @func_renamer('descr_' + opname) def descr_binop(self, space, w_other): if isinstance(w_other, W_IntObject): @@ -279,7 +279,7 @@ return W_LongObject(op(w_other.asbigint(), self.num)) return descr_binop, descr_rbinop - + descr_add, descr_radd = _make_generic_descr_binop('add') descr_sub, descr_rsub = _make_generic_descr_binop_noncommutative('sub') descr_mul, descr_rmul = _make_generic_descr_binop('mul') @@ -321,12 +321,12 @@ except OverflowError: # b too big raise oefmt(space.w_OverflowError, "shift count too large") return W_LongObject(self.num.lshift(shift)) - + def _int_lshift(self, space, w_other): if w_other < 0: raise oefmt(space.w_ValueError, "negative shift count") return W_LongObject(self.num.lshift(w_other)) - + descr_lshift, descr_rlshift = _make_descr_binop(_lshift, _int_lshift) def _rshift(self, space, w_other): @@ -337,7 +337,7 @@ except OverflowError: # b too big # XXX maybe just return 0L instead? raise oefmt(space.w_OverflowError, "shift count too large") return newlong(space, self.num.rshift(shift)) - + def _int_rshift(self, space, w_other): if w_other < 0: raise oefmt(space.w_ValueError, "negative shift count") @@ -352,7 +352,7 @@ raise oefmt(space.w_ZeroDivisionError, "long division or modulo by zero") return newlong(space, z) - + def _floordiv(self, space, w_other): try: z = self.num.floordiv(w_other.asbigint()) @@ -369,7 +369,7 @@ raise oefmt(space.w_ZeroDivisionError, "integer division or modulo by zero") return newlong(space, z) - + def _int_mod(self, space, w_other): try: z = self.num.int_mod(w_other) @@ -404,7 +404,8 @@ while x >= HASH_MODULUS: x -= HASH_MODULUS i -= 1 - return intmask(intmask(x) * v.sign) + h = intmask(intmask(x) * v.sign) + return h - (h == -1) def newlong(space, bigint): diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -82,18 +82,29 @@ def descr_getitem(self, space, w_index): self._check_released(space) start, stop, step, size = space.decode_index4(w_index, self.getlength()) + itemsize = self.buf.getitemsize() + if itemsize > 1: + start *= itemsize + size *= itemsize + stop = start + size + if step == 0: + step = 1 + if stop > self.getlength(): + raise oefmt(space.w_IndexError, 'index out of range') + if step not in (0, 1): + raise oefmt(space.w_NotImplementedError, "") if step == 0: # index only # TODO: this probably isn't very fast - buf = SubBuffer(self.buf, start * self.itemsize, self.itemsize) + buf = SubBuffer(self.buf, start, self.itemsize) fmtiter = UnpackFormatIterator(space, buf) fmtiter.interpret(self.format) return fmtiter.result_w[0] elif step == 1: - buf = SubBuffer(self.buf, start * self.itemsize, - size * self.itemsize) + buf = SubBuffer(self.buf, start, size) return W_MemoryView(buf, self.format, self.itemsize) else: - raise oefmt(space.w_NotImplementedError, "") + buf = SubBuffer(self.buf, start, size) + return W_MemoryView(buf) def descr_setitem(self, space, w_index, w_obj): self._check_released(space) @@ -102,6 +113,21 @@ if space.isinstance_w(w_index, space.w_tuple): raise oefmt(space.w_NotImplementedError, "") start, stop, step, size = space.decode_index4(w_index, self.getlength()) + itemsize = self.buf.getitemsize() + if itemsize > 1: + start *= itemsize + size *= itemsize + stop = start + size + if step == 0: + step = 1 + if stop > self.getlength(): + raise oefmt(space.w_IndexError, 'index out of range') + if step not in (0, 1): + raise oefmt(space.w_NotImplementedError, "") + value = space.buffer_w(w_obj, space.BUF_CONTIG_RO) + if value.getlength() != size: + raise oefmt(space.w_ValueError, + "cannot modify size of memoryview object") if step == 0: # index only # TODO: this probably isn't very fast fmtiter = PackFormatIterator(space, [w_obj], self.itemsize) @@ -111,43 +137,43 @@ raise oefmt(space.w_TypeError, "memoryview: invalid type for format '%s'", self.format) - self.buf.setslice(start * self.itemsize, fmtiter.result.build()) + self.buf.setslice(start, fmtiter.result.build()) elif step == 1: value = space.buffer_w(w_obj, space.BUF_CONTIG_RO) if value.getlength() != size * self.itemsize: raise oefmt(space.w_ValueError, "cannot modify size of memoryview object") - self.buf.setslice(start * self.itemsize, value.as_str()) + self.buf.setslice(start, value.as_str()) else: raise oefmt(space.w_NotImplementedError, "") def descr_len(self, space): self._check_released(space) - return space.wrap(self.getlength()) + return space.wrap(self.buf.getlength()) def w_get_format(self, space): self._check_released(space) - return space.wrap(self.format) + return space.wrap(self.buf.getformat()) def w_get_itemsize(self, space): self._check_released(space) - return space.wrap(self.itemsize) + return space.wrap(self.buf.getitemsize()) def w_get_ndim(self, space): self._check_released(space) - return space.wrap(1) + return space.wrap(self.buf.getndim()) def w_is_readonly(self, space): self._check_released(space) - return space.wrap(self.buf.readonly) + return space.newbool(bool(self.buf.readonly)) def w_get_shape(self, space): self._check_released(space) - return space.newtuple([space.wrap(self.getlength())]) + return space.newtuple([space.wrap(x) for x in self.buf.getshape()]) def w_get_strides(self, space): self._check_released(space) - return space.newtuple([space.wrap(self.itemsize)]) + return space.newtuple([space.wrap(x) for x in self.buf.getstrides()]) def w_get_suboffsets(self, space): self._check_released(space) @@ -237,7 +263,6 @@ self._check_released(space) return _array_to_hexstring(space, self.buf) - W_MemoryView.typedef = TypeDef( "memoryview", __doc__ = """\ diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -265,7 +265,11 @@ result = f1.int(self.space) assert result == f1 -class AppTestInt: +class AppTestInt(object): + def test_hash(self): + assert hash(-1) == (-1).__hash__() == -2 + assert hash(-2) == (-2).__hash__() == -2 + def test_conjugate(self): assert (1).conjugate() == 1 assert (-1).conjugate() == -1 diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -122,6 +122,9 @@ def newlist(self, iterable): return list(iterable) + def newbytes(self, obj): + return bytes(obj) + def call_function(self, func, *args, **kwds): return func(*args, **kwds) diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -261,6 +261,9 @@ "stack based virtual machines (only for backends that support it)", default=True), BoolOption("storesink", "Perform store sinking", default=True), + BoolOption("replace_we_are_jitted", + "Replace we_are_jitted() calls by False", + default=False, cmdline=None), BoolOption("none", "Do not run any backend optimizations", requires=[('translation.backendopt.inline', False), diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -833,9 +833,6 @@ result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) return heaptracker.adr2int(result_adr) - def bh_new_raw_buffer(self, size): - return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') - # vector operations vector_arith_code = """ def bh_vec_{0}_{1}(self, vx, vy, count): diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -757,9 +757,6 @@ self.write_int_at_mem(res, self.vtable_offset, WORD, sizedescr.get_vtable()) return res - def bh_new_raw_buffer(self, size): - return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') - def bh_classof(self, struct): struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct) result_adr = llmemory.cast_ptr_to_adr(struct.typeptr) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -229,8 +229,6 @@ raise NotImplementedError def bh_newunicode(self, length): raise NotImplementedError - def bh_new_raw_buffer(self, size): - raise NotImplementedError def bh_arraylen_gc(self, array, arraydescr): raise NotImplementedError diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -582,6 +582,14 @@ return lltype.malloc(ARRAY, n, flavor='raw', zero=zero, add_memory_pressure=add_memory_pressure, track_allocation=track_allocation) + name = '_ll_1_raw_malloc_varsize' + if zero: + name += '_zero' + if add_memory_pressure: + name += '_mpressure' + if not track_allocation: + name += '_notrack' + _ll_1_raw_malloc_varsize.func_name = name return _ll_1_raw_malloc_varsize return build_ll_1_raw_malloc_varsize @@ -610,6 +618,14 @@ return lltype.malloc(STRUCT, flavor='raw', zero=zero, add_memory_pressure=add_memory_pressure, track_allocation=track_allocation) + name = '_ll_0_raw_malloc_fixedsize' + if zero: + name += '_zero' + if add_memory_pressure: + name += '_mpressure' + if not track_allocation: + name += '_notrack' + _ll_0_raw_malloc_fixedsize.func_name = name return _ll_0_raw_malloc_fixedsize return build_ll_0_raw_malloc_fixedsize diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -367,8 +367,9 @@ class RawBufferPtrInfo(AbstractRawPtrInfo): buffer = None - - def __init__(self, cpu, size=-1): + + def __init__(self, cpu, func, size=-1): + self.func = func self.size = size if self.size != -1: self.buffer = RawBuffer(cpu, None) @@ -425,7 +426,8 @@ @specialize.argtype(1) def visitor_dispatch_virtual_type(self, visitor): buffer = self._get_buffer() - return visitor.visit_vrawbuffer(self.size, + return visitor.visit_vrawbuffer(self.func, + self.size, buffer.offsets[:], buffer.descrs[:]) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1770,7 +1770,7 @@ def test_virtual_raw_malloc_basic(self): ops = """ [i1] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) # 12345 = malloc func guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr) @@ -1787,7 +1787,7 @@ ops = """ [i1] i5 = int_mul(10, 1) - i2 = call_i('malloc', i5, descr=raw_malloc_descr) + i2 = call_i(12345, i5, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr) @@ -1803,7 +1803,7 @@ def test_virtual_raw_malloc_force(self): ops = """ [i1] - i2 = call_i('malloc', 20, descr=raw_malloc_descr) + i2 = call_i(12345, 20, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr_char) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) @@ -1817,7 +1817,7 @@ expected = """ [i1] label(i1) - i2 = call_i('malloc', 20, descr=raw_malloc_descr) + i2 = call_i(12345, 20, descr=raw_malloc_descr) check_memory_error(i2) raw_store(i2, 0, i1, descr=rawarraydescr_char) raw_store(i2, 1, 123, descr=rawarraydescr_char) @@ -1832,7 +1832,7 @@ def test_virtual_raw_malloc_invalid_write_force(self): ops = """ [i1] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label(i1) # we expect the buffer to be forced *after* the label @@ -1843,7 +1843,7 @@ expected = """ [i1] label(i1) - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) check_memory_error(i2) raw_store(i2, 0, i1, descr=rawarraydescr) setarrayitem_raw(i2, 2, 456, descr=rawarraydescr_char) @@ -1855,7 +1855,7 @@ def test_virtual_raw_malloc_invalid_read_force(self): ops = """ [i1] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, i1, descr=rawarraydescr) label(i1) # we expect the buffer to be forced *after* the label @@ -1866,7 +1866,7 @@ expected = """ [i1] label(i1) - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) check_memory_error(i2) raw_store(i2, 0, i1, descr=rawarraydescr) i3 = getarrayitem_raw_i(i2, 0, descr=rawarraydescr_char) @@ -1878,7 +1878,7 @@ def test_virtual_raw_slice(self): ops = """ [i0, i1] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer @@ -1898,7 +1898,7 @@ def test_virtual_raw_slice_of_a_raw_slice(self): ops = """ [i0, i1] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] i3 = int_add(i2, 1) # get a slice of the original buffer i4 = int_add(i3, 1) # get a slice of a slice @@ -1916,7 +1916,7 @@ def test_virtual_raw_slice_force(self): ops = """ [i0, i1] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, 42, descr=rawarraydescr_char) i3 = int_add(i2, 1) # get a slice of the original buffer @@ -1929,7 +1929,7 @@ [i0, i1] label(i0, i1) # these ops are generated by VirtualRawBufferValue._really_force - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) check_memory_error(i2) raw_store(i2, 0, 42, descr=rawarraydescr_char) raw_store(i2, 5, 4242, descr=rawarraydescr_char) @@ -1946,7 +1946,7 @@ i1 = getarrayitem_raw_i(i0, 0, descr=rawarraydescr) i2 = int_add(i1, 1) call_n('free', i0, descr=raw_free_descr) - i3 = call_i('malloc', 10, descr=raw_malloc_descr) + i3 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i3, 0, i2, descr=rawarraydescr) label(i2) @@ -1958,7 +1958,7 @@ i2 = int_add(i1, 1) call_n('free', i0, descr=raw_free_descr) label(i2) - i3 = call_i('malloc', 10, descr=raw_malloc_descr) + i3 = call_i(12345, 10, descr=raw_malloc_descr) check_memory_error(i3) raw_store(i3, 0, i2, descr=rawarraydescr) jump(i3) @@ -1968,7 +1968,7 @@ def test_virtual_raw_store_raw_load(self): ops = """ [i1] - i0 = call_i('malloc', 10, descr=raw_malloc_descr) + i0 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] raw_store(i0, 0, i1, descr=rawarraydescr) i2 = raw_load_i(i0, 0, descr=rawarraydescr) @@ -1986,7 +1986,7 @@ def test_virtual_raw_store_getarrayitem_raw(self): ops = """ [f1] - i0 = call_i('malloc', 16, descr=raw_malloc_descr) + i0 = call_i(12345, 16, descr=raw_malloc_descr) guard_no_exception() [] raw_store(i0, 8, f1, descr=rawarraydescr_float) f2 = getarrayitem_raw_f(i0, 1, descr=rawarraydescr_float) @@ -2004,7 +2004,7 @@ def test_virtual_setarrayitem_raw_raw_load(self): ops = """ [f1] - i0 = call_i('malloc', 16, descr=raw_malloc_descr) + i0 = call_i(12345, 16, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i0, 1, f1, descr=rawarraydescr_float) f2 = raw_load_f(i0, 8, descr=rawarraydescr_float) @@ -2022,7 +2022,7 @@ def test_virtual_raw_buffer_forced_but_slice_not_forced(self): ops = """ [f1] - i0 = call_i('malloc', 16, descr=raw_malloc_descr) + i0 = call_i(12345, 16, descr=raw_malloc_descr) guard_no_exception() [] i1 = int_add(i0, 8) escape_n(i0) @@ -2031,7 +2031,7 @@ """ expected = """ [f1] - i0 = call_i('malloc', 16, descr=raw_malloc_descr) + i0 = call_i(12345, 16, descr=raw_malloc_descr) check_memory_error(i0) escape_n(i0) i1 = int_add(i0, 8) @@ -8886,7 +8886,7 @@ def test_resume_forced_raw_ptr(self): ops = """ [i0] - i = call_i('malloc', 10, descr=raw_malloc_descr) + i = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] is = int_add(i, 8) escape_n(i) @@ -8898,7 +8898,7 @@ """ expected = """ [i0] - i = call_i('malloc', 10, descr=raw_malloc_descr) + i = call_i(12345, 10, descr=raw_malloc_descr) check_memory_error(i) escape_n(i) i1 = int_add(i0, 1) @@ -8966,7 +8966,7 @@ def test_pending_setfield_delayed_malloc(self): ops = """ [i0, p0] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] setarrayitem_raw(i2, 0, 13, descr=rawarraydescr) setfield_gc(p0, i2, descr=valuedescr) @@ -8988,14 +8988,14 @@ def test_raw_buffer_ptr_info_intbounds_bug(self): ops = """ [] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) guard_no_exception() [] guard_value(i2, 12345) [] jump() """ expected = """ [] - i2 = call_i('malloc', 10, descr=raw_malloc_descr) + i2 = call_i(12345, 10, descr=raw_malloc_descr) check_memory_error(i2) guard_value(i2, 12345) [] jump() diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -45,7 +45,8 @@ return opinfo def make_virtual_raw_memory(self, size, source_op): - opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, size) + func = source_op.getarg(0).getint() + opinfo = info.RawBufferPtrInfo(self.optimizer.cpu, func, size) newop = self.replace_op_with(source_op, source_op.getopnum(), args=[source_op.getarg(0), ConstInt(size)]) newop.set_forwarded(opinfo) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -364,8 +364,8 @@ def visit_varraystruct(self, arraydescr, size, fielddescrs): return VArrayStructInfo(arraydescr, size, fielddescrs) - def visit_vrawbuffer(self, size, offsets, descrs): - return VRawBufferInfo(size, offsets, descrs) + def visit_vrawbuffer(self, func, size, offsets, descrs): + return VRawBufferInfo(func, size, offsets, descrs) def visit_vrawslice(self, offset): return VRawSliceInfo(offset) @@ -703,7 +703,8 @@ class VRawBufferInfo(VAbstractRawInfo): - def __init__(self, size, offsets, descrs): + def __init__(self, func, size, offsets, descrs): + self.func = func self.size = size self.offsets = offsets self.descrs = descrs @@ -711,7 +712,7 @@ @specialize.argtype(1) def allocate_int(self, decoder, index): length = len(self.fieldnums) - buffer = decoder.allocate_raw_buffer(self.size) + buffer = decoder.allocate_raw_buffer(self.func, self.size) decoder.virtuals_cache.set_int(index, buffer) for i in range(len(self.offsets)): offset = self.offsets[i] @@ -1130,9 +1131,13 @@ lengthbox) return self.metainterp.execute_new_array(arraydescr, lengthbox) - def allocate_raw_buffer(self, size): + def allocate_raw_buffer(self, func, size): cic = self.metainterp.staticdata.callinfocollection - calldescr, func = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR) + calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR) + # Can't use 'func' from callinfo_for_oopspec(), because we have + # several variants (zero/non-zero, memory-pressure or not, etc.) + # and we have to pick the correct one here; that's why we save + # it in the VRawBufferInfo. return self.metainterp.execute_and_record_varargs( rop.CALL_I, [ConstInt(func), ConstInt(size)], calldescr) @@ -1461,10 +1466,11 @@ def allocate_string(self, length): return self.cpu.bh_newstr(length) - def allocate_raw_buffer(self, size): - buffer = self.cpu.bh_new_raw_buffer(size) - adr = llmemory.cast_ptr_to_adr(buffer) - return llmemory.cast_adr_to_int(adr, "symbolic") + def allocate_raw_buffer(self, func, size): + from rpython.jit.codewriter import heaptracker + cic = self.callinfocollection + calldescr, _ = cic.callinfo_for_oopspec(EffectInfo.OS_RAW_MALLOC_VARSIZE_CHAR) + return self.cpu.bh_call_i(func, [size], None, None, calldescr) def string_setitem(self, str, index, charnum): char = self.decode_int(charnum) diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py --- a/rpython/jit/metainterp/walkvirtual.py +++ b/rpython/jit/metainterp/walkvirtual.py @@ -17,7 +17,7 @@ def visit_varraystruct(self, arraydescr, fielddescrs): raise NotImplementedError("abstract base class") - def visit_vrawbuffer(self, size, offsets, descrs): + def visit_vrawbuffer(self, func, size, offsets, descrs): raise NotImplementedError("abstract base class") def visit_vrawslice(self, offset): diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -452,7 +452,8 @@ merge_if_blocks=True, constfold=True, remove_asserts=True, - really_remove_asserts=True) + really_remove_asserts=True, + replace_we_are_jitted=False) def prejit_optimizations_minimal_inline(self, policy, graphs): from rpython.translator.backendopt.inline import auto_inline_graphs diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py --- a/rpython/rlib/buffer.py +++ b/rpython/rlib/buffer.py @@ -59,6 +59,20 @@ def get_raw_address(self): raise ValueError("no raw buffer") + def getformat(self): + return 'B' + + def getitemsize(self): + return 1 + + def getndim(self): + return 1 + + def getshape(self): + return [self.getlength()] + + def getstrides(self): + return [1] class StringBuffer(Buffer): __slots__ = ['value'] diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -799,8 +799,8 @@ rffi.cast(size_t, map_size), rffi.cast(rffi.INT, use_flag)) else: - def madvice_free(addr, map_size): - "No madvice() on this platform" + def madvise_free(addr, map_size): + "No madvise() on this platform" elif _MS_WINDOWS: def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0): diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -90,6 +90,9 @@ CodeClass._vmprof_unique_id = 0 # default value: "unknown" immut = CodeClass.__dict__.get('_immutable_fields_', []) CodeClass._immutable_fields_ = list(immut) + ['_vmprof_unique_id'] + attrs = CodeClass.__dict__.get('_attrs_', None) + if attrs is not None: + CodeClass._attrs_ = list(attrs) + ['_vmprof_unique_id'] self._code_classes.add(CodeClass) # class WeakCodeObjectList(RWeakListMixin): @@ -189,7 +192,7 @@ def decorated_function(*args): unique_id = get_code_fn(*args)._vmprof_unique_id - unique_id = rffi.cast(lltype.Signed, unique_id) + unique_id = rffi.cast(lltype.Signed, unique_id) # ^^^ removes the "known non-negative" hint for annotation if not jit.we_are_jitted(): x = enter_code(unique_id) diff --git a/rpython/translator/backendopt/all.py b/rpython/translator/backendopt/all.py --- a/rpython/translator/backendopt/all.py +++ b/rpython/translator/backendopt/all.py @@ -2,6 +2,7 @@ from rpython.translator.backendopt import inline from rpython.translator.backendopt.malloc import remove_mallocs from rpython.translator.backendopt.constfold import constant_fold_graph +from rpython.translator.backendopt.constfold import replace_we_are_jitted from rpython.translator.backendopt.stat import print_statistics from rpython.translator.backendopt.merge_if_blocks import merge_if_blocks from rpython.translator import simplify @@ -36,6 +37,7 @@ # inline_threshold, mallocs # merge_if_blocks, constfold, heap2stack # clever_malloc_removal, remove_asserts + # replace_we_are_jitted config = translator.config.translation.backendopt.copy(as_default=True) config.set(**kwds) @@ -49,6 +51,10 @@ print "before optimizations:" print_statistics(translator.graphs[0], translator, "per-graph.txt") + if config.replace_we_are_jitted: + for graph in graphs: + replace_we_are_jitted(graph) + if config.remove_asserts: constfold(config, graphs) remove_asserts(translator, graphs) diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py --- a/rpython/translator/backendopt/constfold.py +++ b/rpython/translator/backendopt/constfold.py @@ -276,3 +276,25 @@ rewire_links(splitblocks, graph) if not diffused and not splitblocks: break # finished + +def replace_symbolic(graph, symbolic, value): + result = False + for block in graph.iterblocks(): + for op in block.operations: + for i, arg in enumerate(op.args): + if isinstance(arg, Constant) and arg.value is symbolic: + op.args[i] = value + result = True + if block.exitswitch is symbolic: + block.exitswitch = value + result = True + return result + +def replace_we_are_jitted(graph): + from rpython.rlib import jit + replacement = Constant(0) + replacement.concretetype = lltype.Signed + did_replacement = replace_symbolic(graph, jit._we_are_jitted, replacement) + if did_replacement: + constant_fold_graph(graph) + return did_replacement diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py --- a/rpython/translator/backendopt/test/test_all.py +++ b/rpython/translator/backendopt/test/test_all.py @@ -289,3 +289,19 @@ llinterp = LLInterpreter(t.rtyper) res = llinterp.eval_graph(later_graph, [10]) assert res == 1 + + def test_replace_we_are_jitted(self): + from rpython.rlib import jit + def f(): + if jit.we_are_jitted(): + return 1 + return 2 + jit.we_are_jitted() + + t = self.translateopt(f, []) + graph = graphof(t, f) + # by default, replace_we_are_jitted is off + assert graph.startblock.operations[0].args[0].value is jit._we_are_jitted + + t = self.translateopt(f, [], replace_we_are_jitted=True) + graph = graphof(t, f) + assert graph.startblock.exits[0].args[0].value == 2 diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py --- a/rpython/translator/backendopt/test/test_constfold.py +++ b/rpython/translator/backendopt/test/test_constfold.py @@ -7,6 +7,7 @@ from rpython.rtyper import rclass from rpython.rlib import objectmodel from rpython.translator.backendopt.constfold import constant_fold_graph +from rpython.translator.backendopt.constfold import replace_we_are_jitted from rpython.conftest import option def get_graph(fn, signature): @@ -343,3 +344,18 @@ merge_if_blocks.merge_if_blocks_once(graph) constant_fold_graph(graph) check_graph(graph, [], 66, t) + +def test_replace_we_are_jitted(): + from rpython.rlib import jit + def fn(): + if jit.we_are_jitted(): + return 1 + return 2 + jit.we_are_jitted() + graph, t = get_graph(fn, []) + result = replace_we_are_jitted(graph) + assert result + checkgraph(graph) + # check shape of graph + assert len(graph.startblock.operations) == 0 + assert graph.startblock.exitswitch is None + assert graph.startblock.exits[0].target.exits[0].args[0].value == 2 diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -381,7 +381,7 @@ """ Run all backend optimizations - lltype version """ from rpython.translator.backendopt.all import backend_optimizations - backend_optimizations(self.translator) + backend_optimizations(self.translator, replace_we_are_jitted=True) STACKCHECKINSERTION = 'stackcheckinsertion_lltype' diff --git a/rpython/translator/test/test_interactive.py b/rpython/translator/test/test_interactive.py --- a/rpython/translator/test/test_interactive.py +++ b/rpython/translator/test/test_interactive.py @@ -78,3 +78,15 @@ dll = ctypes.CDLL(str(t.driver.c_entryp)) f = dll.pypy_g_f assert f(2, 3) == 5 + +def test_check_that_driver_uses_replace_we_are_jitted(): + from rpython.rlib import jit + def f(): + if jit.we_are_jitted(): + return 1 + return 2 + jit.we_are_jitted() + + t = Translation(f, []) + t.backendopt() + graph = t.driver.translator.graphs[0] + assert graph.startblock.exits[0].args[0].value == 2 _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit