Author: Matti Picus <matti.pi...@gmail.com> Branch: Changeset: r96232:750f4840080b Date: 2019-03-10 12:42 +0200 http://bitbucket.org/pypy/pypy/changeset/750f4840080b/
Log: merge newmemoryview-app-level into default diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -124,12 +124,15 @@ ms.n = 0xff00 return repr(ba[:]) + nstruct = dostruct(Native) if sys.byteorder == 'little': - assert dostruct(Native) == dostruct(Little) - assert dostruct(Native) != dostruct(Big) + assert nstruct == dostruct(Little) + assert nstruct != dostruct(Big) + assert Big._fields_[0][1] is not i else: - assert dostruct(Native) == dostruct(Big) - assert dostruct(Native) != dostruct(Little) + assert nstruct == dostruct(Big) + assert nstruct != dostruct(Little) + assert Little._fields_[0][1] is not i def test_from_buffer_copy(): from array import array @@ -190,3 +193,20 @@ assert sizeof(s) == 3 * sizeof(c_int) assert s.a == 4 # 256 + 4 assert s.b == -123 + +def test_memoryview(): + class S(Structure): + _fields_ = [('a', c_int16), + ('b', c_int16), + ] + + S3 = S * 3 + c_array = (2 * S3)( + S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)), + S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)), + ) + + mv = memoryview(c_array) + assert mv.format == 'T{<h:a:<h:b:}' + assert mv.shape == (2, 3) + assert mv.itemsize == 4 diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -4,6 +4,7 @@ from _ctypes.basics import _CData, cdata_from_address, _CDataMeta, sizeof from _ctypes.basics import keepalive_key, store_reference, ensure_objects from _ctypes.basics import CArgObject, as_ffi_pointer +import sys, __pypy__, struct class ArrayMeta(_CDataMeta): def __new__(self, name, cls, typedict): @@ -241,6 +242,24 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + shape = [] + obj = self + while 1: + shape.append(obj._length_) + try: + obj[0]._length_ + except (AttributeError, IndexError): + break + obj = obj[0] + + fmt = get_format_str(obj._type_) + try: + itemsize = struct.calcsize(fmt[1:]) + except: + itemsize = len(buffer(obj[0])) + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt, shape) + ARRAY_CACHE = {} def create_array_type(base, length): @@ -260,3 +279,31 @@ cls = ArrayMeta(name, (Array,), tpdict) ARRAY_CACHE[key] = cls return cls + +byteorder = {'little': '<', 'big': '>'} +swappedorder = {'little': '>', 'big': '<'} + +def get_format_str(typ): + if hasattr(typ, '_fields_'): + if hasattr(typ, '_swappedbytes_'): + bo = swappedorder[sys.byteorder] + else: + bo = byteorder[sys.byteorder] + flds = [] + for name, obj in typ._fields_: + # Trim off the leading '<' or '>' + ch = get_format_str(obj)[1:] + if (ch) == 'B': + flds.append(byteorder[sys.byteorder]) + else: + flds.append(bo) + flds.append(ch) + flds.append(':') + flds.append(name) + flds.append(':') + return 'T{' + ''.join(flds) + '}' + elif hasattr(typ, '_type_'): + ch = typ._type_ + return byteorder[sys.byteorder] + ch + else: + raise ValueError('cannot get format string for %r' % typ) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -2,8 +2,15 @@ from _rawffi import alt as _ffi import sys -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify +except ImportError: + builtinify = lambda f: f + +try: + from __pypy__.bufferable import bufferable +except ImportError: + bufferable = object keepalive_key = str # XXX fix this when provided with test @@ -64,7 +71,7 @@ 'resbuffer' is a _rawffi array of length 1 containing the value, and this returns a general Python object that corresponds. """ - res = object.__new__(self) + res = bufferable.__new__(self) res.__class__ = self res.__dict__['_buffer'] = resbuffer if base is not None: @@ -158,7 +165,7 @@ def __ne__(self, other): return self._obj != other -class _CData(object): +class _CData(bufferable): """ The most basic object for all ctypes types """ __metaclass__ = _CDataMeta diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -7,8 +7,7 @@ from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +from __pypy__ import builtinify, newmemoryview # This cache maps types to pointers to them. _pointer_type_cache = {} @@ -135,6 +134,9 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) + def __buffer__(self, flags): + mv = memoryview(self.getcontents()) + return newmemoryview(mv, mv.itemsize, '&' + mv.format, mv.shape) def _cast_addr(obj, _, tp): if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -2,9 +2,9 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject -from _ctypes.array import Array +from _ctypes.array import Array, get_format_str from _ctypes.pointer import _Pointer -import inspect +import inspect, __pypy__ def names_and_fields(self, _fields_, superclass, anonymous_fields=None): @@ -176,6 +176,11 @@ class StructOrUnionMeta(_CDataMeta): def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) + if hasattr(res, '_swappedbytes_') and '_fields_' in typedict: + # Activate the stdlib ctypes._swapped_meta.__setattr__ to convert fields + tmp = res._fields_ + delattr(res, '_fields_') + setattr(res, '_fields_', tmp) if "_abstract_" in typedict: return res cls = cls or (object,) @@ -254,17 +259,7 @@ or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - fields = [None] * len(cls._fields_) - for i in range(len(cls._fields_)): - if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): - swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) - else: - swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) - if len(cls._fields_[i]) < 3: - fields[i] = (cls._fields_[i][0], swapped) - else: - fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) - names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + names_and_fields(cls, cls._fields_, _CData, cls.__dict__.get('_anonymous_', None)) self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) @@ -304,6 +299,10 @@ def _to_ffi_param(self): return self._buffer + def __buffer__(self, flags): + fmt = get_format_str(self) + itemsize = type(self)._sizeofinstances() + return __pypy__.newmemoryview(memoryview(self._buffer), itemsize, fmt) class StructureMeta(StructOrUnionMeta): _is_union = False diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst --- a/pypy/doc/__pypy__-module.rst +++ b/pypy/doc/__pypy__-module.rst @@ -13,7 +13,6 @@ ``if platform.python_implementation == 'PyPy'`` block or otherwise hidden from the CPython interpreter. - Generally available functionality --------------------------------- @@ -25,13 +24,14 @@ - ``attach_gdb()``: start a GDB at the interpreter-level (or a PDB before translation). - - ``identity_dict(object)``: A dictionary that considers keys by object identity. - Distinct objects will have separate entries even if they compare equal. - All objects can be used as keys, even non-hashable ones --- but avoid using - immutable objects like integers: two int objects 42 may or may not be - internally the same object. + - ``newmemoryview(buffer, itemsize, format, shape=None, strides=None)``: + create a `memoryview` instance with the data from ``buffer`` and the + specified itemsize, format, and optional shape and strides. - - ``set_debug`` + - ``bufferable``: a base class that provides a ``__buffer__(self, flags)`` + method for subclasses to override. This method should return a memoryview + instance of the class instance. It is called by the C-API's ``tp_as_buffer. + bf_getbuffer``. - ``builtinify(func)``: To implement at app-level modules that are, in CPython, implemented in C: this decorator protects a function from being ever bound diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -37,3 +37,9 @@ Use utf8 internally to represent unicode, with the goal of never using rpython-level unicode +.. branch: newmemoryview-app-level + +Since _ctypes is implemented in pure python over libffi, add interfaces and +methods to support the buffer interface from python. Specifically, add a +``__pypy__.newmemoryview`` function to create a memoryview and extend the use +of the PyPy-specific ``__buffer__`` class method. diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -62,11 +62,18 @@ class PyPyDateTime(MixedModule): appleveldefs = {} interpleveldefs = { - 'dateinterop': 'interp_pypydatetime.W_DateTime_Date', - 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', - 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', + 'dateinterop' : 'interp_pypydatetime.W_DateTime_Date', + 'timeinterop' : 'interp_pypydatetime.W_DateTime_Time', + 'deltainterop' : 'interp_pypydatetime.W_DateTime_Delta', } +class PyPyBufferable(MixedModule): + appleveldefs = {} + interpleveldefs = { + 'bufferable': 'interp_buffer.W_Bufferable', + } + + class Module(MixedModule): """ PyPy specific "magic" functions. A lot of them are experimental and subject to change, many are internal. """ @@ -110,6 +117,7 @@ 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', 'pyos_inputhook' : 'interp_magic.pyos_inputhook', + 'newmemoryview' : 'interp_buffer.newmemoryview', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' @@ -121,6 +129,7 @@ "intop": IntOpModule, "os": OsModule, '_pypydatetime': PyPyDateTime, + 'bufferable': PyPyBufferable, } def setup_after_space_initialization(self): diff --git a/pypy/module/__pypy__/interp_buffer.py b/pypy/module/__pypy__/interp_buffer.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/interp_buffer.py @@ -0,0 +1,100 @@ +# +# An app-level interface to tp_as_buffer->bf_getbuffer. +# + +from pypy.interpreter.error import oefmt +from pypy.interpreter.gateway import unwrap_spec, interp2app +from pypy.objspace.std.memoryobject import BufferViewND +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef, generic_new_descr + +class W_Bufferable(W_Root): + def __init__(self, space): + pass + + def descr_buffer(self, space, w_flags): + if type(self) is W_Bufferable: + raise oefmt(space.w_ValueError, "override __buffer__ in a subclass") + return space.call_method(self, '__buffer__', w_flags) + + def readbuf_w(self, space): + mv = space.call_method(self, '__buffer__', space.newint(0)) + return mv.buffer_w(space, 0).as_readbuf() + +W_Bufferable.typedef = TypeDef("Bufferable", None, None, 'read-write', + __doc__ = """a helper class for a app-level class (like _ctypes.Array) +that want to support tp_as_buffer.bf_getbuffer via a __buffer__ method""", + __new__ = generic_new_descr(W_Bufferable), + __buffer__ = interp2app(W_Bufferable.descr_buffer), +) + +@unwrap_spec(itemsize=int, format='text') +def newmemoryview(space, w_obj, itemsize, format, w_shape=None, w_strides=None): + ''' + newmemoryview(buf, itemsize, format, shape=None, strides=None) + ''' + if not space.isinstance_w(w_obj, space.w_memoryview): + raise oefmt(space.w_ValueError, "memoryview expected") + # minimal error checking + lgt = space.len_w(w_obj) + old_size = w_obj.getitemsize() + nbytes = lgt * old_size + if w_shape: + tot = 1 + shape = [] + for w_v in space.listview(w_shape): + v = space.int_w(w_v) + shape.append(v) + tot *= v + if tot * itemsize != nbytes: + raise oefmt(space.w_ValueError, + "shape/itemsize %s/%d does not match obj len/itemsize %d/%d", + str(shape), itemsize, lgt, old_size) + else: + if nbytes % itemsize != 0: + raise oefmt(space.w_ValueError, + "itemsize %d does not match obj len/itemsize %d/%d", + itemsize, lgt, old_size) + shape = [nbytes / itemsize,] + ndim = len(shape) + if w_strides: + strides = [] + for w_v in space.listview(w_strides): + v = space.int_w(w_v) + strides.append(v) + if not w_shape and len(strides) != 1: + raise oefmt(space.w_ValueError, + "strides must have one value if shape not provided") + if len(strides) != ndim: + raise oefmt(space.w_ValueError, + "shape %s does not match strides %s", + str(shape), str(strides)) + else: + # start from the right, c-order layout + strides = [itemsize] * ndim + for v in range(ndim - 2, -1, -1): + strides[v] = strides[v + 1] * shape[v + 1] + # check that the strides are not too big + for i in range(ndim): + if strides[i] * shape[i] > nbytes: + raise oefmt(space.w_ValueError, + "shape %s and strides %s exceed object size %d", + shape, strides, nbytes) + view = space.buffer_w(w_obj, 0) + return space.newmemoryview(FormatBufferViewND(view, itemsize, format, ndim, + shape, strides)) + +class FormatBufferViewND(BufferViewND): + _immutable_ = True + _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides', + 'format', 'itemsize'] + def __init__(self, parent, itemsize, format, ndim, shape, strides): + BufferViewND.__init__(self, parent, ndim, shape, strides) + self.format = format + self.itemsize = itemsize + + def getformat(self): + return self.format + + def getitemsize(self): + return self.itemsize diff --git a/pypy/module/__pypy__/test/test_newmemoryview.py b/pypy/module/__pypy__/test/test_newmemoryview.py new file mode 100644 --- /dev/null +++ b/pypy/module/__pypy__/test/test_newmemoryview.py @@ -0,0 +1,32 @@ + + +class AppTestMinimal: + spaceconfig = dict(usemodules=['__pypy__']) + + def test_newmemoryview(self): + from __pypy__ import newmemoryview + b = bytearray(12) + # The format can be anything, we only verify shape, strides, and itemsize + m = newmemoryview(memoryview(b), 2, 'T{<h:a}', shape=(2, 3)) + assert m.strides == (6, 2) + m = newmemoryview(memoryview(b), 2, 'T{<h:a}', shape=(2, 3), + strides=(6, 2)) + assert m.strides == (6, 2) + assert m.format == 'T{<h:a}' + assert m.itemsize == 2 + + def test_bufferable(self): + from __pypy__ import bufferable, newmemoryview + class B(bufferable.bufferable): + def __init__(self): + self.data = bytearray('abc') + + def __buffer__(self, flags): + return newmemoryview(memoryview(self.data), 1, 'B') + + + obj = B() + buf = buffer(obj) + v = obj.data[2] + assert ord(buf[2]) == v + diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -29,6 +29,7 @@ from pypy.module.__builtin__.descriptor import W_Property from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.micronumpy.base import W_NDimArray +from pypy.module.__pypy__.interp_buffer import W_Bufferable from rpython.rlib.entrypoint import entrypoint_lowlevel from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable @@ -719,6 +720,7 @@ 'PyMemberDescr_Type': 'space.gettypeobject(cpyext.typeobject.W_MemberDescr.typedef)', 'PyMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)', 'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)', + 'PyBufferable_Type': 'space.gettypeobject(W_Bufferable.typedef)', }.items(): register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl) diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -41,7 +41,9 @@ fill_Py_buffer(space, w_obj.view, view) try: view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address()) - view.c_obj = make_ref(space, w_userdata) + # not used in PyPy to keep something alive, + # but some c-extensions check the type without checking for NULL + view.c_obj = make_ref(space, space.w_None) rffi.setintfield(view, 'c_readonly', w_obj.view.readonly) except ValueError: w_s = w_obj.descr_tobytes(space) diff --git a/pypy/module/cpyext/parse/cpyext_memoryobject.h b/pypy/module/cpyext/parse/cpyext_memoryobject.h --- a/pypy/module/cpyext/parse/cpyext_memoryobject.h +++ b/pypy/module/cpyext/parse/cpyext_memoryobject.h @@ -1,6 +1,12 @@ /* The struct is declared here but it shouldn't be considered public. Don't access those fields directly, use the functions instead! */ + + +/* this is wrong, PyMemoryViewObject should use PyObject_VAR_HEAD, and use + ob_data[1] to hold the shapes, strides, and offsets for the view. Then + we should use specialized allocators (that break the cpyext model) to + allocate ob_data = malloc(sizeof(Py_ssize_t) * view.ndims * 3) */ typedef struct { PyObject_HEAD Py_buffer view; diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -68,7 +68,8 @@ typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); /* Py3k buffer interface, adapted for PyPy */ -#define Py_MAX_NDIMS 32 +/* XXX remove this constant, us a PyObject_VAR_HEAD instead */ +#define Py_MAX_NDIMS 36 #define Py_MAX_FMT 128 typedef struct bufferinfo { void *buf; diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -34,6 +34,23 @@ decref(space, ref) decref(space, c_memoryview) + def test_class_with___buffer__(self, space, api): + w_obj = space.appexec([], """(): + from __pypy__.bufferable import bufferable + class B(bufferable): + def __init__(self): + self.buf = bytearray(10) + + def __buffer__(self, flags): + return memoryview(self.buf) + return B()""") + py_obj = make_ref(space, w_obj) + assert py_obj.c_ob_type.c_tp_as_buffer + assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer + assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getreadbuffer + assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getwritebuffer + + class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase): def test_fillWithObject(self): module = self.import_extension('foo', [ diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -309,3 +309,86 @@ return (_IsCContiguous(ndim, shape, strides, itemsize) or _IsFortranContiguous(ndim, shape, strides, itemsize)) return 0 + + +class IndirectView(BufferView): + """Base class for views into another BufferView""" + _immutable_ = True + _attrs_ = ['readonly', 'parent'] + + def getlength(self): + return self.parent.getlength() + + def as_str(self): + return self.parent.as_str() + + def as_str_and_offset_maybe(self): + return self.parent.as_str_and_offset_maybe() + + def getbytes(self, start, size): + return self.parent.getbytes(start, size) + + def setbytes(self, start, string): + self.parent.setbytes(start, string) + + def get_raw_address(self): + return self.parent.get_raw_address() + + def as_readbuf(self): + return self.parent.as_readbuf() + + def as_writebuf(self): + return self.parent.as_writebuf() + +class BufferView1D(IndirectView): + _immutable_ = True + _attrs_ = ['readonly', 'parent', 'format', 'itemsize'] + + def __init__(self, parent, format, itemsize): + self.parent = parent + self.readonly = parent.readonly + self.format = format + self.itemsize = itemsize + + def getformat(self): + return self.format + + def getitemsize(self): + return self.itemsize + + def getndim(self): + return 1 + + def getshape(self): + return [self.getlength() // self.itemsize] + + def getstrides(self): + return [self.itemsize] + +class BufferViewND(IndirectView): + _immutable_ = True + _attrs_ = ['readonly', 'parent', 'ndim', 'shape', 'strides'] + + def __init__(self, parent, ndim, shape, strides): + assert parent.getndim() == 1 + assert len(shape) == len(strides) == ndim + self.parent = parent + self.readonly = parent.readonly + self.ndim = ndim + self.shape = shape + self.strides = strides + + def getformat(self): + return self.parent.getformat() + + def getitemsize(self): + return self.parent.getitemsize() + + def getndim(self): + return self.ndim + + def getshape(self): + return self.shape + + def getstrides(self): + return self.strides _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit