Author: Antonio Cuni <[email protected]>
Branch:
Changeset: r68359:71a0ee32f321
Date: 2013-12-02 17:49 +0100
http://bitbucket.org/pypy/pypy/changeset/71a0ee32f321/
Log: merge the ndarray-buffer branch, which adds support for the buffer=
argument to the ndarray ctor. This is needed e.g. to use
numpy.memmap()
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -47,6 +47,9 @@
def get_raw_address(self):
raise ValueError("no raw buffer")
+ def is_writable(self):
+ return False
+
# __________ app-level support __________
def descr_len(self, space):
@@ -135,6 +138,9 @@
__slots__ = () # no extra slot here
+ def is_writable(self):
+ return True
+
def setitem(self, index, char):
"Write a character into the buffer."
raise NotImplementedError # Must be overriden. No bounds checks.
diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py
b/pypy/module/micronumpy/arrayimpl/concrete.py
--- a/pypy/module/micronumpy/arrayimpl/concrete.py
+++ b/pypy/module/micronumpy/arrayimpl/concrete.py
@@ -392,6 +392,21 @@
def __del__(self):
free_raw_storage(self.storage, track_allocation=False)
+class ConcreteArrayWithBase(ConcreteArrayNotOwning):
+ def __init__(self, shape, dtype, order, strides, backstrides, storage,
orig_base):
+ ConcreteArrayNotOwning.__init__(self, shape, dtype, order,
+ strides, backstrides, storage)
+ self.orig_base = orig_base
+
+ def base(self):
+ return self.orig_base
+
+
+class ConcreteNonWritableArrayWithBase(ConcreteArrayWithBase):
+ def descr_setitem(self, space, orig_array, w_index, w_value):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "assignment destination is read-only"))
+
class NonWritableArray(ConcreteArray):
def descr_setitem(self, space, orig_array, w_index, w_value):
diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -49,11 +49,24 @@
return W_NDimArray(impl)
@staticmethod
- def from_shape_and_storage(space, shape, storage, dtype, order='C',
owning=False, w_subtype=None):
+ def from_shape_and_storage(space, shape, storage, dtype, order='C',
owning=False,
+ w_subtype=None, w_base=None, writable=True):
from pypy.module.micronumpy.arrayimpl import concrete
assert shape
strides, backstrides = calc_strides(shape, dtype, order)
- if owning:
+ if w_base is not None:
+ if owning:
+ raise OperationError(space.w_ValueError,
+ space.wrap("Cannot have owning=True when specifying a
buffer"))
+ if writable:
+ impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
strides,
+ backstrides, storage,
w_base)
+ else:
+ impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype,
order,
+ strides,
backstrides,
+ storage,
w_base)
+
+ elif owning:
# Will free storage when GCd
impl = concrete.ConcreteArray(shape, dtype, order, strides,
backstrides, storage=storage)
diff --git a/pypy/module/micronumpy/interp_numarray.py
b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,3 +1,5 @@
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rawstorage import RAW_STORAGE_PTR
from pypy.interpreter.error import operationerrfmt, OperationError
from pypy.interpreter.typedef import TypeDef, GetSetProperty,
make_weakref_descr
from pypy.interpreter.gateway import interp2app, unwrap_spec, applevel, \
@@ -20,6 +22,7 @@
from rpython.rlib.rstring import StringBuilder
from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation
from pypy.module.micronumpy.conversion_utils import order_converter,
multi_axis_converter
+from pypy.module.micronumpy import support
from pypy.module.micronumpy.constants import *
def _find_shape(space, w_size, dtype):
@@ -1067,13 +1070,35 @@
offset=0, w_strides=None, order='C'):
from pypy.module.micronumpy.arrayimpl.concrete import ConcreteArray
from pypy.module.micronumpy.support import calc_strides
- if (offset != 0 or not space.is_none(w_strides) or
- not space.is_none(w_buffer)):
- raise OperationError(space.w_NotImplementedError,
- space.wrap("unsupported param"))
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
shape = _find_shape(space, w_shape, dtype)
+
+ if not space.is_none(w_buffer):
+ if (not space.is_none(w_strides)):
+ raise OperationError(space.w_NotImplementedError,
+ space.wrap("unsupported param"))
+
+ buf = space.buffer_w(w_buffer)
+ try:
+ raw_ptr = buf.get_raw_address()
+ except ValueError:
+ raise OperationError(space.w_TypeError, space.wrap(
+ "Only raw buffers are supported"))
+ if not shape:
+ raise OperationError(space.w_TypeError, space.wrap(
+ "numpy scalars from buffers not supported yet"))
+ totalsize = support.product(shape) * dtype.get_size()
+ if totalsize+offset > buf.getlength():
+ raise OperationError(space.w_TypeError, space.wrap(
+ "buffer is too small for requested array"))
+ storage = rffi.cast(RAW_STORAGE_PTR, raw_ptr)
+ storage = rffi.ptradd(storage, offset)
+ return W_NDimArray.from_shape_and_storage(space, shape, storage, dtype,
+ w_subtype=w_subtype,
+ w_base=w_buffer,
+ writable=buf.is_writable())
+
if not shape:
return W_NDimArray.new_scalar(space, dtype)
if space.is_w(w_subtype, space.gettypefor(W_NDimArray)):
@@ -1093,8 +1118,6 @@
Create an array from an existing buffer, given its address as int.
PyPy-only implementation detail.
"""
- from rpython.rtyper.lltypesystem import rffi
- from rpython.rlib.rawstorage import RAW_STORAGE_PTR
storage = rffi.cast(RAW_STORAGE_PTR, addr)
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype),
diff --git a/pypy/module/micronumpy/test/test_numarray.py
b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -219,6 +219,7 @@
class AppTestNumArray(BaseNumpyAppTest):
spaceconfig = dict(usemodules=["micronumpy", "struct", "binascii"])
+
def w_CustomIndexObject(self, index):
class CustomIndexObject(object):
def __init__(self, index):
@@ -2089,6 +2090,69 @@
a = np.ndarray([1], dtype=bool)
assert a[0] == True
+
+class AppTestNumArrayFromBuffer(BaseNumpyAppTest):
+ spaceconfig = dict(usemodules=["micronumpy", "array", "mmap"])
+
+ def setup_class(cls):
+ from rpython.tool.udir import udir
+ BaseNumpyAppTest.setup_class.im_func(cls)
+ cls.w_tmpname = cls.space.wrap(str(udir.join('mmap-')))
+
+ def test_ndarray_from_buffer(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*2*3)
+ a = np.ndarray((3,), buffer=buf, dtype='i2')
+ a[0] = ord('b')
+ a[1] = ord('a')
+ a[2] = ord('r')
+ assert list(buf) == ['b', '\x00', 'a', '\x00', 'r', '\x00']
+ assert a.base is buf
+
+ def test_ndarray_subclass_from_buffer(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*2*3)
+ class X(np.ndarray):
+ pass
+ a = X((3,), buffer=buf, dtype='i2')
+ assert type(a) is X
+
+ def test_ndarray_from_buffer_and_offset(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*7)
+ buf[0] = 'X'
+ a = np.ndarray((3,), buffer=buf, offset=1, dtype='i2')
+ a[0] = ord('b')
+ a[1] = ord('a')
+ a[2] = ord('r')
+ assert list(buf) == ['X', 'b', '\x00', 'a', '\x00', 'r', '\x00']
+
+ def test_ndarray_from_buffer_out_of_bounds(self):
+ import numpypy as np
+ import array
+ buf = array.array('c', ['\x00']*2*10) # 20 bytes
+ info = raises(TypeError, "np.ndarray((11,), buffer=buf, dtype='i2')")
+ assert str(info.value).startswith('buffer is too small')
+ info = raises(TypeError, "np.ndarray((5,), buffer=buf, offset=15,
dtype='i2')")
+ assert str(info.value).startswith('buffer is too small')
+
+ def test_ndarray_from_readonly_buffer(self):
+ import numpypy as np
+ from mmap import mmap, ACCESS_READ
+ f = open(self.tmpname, "w+")
+ f.write("hello")
+ f.flush()
+ buf = mmap(f.fileno(), 5, access=ACCESS_READ)
+ a = np.ndarray((5,), buffer=buf, dtype='c')
+ raises(ValueError, "a[0] = 'X'")
+ buf.close()
+ f.close()
+
+
+
class AppTestMultiDim(BaseNumpyAppTest):
def test_init(self):
import numpypy
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -314,6 +314,14 @@
self.check_valid_writeable()
self.mmap.setslice(start, string)
+ def is_writable(self):
+ try:
+ self.mmap.check_writeable()
+ except RMMapError:
+ return False
+ else:
+ return True
+
def get_raw_address(self):
self.check_valid()
return self.mmap.data
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
