Author: Stefan H. Muller <[email protected]>
Branch: pypy-pyarray
Changeset: r65896:afe17c743cf7
Date: 2013-07-28 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/afe17c743cf7/
Log: - Add cpyext implementation of Numpy PyArray_* C-API
* pypy/module/cpyext/include/numpy/arrayobject.h
* pypy/module/cpyext/ndarrayobject.py
* pypy/module/cpyext/test/test_ndarrayobject.py
- pypy/module/cpyext/api.py: copy_header_files() now copies the numpy
subdirectory as well.
- pypy/module/micronumpy/interp_dtype.py: DtypeCache.dtypes_by_num:
* Keep in dictionary form, since otherwise not all dtypes can be
reached.
- lib_pypy/numpy.py, lib_pypy/numpypy/__init__.py:
* "import numpy" now displays a warning but falls back to "import
numpypy as numpy" *without* raising an ImportError.
- pypy/module/cpyext/include/boolobject.h and complexobject.h:
* Add #define's for PyIntObject and PyComplexObject.
diff --git a/lib_pypy/numpy.py b/lib_pypy/numpy.py
--- a/lib_pypy/numpy.py
+++ b/lib_pypy/numpy.py
@@ -1,5 +1,14 @@
-raise ImportError(
+import warnings
+
+warnings.warn(
"The 'numpy' module of PyPy is in-development and not complete. "
- "To try it out anyway, you can either import from 'numpypy', "
- "or just write 'import numpypy' first in your program and then "
- "import from 'numpy' as usual.")
+ "To avoid this warning, write 'import numpypy as numpy'. ")
+
+from numpypy import *
+
+import os
+
+def get_include():
+ head, tail = os.path.split(os.path.dirname(os.path.abspath(__file__)))
+ return os.path.join(head, 'include')
+
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -10,5 +10,5 @@
__all__ += core.__all__
__all__ += lib.__all__
-import sys
-sys.modules.setdefault('numpy', sys.modules['numpypy'])
+#import sys
+#sys.modules.setdefault('numpy', sys.modules['numpypy'])
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -36,6 +36,7 @@
import pypy.module.cpyext.object
import pypy.module.cpyext.stringobject
import pypy.module.cpyext.tupleobject
+import pypy.module.cpyext.ndarrayobject
import pypy.module.cpyext.setobject
import pypy.module.cpyext.dictobject
import pypy.module.cpyext.intobject
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -130,11 +130,7 @@
udir.join('pypy_macros.h').write("/* Will be filled later */\n")
globals().update(rffi_platform.configure(CConfig_constants))
-def copy_header_files(dstdir):
- assert dstdir.check(dir=True)
- headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
- for name in ("pypy_decl.h", "pypy_macros.h"):
- headers.append(udir.join(name))
+def _copy_header_files(headers, dstdir):
for header in headers:
target = dstdir.join(header.basename)
try:
@@ -145,6 +141,25 @@
target.chmod(0444) # make the file read-only, to make sure that nobody
# edits it by mistake
+def copy_header_files(dstdir):
+ # XXX: 20 lines of code to recursively copy a directory, really??
+ assert dstdir.check(dir=True)
+ headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl')
+ for name in ("pypy_decl.h", "pypy_macros.h"):
+ headers.append(udir.join(name))
+ _copy_header_files(headers, dstdir)
+
+ try:
+ dstdir.mkdir('numpy')
+ except py.error.EEXIST:
+ pass
+ numpy_dstdir = dstdir / 'numpy'
+
+ numpy_include_dir = include_dir / 'numpy'
+ numpy_headers = numpy_include_dir.listdir('*.h') +
numpy_include_dir.listdir('*.inl')
+ _copy_header_files(numpy_headers, numpy_dstdir)
+
+
class NotSpecified(object):
pass
_NOT_SPECIFIED = NotSpecified()
diff --git a/pypy/module/cpyext/include/boolobject.h
b/pypy/module/cpyext/include/boolobject.h
--- a/pypy/module/cpyext/include/boolobject.h
+++ b/pypy/module/cpyext/include/boolobject.h
@@ -7,6 +7,8 @@
extern "C" {
#endif
+#define PyBoolObject PyIntObject
+
#define Py_False ((PyObject *) &_Py_ZeroStruct)
#define Py_True ((PyObject *) &_Py_TrueStruct)
diff --git a/pypy/module/cpyext/include/complexobject.h
b/pypy/module/cpyext/include/complexobject.h
--- a/pypy/module/cpyext/include/complexobject.h
+++ b/pypy/module/cpyext/include/complexobject.h
@@ -6,6 +6,9 @@
extern "C" {
#endif
+/* fake PyComplexObject so that code that doesn't do direct field access works
*/
+#define PyComplexObject PyObject
+
typedef struct Py_complex_t {
double real;
double imag;
diff --git a/pypy/module/cpyext/include/numpy/arrayobject.h
b/pypy/module/cpyext/include/numpy/arrayobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/numpy/arrayobject.h
@@ -0,0 +1,52 @@
+
+/* NDArray object interface - S. H. Muller, 2013/07/26 */
+
+#ifndef Py_NDARRAYOBJECT_H
+#define Py_NDARRAYOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* fake PyArrayObject so that code that doesn't do direct field access works */
+#define PyArrayObject PyObject
+
+#ifndef npy_intp
+#define npy_intp long
+#endif
+#ifndef import_array
+#define import_array()
+#endif
+
+/* copied from numpy/ndarraytypes.h
+ * keep numbers in sync with micronumpy.interp_dtype.DTypeCache
+ */
+enum NPY_TYPES { NPY_BOOL=0,
+ NPY_BYTE, NPY_UBYTE,
+ NPY_SHORT, NPY_USHORT,
+ NPY_INT, NPY_UINT,
+ NPY_LONG, NPY_ULONG,
+ NPY_LONGLONG, NPY_ULONGLONG,
+ NPY_FLOAT, NPY_DOUBLE, NPY_LONGDOUBLE,
+ NPY_CFLOAT, NPY_CDOUBLE, NPY_CLONGDOUBLE,
+ NPY_OBJECT=17,
+ NPY_STRING, NPY_UNICODE,
+ NPY_VOID,
+ /*
+ * New 1.6 types appended, may be integrated
+ * into the above in 2.0.
+ */
+ NPY_DATETIME, NPY_TIMEDELTA, NPY_HALF,
+
+ NPY_NTYPES,
+ NPY_NOTYPE,
+ NPY_CHAR, /* special flag */
+ NPY_USERDEF=256, /* leave room for characters */
+
+ /* The number of types not including the new 1.6 types */
+ NPY_NTYPES_ABI_COMPATIBLE=21
+};
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_NDARRAYOBJECT_H */
diff --git a/pypy/module/cpyext/ndarrayobject.py
b/pypy/module/cpyext/ndarrayobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/ndarrayobject.py
@@ -0,0 +1,106 @@
+"""
+Numpy C-API for PyPy - S. H. Muller, 2013/07/26
+"""
+
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.api import cpython_api, Py_ssize_t, CANNOT_FAIL
+from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.micronumpy.interp_numarray import W_NDimArray,
convert_to_array
+from pypy.module.micronumpy.interp_dtype import get_dtype_cache
+from pypy.module.micronumpy.arrayimpl.scalar import Scalar
+from rpython.rlib.rawstorage import RAW_STORAGE_PTR
+
+# the asserts are needed, otherwise the translation fails
+
+@cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+def PyArray_NDIM(space, w_array):
+ assert isinstance(w_array, W_NDimArray)
+ return len(w_array.get_shape())
+
+@cpython_api([PyObject, Py_ssize_t], Py_ssize_t, error=CANNOT_FAIL)
+def PyArray_DIM(space, w_array, n):
+ assert isinstance(w_array, W_NDimArray)
+ return w_array.get_shape()[n]
+
+@cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+def PyArray_SIZE(space, w_array):
+ assert isinstance(w_array, W_NDimArray)
+ return w_array.get_size()
+
+@cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+def PyArray_ITEMSIZE(space, w_array):
+ assert isinstance(w_array, W_NDimArray)
+ return w_array.get_dtype().get_size()
+
+@cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+def PyArray_NBYTES(space, w_array):
+ assert isinstance(w_array, W_NDimArray)
+ return w_array.get_size() * w_array.get_dtype().get_size()
+
+@cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
+def PyArray_TYPE(space, w_array):
+ assert isinstance(w_array, W_NDimArray)
+ return w_array.get_dtype().num
+
+
+@cpython_api([PyObject], rffi.VOIDP, error=CANNOT_FAIL)
+def PyArray_DATA(space, w_array):
+ # fails on scalars - see PyArray_FromAny()
+ assert isinstance(w_array, W_NDimArray)
+ return rffi.cast(rffi.VOIDP, w_array.implementation.storage)
+
+
+@cpython_api([PyObject, rffi.VOIDP, Py_ssize_t, Py_ssize_t, Py_ssize_t,
rffi.VOIDP],
+ PyObject)
+def PyArray_FromAny(space, w_obj, dtype, min_depth, max_depth, requirements,
context):
+ # ignore all additional arguments for now
+ w_array = convert_to_array(space, w_obj)
+ if w_array.is_scalar():
+ # since PyArray_DATA() fails on scalars, create a 1D array and set
empty
+ # shape. So the following combination works for *reading* scalars:
+ # PyObject *arr = PyArray_FromAny(obj);
+ # int nd = PyArray_NDIM(arr);
+ # void *data = PyArray_DATA(arr);
+ impl = w_array.implementation
+ w_array = W_NDimArray.from_shape(space, [1], impl.dtype)
+ w_array.implementation.setitem(0, impl.value)
+ w_array.implementation.shape = []
+ return w_array
+
+
+@cpython_api([Py_ssize_t, rffi.LONGP, Py_ssize_t], PyObject)
+def PyArray_SimpleNew(space, nd, dims, typenum):
+ dtype = get_dtype_cache(space).dtypes_by_num[typenum]
+ shape = []
+ for i in range(nd):
+ # back-and-forth wrapping needed to translate
+ shape.append(space.int_w(space.wrap(dims[i])))
+
+ return W_NDimArray.from_shape(space, shape, dtype)
+
+
+def simple_new_from_data(space, nd, dims, typenum, data, owning):
+ dtype = get_dtype_cache(space).dtypes_by_num[typenum]
+ storage = rffi.cast(RAW_STORAGE_PTR, data)
+ if nd == 0:
+ w_val = dtype.itemtype.box_raw_data(storage)
+ return W_NDimArray(Scalar(dtype, w_val))
+ else:
+ shape = []
+ for i in range(nd):
+ # back-and-forth wrapping needed to translate
+ shape.append(space.int_w(space.wrap(dims[i])))
+
+ return W_NDimArray.from_shape_and_storage(space, shape, storage,
dtype, owning=owning)
+
+@cpython_api([Py_ssize_t, rffi.LONGP, Py_ssize_t, rffi.VOIDP], PyObject)
+def PyArray_SimpleNewFromData(space, nd, dims, typenum, data):
+ return simple_new_from_data(space, nd, dims, typenum, data, owning=False)
+
+@cpython_api([Py_ssize_t, rffi.LONGP, Py_ssize_t, rffi.VOIDP], PyObject)
+def PyArray_SimpleNewFromDataOwning(space, nd, dims, typenum, data):
+ # Variant to take over ownership of the memory, equivalent to:
+ # PyObject *arr = PyArray_SimpleNewFromData(nd, dims, typenum, data);
+ # ((PyArrayObject*)arr)->flags |= NPY_OWNDATA;
+ return simple_new_from_data(space, nd, dims, typenum, data, owning=True)
+
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py
b/pypy/module/cpyext/test/test_ndarrayobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -0,0 +1,179 @@
+import py
+
+from pypy.module.cpyext.pyobject import PyObject
+from pypy.module.cpyext.test.test_api import BaseApiTest
+from rpython.rtyper.lltypesystem import rffi, lltype
+
+from pypy.module.micronumpy.interp_numarray import W_NDimArray
+from pypy.module.micronumpy.interp_dtype import get_dtype_cache
+
+def scalar(space):
+ dtype = get_dtype_cache(space).w_float64dtype
+ return W_NDimArray.new_scalar(space, dtype, space.wrap(10.))
+
+def array(space, shape):
+ dtype = get_dtype_cache(space).w_float64dtype
+ return W_NDimArray.from_shape(space, shape, dtype, order='C')
+
+def iarray(space, shape):
+ dtype = get_dtype_cache(space).w_int64dtype
+ return W_NDimArray.from_shape(space, shape, dtype, order='C')
+
+
+NULL = lltype.nullptr(rffi.VOIDP.TO)
+
+class TestNDArrayObject(BaseApiTest):
+
+ def test_NDIM(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_NDIM(a) == 3
+
+ def test_DIM(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_DIM(a, 1) == 5
+
+ def test_SIZE(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_SIZE(a) == 150
+
+ def test_ITEMSIZE(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_ITEMSIZE(a) == 8
+
+ def test_NBYTES(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_NBYTES(a) == 1200
+
+ def test_TYPE(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_TYPE(a) == 12
+
+ def test_DATA(self, space, api):
+ a = array(space, [10, 5, 3])
+ addr = api.PyArray_DATA(a)
+ addr2 = rffi.cast(rffi.VOIDP, a.implementation.storage)
+ assert addr == addr2
+
+ def test_FromAny_scalar(self, space, api):
+ a0 = scalar(space)
+ assert a0.implementation.get_scalar_value().value == 10.
+
+ a = api.PyArray_FromAny(a0, NULL, 0, 0, 0, NULL)
+ assert api.PyArray_NDIM(a) == 0
+
+ ptr = rffi.cast(rffi.DOUBLEP, api.PyArray_DATA(a))
+ assert ptr[0] == 10.
+
+ def test_FromAny(self, space, api):
+ a = array(space, [10, 5, 3])
+ assert api.PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a
+
+ def test_list_from_fixedptr(self, space, api):
+ A = lltype.GcArray(lltype.Float)
+ ptr = lltype.malloc(A, 3)
+ assert isinstance(ptr, lltype._ptr)
+ ptr[0] = 10.
+ ptr[1] = 5.
+ ptr[2] = 3.
+ l = list(ptr)
+ assert l == [10., 5., 3.]
+
+ def test_list_from_openptr(self, space, api):
+ nd = 3
+ a = array(space, [nd])
+ ptr = rffi.cast(rffi.DOUBLEP, api.PyArray_DATA(a))
+ ptr[0] = 10.
+ ptr[1] = 5.
+ ptr[2] = 3.
+ l = []
+ for i in range(nd):
+ l.append(ptr[i])
+ assert l == [10., 5., 3.]
+
+ def test_SimpleNew_scalar(self, space, api):
+ ptr_s = lltype.nullptr(rffi.LONGP.TO)
+ a = api.PyArray_SimpleNew(0, ptr_s, 12)
+
+ dtype = get_dtype_cache(space).w_float64dtype
+
+ a.set_scalar_value(dtype.itemtype.box(10.))
+ assert a.get_scalar_value().value == 10.
+
+ def test_SimpleNewFromData_scalar(self, space, api):
+ a = array(space, [1])
+ num = api.PyArray_TYPE(a)
+ ptr_a = api.PyArray_DATA(a)
+
+ x = rffi.cast(rffi.DOUBLEP, ptr_a)
+ x[0] = float(10.)
+
+ ptr_s = lltype.nullptr(rffi.LONGP.TO)
+
+ res = api.PyArray_SimpleNewFromData(0, ptr_s, num, ptr_a)
+ assert res.is_scalar()
+ assert res.get_scalar_value().value == 10.
+
+ def test_SimpleNew(self, space, api):
+ shape = [10, 5, 3]
+ nd = len(shape)
+
+ s = iarray(space, [nd])
+ ptr_s = rffi.cast(rffi.LONGP, api.PyArray_DATA(s))
+ ptr_s[0] = 10
+ ptr_s[1] = 5
+ ptr_s[2] = 3
+
+ a = api.PyArray_SimpleNew(nd, ptr_s, 12)
+
+ #assert list(api.PyArray_DIMS(a))[:3] == shape
+
+ ptr_a = api.PyArray_DATA(a)
+
+ x = rffi.cast(rffi.DOUBLEP, ptr_a)
+ for i in range(150):
+ x[i] = float(i)
+
+ for i in range(150):
+ assert x[i] == float(i)
+
+ def test_SimpleNewFromData(self, space, api):
+ shape = [10, 5, 3]
+ nd = len(shape)
+
+ s = iarray(space, [nd])
+ ptr_s = rffi.cast(rffi.LONGP, api.PyArray_DATA(s))
+ ptr_s[0] = 10
+ ptr_s[1] = 5
+ ptr_s[2] = 3
+
+ a = array(space, shape)
+ num = api.PyArray_TYPE(a)
+ ptr_a = api.PyArray_DATA(a)
+
+ x = rffi.cast(rffi.DOUBLEP, ptr_a)
+ for i in range(150):
+ x[i] = float(i)
+
+ res = api.PyArray_SimpleNewFromData(nd, ptr_s, num, ptr_a)
+ assert api.PyArray_TYPE(res) == num
+ assert api.PyArray_DATA(res) == ptr_a
+ for i in range(nd):
+ assert api.PyArray_DIM(res, i) == shape[i]
+ ptr_r = rffi.cast(rffi.DOUBLEP, api.PyArray_DATA(res))
+ for i in range(150):
+ assert ptr_r[i] == float(i)
+
+ def test_SimpleNewFromData_complex(self, space, api):
+ a = array(space, [2])
+ ptr_a = api.PyArray_DATA(a)
+
+ x = rffi.cast(rffi.DOUBLEP, ptr_a)
+ x[0] = 3.
+ x[1] = 4.
+
+ ptr_s = lltype.nullptr(rffi.LONGP.TO)
+
+ res = api.PyArray_SimpleNewFromData(0, ptr_s, 15, ptr_a)
+ assert res.get_scalar_value().real == 3.
+ assert res.get_scalar_value().imag == 4.
+
diff --git a/pypy/module/micronumpy/interp_dtype.py
b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -804,10 +804,12 @@
self.dtypes_by_name[alias] = dtype
self.dtypes_by_name[dtype.char] = dtype
- self.dtypes_by_num = [dtype for dtype in
- sorted(self.dtypes_by_num.values(), key=lambda dtype:
dtype.num)
- if dtype.num <= self.w_float64dtype.num]
- assert len(self.dtypes_by_num) == self.w_float64dtype.num + 1
+ # shmuller 2013/07/22: Cannot find complex data types after conversion
to
+ # list! Solution: Keep in dictionary form.
+ #self.dtypes_by_num = [dtype for dtype in
+ # sorted(self.dtypes_by_num.values(), key=lambda dtype:
dtype.num)
+ # if dtype.num <= self.w_float64dtype.num]
+ #assert len(self.dtypes_by_num) == self.w_float64dtype.num + 1
typeinfo_full = {
'LONGLONG': self.w_int64dtype,
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
