Author: mattip <[email protected]>
Branch:
Changeset: r78690:c1b066339482
Date: 2015-07-27 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/c1b066339482/
Log: merge nditer-revisited which provides 'buffered' flag and other
small improvements
diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py
--- a/pypy/module/micronumpy/nditer.py
+++ b/pypy/module/micronumpy/nditer.py
@@ -9,7 +9,8 @@
from pypy.module.micronumpy.iterators import ArrayIter
from pypy.module.micronumpy.strides import (calculate_broadcast_strides,
shape_agreement,
shape_agreement_multiple)
-from pypy.module.micronumpy.casting import find_binop_result_dtype
+from pypy.module.micronumpy.casting import (find_binop_result_dtype,
+ can_cast_array, can_cast_type)
def parse_op_arg(space, name, w_op_flags, n, parse_one_arg):
@@ -108,9 +109,7 @@
if item == 'external_loop':
nditer.external_loop = True
elif item == 'buffered':
- raise oefmt(space.w_NotImplementedError,
- 'nditer buffered not implemented yet')
- # For numpy compatability
+ # Each iterator should be 1d
nditer.buffered = True
elif item == 'c_index':
nditer.tracked_index = 'C'
@@ -213,30 +212,6 @@
return arr
-def get_iter(space, order, arr, shape, dtype, op_flags, base):
- imp = arr.implementation
- backward = is_backward(imp, order)
- if arr.is_scalar():
- return ConcreteIter(imp, 1, [], [], [], op_flags, base)
- if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
- (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward):
- # flip the strides. Is this always true for multidimension?
- strides = imp.strides[:]
- backstrides = imp.backstrides[:]
- shape = imp.shape[:]
- strides.reverse()
- backstrides.reverse()
- shape.reverse()
- else:
- strides = imp.strides
- backstrides = imp.backstrides
- r = calculate_broadcast_strides(strides, backstrides, imp.shape,
- shape, backward)
- if len(shape) != len(r[0]):
- # shape can be shorter when using an external loop, just return a view
- return ConcreteIter(imp, imp.get_size(), imp.shape, r[0], r[1],
op_flags, base)
- return ConcreteIter(imp, imp.get_size(), shape, r[0], r[1], op_flags, base)
-
def calculate_ndim(op_in, oa_ndim):
if oa_ndim >=0:
return oa_ndim
@@ -308,6 +283,8 @@
of shape (4,3) by setting the offset to the beginning of the data at each
iteration
'''
shape = [s+1 for s in old_iter.shape_m1]
+ if len(shape) < 1:
+ return old_iter
strides = old_iter.strides
backstrides = old_iter.backstrides
if order == 'F':
@@ -328,6 +305,8 @@
_shape = [shape[-1]] + old_iter.slice_shape
_backstride = [(shape[-1] - 1) * strides[-1]] +
old_iter.slice_backstride
fastest = shape[-1]
+ if fastest == 0:
+ return old_iter
if flat:
_shape = [support.product(_shape)]
if len(_stride) > 1:
@@ -383,6 +362,10 @@
self.done = False
self.first_next = True
self.op_axes = []
+ if not space.is_w(w_casting, space.w_None):
+ self.casting = space.str_w(w_casting)
+ else:
+ self.casting = 'safe'
# convert w_seq operands to a list of W_NDimArray
if space.isinstance_w(w_seq, space.w_tuple) or \
space.isinstance_w(w_seq, space.w_list):
@@ -445,8 +428,15 @@
self.seq[i] = W_NDimArray.from_shape(space, self.shape,
out_dtype)
else:
if not self.op_flags[i].broadcast:
- # Raises if ooutput cannot be broadcast
- shape_agreement(space, self.shape, self.seq[i], False)
+ # Raises if output cannot be broadcast
+ try:
+ shape_agreement(space, self.shape, self.seq[i],
False)
+ except OperationError as e:
+ raise oefmt(space.w_ValueError, "non-broadcastable"
+ " output operand with shape %s doesn't match "
+ "the broadcast shape %s",
+ str(self.seq[i].get_shape()),
+ str(self.shape))
if self.tracked_index != "":
if self.order == "K":
@@ -465,14 +455,41 @@
if not self_d:
self.dtypes[i] = seq_d
elif self_d != seq_d:
- if not 'r' in self.op_flags[i].tmp_copy:
- raise oefmt(space.w_TypeError,
- "Iterator operand required copying or "
- "buffering for operand %d", i)
- impl = self.seq[i].implementation
- order = support.get_order_as_CF(impl.order, self.order)
- new_impl = impl.astype(space, self_d, order)
- self.seq[i] = W_NDimArray(new_impl)
+ impl = self.seq[i].implementation
+ order = support.get_order_as_CF(impl.order, self.order)
+ if self.buffered or 'r' in self.op_flags[i].tmp_copy:
+ if not can_cast_array(
+ space, self.seq[i], self_d, self.casting):
+ raise oefmt(space.w_TypeError, "Iterator
operand %d"
+ " dtype could not be cast from %s to %s"
+ " according to the rule '%s'", i,
+ space.str_w(seq_d.descr_repr(space)),
+ space.str_w(self_d.descr_repr(space)),
+ self.casting)
+
+ new_impl = impl.astype(space, self_d,
order).copy(space)
+ self.seq[i] = W_NDimArray(new_impl)
+ else:
+ raise oefmt(space.w_TypeError, "Iterator "
+ "operand required copying or buffering, "
+ "but neither copying nor buffering was "
+ "enabled")
+ if 'w' in self.op_flags[i].rw:
+ if not can_cast_type(
+ space, self_d, seq_d, self.casting):
+ raise oefmt(space.w_TypeError, "Iterator"
+ " requested dtype could not be cast from "
+ " %s to %s, the operand %d dtype, accord"
+ "ing to the rule '%s'",
+ space.str_w(self_d.descr_repr(space)),
+ space.str_w(seq_d.descr_repr(space)),
+ i, self.casting)
+ elif self.buffered:
+ for i in range(len(self.seq)):
+ if i not in outargs:
+ self.seq[i] = self.seq[i].descr_copy(space,
+ w_order=space.wrap(self.order))
+ self.dtypes = [s.get_dtype() for s in self.seq]
else:
#copy them from seq
self.dtypes = [s.get_dtype() for s in self.seq]
@@ -480,14 +497,43 @@
# create an iterator for each operand
self.iters = []
for i in range(len(self.seq)):
- it = get_iter(space, self.order, self.seq[i], self.shape,
- self.dtypes[i], self.op_flags[i], self)
+ it = self.get_iter(space, i)
it.contiguous = False
self.iters.append((it, it.reset()))
if self.external_loop:
coalesce_axes(self, space)
+ def get_iter(self, space, i):
+ arr = self.seq[i]
+ dtype = self.dtypes[i]
+ shape = self.shape
+ imp = arr.implementation
+ backward = is_backward(imp, self.order)
+ if arr.is_scalar():
+ return ConcreteIter(imp, 1, [], [], [], self.op_flags[i], self)
+ if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
+ (abs(imp.strides[0]) > abs(imp.strides[-1]) and backward):
+ # flip the strides. Is this always true for multidimension?
+ strides = imp.strides[:]
+ backstrides = imp.backstrides[:]
+ shape = imp.shape[:]
+ strides.reverse()
+ backstrides.reverse()
+ shape.reverse()
+ else:
+ strides = imp.strides
+ backstrides = imp.backstrides
+ r = calculate_broadcast_strides(strides, backstrides, imp.shape,
+ shape, backward)
+ iter_shape = shape
+ if len(shape) != len(r[0]):
+ # shape can be shorter when using an external loop, just return a
view
+ iter_shape = imp.shape
+ return ConcreteIter(imp, imp.get_size(), iter_shape, r[0], r[1],
+ self.op_flags[i], self)
+
+
def set_op_axes(self, space, w_op_axes):
if space.len_w(w_op_axes) != len(self.seq):
raise oefmt(space.w_ValueError,
@@ -520,8 +566,8 @@
return space.wrap(self)
def getitem(self, it, st):
- res = it.getoperand(st)
- return W_NDimArray(res)
+ w_res = W_NDimArray(it.getoperand(st))
+ return w_res
def descr_getitem(self, space, w_idx):
idx = space.int_w(w_idx)
diff --git a/pypy/module/micronumpy/test/dummy_module.py
b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -38,3 +38,6 @@
a = zeros(*args, **kwargs)
a.fill(1)
return a
+
+def isscalar(a):
+ return type(a) in [typeinfo[t] for t in types]
diff --git a/pypy/module/micronumpy/test/test_nditer.py
b/pypy/module/micronumpy/test/test_nditer.py
--- a/pypy/module/micronumpy/test/test_nditer.py
+++ b/pypy/module/micronumpy/test/test_nditer.py
@@ -64,18 +64,15 @@
a = arange(24).reshape(2, 3, 4)
import sys
r = []
- n = 0
for x in nditer(a, flags=['external_loop']):
r.append(x)
- n += 1
- assert n == 1
+ assert len(r) == 1
+ assert r[0].shape == (24,)
assert (array(r) == range(24)).all()
r = []
- n = 0
for x in nditer(a, flags=['external_loop'], order='F'):
r.append(x)
- n += 1
- assert n == 12
+ assert len(r) == 12
assert (array(r) == [[ 0, 12], [ 4, 16], [ 8, 20], [ 1, 13], [ 5, 17],
[ 9, 21],
[ 2, 14], [ 6, 18], [10, 22], [ 3, 15], [ 7, 19],
[11, 23],
]).all()
@@ -149,19 +146,60 @@
# assert str(exc.value).startswith("Iterator flag EXTERNAL_LOOP
cannot")
def test_buffered(self):
- from numpy import arange, nditer, array
- a = arange(6).reshape(2,3)
- import sys
- if '__pypy__' in sys.builtin_module_names:
- raises(NotImplementedError, nditer, a, flags=['buffered'])
- skip('nditer buffered not implmented')
+ from numpy import arange, nditer, array, isscalar
+ a = arange(24).reshape(2, 3, 4)
+ r = []
+ for x in nditer(a, flags=['external_loop'], order='F'):
+ r.append(x)
+ array_r = array(r)
+ assert len(array_r.shape) == 2
+ assert array_r.shape == (12, 2)
+ assert (array_r == [[0, 12], [4, 16], [8, 20], [1, 13], [5, 17], [9,
21],
+ [2, 14], [6, 18], [10, 22], [3, 15], [7, 19], [11,
23]]).all
+ assert (a == arange(24).reshape(2, 3, 4)).all()
+ a[0,0,0] = 100
+ assert r[0][0] == 100
+
+ r = []
+ try:
+ it = nditer(a, flags=['buffered'], order='F')
+ except NotImplementedError as e:
+ assert 'unsupported value for order' in str(e)
+ skip('buffered with order="F" requires fortran tmp array creation')
+ for x in it:
+ r.append(x)
+ array_r = array(r)
+ assert len(array_r.shape) == 1
+ assert array_r.shape == (24,)
+ assert r[0].shape == ()
+ assert not isscalar(r[0])
+ assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
+ 2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
+ assert a.shape == (2, 3, 4)
+ a[0,0,0] = 0
+ # buffered copies the data into a tmp array
+ assert r[0] == 100
+ assert (a == arange(24).reshape(2, 3, 4)).all()
+
r = []
for x in nditer(a, flags=['external_loop', 'buffered'], order='F'):
r.append(x)
- array_r = array(r)
- assert len(array_r.shape) == 2
- assert array_r.shape == (1, 6)
- assert (array_r == [0, 3, 1, 4, 2, 5]).all()
+ assert r[0].shape == (24,)
+ assert (array_r == [0, 12, 4, 16, 8, 20, 1, 13, 5, 17, 9, 21,
+ 2, 14, 6, 18, 10, 22, 3, 15, 7, 19, 11, 23]).all
+ assert a.shape == (2, 3, 4)
+ assert (a == arange(24).reshape(2, 3, 4)).all()
+
+ def test_zerosize(self):
+ from numpy import nditer, array
+ for a in [ array([]), array([1]), array([1, 2]) ]:
+ buffersize = max(16 * 1024 ** 2 // a.itemsize, 1)
+ r = []
+ for chunk in nditer(a,
+ flags=['external_loop', 'buffered', 'zerosize_ok'],
+ buffersize=buffersize, order='C'):
+ r.append(chunk)
+ assert (r == a).all()
def test_op_dtype(self):
from numpy import arange, nditer, sqrt, array
@@ -188,11 +226,10 @@
from numpy import arange, nditer
import sys
a = arange(6.)
- if '__pypy__' in sys.builtin_module_names:
- raises(NotImplementedError, nditer, a, flags=['buffered'],
op_dtypes=['float32'])
- skip('nditer casting not implemented yet')
exc = raises(TypeError, nditer, a, flags=['buffered'],
op_dtypes=['float32'])
- assert str(exc.value).startswith("Iterator operand 0 dtype could not
be cast")
+ assert str(exc.value) == "Iterator operand 0 dtype could not be " + \
+ "cast from dtype('float64') to dtype('float32') according to the"
+\
+ " rule 'safe'"
r = []
for x in nditer(a, flags=['buffered'], op_dtypes=['float32'],
casting='same_kind'):
@@ -232,9 +269,6 @@
return it.operands[1]
assert (square1([1, 2, 3]) == [1, 4, 9]).all()
- if '__pypy__' in sys.builtin_module_names:
- raises(NotImplementedError, nditer, [1, 2], flags=['buffered'])
- skip('nditer buffered not implmented')
def square2(a, out=None):
it = nditer([a, out], flags=['external_loop', 'buffered'],
op_flags=[['readonly'],
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
