Author: Maciej Fijalkowski <[email protected]>
Branch: numpypy-axisops
Changeset: r51260:d15dcd1e48b1
Date: 2012-01-12 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/d15dcd1e48b1/
Log: in-progress. Now I understand what's exactly missing from making it
nicely work with lazy evaluation. breaks tests
diff --git a/pypy/module/micronumpy/interp_iter.py
b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -22,9 +22,6 @@
def done(self):
raise NotImplementedError
- def axis_done(self):
- raise NotImplementedError
-
class ArrayIterator(BaseIterator):
def __init__(self, size):
self.offset = 0
@@ -106,74 +103,56 @@
def next(self, shapelen):
return self
-def axis_iter_from_arr(arr, dim=-1):
- # The assert is needed for zjit tests
- from pypy.module.micronumpy.interp_numarray import ConcreteArray
- assert isinstance(arr, ConcreteArray)
- return AxisIterator(arr.start, arr.strides, arr.backstrides, arr.shape,
- dim)
-
class AxisIterator(BaseIterator):
""" Accept an addition argument dim
Redorder the dimensions to iterate over dim most often.
Set a flag at the end of each run over dim.
"""
- def __init__(self, arr_start, strides, backstrides, shape, dim):
+ def __init__(self, dim, shape, strides, backstrides):
self.shape = shape
- self.shapelen = len(shape)
self.indices = [0] * len(shape)
self._done = False
- self._axis_done = False
- self.offset = arr_start
+ self.axis_done = False
+ self.offset = -1
self.dim = dim
- self.dim_order = []
- if self.dim >= 0:
- self.dim_order.append(self.dim)
- for i in range(self.shapelen - 1, -1, -1):
- if i == self.dim:
- continue
- self.dim_order.append(i)
- self.strides = strides
- self.backstrides = backstrides
+ self.strides = strides[:dim] + [0] + strides[dim:]
+ self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
+ self.dim_order = [dim]
+ for i in range(len(shape) - 1, -1, -1):
+ if i != self.dim:
+ self.dim_order.append(i)
def done(self):
return self._done
- def axis_done(self):
- return self._axis_done
-
@jit.unroll_safe
def next(self, shapelen):
- #shapelen will always be one less than self.shapelen
offset = self.offset
- _axis_done = False
done = False
- #indices = [0] * self.shapelen
- #for i in range(self.shapelen):
- # indices[i] = self.indices[i]
- indices = self.indices
+ indices = [0] * shapelen
+ for i in range(shapelen):
+ indices[i] = self.indices[i]
+ axis_done = False
for i in self.dim_order:
if indices[i] < self.shape[i] - 1:
indices[i] += 1
- offset += self.strides[i]
break
else:
if i == self.dim:
- _axis_done = True
+ axis_done = True
+ offset += 1
indices[i] = 0
- offset -= self.backstrides[i]
else:
done = True
res = instantiate(AxisIterator)
- res._axis_done = _axis_done
+ res.axis_done = axis_done
+ res.strides = self.strides
+ res.backstrides = self.backstrides
res.offset = offset
res.indices = indices
- res.strides = self.strides
+ res.shape = self.shape
+ res.dim = self.dim
res.dim_order = self.dim_order
- res.backstrides = self.backstrides
- res.shape = self.shape
- res.shapelen = self.shapelen
- res.dim = self.dim
res._done = done
return res
diff --git a/pypy/module/micronumpy/interp_numarray.py
b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -9,7 +9,7 @@
from pypy.tool.sourcetools import func_with_new_name
from pypy.rlib.rstring import StringBuilder
from pypy.module.micronumpy.interp_iter import ArrayIterator, OneDimIterator,\
- view_iter_from_arr, SkipLastAxisIterator
+ view_iter_from_arr, SkipLastAxisIterator, AxisIterator
numpy_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
@@ -36,13 +36,6 @@
get_printable_location=signature.new_printable_location('slice'),
)
-axisreduce_driver = jit.JitDriver(
- greens=['shapelen', 'sig'],
- virtualizables=['frame'],
- reds=['identity', 'self','result', 'ri', 'frame', 'dtype', 'value'],
- get_printable_location=signature.new_printable_location('axisreduce'),
-)
-
def _find_shape_and_elems(space, w_iterable):
shape = [space.len_w(w_iterable)]
@@ -695,7 +688,6 @@
# to allow garbage-collecting them
raise NotImplementedError
- @jit.unroll_safe
def compute(self):
result = W_NDimArray(self.size, self.shape, self.find_dtype())
shapelen = len(self.shape)
@@ -760,74 +752,6 @@
self.child = None
-class Reduce(VirtualArray):
- def __init__(self, binfunc, name, dim, res_dtype, values, identity=None):
- shape = values.shape[0:dim] + values.shape[dim + 1:len(values.shape)]
- VirtualArray.__init__(self, name, shape, res_dtype)
- self.values = values
- self.size = 1
- for s in shape:
- self.size *= s
- self.binfunc = binfunc
- self.dtype = res_dtype
- self.dim = dim
- self.identity = identity
-
- def _del_sources(self):
- self.values = None
-
- def create_sig(self, res_shape):
- if self.forced_result is not None:
- return self.forced_result.create_sig(res_shape)
- return signature.AxisReduceSignature(self.binfunc, self.name,
self.dtype,
- signature.ViewSignature(self.dtype),
- self.values.create_sig(res_shape))
-
- def get_identity(self, sig, frame, shapelen):
- #XXX does this allocate? Yes :(
- #XXX is this inlinable? Yes :)
- if self.identity is None:
- value = sig.eval(frame, self.values).convert_to(self.dtype)
- frame.next(shapelen)
- else:
- value = self.identity.convert_to(self.dtype)
- return value
-
- @jit.unroll_safe
- def compute(self):
- dtype = self.dtype
- result = W_NDimArray(self.size, self.shape, dtype)
- self.values = self.values.get_concrete()
- shapelen = len(result.shape)
- identity = self.identity
- sig = self.find_sig(res_shape=result.shape, arr=self.values)
- ri = ArrayIterator(result.size)
- frame = sig.create_frame(self.values, dim=self.dim)
- value = self.get_identity(sig, frame, shapelen)
- assert isinstance(sig, signature.AxisReduceSignature)
- while not frame.done():
- axisreduce_driver.jit_merge_point(frame=frame, self=self,
- value=value, sig=sig,
- shapelen=shapelen, ri=ri,
- dtype=dtype,
- identity=identity,
- result=result)
- if frame.axis_done():
- result.dtype.setitem(result.storage, ri.offset, value)
- if identity is None:
- value = sig.eval(frame, self.values).convert_to(dtype)
- frame.next(shapelen)
- else:
- value = identity.convert_to(dtype)
- ri = ri.next(shapelen)
- value = self.binfunc(dtype, value,
- sig.eval(frame, self.values).convert_to(dtype))
- frame.next(shapelen)
- assert ri.done
- result.dtype.setitem(result.storage, ri.offset, value)
- return result
-
-
class Call1(VirtualArray):
def __init__(self, ufunc, name, shape, res_dtype, values):
VirtualArray.__init__(self, name, shape, res_dtype)
@@ -869,6 +793,16 @@
self.left.create_sig(res_shape),
self.right.create_sig(res_shape))
+class AxisReduce(Call2):
+ """ NOTE: this is only used as a container, you should never
+ encounter such things in the wild. Remove this comment
+ when we'll make AxisReduce lazy
+ """
+ def __init__(self, ufunc, name, shape, dtype, left, right, dim):
+ Call2.__init__(self, ufunc, name, shape, dtype, dtype,
+ left, right)
+ self.dim = dim
+
class ConcreteArray(BaseArray):
""" An array that have actual storage, whether owned or not
"""
diff --git a/pypy/module/micronumpy/interp_ufuncs.py
b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,8 +3,8 @@
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.typedef import TypeDef, GetSetProperty,
interp_attrproperty
from pypy.module.micronumpy import interp_boxes, interp_dtype
-from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature,\
- find_sig, new_printable_location
+from pypy.module.micronumpy.signature import ReduceSignature,\
+ find_sig, new_printable_location, AxisReduceSignature, ScalarSignature
from pypy.rlib import jit
from pypy.rlib.rarithmetic import LONG_BIT
from pypy.tool.sourcetools import func_with_new_name
@@ -16,6 +16,14 @@
get_printable_location=new_printable_location('reduce'),
)
+axisreduce_driver = jit.JitDriver(
+ greens=['shapelen', 'sig'],
+ virtualizables=['frame'],
+ reds=['self','arr', 'frame', 'shapelen'],
+# name='axisreduce',
+ get_printable_location=new_printable_location('axisreduce'),
+)
+
class W_Ufunc(Wrappable):
_attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
@@ -129,14 +137,13 @@
raise operationerrfmt(space.w_ValueError, "zero-size array to "
"%s.reduce without identity", self.name)
if shapelen > 1 and dim >= 0:
- from pypy.module.micronumpy.interp_numarray import Reduce
- res = Reduce(self.func, self.name, dim, dtype, obj, self.identity)
- obj.add_invalidates(res)
+ res = self.do_axis_reduce(obj, dtype, dim)
return space.wrap(res)
+ scalarsig = ScalarSignature(dtype)
sig = find_sig(ReduceSignature(self.func, self.name, dtype,
- ScalarSignature(dtype),
+ scalarsig,
obj.create_sig(obj.shape)), obj)
- frame = sig.create_frame(obj, dim=-1)
+ frame = sig.create_frame(obj)
if self.identity is None:
value = sig.eval(frame, obj).convert_to(dtype)
frame.next(shapelen)
@@ -144,6 +151,46 @@
value = self.identity.convert_to(dtype)
return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
+ def do_axis_reduce(self, obj, dtype, dim):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+ W_NDimArray
+
+ shape = obj.shape[0:dim] + obj.shape[dim + 1:len(obj.shape)]
+ size = 1
+ for s in shape:
+ size *= s
+ result = W_NDimArray(size, shape, dtype)
+ rightsig = obj.create_sig(obj.shape)
+ # note - this is just a wrapper so signature can fetch
+ # both left and right, nothing more, especially
+ # this is not a true virtual array, because shapes
+ # don't quite match
+ arr = AxisReduce(self.func, self.name, shape, dtype,
+ result, obj, dim)
+ scalarsig = ScalarSignature(dtype)
+ sig = find_sig(AxisReduceSignature(self.func, self.name, dtype,
+ scalarsig, rightsig), arr)
+ frame = sig.create_frame(arr)
+ shapelen = len(obj.shape)
+ if self.identity is None:
+ frame.identity = sig.eval(frame, arr).convert_to(dtype)
+ frame.next(shapelen)
+ else:
+ frame.identity = self.identity.convert_to(dtype)
+ frame.value = frame.identity
+ self.reduce_axis_loop(frame, sig, shapelen, arr)
+ return result
+
+ def reduce_axis_loop(self, frame, sig, shapelen, arr):
+ while not frame.done():
+ axisreduce_driver.jit_merge_point(frame=frame, self=self,
+ sig=sig,
+ shapelen=shapelen, arr=arr)
+ sig.eval(frame, arr)
+ frame.next(shapelen)
+ # store the last value, when everything is done
+ arr.left.setitem(frame.iterators[0].offset, frame.value)
+
def reduce_loop(self, shapelen, sig, frame, value, obj, dtype):
while not frame.done():
reduce_driver.jit_merge_point(sig=sig,
diff --git a/pypy/module/micronumpy/signature.py
b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -1,7 +1,7 @@
from pypy.rlib.objectmodel import r_dict, compute_identity_hash, compute_hash
from pypy.rlib.rarithmetic import intmask
from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \
- OneDimIterator, ConstantIterator, axis_iter_from_arr
+ OneDimIterator, ConstantIterator, AxisIterator
from pypy.module.micronumpy.strides import calculate_slice_strides
from pypy.rlib.jit import hint, unroll_safe, promote
@@ -33,7 +33,8 @@
return sig
class NumpyEvalFrame(object):
- _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]']
+ _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]',
+ 'value', 'identity']
@unroll_safe
def __init__(self, iterators, arrays):
@@ -47,6 +48,8 @@
break
else:
self.final_iter = -1
+ self.value = None
+ self.identity = None
def done(self):
final_iter = promote(self.final_iter)
@@ -57,13 +60,7 @@
@unroll_safe
def next(self, shapelen):
for i in range(len(self.iterators)):
- self.iterators[i] = self.iterators[i].next(shapelen)
-
- def axis_done(self):
- final_iter = promote(self.final_iter)
- if final_iter < 0:
- return False
- return self.iterators[final_iter].axis_done()
+ self.iterators[i] = self.iterators[i].next(shapelen)
def _add_ptr_to_cache(ptr, cache):
i = 0
@@ -101,13 +98,13 @@
allnumbers.append(no)
self.iter_no = no
- def create_frame(self, arr, res_shape=None, chunks=None, dim=-1):
+ def create_frame(self, arr, res_shape=None, chunks=None):
if chunks is None:
chunks = []
res_shape = res_shape or arr.shape
iterlist = []
arraylist = []
- self._create_iter(iterlist, arraylist, arr, res_shape, chunks, dim)
+ self._create_iter(iterlist, arraylist, arr, res_shape, chunks)
return NumpyEvalFrame(iterlist, arraylist)
@@ -150,7 +147,7 @@
assert concr.dtype is self.dtype
self.array_no = _add_ptr_to_cache(concr.storage, cache)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
from pypy.module.micronumpy.interp_numarray import ConcreteArray
concr = arr.get_concrete()
assert isinstance(concr, ConcreteArray)
@@ -178,7 +175,7 @@
def _invent_array_numbering(self, arr, cache):
pass
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
if self.iter_no >= len(iterlist):
iter = ConstantIterator()
iterlist.append(iter)
@@ -219,12 +216,12 @@
assert isinstance(other, VirtualSliceSignature)
return self.child.eq(other.child, compare_array_no)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
from pypy.module.micronumpy.interp_numarray import VirtualSlice
assert isinstance(arr, VirtualSlice)
chunklist.append(arr.chunks)
self.child._create_iter(iterlist, arraylist, arr.child, res_shape,
- chunklist, dim)
+ chunklist)
def eval(self, frame, arr):
from pypy.module.micronumpy.interp_numarray import VirtualSlice
@@ -260,11 +257,11 @@
assert isinstance(arr, Call1)
self.child._invent_array_numbering(arr.values, cache)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
from pypy.module.micronumpy.interp_numarray import Call1
assert isinstance(arr, Call1)
self.child._create_iter(iterlist, arraylist, arr.values, res_shape,
- chunklist, dim)
+ chunklist)
def eval(self, frame, arr):
from pypy.module.micronumpy.interp_numarray import Call1
@@ -305,14 +302,14 @@
self.left._invent_numbering(cache, allnumbers)
self.right._invent_numbering(cache, allnumbers)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
from pypy.module.micronumpy.interp_numarray import Call2
assert isinstance(arr, Call2)
self.left._create_iter(iterlist, arraylist, arr.left, res_shape,
- chunklist, dim)
+ chunklist)
self.right._create_iter(iterlist, arraylist, arr.right, res_shape,
- chunklist, dim)
+ chunklist)
def eval(self, frame, arr):
from pypy.module.micronumpy.interp_numarray import Call2
@@ -327,9 +324,9 @@
class ReduceSignature(Call2):
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
self.right._create_iter(iterlist, arraylist, arr, res_shape,
- chunklist, dim)
+ chunklist)
def _invent_numbering(self, cache, allnumbers):
self.right._invent_numbering(cache, allnumbers)
@@ -341,33 +338,39 @@
return self.right.eval(frame, arr)
def debug_repr(self):
- return 'ReduceSig(%s, %s, %s)' % (self.name, self.left.debug_repr(),
- self.right.debug_repr())
+ return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
class AxisReduceSignature(Call2):
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist,
dim):
- from pypy.module.micronumpy.interp_numarray import ConcreteArray
- concr = arr.get_concrete()
- assert isinstance(concr, ConcreteArray)
- storage = concr.storage
- if self.iter_no >= len(iterlist):
- _iter = axis_iter_from_arr(concr, dim)
- from interp_iter import AxisIterator
- assert isinstance(_iter, AxisIterator)
- iterlist.append(_iter)
- if self.array_no >= len(arraylist):
- arraylist.append(storage)
+ def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+ assert isinstance(arr, AxisReduce)
+ assert not iterlist # we assume that later in eval
+ iterlist.append(AxisIterator(arr.dim, arr.right.shape,
+ arr.left.strides,
+ arr.left.backstrides))
+ self.right._create_iter(iterlist, arraylist, arr.right,
arr.right.shape,
+ chunklist)
def _invent_numbering(self, cache, allnumbers):
+ no = len(allnumbers)
+ allnumbers.append(no)
self.right._invent_numbering(cache, allnumbers)
def _invent_array_numbering(self, arr, cache):
- self.right._invent_array_numbering(arr, cache)
+ from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+ assert isinstance(arr, AxisReduce)
+ self.right._invent_array_numbering(arr.right, cache)
def eval(self, frame, arr):
- return self.right.eval(frame, arr)
+ if frame.iterators[0].axis_done:
+ arr.left.setitem(frame.iterators[0].offset, frame.value)
+ frame.value = frame.identity
+ v = self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+ print v.value, frame.value.value
+ frame.value = self.binfunc(self.calc_dtype, frame.value, v)
+ return frame.value
def debug_repr(self):
- return 'AxisReduceSig(%s, %s, %s)' % (self.name,
self.left.debug_repr(),
- self.right.debug_repr())
-
+ return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_numarray.py
b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -745,7 +745,7 @@
raises(TypeError, 'a.sum(2, 3)')
- def test_reduceND(self):
+ def test_reduce_nd(self):
from numpypy import arange
a = arange(15).reshape(5, 3)
assert a.sum() == 105
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
