Author: Armin Rigo <ar...@tunes.org>
Branch:
Changeset: r69542:339fe18d37ef
Date: 2014-02-28 10:55 +0100
http://bitbucket.org/pypy/pypy/changeset/339fe18d37ef/
Log: merge heads
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -30,7 +30,8 @@
it you would not be able to find the standard library (and run pretty much
nothing). Arguments:
- * ``home``: null terminated path
+ * ``home``: null terminated path to an executable inside the pypy directory
+ (can be a .so name, can be made up)
* ``verbose``: if non-zero, would print error messages to stderr
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -85,3 +85,6 @@
.. branch: remove-intlong-smm
kills int/long/smalllong/bool multimethods
+
+.. branch: numpy-refactor
+Cleanup micronumpy module
diff --git a/pypy/module/micronumpy/bench/dot.py
b/pypy/module/micronumpy/bench/dot.py
--- a/pypy/module/micronumpy/bench/dot.py
+++ b/pypy/module/micronumpy/bench/dot.py
@@ -1,28 +1,32 @@
+import sys
import time
try:
- import numpypy
+ import numpypy as numpy
except ImportError:
- pass
+ import numpy
-import numpy
-
-def get_matrix():
+def get_matrix(n):
import random
- n = 502
x = numpy.zeros((n,n), dtype=numpy.float64)
for i in range(n):
for j in range(n):
x[i][j] = random.random()
return x
-def main():
- x = get_matrix()
- y = get_matrix()
+def main(n, r):
+ x = get_matrix(n)
+ y = get_matrix(n)
a = time.time()
- #z = numpy.dot(x, y) # uses numpy possibly-blas-lib dot
- z = numpy.core.multiarray.dot(x, y) # uses strictly numpy C dot
+ for _ in xrange(r):
+ #z = numpy.dot(x, y) # uses numpy possibly-blas-lib dot
+ z = numpy.core.multiarray.dot(x, y) # uses strictly numpy C dot
b = time.time()
- print '%.2f seconds' % (b-a)
+ print '%d runs, %.2f seconds' % (r, b-a)
-main()
+n = int(sys.argv[1])
+try:
+ r = int(sys.argv[2])
+except IndexError:
+ r = 1
+main(n, r)
diff --git a/pypy/module/micronumpy/compile.py
b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -9,7 +9,7 @@
from rpython.rlib.nonconst import NonConstant
from pypy.module.micronumpy import boxes, ufuncs
from pypy.module.micronumpy.arrayops import where
-from pypy.module.micronumpy.base import W_NDimArray
+from pypy.module.micronumpy.ndarray import W_NDimArray
from pypy.module.micronumpy.ctors import array
from pypy.module.micronumpy.descriptor import get_dtype_cache
diff --git a/pypy/module/micronumpy/concrete.py
b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -5,15 +5,18 @@
from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
raw_storage_getitem, raw_storage_setitem, RAW_STORAGE
from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.micronumpy import support, loop, iter
+from pypy.module.micronumpy import support, loop
from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \
ArrayArgumentException
+from pypy.module.micronumpy.iterators import ArrayIter
from pypy.module.micronumpy.strides import (Chunk, Chunks, NewAxisChunk,
RecordChunk, calc_strides, calc_new_strides, shape_agreement,
- calculate_broadcast_strides, calculate_dot_strides)
+ calculate_broadcast_strides)
class BaseConcreteArray(object):
+ _immutable_fields_ = ['dtype?', 'storage', 'start', 'size', 'shape[*]',
+ 'strides[*]', 'backstrides[*]', 'order']
start = 0
parent = None
@@ -283,17 +286,9 @@
self.get_backstrides(),
self.get_shape(), shape,
backward_broadcast)
- return iter.MultiDimViewIterator(self, self.start,
- r[0], r[1], shape)
- return iter.ArrayIterator(self)
-
- def create_axis_iter(self, shape, dim, cum):
- return iter.AxisIterator(self, shape, dim, cum)
-
- def create_dot_iter(self, shape, skip):
- r = calculate_dot_strides(self.get_strides(), self.get_backstrides(),
- shape, skip)
- return iter.MultiDimViewIterator(self, self.start, r[0], r[1], shape)
+ return ArrayIter(self, support.product(shape), shape, r[0], r[1])
+ return ArrayIter(self, self.get_size(), self.shape,
+ self.strides, self.backstrides)
def swapaxes(self, space, orig_arr, axis1, axis2):
shape = self.get_shape()[:]
@@ -357,6 +352,8 @@
orig_array)
def set_dtype(self, space, dtype):
+ # size/shape/strides shouldn't change
+ assert dtype.elsize == self.dtype.elsize
self.dtype = dtype
def argsort(self, space, w_axis):
diff --git a/pypy/module/micronumpy/iter.py b/pypy/module/micronumpy/iter.py
deleted file mode 100644
--- a/pypy/module/micronumpy/iter.py
+++ /dev/null
@@ -1,217 +0,0 @@
-""" This is a mini-tutorial on iterators, strides, and
-memory layout. It assumes you are familiar with the terms, see
-http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html
-for a more gentle introduction.
-
-Given an array x: x.shape == [5,6], where each element occupies one byte
-
-At which byte in x.data does the item x[3,4] begin?
-if x.strides==[1,5]:
- pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0])
- pData = x.pData + (x.start + 24) * sizeof(x.pData[0])
-so the offset of the element is 24 elements after the first
-
-What is the next element in x after coordinates [3,4]?
-if x.order =='C':
- next == [3,5] => offset is 28
-if x.order =='F':
- next == [4,4] => offset is 24
-so for the strides [1,5] x is 'F' contiguous
-likewise, for the strides [6,1] x would be 'C' contiguous.
-
-Iterators have an internal representation of the current coordinates
-(indices), the array, strides, and backstrides. A short digression to
-explain backstrides: what is the coordinate and offset after [3,5] in
-the example above?
-if x.order == 'C':
- next == [4,0] => offset is 4
-if x.order == 'F':
- next == [4,5] => offset is 25
-Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a
-shape dimension
- which is back 25 and forward 1,
- which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
-so if we precalculate the overflow backstride as
-[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
-we can go faster.
-All the calculations happen in next()
-
-next_skip_x(steps) tries to do the iteration for a number of steps at once,
-but then we cannot guarantee that we only overflow one single shape
-dimension, perhaps we could overflow times in one big step.
-"""
-from rpython.rlib import jit
-from pypy.module.micronumpy import support
-from pypy.module.micronumpy.base import W_NDimArray
-
-
-class PureShapeIterator(object):
- def __init__(self, shape, idx_w):
- self.shape = shape
- self.shapelen = len(shape)
- self.indexes = [0] * len(shape)
- self._done = False
- self.idx_w = [None] * len(idx_w)
- for i, w_idx in enumerate(idx_w):
- if isinstance(w_idx, W_NDimArray):
- self.idx_w[i] = w_idx.create_iter(shape)
-
- def done(self):
- return self._done
-
- @jit.unroll_safe
- def next(self):
- for w_idx in self.idx_w:
- if w_idx is not None:
- w_idx.next()
- for i in range(self.shapelen - 1, -1, -1):
- if self.indexes[i] < self.shape[i] - 1:
- self.indexes[i] += 1
- break
- else:
- self.indexes[i] = 0
- else:
- self._done = True
-
- @jit.unroll_safe
- def get_index(self, space, shapelen):
- return [space.wrap(self.indexes[i]) for i in range(shapelen)]
-
-
-class ArrayIterator(object):
- def __init__(self, array):
- self.array = array
- self.start = array.start
- self.size = array.get_size()
- self.ndim_m1 = len(array.shape) - 1
- self.shape_m1 = [s - 1 for s in array.shape]
- self.strides = array.strides[:]
- self.backstrides = array.backstrides[:]
- self.reset()
-
- def reset(self):
- self.index = 0
- self.indices = [0] * (self.ndim_m1 + 1)
- self.offset = self.start
-
- @jit.unroll_safe
- def next(self):
- self.index += 1
- for i in xrange(self.ndim_m1, -1, -1):
- if self.indices[i] < self.shape_m1[i]:
- self.indices[i] += 1
- self.offset += self.strides[i]
- break
- else:
- self.indices[i] = 0
- self.offset -= self.backstrides[i]
-
- def next_skip_x(self, step):
- # XXX implement
- for _ in range(step):
- self.next()
-
- def done(self):
- return self.index >= self.size
-
- def getitem(self):
- return self.array.getitem(self.offset)
-
- def getitem_bool(self):
- return self.array.getitem_bool(self.offset)
-
- def setitem(self, elem):
- self.array.setitem(self.offset, elem)
-
-
-class MultiDimViewIterator(ArrayIterator):
- def __init__(self, array, start, strides, backstrides, shape):
- self.indexes = [0] * len(shape)
- self.array = array
- self.shape = shape
- self.offset = start
- self.shapelen = len(shape)
- self._done = self.shapelen == 0 or support.product(shape) == 0
- self.strides = strides
- self.backstrides = backstrides
- self.size = array.size
-
- @jit.unroll_safe
- def next(self):
- offset = self.offset
- for i in range(self.shapelen - 1, -1, -1):
- if self.indexes[i] < self.shape[i] - 1:
- self.indexes[i] += 1
- offset += self.strides[i]
- break
- else:
- self.indexes[i] = 0
- offset -= self.backstrides[i]
- else:
- self._done = True
- self.offset = offset
-
- @jit.unroll_safe
- def next_skip_x(self, step):
- for i in range(len(self.shape) - 1, -1, -1):
- if self.indexes[i] < self.shape[i] - step:
- self.indexes[i] += step
- self.offset += self.strides[i] * step
- break
- else:
- remaining_step = (self.indexes[i] + step) // self.shape[i]
- this_i_step = step - remaining_step * self.shape[i]
- self.offset += self.strides[i] * this_i_step
- self.indexes[i] = self.indexes[i] + this_i_step
- step = remaining_step
- else:
- self._done = True
-
- def done(self):
- return self._done
-
- def reset(self):
- self.offset %= self.size
-
-
-class AxisIterator(ArrayIterator):
- def __init__(self, array, shape, dim, cumulative):
- self.shape = shape
- strides = array.get_strides()
- backstrides = array.get_backstrides()
- if cumulative:
- self.strides = strides
- self.backstrides = backstrides
- elif len(shape) == len(strides):
- # keepdims = True
- self.strides = strides[:dim] + [0] + strides[dim + 1:]
- self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:]
- else:
- self.strides = strides[:dim] + [0] + strides[dim:]
- self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
- self.first_line = True
- self.indices = [0] * len(shape)
- self._done = array.get_size() == 0
- self.offset = array.start
- self.dim = dim
- self.array = array
-
- @jit.unroll_safe
- def next(self):
- for i in range(len(self.shape) - 1, -1, -1):
- if self.indices[i] < self.shape[i] - 1:
- if i == self.dim:
- self.first_line = False
- self.indices[i] += 1
- self.offset += self.strides[i]
- break
- else:
- if i == self.dim:
- self.first_line = True
- self.indices[i] = 0
- self.offset -= self.backstrides[i]
- else:
- self._done = True
-
- def done(self):
- return self._done
diff --git a/pypy/module/micronumpy/iterators.py
b/pypy/module/micronumpy/iterators.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/iterators.py
@@ -0,0 +1,167 @@
+""" This is a mini-tutorial on iterators, strides, and
+memory layout. It assumes you are familiar with the terms, see
+http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html
+for a more gentle introduction.
+
+Given an array x: x.shape == [5,6], where each element occupies one byte
+
+At which byte in x.data does the item x[3,4] begin?
+if x.strides==[1,5]:
+ pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0])
+ pData = x.pData + (x.start + 24) * sizeof(x.pData[0])
+so the offset of the element is 24 elements after the first
+
+What is the next element in x after coordinates [3,4]?
+if x.order =='C':
+ next == [3,5] => offset is 28
+if x.order =='F':
+ next == [4,4] => offset is 24
+so for the strides [1,5] x is 'F' contiguous
+likewise, for the strides [6,1] x would be 'C' contiguous.
+
+Iterators have an internal representation of the current coordinates
+(indices), the array, strides, and backstrides. A short digression to
+explain backstrides: what is the coordinate and offset after [3,5] in
+the example above?
+if x.order == 'C':
+ next == [4,0] => offset is 4
+if x.order == 'F':
+ next == [4,5] => offset is 25
+Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a
+shape dimension
+ which is back 25 and forward 1,
+ which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
+so if we precalculate the overflow backstride as
+[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
+we can go faster.
+All the calculations happen in next()
+
+next_skip_x(steps) tries to do the iteration for a number of steps at once,
+but then we cannot guarantee that we only overflow one single shape
+dimension, perhaps we could overflow times in one big step.
+"""
+from rpython.rlib import jit
+from pypy.module.micronumpy import support
+from pypy.module.micronumpy.base import W_NDimArray
+
+
+class PureShapeIter(object):
+ def __init__(self, shape, idx_w):
+ self.shape = shape
+ self.shapelen = len(shape)
+ self.indexes = [0] * len(shape)
+ self._done = False
+ self.idx_w = [None] * len(idx_w)
+ for i, w_idx in enumerate(idx_w):
+ if isinstance(w_idx, W_NDimArray):
+ self.idx_w[i] = w_idx.create_iter(shape)
+
+ def done(self):
+ return self._done
+
+ @jit.unroll_safe
+ def next(self):
+ for w_idx in self.idx_w:
+ if w_idx is not None:
+ w_idx.next()
+ for i in range(self.shapelen - 1, -1, -1):
+ if self.indexes[i] < self.shape[i] - 1:
+ self.indexes[i] += 1
+ break
+ else:
+ self.indexes[i] = 0
+ else:
+ self._done = True
+
+ @jit.unroll_safe
+ def get_index(self, space, shapelen):
+ return [space.wrap(self.indexes[i]) for i in range(shapelen)]
+
+
+class ArrayIter(object):
+ _immutable_fields_ = ['array', 'size', 'indices', 'shape[*]',
+ 'strides[*]', 'backstrides[*]']
+
+ def __init__(self, array, size, shape, strides, backstrides):
+ assert len(shape) == len(strides) == len(backstrides)
+ self.array = array
+ self.size = size
+ self.indices = [0] * len(shape)
+ self.shape = shape
+ self.strides = strides
+ self.backstrides = backstrides
+ self.reset()
+
+ @jit.unroll_safe
+ def reset(self):
+ self.index = 0
+ for i in xrange(len(self.shape)):
+ self.indices[i] = 0
+ self.offset = self.array.start
+
+ @jit.unroll_safe
+ def next(self):
+ self.index += 1
+ for i in xrange(len(self.shape) - 1, -1, -1):
+ if self.indices[i] < self.shape[i] - 1:
+ self.indices[i] += 1
+ self.offset += self.strides[i]
+ break
+ else:
+ self.indices[i] = 0
+ self.offset -= self.backstrides[i]
+
+ @jit.unroll_safe
+ def next_skip_x(self, step):
+ assert step >= 0
+ if step == 0:
+ return
+ self.index += step
+ for i in xrange(len(self.shape) - 1, -1, -1):
+ if self.indices[i] < self.shape[i] - step:
+ self.indices[i] += step
+ self.offset += self.strides[i] * step
+ break
+ else:
+ remaining_step = (self.indices[i] + step) // self.shape[i]
+ this_i_step = step - remaining_step * self.shape[i]
+ self.indices[i] = self.indices[i] + this_i_step
+ self.offset += self.strides[i] * this_i_step
+ step = remaining_step
+ assert step > 0
+
+ def done(self):
+ return self.index >= self.size
+
+ def getitem(self):
+ return self.array.getitem(self.offset)
+
+ def getitem_bool(self):
+ return self.array.getitem_bool(self.offset)
+
+ def setitem(self, elem):
+ self.array.setitem(self.offset, elem)
+
+
+def AxisIter(array, shape, axis, cumulative):
+ strides = array.get_strides()
+ backstrides = array.get_backstrides()
+ if not cumulative:
+ if len(shape) == len(strides):
+ # keepdims = True
+ strides = strides[:axis] + [0] + strides[axis + 1:]
+ backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
+ else:
+ strides = strides[:axis] + [0] + strides[axis:]
+ backstrides = backstrides[:axis] + [0] + backstrides[axis:]
+ return ArrayIter(array, support.product(shape), shape, strides,
backstrides)
+
+
+def AllButAxisIter(array, axis):
+ size = array.get_size()
+ shape = array.get_shape()[:]
+ backstrides = array.backstrides[:]
+ if size:
+ size /= shape[axis]
+ shape[axis] = backstrides[axis] = 0
+ return ArrayIter(array, size, shape, array.strides, backstrides)
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -8,7 +8,8 @@
from rpython.rtyper.lltypesystem import lltype, rffi
from pypy.module.micronumpy import support, constants as NPY
from pypy.module.micronumpy.base import W_NDimArray
-from pypy.module.micronumpy.iter import PureShapeIterator
+from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \
+ AllButAxisIter
call2_driver = jit.JitDriver(name='numpy_call2',
@@ -203,9 +204,9 @@
def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity,
cumulative,
temp):
- out_iter = out.create_axis_iter(arr.get_shape(), axis, cumulative)
+ out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative)
if cumulative:
- temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False)
+ temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
else:
temp_iter = out_iter # hack
arr_iter = arr.create_iter()
@@ -215,16 +216,14 @@
while not out_iter.done():
axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func,
dtype=dtype)
- if arr_iter.done():
- w_val = identity
+ assert not arr_iter.done()
+ w_val = arr_iter.getitem().convert_to(space, dtype)
+ if out_iter.indices[axis] == 0:
+ if identity is not None:
+ w_val = func(dtype, identity, w_val)
else:
- w_val = arr_iter.getitem().convert_to(space, dtype)
- if out_iter.first_line:
- if identity is not None:
- w_val = func(dtype, identity, w_val)
- else:
- cur = temp_iter.getitem()
- w_val = func(dtype, cur, w_val)
+ cur = temp_iter.getitem()
+ w_val = func(dtype, cur, w_val)
out_iter.setitem(w_val)
if cumulative:
temp_iter.setitem(w_val)
@@ -261,7 +260,6 @@
argmin = _new_argmin_argmax('min')
argmax = _new_argmin_argmax('max')
-# note that shapelen == 2 always
dot_driver = jit.JitDriver(name = 'numpy_dot',
greens = ['dtype'],
reds = 'auto')
@@ -282,25 +280,27 @@
'''
left_shape = left.get_shape()
right_shape = right.get_shape()
- broadcast_shape = left_shape[:-1] + right_shape
- left_skip = [len(left_shape) - 1 + i for i in range(len(right_shape))
- if i != right_critical_dim]
- right_skip = range(len(left_shape) - 1)
- result_skip = [len(result.get_shape()) - (len(right_shape) > 1)]
+ assert left_shape[-1] == right_shape[right_critical_dim]
assert result.get_dtype() == dtype
- outi = result.create_dot_iter(broadcast_shape, result_skip)
- lefti = left.create_dot_iter(broadcast_shape, left_skip)
- righti = right.create_dot_iter(broadcast_shape, right_skip)
- while not outi.done():
- dot_driver.jit_merge_point(dtype=dtype)
- lval = lefti.getitem().convert_to(space, dtype)
- rval = righti.getitem().convert_to(space, dtype)
- outval = outi.getitem()
- v = dtype.itemtype.mul(lval, rval)
- v = dtype.itemtype.add(v, outval)
- outi.setitem(v)
- outi.next()
- righti.next()
+ outi = result.create_iter()
+ lefti = AllButAxisIter(left.implementation, len(left_shape) - 1)
+ righti = AllButAxisIter(right.implementation, right_critical_dim)
+ while not lefti.done():
+ while not righti.done():
+ oval = outi.getitem()
+ i1 = lefti.offset
+ i2 = righti.offset
+ for _ in xrange(left.implementation.shape[-1]):
+ dot_driver.jit_merge_point(dtype=dtype)
+ lval = left.implementation.getitem(i1).convert_to(space, dtype)
+ rval = right.implementation.getitem(i2).convert_to(space,
dtype)
+ oval = dtype.itemtype.add(oval, dtype.itemtype.mul(lval, rval))
+ i1 += left.implementation.strides[-1]
+ i2 += right.implementation.strides[right_critical_dim]
+ outi.setitem(oval)
+ outi.next()
+ righti.next()
+ righti.reset()
lefti.next()
return result
@@ -478,7 +478,7 @@
prefixlen = len(prefix_w)
indexlen = len(indexes_w)
dtype = arr.get_dtype()
- iter = PureShapeIterator(iter_shape, indexes_w)
+ iter = PureShapeIter(iter_shape, indexes_w)
indexlen = len(indexes_w)
while not iter.done():
getitem_int_driver.jit_merge_point(shapelen=shapelen,
indexlen=indexlen,
@@ -507,7 +507,7 @@
indexlen = len(indexes_w)
prefixlen = len(prefix_w)
dtype = arr.get_dtype()
- iter = PureShapeIterator(iter_shape, indexes_w)
+ iter = PureShapeIter(iter_shape, indexes_w)
while not iter.done():
setitem_int_driver.jit_merge_point(shapelen=shapelen,
indexlen=indexlen,
dtype=dtype, prefixlen=prefixlen)
@@ -632,7 +632,7 @@
def diagonal_array(space, arr, out, offset, axis1, axis2, shape):
out_iter = out.create_iter()
- iter = PureShapeIterator(shape, [])
+ iter = PureShapeIter(shape, [])
shapelen_minus_1 = len(shape) - 1
assert shapelen_minus_1 >= 0
if axis1 < axis2:
diff --git a/pypy/module/micronumpy/ndarray.py
b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -286,12 +286,6 @@
return self.implementation.create_iter(
shape=shape, backward_broadcast=backward_broadcast)
- def create_axis_iter(self, shape, dim, cum):
- return self.implementation.create_axis_iter(shape, dim, cum)
-
- def create_dot_iter(self, shape, skip):
- return self.implementation.create_dot_iter(shape, skip)
-
def is_scalar(self):
return len(self.get_shape()) == 0
diff --git a/pypy/module/micronumpy/sort.py b/pypy/module/micronumpy/sort.py
--- a/pypy/module/micronumpy/sort.py
+++ b/pypy/module/micronumpy/sort.py
@@ -1,6 +1,3 @@
-""" This is the implementation of various sorting routines in numpy. It's here
-because it only makes sense on a concrete array
-"""
from pypy.interpreter.error import OperationError, oefmt
from rpython.rlib.listsort import make_timsort_class
from rpython.rlib.objectmodel import specialize
@@ -11,10 +8,15 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.module.micronumpy import descriptor, types, constants as NPY
from pypy.module.micronumpy.base import W_NDimArray
-from pypy.module.micronumpy.iter import AxisIterator
+from pypy.module.micronumpy.iterators import AllButAxisIter
INT_SIZE = rffi.sizeof(lltype.Signed)
+all_types = (types.all_float_types + types.all_complex_types +
+ types.all_int_types)
+all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
+all_types = unrolling_iterable(all_types)
+
def make_argsort_function(space, itemtype, comp_type, count=1):
TP = itemtype.T
@@ -146,21 +148,20 @@
if axis < 0 or axis >= len(shape):
raise OperationError(space.w_IndexError, space.wrap(
"Wrong axis %d" % axis))
- iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
- iter = AxisIterator(arr, iterable_shape, axis, False)
+ arr_iter = AllButAxisIter(arr, axis)
index_impl = index_arr.implementation
- index_iter = AxisIterator(index_impl, iterable_shape, axis, False)
+ index_iter = AllButAxisIter(index_impl, axis)
stride_size = arr.strides[axis]
index_stride_size = index_impl.strides[axis]
axis_size = arr.shape[axis]
- while not iter.done():
+ while not arr_iter.done():
for i in range(axis_size):
raw_storage_setitem(storage, i * index_stride_size +
index_iter.offset, i)
r = Repr(index_stride_size, stride_size, axis_size,
- arr.get_storage(), storage, index_iter.offset,
iter.offset)
+ arr.get_storage(), storage, index_iter.offset,
arr_iter.offset)
ArgSort(r).sort()
- iter.next()
+ arr_iter.next()
index_iter.next()
return index_arr
@@ -292,14 +293,13 @@
if axis < 0 or axis >= len(shape):
raise OperationError(space.w_IndexError, space.wrap(
"Wrong axis %d" % axis))
- iterable_shape = shape[:axis] + [0] + shape[axis + 1:]
- iter = AxisIterator(arr, iterable_shape, axis, False)
+ arr_iter = AllButAxisIter(arr, axis)
stride_size = arr.strides[axis]
axis_size = arr.shape[axis]
- while not iter.done():
- r = Repr(stride_size, axis_size, arr.get_storage(),
iter.offset)
+ while not arr_iter.done():
+ r = Repr(stride_size, axis_size, arr.get_storage(),
arr_iter.offset)
ArgSort(r).sort()
- iter.next()
+ arr_iter.next()
return sort
@@ -319,11 +319,6 @@
"sorting of non-numeric types '%s' is not implemented",
arr.dtype.get_name())
-all_types = (types.all_float_types + types.all_complex_types +
- types.all_int_types)
-all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
-all_types = unrolling_iterable(all_types)
-
class ArgSortCache(object):
built = False
diff --git a/pypy/module/micronumpy/strides.py
b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -421,18 +421,3 @@
n_old_elems_to_use *= old_shape[oldI]
assert len(new_strides) == len(new_shape)
return new_strides[:]
-
-
-def calculate_dot_strides(strides, backstrides, res_shape, skip_dims):
- rstrides = [0] * len(res_shape)
- rbackstrides = [0] * len(res_shape)
- j = 0
- for i in range(len(res_shape)):
- if i in skip_dims:
- rstrides[i] = 0
- rbackstrides[i] = 0
- else:
- rstrides[i] = strides[j]
- rbackstrides[i] = backstrides[j]
- j += 1
- return rstrides, rbackstrides
diff --git a/pypy/module/micronumpy/test/test_arrayops.py
b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -41,8 +41,7 @@
a[0] = 0
assert (b == [1, 1, 1, 0, 0]).all()
-
- def test_dot(self):
+ def test_dot_basic(self):
from numpypy import array, dot, arange
a = array(range(5))
assert dot(a, a) == 30.0
@@ -69,7 +68,7 @@
assert b.shape == (4, 3)
c = dot(a, b)
assert (c == [[[14, 38, 62], [38, 126, 214], [62, 214, 366]],
- [[86, 302, 518], [110, 390, 670], [134, 478, 822]]]).all()
+ [[86, 302, 518], [110, 390, 670], [134, 478,
822]]]).all()
c = dot(a, b[:, 2])
assert (c == [[62, 214, 366], [518, 670, 822]]).all()
a = arange(3*2*6).reshape((3,2,6))
diff --git a/pypy/module/micronumpy/test/test_iter.py
b/pypy/module/micronumpy/test/test_iter.py
deleted file mode 100644
--- a/pypy/module/micronumpy/test/test_iter.py
+++ /dev/null
@@ -1,93 +0,0 @@
-from pypy.module.micronumpy.iter import MultiDimViewIterator
-
-
-class MockArray(object):
- size = 1
-
-
-class TestIterDirect(object):
- def test_C_viewiterator(self):
- #Let's get started, simple iteration in C order with
- #contiguous layout => strides[-1] is 1
- start = 0
- shape = [3, 5]
- strides = [5, 1]
- backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
- assert backstrides == [10, 4]
- i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape)
- i.next()
- i.next()
- i.next()
- assert i.offset == 3
- assert not i.done()
- assert i.indexes == [0,3]
- #cause a dimension overflow
- i.next()
- i.next()
- assert i.offset == 5
- assert i.indexes == [1,0]
-
- #Now what happens if the array is transposed? strides[-1] != 1
- # therefore layout is non-contiguous
- strides = [1, 3]
- backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
- assert backstrides == [2, 12]
- i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape)
- i.next()
- i.next()
- i.next()
- assert i.offset == 9
- assert not i.done()
- assert i.indexes == [0,3]
- #cause a dimension overflow
- i.next()
- i.next()
- assert i.offset == 1
- assert i.indexes == [1,0]
-
- def test_C_viewiterator_step(self):
- #iteration in C order with #contiguous layout => strides[-1] is 1
- #skip less than the shape
- start = 0
- shape = [3, 5]
- strides = [5, 1]
- backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
- assert backstrides == [10, 4]
- i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape)
- i.next_skip_x(2)
- i.next_skip_x(2)
- i.next_skip_x(2)
- assert i.offset == 6
- assert not i.done()
- assert i.indexes == [1,1]
- #And for some big skips
- i.next_skip_x(5)
- assert i.offset == 11
- assert i.indexes == [2,1]
- i.next_skip_x(5)
- # Note: the offset does not overflow but recycles,
- # this is good for broadcast
- assert i.offset == 1
- assert i.indexes == [0,1]
- assert i.done()
-
- #Now what happens if the array is transposed? strides[-1] != 1
- # therefore layout is non-contiguous
- strides = [1, 3]
- backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
- assert backstrides == [2, 12]
- i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape)
- i.next_skip_x(2)
- i.next_skip_x(2)
- i.next_skip_x(2)
- assert i.offset == 4
- assert i.indexes == [1,1]
- assert not i.done()
- i.next_skip_x(5)
- assert i.offset == 5
- assert i.indexes == [2,1]
- assert not i.done()
- i.next_skip_x(5)
- assert i.indexes == [0,1]
- assert i.offset == 3
- assert i.done()
diff --git a/pypy/module/micronumpy/test/test_iterators.py
b/pypy/module/micronumpy/test/test_iterators.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_iterators.py
@@ -0,0 +1,96 @@
+from pypy.module.micronumpy import support
+from pypy.module.micronumpy.iterators import ArrayIter
+
+
+class MockArray(object):
+ start = 0
+
+
+class TestIterDirect(object):
+ def test_iterator_basic(self):
+ #Let's get started, simple iteration in C order with
+ #contiguous layout => strides[-1] is 1
+ shape = [3, 5]
+ strides = [5, 1]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [10, 4]
+ i = ArrayIter(MockArray, support.product(shape), shape,
+ strides, backstrides)
+ i.next()
+ i.next()
+ i.next()
+ assert i.offset == 3
+ assert not i.done()
+ assert i.indices == [0,3]
+ #cause a dimension overflow
+ i.next()
+ i.next()
+ assert i.offset == 5
+ assert i.indices == [1,0]
+
+ #Now what happens if the array is transposed? strides[-1] != 1
+ # therefore layout is non-contiguous
+ strides = [1, 3]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [2, 12]
+ i = ArrayIter(MockArray, support.product(shape), shape,
+ strides, backstrides)
+ i.next()
+ i.next()
+ i.next()
+ assert i.offset == 9
+ assert not i.done()
+ assert i.indices == [0,3]
+ #cause a dimension overflow
+ i.next()
+ i.next()
+ assert i.offset == 1
+ assert i.indices == [1,0]
+
+ def test_iterator_step(self):
+ #iteration in C order with #contiguous layout => strides[-1] is 1
+ #skip less than the shape
+ shape = [3, 5]
+ strides = [5, 1]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [10, 4]
+ i = ArrayIter(MockArray, support.product(shape), shape,
+ strides, backstrides)
+ i.next_skip_x(2)
+ i.next_skip_x(2)
+ i.next_skip_x(2)
+ assert i.offset == 6
+ assert not i.done()
+ assert i.indices == [1,1]
+ #And for some big skips
+ i.next_skip_x(5)
+ assert i.offset == 11
+ assert i.indices == [2,1]
+ i.next_skip_x(5)
+ # Note: the offset does not overflow but recycles,
+ # this is good for broadcast
+ assert i.offset == 1
+ assert i.indices == [0,1]
+ assert i.done()
+
+ #Now what happens if the array is transposed? strides[-1] != 1
+ # therefore layout is non-contiguous
+ strides = [1, 3]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [2, 12]
+ i = ArrayIter(MockArray, support.product(shape), shape,
+ strides, backstrides)
+ i.next_skip_x(2)
+ i.next_skip_x(2)
+ i.next_skip_x(2)
+ assert i.offset == 4
+ assert i.indices == [1,1]
+ assert not i.done()
+ i.next_skip_x(5)
+ assert i.offset == 5
+ assert i.indices == [2,1]
+ assert not i.done()
+ i.next_skip_x(5)
+ assert i.indices == [0,1]
+ assert i.offset == 3
+ assert i.done()
diff --git a/pypy/module/micronumpy/test/test_ndarray.py
b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1740,10 +1740,11 @@
a = array([(1, 2)], dtype=[('a', 'int64'), ('b', 'int64')])[0]
assert a.shape == ()
assert a.view('S16') == '\x01' + '\x00' * 7 + '\x02'
- a = array(2, dtype='int64')
- b = a.view('complex64')
+ a = array(2, dtype='<i8')
+ b = a.view('<c8')
assert 0 < b.real < 1
- assert b.imag == 0
+ assert b.real.tostring() == '\x02\x00\x00\x00'
+ assert b.imag.tostring() == '\x00' * 4
def test_array_view(self):
from numpypy import array, dtype
diff --git a/pypy/module/micronumpy/test/test_zjit.py
b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -59,6 +59,7 @@
if self.graph is None:
interp, graph = self.meta_interp(f, [0],
listops=True,
+ listcomp=True,
backendopt=True,
graph_and_interp_only=True)
self.__class__.interp = interp
@@ -69,7 +70,6 @@
reset_jit()
i = self.code_mapping[name]
retval = self.interp.eval_graph(self.graph, [i])
- py.test.skip("don't run for now")
return retval
def define_add():
@@ -81,6 +81,7 @@
def test_add(self):
result = self.run("add")
+ py.test.skip("don't run for now")
self.check_simple_loop({'raw_load': 2, 'float_add': 1,
'raw_store': 1, 'int_add': 1,
'int_ge': 1, 'guard_false': 1, 'jump': 1,
@@ -96,6 +97,7 @@
def test_floatadd(self):
result = self.run("float_add")
assert result == 3 + 3
+ py.test.skip("don't run for now")
self.check_simple_loop({"raw_load": 1, "float_add": 1,
"raw_store": 1, "int_add": 1,
"int_ge": 1, "guard_false": 1, "jump": 1,
@@ -111,6 +113,7 @@
def test_sum(self):
result = self.run("sum")
assert result == 2 * sum(range(30))
+ py.test.skip("don't run for now")
self.check_simple_loop({"raw_load": 2, "float_add": 2,
"int_add": 1, "int_ge": 1, "guard_false": 1,
"jump": 1, 'arraylen_gc': 1})
@@ -125,6 +128,7 @@
def test_axissum(self):
result = self.run("axissum")
assert result == 30
+ py.test.skip("don't run for now")
# XXX note - the bridge here is fairly crucial and yet it's pretty
# bogus. We need to improve the situation somehow.
self.check_simple_loop({'raw_load': 2,
@@ -177,6 +181,7 @@
for i in range(30):
expected *= i * 2
assert result == expected
+ py.test.skip("don't run for now")
self.check_simple_loop({"raw_load": 2, "float_add": 1,
"float_mul": 1, "int_add": 1,
"int_ge": 1, "guard_false": 1, "jump": 1,
@@ -222,6 +227,7 @@
def test_any(self):
result = self.run("any")
assert result == 1
+ py.test.skip("don't run for now")
self.check_simple_loop({"raw_load": 2, "float_add": 1,
"int_and": 1, "int_add": 1,
'cast_float_to_int': 1,
@@ -263,6 +269,7 @@
def test_ufunc(self):
result = self.run("ufunc")
assert result == -6
+ py.test.skip("don't run for now")
self.check_simple_loop({"raw_load": 2, "float_add": 1,
"float_neg": 1,
"raw_store": 1, "int_add": 1,
@@ -291,6 +298,7 @@
def test_specialization(self):
self.run("specialization")
+ py.test.skip("don't run for now")
# This is 3, not 2 because there is a bridge for the exit.
self.check_trace_count(3)
@@ -305,6 +313,7 @@
def test_slice(self):
result = self.run("slice")
assert result == 18
+ py.test.skip("don't run for now")
self.check_simple_loop({'raw_load': 2,
'float_add': 1,
'raw_store': 1,
@@ -345,6 +354,7 @@
def test_multidim(self):
result = self.run('multidim')
assert result == 8
+ py.test.skip("don't run for now")
# int_add might be 1 here if we try slightly harder with
# reusing indexes or some optimization
self.check_simple_loop({'float_add': 1, 'raw_load': 2,
@@ -395,6 +405,7 @@
def test_setslice(self):
result = self.run("setslice")
assert result == 11.0
+ py.test.skip("don't run for now")
self.check_trace_count(1)
self.check_simple_loop({'raw_load': 2, 'float_add': 1,
'raw_store': 1, 'int_add': 2,
@@ -412,6 +423,7 @@
def test_virtual_slice(self):
result = self.run("virtual_slice")
assert result == 4
+ py.test.skip("don't run for now")
self.check_trace_count(1)
self.check_simple_loop({'raw_load': 2, 'float_add': 1,
'raw_store': 1, 'int_add': 1,
@@ -428,6 +440,7 @@
def test_flat_iter(self):
result = self.run("flat_iter")
assert result == 6
+ py.test.skip("don't run for now")
self.check_trace_count(1)
self.check_simple_loop({'raw_load': 2, 'float_add': 1,
'raw_store': 1, 'int_add': 2,
@@ -444,6 +457,7 @@
def test_flat_getitem(self):
result = self.run("flat_getitem")
assert result == 10.0
+ py.test.skip("don't run for now")
self.check_trace_count(1)
self.check_simple_loop({'raw_load': 1,
'raw_store': 1,
@@ -466,6 +480,7 @@
def test_flat_setitem(self):
result = self.run("flat_setitem")
assert result == 1.0
+ py.test.skip("don't run for now")
self.check_trace_count(1)
# XXX not ideal, but hey, let's ignore it for now
self.check_simple_loop({'raw_load': 1,
@@ -494,18 +509,40 @@
def test_dot(self):
result = self.run("dot")
assert result == 184
- self.check_simple_loop({'arraylen_gc': 9,
- 'float_add': 1,
+ self.check_simple_loop({'float_add': 1,
'float_mul': 1,
- 'raw_load': 3,
- 'guard_false': 3,
- 'guard_true': 3,
- 'int_add': 6,
- 'int_lt': 6,
- 'int_sub': 3,
+ 'guard_not_invalidated': 1,
+ 'guard_false': 1,
+ 'int_add': 3,
+ 'int_ge': 1,
'jump': 1,
- 'raw_store': 1})
-
+ 'raw_load': 2,
+ 'setfield_gc': 1})
+ self.check_resops({'arraylen_gc': 4,
+ 'float_add': 2,
+ 'float_mul': 2,
+ 'getarrayitem_gc': 11,
+ 'getarrayitem_gc_pure': 15,
+ 'getfield_gc': 26,
+ 'getfield_gc_pure': 32,
+ 'guard_class': 4,
+ 'guard_false': 18,
+ 'guard_not_invalidated': 2,
+ 'guard_true': 9,
+ 'int_add': 25,
+ 'int_ge': 8,
+ 'int_le': 8,
+ 'int_lt': 7,
+ 'int_sub': 15,
+ 'jump': 3,
+ 'new': 1,
+ 'new_with_vtable': 1,
+ 'raw_load': 6,
+ 'raw_store': 1,
+ 'same_as': 2,
+ 'setarrayitem_gc': 10,
+ 'setfield_gc': 19})
+
def define_argsort():
return """
a = |30|
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -164,13 +164,13 @@
def reduce(self, space, w_obj, w_axis, keepdims=False, out=None,
dtype=None,
cumulative=False):
if self.argcount != 2:
- raise OperationError(space.w_ValueError, space.wrap("reduce only "
- "supported for binary functions"))
+ raise oefmt(space.w_ValueError,
+ "reduce only supported for binary functions")
assert isinstance(self, W_Ufunc2)
obj = convert_to_array(space, w_obj)
if obj.get_dtype().is_flexible():
- raise OperationError(space.w_TypeError,
- space.wrap('cannot perform reduce with flexible type'))
+ raise oefmt(space.w_TypeError,
+ "cannot perform reduce with flexible type")
obj_shape = obj.get_shape()
if obj.is_scalar():
return obj.get_scalar_value()
@@ -210,7 +210,7 @@
if out:
dtype = out.get_dtype()
temp = W_NDimArray.from_shape(space, temp_shape, dtype,
- w_instance=obj)
+ w_instance=obj)
elif keepdims:
shape = obj_shape[:axis] + [1] + obj_shape[axis + 1:]
else:
@@ -236,21 +236,28 @@
)
dtype = out.get_dtype()
else:
- out = W_NDimArray.from_shape(space, shape, dtype,
w_instance=obj)
- return loop.do_axis_reduce(space, shape, self.func, obj, dtype,
axis, out,
- self.identity, cumulative, temp)
+ out = W_NDimArray.from_shape(space, shape, dtype,
+ w_instance=obj)
+ if obj.get_size() == 0:
+ if self.identity is not None:
+ out.fill(space, self.identity.convert_to(space, dtype))
+ return out
+ return loop.do_axis_reduce(space, shape, self.func, obj, dtype,
+ axis, out, self.identity, cumulative,
+ temp)
if cumulative:
if out:
if out.get_shape() != [obj.get_size()]:
raise OperationError(space.w_ValueError, space.wrap(
"out of incompatible size"))
else:
- out = W_NDimArray.from_shape(space, [obj.get_size()], dtype,
w_instance=obj)
+ out = W_NDimArray.from_shape(space, [obj.get_size()], dtype,
+ w_instance=obj)
loop.compute_reduce_cumulative(space, obj, out, dtype, self.func,
- self.identity)
+ self.identity)
return out
if out:
- if len(out.get_shape())>0:
+ if len(out.get_shape()) > 0:
raise oefmt(space.w_ValueError,
"output parameter for reduction operation %s has "
"too many dimensions", self.name)
@@ -262,7 +269,8 @@
return out
if keepdims:
shape = [1] * len(obj_shape)
- out = W_NDimArray.from_shape(space, [1] * len(obj_shape), dtype,
w_instance=obj)
+ out = W_NDimArray.from_shape(space, [1] * len(obj_shape), dtype,
+ w_instance=obj)
out.implementation.setitem(0, res)
return out
return res
@@ -274,6 +282,7 @@
raise OperationError(space.w_ValueError, space.wrap(
"outer product only supported for binary functions"))
+
class W_Ufunc1(W_Ufunc):
_immutable_fields_ = ["func", "bool_result"]
argcount = 1
diff --git a/rpython/rlib/entrypoint.py b/rpython/rlib/entrypoint.py
--- a/rpython/rlib/entrypoint.py
+++ b/rpython/rlib/entrypoint.py
@@ -78,7 +78,8 @@
# registered
RPython_StartupCode = rffi.llexternal('RPython_StartupCode', [], lltype.Void,
- _nowrapper=True)
+ _nowrapper=True,
+ random_effects_on_gcobjs=True)
@entrypoint('main', [], c_name='rpython_startup_code')
def rpython_startup_code():
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
