Author: Alex Gaynor <[email protected]>
Branch: numpy-dtype-alt
Changeset: r46341:23225085fd60
Date: 2011-08-06 17:29 -0700
http://bitbucket.org/pypy/pypy/changeset/23225085fd60/
Log: Started implementing dtypes, currently fails due to some mess with
casting to and from VOIDP.
diff --git a/pypy/module/micronumpy/__init__.py
b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -7,6 +7,8 @@
interpleveldefs = {
'array': 'interp_numarray.SingleDimArray',
+ 'dtype': 'interp_dtype.W_Dtype',
+
'zeros': 'interp_numarray.zeros',
'empty': 'interp_numarray.zeros',
'ones': 'interp_numarray.ones',
diff --git a/pypy/module/micronumpy/interp_dtype.py
b/pypy/module/micronumpy/interp_dtype.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -0,0 +1,124 @@
+from pypy.interpreter.baseobjspace import Wrappable
+from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.rlib.unroll import unrolling_iterable
+from pypy.rpython.lltypesystem import lltype, rffi
+
+
+SIGNEDLTR = "i"
+
+class W_Dtype(Wrappable):
+ aliases = []
+ applevel_types = []
+
+ def __init__(self, space):
+ pass
+
+ def descr__new__(space, w_subtype, w_dtype):
+ if space.isinstance_w(w_dtype, space.w_str):
+ dtype = space.str_w(w_dtype)
+ for alias, dtype_class in dtypes_by_alias:
+ if alias == dtype:
+ return space.fromcache(dtype_class)
+ elif isinstance(space.interpclass_w(w_dtype), W_Dtype):
+ return w_dtype
+ elif space.isinstance_w(w_dtype, space.w_type):
+ for typename, dtype_class in dtypes_by_apptype:
+ if space.is_w(getattr(space, "w_%s" % typename), w_dtype):
+ return space.fromcache(dtype_class)
+ assert False
+
+ def descr_repr(self, space):
+ return space.wrap("dtype('%s')" % self.name)
+
+ def descr_str(self, space):
+ return space.wrap(self.name)
+
+class LowLevelDtype(object):
+ _mixin_ = True
+
+ def erase(self, storage):
+ return rffi.cast(rffi.VOIDP, storage)
+ def unerase(self, storage):
+ return rffi.cast(self.TP, storage)
+
+ def malloc(self, size):
+ # XXX find out why test_zjit explodes with tracking of allocations
+ return self.erase(lltype.malloc(self.TP, size,
+ zero=True, flavor="raw",
+ track_allocation=False, add_memory_pressure=True
+ ))
+
+ def getitem(self, storage, i):
+ return self.unerase(storage)[i]
+
+ def setitem(self, storage, i, item):
+ self.unerase(storage)[i] = item
+
+ def setitem_w(self, space, storage, i, w_item):
+ self.setitem(storage, i, self.unwrap(space, w_item))
+
+
+class W_Int8Dtype(W_Dtype, LowLevelDtype):
+ num = 1
+ kind = SIGNEDLTR
+ aliases = ["int8"]
+
+class W_Int32Dtype(W_Dtype, LowLevelDtype):
+ num = 5
+ kind = SIGNEDLTR
+ aliases = ["i"]
+
+class W_Int64Dtype(W_Dtype, LowLevelDtype):
+ num = 9
+ applevel_types = ["long"]
+
+class W_LongDtype(W_Dtype, LowLevelDtype):
+ num = 7
+ kind = SIGNEDLTR
+ aliases = ["l"]
+ applevel_types = ["int"]
+
+class W_BoolDtype(W_Dtype, LowLevelDtype):
+ num = 0
+ name = "bool"
+ aliases = ["?"]
+ applevel_types = ["bool"]
+ TP = lltype.Array(lltype.Bool, hints={'nolength': True})
+
+ def unwrap(self, space, w_item):
+ return space.is_true(w_item)
+
+class W_Float64Dtype(W_Dtype, LowLevelDtype):
+ num = 12
+ applevel_types = ["float"]
+ TP = lltype.Array(lltype.Float, hints={'nolength': True})
+
+ def unwrap(self, space, w_item):
+ return space.float_w(space.float(w_item))
+
+
+ALL_DTYPES = [
+ W_Int8Dtype, W_Int32Dtype, W_Int64Dtype, W_LongDtype, W_BoolDtype,
W_Float64Dtype
+]
+
+dtypes_by_alias = unrolling_iterable([
+ (alias, dtype)
+ for dtype in ALL_DTYPES
+ for alias in dtype.aliases
+])
+dtypes_by_apptype = unrolling_iterable([
+ (apptype, dtype)
+ for dtype in ALL_DTYPES
+ for apptype in dtype.applevel_types
+])
+
+W_Dtype.typedef = TypeDef("dtype",
+ __new__ = interp2app(W_Dtype.descr__new__.im_func),
+
+ __repr__ = interp2app(W_Dtype.descr_repr),
+ __str__ = interp2app(W_Dtype.descr_str),
+
+ num = interp_attrproperty("num", cls=W_Dtype),
+ kind = interp_attrproperty("kind", cls=W_Dtype),
+)
\ No newline at end of file
diff --git a/pypy/module/micronumpy/interp_numarray.py
b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,17 +1,17 @@
+import math
+
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.module.micronumpy import interp_ufuncs, interp_dtype
from pypy.module.micronumpy.interp_support import Signature
-from pypy.module.micronumpy import interp_ufuncs
from pypy.objspace.std.floatobject import float2string as float2string_orig
from pypy.rlib import jit
from pypy.rlib.rfloat import DTSF_STR_PRECISION
from pypy.rpython.lltypesystem import lltype
from pypy.tool.sourcetools import func_with_new_name
-import math
-TP = lltype.Array(lltype.Float, hints={'nolength': True})
numpy_driver = jit.JitDriver(greens = ['signature'],
reds = ['result_size', 'i', 'self', 'result'])
@@ -45,6 +45,20 @@
arr.force_if_needed()
del self.invalidates[:]
+ def descr__new__(space, w_subtype, w_size_or_iterable,
w_dtype=NoneNotWrapped):
+ if w_dtype is None:
+ w_dtype = space.w_float
+ dtype = space.interp_w(interp_dtype.W_Dtype,
+ space.call_function(space.gettypefor(interp_dtype.W_Dtype),
w_dtype)
+ )
+ l = space.listview(w_size_or_iterable)
+ arr = SingleDimArray(len(l), dtype=dtype)
+ i = 0
+ for w_elem in l:
+ dtype.setitem_w(space, arr.storage, i, w_elem)
+ i += 1
+ return arr
+
def _unaryop_impl(w_ufunc):
def impl(self, space):
return w_ufunc(space, self)
@@ -206,7 +220,7 @@
raise NotImplementedError
def descr_copy(self, space):
- return new_numarray(space, self)
+ return space.call_function(space.gettypefor(BaseArray), self)
def descr_get_shape(self, space):
return space.newtuple([self.descr_len(space)])
@@ -242,18 +256,18 @@
self.find_size())
if step == 0:
# Single index
- self.get_concrete().setitem(start, space.float_w(w_value))
+ self.get_concrete().setitem(space, start, w_value)
else:
concrete = self.get_concrete()
if isinstance(w_value, BaseArray):
- # for now we just copy if setting part of an array from
+ # for now we just copy if setting part of an array from
# part of itself. can be improved.
if (concrete.get_root_storage() ==
w_value.get_concrete().get_root_storage()):
- w_value = new_numarray(space, w_value)
+ w_value = space.call_function(space.gettypefor(BaseArray),
w_value)
else:
w_value = convert_to_array(space, w_value)
- concrete.setslice(space, start, stop, step,
+ concrete.setslice(space, start, stop, step,
slice_length, w_value)
def descr_mean(self, space):
@@ -266,7 +280,7 @@
slice_driver1.jit_merge_point(signature=source.signature,
step=step, stop=stop, i=i, j=j, source=source,
dest=dest)
- dest.storage[i] = source.eval(j)
+ dest.dtype.setitem(dest.storage, i, source.eval(j))
j += 1
i += step
@@ -277,7 +291,7 @@
slice_driver2.jit_merge_point(signature=source.signature,
step=step, stop=stop, i=i, j=j, source=source,
dest=dest)
- dest.storage[i] = source.eval(j)
+ dest.dtype.setitem(dest.storage, i, source.eval(j))
j += 1
i += step
@@ -286,7 +300,7 @@
return w_obj
elif space.issequence_w(w_obj):
# Convert to array.
- return new_numarray(space, w_obj)
+ return space.call_function(space.gettypefor(BaseArray), w_obj)
else:
# If it's a scalar
return FloatWrapper(space.float_w(w_obj))
@@ -305,6 +319,9 @@
def find_size(self):
raise ValueError
+ def find_dtype(self):
+ raise ValueError
+
def eval(self, i):
return self.float_value
@@ -325,12 +342,12 @@
i = 0
signature = self.signature
result_size = self.find_size()
- result = SingleDimArray(result_size)
+ result = SingleDimArray(result_size, self.find_dtype())
while i < result_size:
numpy_driver.jit_merge_point(signature=signature,
result_size=result_size, i=i,
self=self, result=result)
- result.storage[i] = self.eval(i)
+ result.dtype.setitem(result.storage, i, self.eval(i))
i += 1
return result
@@ -354,6 +371,11 @@
return self.forced_result.find_size()
return self._find_size()
+ def find_dtype(self):
+ if self.forced_result is not None:
+ return self.forced_result.find_dtype()
+ return self._find_dtype()
+
class Call1(VirtualArray):
_immutable_fields_ = ["function", "values"]
@@ -369,6 +391,9 @@
def _find_size(self):
return self.values.find_size()
+ def _find_dtype(self):
+ return self.values.find_dtype()
+
def _eval(self, i):
return self.function(self.values.eval(i))
@@ -399,6 +424,27 @@
lhs, rhs = self.left.eval(i), self.right.eval(i)
return self.function(lhs, rhs)
+ def _find_dtype(self):
+ lhs_dtype = None
+ rhs_dtype = None
+ try:
+ lhs_dtype = self.left.find_dtype()
+ except ValueError:
+ pass
+ try:
+ rhs_dtype = self.right.find_dtype()
+ except ValueError:
+ pass
+ if lhs_dtype is not None and rhs_dtype is not None:
+ assert lhs_dtype is rhs_dtype
+ return lhs_dtype
+ elif lhs_dtype is not None:
+ return lhs_dtype
+ elif rhs_dtype is not None:
+ return rhs_dtype
+ else:
+ raise ValueError
+
class ViewArray(BaseArray):
"""
Class for representing views of arrays, they will reflect changes of parent
@@ -422,9 +468,9 @@
def eval(self, i):
return self.parent.eval(self.calc_index(i))
- @unwrap_spec(item=int, value=float)
- def setitem(self, item, value):
- return self.parent.setitem(self.calc_index(item), value)
+ @unwrap_spec(item=int)
+ def setitem(self, space, item, w_value):
+ return self.parent.setitem(space, self.calc_index(item), w_value)
def descr_len(self, space):
return space.wrap(self.find_size())
@@ -456,6 +502,9 @@
def find_size(self):
return self.size
+ def find_dtype(self):
+ return self.parent.find_dtype()
+
def setslice(self, space, start, stop, step, slice_length, arr):
start = self.calc_index(start)
if stop != -1:
@@ -473,13 +522,11 @@
class SingleDimArray(BaseArray):
signature = Signature()
- def __init__(self, size):
+ def __init__(self, size, dtype):
BaseArray.__init__(self)
self.size = size
- self.storage = lltype.malloc(TP, size, zero=True,
- flavor='raw', track_allocation=False,
- add_memory_pressure=True)
- # XXX find out why test_zjit explodes with trackign of allocations
+ self.dtype = dtype
+ self.storage = dtype.malloc(size)
def get_concrete(self):
return self
@@ -490,15 +537,18 @@
def find_size(self):
return self.size
+ def find_dtype(self):
+ return self.dtype
+
def eval(self, i):
- return self.storage[i]
+ return self.dtype.getitem(self.storage, i)
def descr_len(self, space):
return space.wrap(self.size)
- def setitem(self, item, value):
+ def setitem(self, space, item, w_value):
self.invalidated()
- self.storage[item] = value
+ self.dtype.setitem_w(space, self.storage, item, w_value)
def setslice(self, space, start, stop, step, slice_length, arr):
if step > 0:
@@ -509,32 +559,20 @@
def __del__(self):
lltype.free(self.storage, flavor='raw', track_allocation=False)
-def new_numarray(space, w_size_or_iterable):
- l = space.listview(w_size_or_iterable)
- arr = SingleDimArray(len(l))
- i = 0
- for w_elem in l:
- arr.storage[i] = space.float_w(space.float(w_elem))
- i += 1
- return arr
-
-def descr_new_numarray(space, w_type, w_size_or_iterable):
- return space.wrap(new_numarray(space, w_size_or_iterable))
-
@unwrap_spec(size=int)
def zeros(space, size):
- return space.wrap(SingleDimArray(size))
+ return space.wrap(SingleDimArray(size,
dtype=space.fromcache(interp_dtype.W_Float64Dtype)))
@unwrap_spec(size=int)
def ones(space, size):
- arr = SingleDimArray(size)
+ arr = SingleDimArray(size,
dtype=space.fromcache(interp_dtype.W_Float64Dtype))
for i in xrange(size):
- arr.storage[i] = 1.0
+ arr.dtype.setitem(arr.storage, i, 1.0)
return space.wrap(arr)
BaseArray.typedef = TypeDef(
'numarray',
- __new__ = interp2app(descr_new_numarray),
+ __new__ = interp2app(BaseArray.descr__new__.im_func),
copy = interp2app(BaseArray.descr_copy),
shape = GetSetProperty(BaseArray.descr_get_shape),
diff --git a/pypy/module/micronumpy/interp_support.py
b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -1,7 +1,8 @@
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
from pypy.rlib.rstruct.runpack import runpack
from pypy.rpython.lltypesystem import lltype, rffi
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import unwrap_spec
FLOAT_SIZE = rffi.sizeof(lltype.Float)
@@ -17,14 +18,14 @@
raise OperationError(space.w_ValueError, space.wrap(
"string length %d not divisable by %d" % (length, FLOAT_SIZE)))
- a = SingleDimArray(number)
+ a = SingleDimArray(number, dtype=space.fromcache(W_Float64Dtype))
start = 0
end = FLOAT_SIZE
i = 0
while i < number:
part = s[start:end]
- a.storage[i] = runpack('d', part)
+ a.dtype.setitem(a.storage, i, runpack('d', part))
i += 1
start += FLOAT_SIZE
end += FLOAT_SIZE
diff --git a/pypy/module/micronumpy/test/test_base.py
b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,13 +1,15 @@
from pypy.conftest import gettestobjspace
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
from pypy.module.micronumpy.interp_numarray import SingleDimArray, FloatWrapper
+
class BaseNumpyAppTest(object):
def setup_class(cls):
cls.space = gettestobjspace(usemodules=('micronumpy',))
class TestSignature(object):
def test_binop_signature(self, space):
- ar = SingleDimArray(10)
+ ar = SingleDimArray(10, dtype=space.fromcache(W_Float64Dtype))
v1 = ar.descr_add(space, ar)
v2 = ar.descr_add(space, FloatWrapper(2.0))
assert v1.signature is not v2.signature
@@ -17,7 +19,7 @@
assert v1.signature is v4.signature
def test_slice_signature(self, space):
- ar = SingleDimArray(10)
+ ar = SingleDimArray(10, dtype=space.fromcache(W_Float64Dtype))
v1 = ar.descr_getitem(space, space.wrap(slice(1, 5, 1)))
v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
assert v1.signature is v2.signature
diff --git a/pypy/module/micronumpy/test/test_dtypes.py
b/pypy/module/micronumpy/test/test_dtypes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -0,0 +1,37 @@
+import py
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+
+class AppTestDtypes(BaseNumpyAppTest):
+ def test_dtype(self):
+ from numpy import dtype
+
+ d = dtype('l')
+ assert d.num == 7
+ assert d.kind == 'i'
+ assert dtype('int8').num == 1
+ assert dtype(d) is d
+
+ def test_dtype_with_types(self):
+ from numpy import dtype
+
+ assert dtype(bool).num == 0
+ assert dtype(int).num == 7
+ assert dtype(long).num == 9
+ assert dtype(float).num == 12
+
+ def test_repr_str(self):
+ from numpy import dtype
+
+ d = dtype('?')
+ assert repr(d) == "dtype('bool')"
+ assert str(d) == "bool"
+
+ def test_bool_array(self):
+ from numpy import array
+
+ a = array([0, 1, 2, 2.5], dtype='?')
+ assert a[0] is False
+ for i in xrange(1, 4):
+ assert a[i] is True
diff --git a/pypy/module/micronumpy/test/test_zjit.py
b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,11 +1,13 @@
from pypy.jit.metainterp.test.support import LLJitMixin
-from pypy.rpython.test.test_llinterp import interpret
+from pypy.module.micronumpy.compile import numpy_compile
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
from pypy.module.micronumpy.interp_numarray import (SingleDimArray, Signature,
FloatWrapper, Call2, SingleDimSlice, add, mul, Call1)
from pypy.module.micronumpy.interp_ufuncs import negative
-from pypy.module.micronumpy.compile import numpy_compile
+from pypy.rlib.nonconst import NonConstant
from pypy.rlib.objectmodel import specialize
-from pypy.rlib.nonconst import NonConstant
+from pypy.rpython.test.test_llinterp import interpret
+
class FakeSpace(object):
w_ValueError = None
@@ -23,12 +25,14 @@
class TestNumpyJIt(LLJitMixin):
def setup_class(cls):
cls.space = FakeSpace()
+ cls.float64_dtype = W_Float64Dtype(cls.space)
def test_add(self):
def f(i):
- ar = SingleDimArray(i)
+ ar = SingleDimArray(i, dtype=self.float64_dtype)
v = Call2(add, ar, ar, Signature())
- return v.get_concrete().storage[3]
+ concrete = v.get_concrete()
+ return concrete.dtype.getitem(concrete.storage, 3)
result = self.meta_interp(f, [5], listops=True, backendopt=True)
self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
@@ -40,7 +44,7 @@
def f(i):
ar = SingleDimArray(i)
v = Call2(add, ar, FloatWrapper(4.5), Signature())
- return v.get_concrete().storage[3]
+ return v.dtype.getitem(v.get_concrete().storage, 3)
result = self.meta_interp(f, [5], listops=True, backendopt=True)
self.check_loops({"getarrayitem_raw": 1, "float_add": 1,
@@ -88,7 +92,7 @@
result = self.meta_interp(f, [5], listops=True, backendopt=True)
self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
"float_gt": 1, "int_add": 1,
- "int_lt": 1, "guard_true": 1,
+ "int_lt": 1, "guard_true": 1,
"guard_false": 1, "jump": 1})
assert result == f(5)
@@ -269,10 +273,10 @@
assert x.size == 10
assert x.storage[0] == 0
assert x.storage[1] == ((1 + 1) * 1.2) / 1.2 - 1
-
+
def test_translation(self):
# we import main to check if the target compiles
from pypy.translator.goal.targetnumpystandalone import main
from pypy.rpython.annlowlevel import llstr
-
+
interpret(main, [llstr('af+'), 100])
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -117,6 +117,10 @@
return erase, unerase
+def new_static_erasing_pair(name):
+ erase, unerase = new_erasing_pair(name)
+ return staticmethod(erase), staticmethod(unerase)
+
# ---------- implementation-specific ----------
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
