Author: Jeff Terrace <jterr...@gmail.com>
Branch: numpy-full-fromstring
Changeset: r50475:8c97ceda4c34
Date: 2011-12-13 15:51 -0500
http://bitbucket.org/pypy/pypy/changeset/8c97ceda4c34/
Log: Wrote some really ugly code to deal with all the corner cases of
numpy fromstring error handling
diff --git a/pypy/module/micronumpy/interp_support.py
b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -2,14 +2,17 @@
from pypy.interpreter.gateway import unwrap_spec
from pypy.rpython.lltypesystem import lltype, rffi
from pypy.module.micronumpy import interp_dtype
+from pypy.objspace.std.strutil import strip_spaces
FLOAT_SIZE = rffi.sizeof(lltype.Float)
def _fromstring_text(space, s, count, sep, length, dtype):
- import string
from pypy.module.micronumpy.interp_numarray import W_NDimArray
+ sep_stripped = strip_spaces(sep)
+ skip_bad_vals = True if len(sep_stripped) == 0 else False
+
A = []
num_items = 0
ptr = 0
@@ -18,10 +21,28 @@
nextptr = s.find(sep, ptr)
if nextptr < 0:
nextptr = length
- piece = s[ptr:nextptr]
- #FIXME: need to check piece.isspace() also, but does not translate
- if len(piece) > 0:
- val = dtype.coerce(space, space.wrap(piece))
+ piece = strip_spaces(s[ptr:nextptr])
+ if len(piece) > 0 or not skip_bad_vals:
+ if len(piece) == 0 and not skip_bad_vals:
+ val = dtype.itemtype.default_fromstring(space)
+ else:
+ try:
+ val = dtype.coerce(space, space.wrap(piece))
+ except OperationError, e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ gotit = False
+ while not gotit and len(piece) > 0:
+ piece = piece[:-1]
+ try:
+ val = dtype.coerce(space, space.wrap(piece))
+ gotit = True
+ except OperationError, e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ if not gotit:
+ val = dtype.itemtype.default_fromstring(space)
+ nextptr = length
A.append(val)
num_items += 1
ptr = nextptr + 1
diff --git a/pypy/module/micronumpy/test/test_numarray.py
b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1173,7 +1173,7 @@
cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
def test_fromstring(self):
- from numpypy import fromstring, uint8, float32, int32
+ from numpypy import fromstring, array, uint8, float32, int32
a = fromstring(self.data)
for i in range(4):
assert a[i] == i + 1
@@ -1201,12 +1201,28 @@
assert g[0] == 1
assert g[1] == 2
assert g[2] == 3
- #FIXME: below should work
- #h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
- #assert len(h) == 3
- #assert h[0] == 1
- #assert h[1] == 2
- #assert h[2] == 3
+ h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
+ assert h.tolist() == [1,0,2,3]
+ i = fromstring("1 2 3", dtype=uint8, sep=" ")
+ assert i.tolist() == [1,2,3]
+ j = fromstring("1\t\t\t\t2\t3", dtype=uint8, sep="\t")
+ assert j.tolist() == [1,2,3]
+ k = fromstring("1,x,2,3", dtype=uint8, sep=",")
+ assert k.tolist() == [1,0]
+ l = fromstring("1,x,2,3", dtype='float32', sep=",")
+ assert l.tolist() == [1.0,-1.0]
+ m = fromstring("1,,2,3", sep=",")
+ assert m.tolist() == [1.0,-1.0,2.0,3.0]
+ n = fromstring("3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
+ assert n.tolist() == [3]
+ o = fromstring("1.0 2f.0f 3.8 2.2", dtype=float32, sep=" ")
+ assert len(o) == 2
+ assert o[0] == 1.0
+ assert o[1] == 2.0
+ p = fromstring("1.0,,2.0,3.0", sep=",")
+ assert p.tolist() == [1.0, -1.0, 2.0, 3.0]
+ q = fromstring("1.0,,2.0,3.0", sep=" ")
+ assert q.tolist() == [1.0]
def test_fromstring_types(self):
from numpypy import fromstring
@@ -1241,8 +1257,6 @@
raises(ValueError, fromstring, "\x01\x03\x03", dtype=uint16)
#5 bytes is larger than 3 bytes
raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8)
- #can't cast floats to ints with fromstring
- raises(ValueError, fromstring, "3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
class AppTestRepr(BaseNumpyAppTest):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -87,6 +87,9 @@
def _coerce(self, space, w_item):
raise NotImplementedError
+ def default_fromstring(self, space):
+ raise NotImplementedError
+
def read(self, storage, width, i, offset):
return
self.box(libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
width, storage, i, offset
@@ -213,6 +216,9 @@
def for_computation(self, v):
return widen(v)
+
+ def default_fromstring(self, space):
+ return self._coerce(space, space.wrap(0))
@simple_binary_op
def div(self, v1, v2):
@@ -320,6 +326,9 @@
def for_computation(self, v):
return float(v)
+ def default_fromstring(self, space):
+ return self._coerce(space, space.wrap(-1.0))
+
@simple_binary_op
def div(self, v1, v2):
try:
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit
