Author: mattip <[email protected]>
Branch:
Changeset: r80593:fd1672bd10fe
Date: 2015-11-08 19:57 +0200
http://bitbucket.org/pypy/pypy/changeset/fd1672bd10fe/
Log: merge propogate-nans, which tries harder to preserve nan mantissa
values across float conversions
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -18,3 +18,10 @@
.. branch: int_0/i-need-this-library-to-build-on-ubuntu-1-1446717626227
Document that libgdbm-dev is required for translation/packaging
+
+.. branch: propogate-nans
+
+Ensure that ndarray conversion from int16->float16->float32->float16->int16
+preserves all int16 values, even across nan conversions. Also fix argmax,
argmin
+for nan comparisons
+
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -534,10 +534,10 @@
while not inner_iter.done(inner_state):
arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
w_val = inner_iter.getitem(inner_state)
- new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
- if dtype.itemtype.ne(new_best, cur_best):
+ old_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
+ if not old_best:
result = idx
- cur_best = new_best
+ cur_best = w_val
inner_state = inner_iter.next(inner_state)
idx += 1
result = get_dtype_cache(space).w_longdtype.box(result)
@@ -557,17 +557,17 @@
while not iter.done(state):
arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
w_val = iter.getitem(state)
- new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
- if dtype.itemtype.ne(new_best, cur_best):
+ old_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
+ if not old_best:
result = idx
- cur_best = new_best
+ cur_best = w_val
state = iter.next(state)
idx += 1
return result
return argmin_argmax, argmin_argmax_flat
-argmin, argmin_flat = _new_argmin_argmax('min')
-argmax, argmax_flat = _new_argmin_argmax('max')
+argmin, argmin_flat = _new_argmin_argmax('argmin')
+argmax, argmax_flat = _new_argmin_argmax('argmax')
dot_driver = jit.JitDriver(name = 'numpy_dot',
greens = ['dtype'],
diff --git a/pypy/module/micronumpy/test/test_ndarray.py
b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1852,6 +1852,24 @@
a = array([(1, 2)], dtype=[('a', 'int64'), ('b', 'int64')])
assert a.view('S16')[0] == '\x01' + '\x00' * 7 + '\x02'
+ def test_half_conversions(self):
+ from numpy import array, arange
+ from math import isnan, isinf
+ e = array([0, -1, -float('inf'), float('nan'), 6], dtype='float16')
+ assert map(isnan, e) == [False, False, False, True, False]
+ assert map(isinf, e) == [False, False, True, False, False]
+ assert e.argmax() == 3
+ # numpy preserves value for uint16 -> cast_as_float16 ->
+ # convert_to_float64 -> convert_to_float16 -> uint16
+ # even for float16 various float16 nans
+ all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
+ all_f16.dtype = 'float16'
+ all_f32 = array(all_f16, dtype='float32')
+ b = array(all_f32, dtype='float16')
+ c = b.view(dtype='uint16')
+ d = all_f16.view(dtype='uint16')
+ assert (c == d).all()
+
def test_ndarray_view_empty(self):
from numpy import array, dtype
x = array([], dtype=[('a', 'int8'), ('b', 'int8')])
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -345,6 +345,14 @@
def min(self, v1, v2):
return min(v1, v2)
+ @raw_binary_op
+ def argmax(self, v1, v2):
+ return v1 >= v2
+
+ @raw_binary_op
+ def argmin(self, v1, v2):
+ return v1 <= v2
+
@raw_unary_op
def rint(self, v):
float64 = Float64(self.space)
@@ -820,6 +828,14 @@
def min(self, v1, v2):
return v1 if v1 <= v2 or rfloat.isnan(v1) else v2
+ @raw_binary_op
+ def argmax(self, v1, v2):
+ return v1 >= v2 or rfloat.isnan(v1)
+
+ @raw_binary_op
+ def argmin(self, v1, v2):
+ return v1 <= v2 or rfloat.isnan(v1)
+
@simple_binary_op
def fmax(self, v1, v2):
return v1 if v1 >= v2 or rfloat.isnan(v2) else v2
@@ -1407,6 +1423,16 @@
return v1
return v2
+ def argmin(self, v1, v2):
+ if self.le(v1, v2) or self.isnan(v1):
+ return True
+ return False
+
+ def argmax(self, v1, v2):
+ if self.ge(v1, v2) or self.isnan(v1):
+ return True
+ return False
+
@complex_binary_op
def floordiv(self, v1, v2):
(r1, i1), (r2, i2) = v1, v2
@@ -1927,6 +1953,18 @@
return v1
return v2
+ @raw_binary_op
+ def argmax(self, v1, v2):
+ if self.space.is_true(self.space.ge(v1, v2)):
+ return True
+ return False
+
+ @raw_binary_op
+ def argmin(self, v1, v2):
+ if self.space.is_true(self.space.le(v1, v2)):
+ return True
+ return False
+
@raw_unary_op
def bool(self,v):
return self._obool(v)
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -5,7 +5,8 @@
import math
from rpython.rlib import rarithmetic, rfloat, objectmodel, jit
-from rpython.rlib.rarithmetic import r_ulonglong
+from rpython.rtyper.lltypesystem.rffi import r_ulonglong, r_longlong,
LONGLONG, ULONGLONG, cast
+from rpython.rlib.longlong2float import longlong2float, float2longlong
def round_to_nearest(x):
"""Python 3 style round: round a float x to the nearest int, but
@@ -60,7 +61,16 @@
if exp == MAX_EXP - MIN_EXP + 2:
# nan or infinity
- result = rfloat.NAN if mant else rfloat.INFINITY
+ if mant == 0:
+ result = rfloat.INFINITY
+ else:
+ # preserve at most 52 bits of mant value, but pad w/zeros
+ exp = r_ulonglong(0x7ff) << 52
+ sign = r_ulonglong(sign) << 63
+ mant = r_ulonglong(mant) << (52 - MANT_DIG)
+ uint = exp | mant | sign
+ result = longlong2float(cast(LONGLONG, uint))
+ return result
elif exp == 0:
# subnormal or zero
result = math.ldexp(mant, MIN_EXP - MANT_DIG)
@@ -72,7 +82,7 @@
def float_unpack80(QQ, size):
'''Unpack a (mant, exp) tuple of r_ulonglong in 80-bit extended format
- into a long double float
+ into a python float (a double)
'''
if size == 10 or size == 12 or size == 16:
MIN_EXP = -16381
@@ -100,7 +110,13 @@
if exp == MAX_EXP - MIN_EXP + 2:
# nan or infinity
- result = rfloat.NAN if mant &((one << MANT_DIG - 1) - 1) else
rfloat.INFINITY
+ if mant == 0:
+ result = rfloat.INFINITY
+ else:
+ exp = r_ulonglong(0x7ff) << 52
+ uint = exp | r_ulonglong(mant) | r_ulonglong(sign)
+ result = longlong2float(cast(LONGLONG, uint))
+ return result
else:
# normal
result = math.ldexp(mant, exp + MIN_EXP - MANT_DIG - 1)
@@ -128,13 +144,19 @@
raise ValueError("invalid size value")
sign = rfloat.copysign(1.0, x) < 0.0
- if not rfloat.isfinite(x):
- if rfloat.isinf(x):
- mant = r_ulonglong(0)
- exp = MAX_EXP - MIN_EXP + 2
- else: # rfloat.isnan(x):
- mant = r_ulonglong(1) << (MANT_DIG-2) # other values possible
- exp = MAX_EXP - MIN_EXP + 2
+ if rfloat.isinf(x):
+ mant = r_ulonglong(0)
+ exp = MAX_EXP - MIN_EXP + 2
+ elif rfloat.isnan(x):
+ asint = cast(ULONGLONG, float2longlong(x))
+ mant = asint & ((r_ulonglong(1) << 51) - 1)
+ sign = asint >> 63
+ # shift off lower bits, perhaps losing data
+ if MANT_DIG <= 52:
+ mant = mant >> (52 - MANT_DIG)
+ if mant == 0:
+ mant = r_ulonglong(1) << (MANT_DIG - 2) - 1
+ exp = MAX_EXP - MIN_EXP + 2
elif x == 0.0:
mant = r_ulonglong(0)
exp = 0
@@ -167,7 +189,7 @@
# check constraints
if not objectmodel.we_are_translated():
- assert 0 <= mant < 1 << MANT_DIG - 1
+ assert 0 <= mant <= (1 << MANT_DIG) - 1
assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
assert 0 <= sign <= 1
exp = r_ulonglong(exp)
@@ -187,13 +209,16 @@
raise ValueError("invalid size value")
sign = rfloat.copysign(1.0, x) < 0.0
- if not rfloat.isfinite(x):
- if rfloat.isinf(x):
- mant = r_ulonglong(0)
- exp = MAX_EXP - MIN_EXP + 2
- else: # rfloat.isnan(x):
- mant = (r_ulonglong(1) << (MANT_DIG-2)) - 1 # other values possible
- exp = MAX_EXP - MIN_EXP + 2
+ if rfloat.isinf(x):
+ mant = r_ulonglong(0)
+ exp = MAX_EXP - MIN_EXP + 2
+ elif rfloat.isnan(x): # rfloat.isnan(x):
+ asint = cast(ULONGLONG, float2longlong(x))
+ mant = asint & ((r_ulonglong(1) << 51) - 1)
+ if mant == 0:
+ mant = r_ulonglong(1) << (MANT_DIG-2) - 1
+ sign = asint < 0
+ exp = MAX_EXP - MIN_EXP + 2
elif x == 0.0:
mant = r_ulonglong(0)
exp = 0
@@ -221,12 +246,12 @@
if exp >= MAX_EXP - MIN_EXP + 2:
raise OverflowError("float too large to pack in this format")
+ mant = mant << 1
# check constraints
if not objectmodel.we_are_translated():
- assert 0 <= mant < 1 << MANT_DIG - 1
+ assert 0 <= mant <= (1 << MANT_DIG) - 1
assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
assert 0 <= sign <= 1
- mant = mant << 1
exp = r_ulonglong(exp)
sign = r_ulonglong(sign)
return (mant, (sign << BITS - MANT_DIG - 1) | exp)
diff --git a/rpython/rlib/rstruct/test/test_ieee.py
b/rpython/rlib/rstruct/test/test_ieee.py
--- a/rpython/rlib/rstruct/test/test_ieee.py
+++ b/rpython/rlib/rstruct/test/test_ieee.py
@@ -168,15 +168,31 @@
def test_random(self):
# construct a Python float from random integer, using struct
+ mantissa_mask = (1 << 53) - 1
for _ in xrange(10000):
Q = random.randrange(2**64)
x = struct.unpack('<d', struct.pack('<Q', Q))[0]
# nans are tricky: we can't hope to reproduce the bit
- # pattern exactly, so check_float will fail for a random nan.
- if isnan(x):
+ # pattern exactly, so check_float will fail for a nan
+ # whose mantissa does not fit into float16's mantissa.
+ if isnan(x) and (Q & mantissa_mask) >= 1 << 11:
continue
self.check_float(x)
+ def test_various_nans(self):
+ # check patterns that should preserve the mantissa across nan
conversions
+ maxmant64 = (1 << 52) - 1 # maximum double mantissa
+ maxmant16 = (1 << 10) - 1 # maximum float16 mantissa
+ assert maxmant64 >> 42 == maxmant16
+ exp = 0xfff << 52
+ for i in range(20):
+ val_to_preserve = exp | ((maxmant16 - i) << 42)
+ a = ieee.float_unpack(val_to_preserve, 8)
+ assert isnan(a), 'i %d, maxmant %s' % (i, hex(val_to_preserve))
+ b = ieee.float_pack(a, 8)
+ assert b == val_to_preserve, 'i %d, val %s b %s' % (i,
hex(val_to_preserve), hex(b))
+ b = ieee.float_pack(a, 2)
+ assert b == 0xffff - i, 'i %d, b%s' % (i, hex(b))
class TestCompiled:
def test_pack_float(self):
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
