Author: Hakan Ardo <ha...@debian.org>
Branch: jit-improve-nested-loops
Changeset: r50823:f637da0553fb
Date: 2011-12-22 16:04 +0100
http://bitbucket.org/pypy/pypy/changeset/f637da0553fb/
Log: hg merge default
diff --git a/pypy/jit/backend/x86/jump.py b/pypy/jit/backend/x86/jump.py
--- a/pypy/jit/backend/x86/jump.py
+++ b/pypy/jit/backend/x86/jump.py
@@ -17,7 +17,10 @@
key = src._getregkey()
if key in srccount:
if key == dst_locations[i]._getregkey():
- srccount[key] = -sys.maxint # ignore a move "x = x"
+ # ignore a move "x = x"
+ # setting any "large enough" negative value is ok, but
+ # be careful of overflows, don't use -sys.maxint
+ srccount[key] = -len(dst_locations) - 1
pending_dests -= 1
else:
srccount[key] += 1
diff --git a/pypy/jit/backend/x86/test/test_jump.py
b/pypy/jit/backend/x86/test/test_jump.py
--- a/pypy/jit/backend/x86/test/test_jump.py
+++ b/pypy/jit/backend/x86/test/test_jump.py
@@ -385,3 +385,32 @@
assert read(loc, WORD) == src_values1[i]
for i, loc in enumerate(dst_locations2):
assert read(loc, 8) == src_values2[i]
+
+
+def test_overflow_bug():
+ CASE = [
+ (-144, -248), # \ cycle
+ (-248, -144), # /
+ (-488, -416), # \ two usages of -488
+ (-488, -480), # /
+ (-488, -488), # - one self-application of -488
+ ]
+ class FakeAssembler:
+ def regalloc_mov(self, src, dst):
+ print "mov", src, dst
+ def regalloc_push(self, x):
+ print "push", x
+ def regalloc_pop(self, x):
+ print "pop", x
+ def regalloc_immedmem2mem(self, x, y):
+ print "?????????????????????????"
+ def main():
+ srclocs = [StackLoc(9999, x, 'i') for x,y in CASE]
+ dstlocs = [StackLoc(9999, y, 'i') for x,y in CASE]
+ remap_frame_layout(FakeAssembler(), srclocs, dstlocs, eax)
+ # it works when run directly
+ main()
+ # but it used to crash when translated,
+ # because of a -sys.maxint-2 overflowing to sys.maxint
+ from pypy.rpython.test.test_llinterp import interpret
+ interpret(main, [])
diff --git a/pypy/module/_multiprocessing/interp_connection.py
b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -6,7 +6,7 @@
OperationError, wrap_oserror, operationerrfmt)
from pypy.rpython.lltypesystem import rffi, lltype
from pypy.rlib.rarithmetic import intmask
-from pypy.rlib import rpoll
+from pypy.rlib import rpoll, rsocket
import sys
READABLE = 1
@@ -252,7 +252,9 @@
# "header" and the "body" of the message and send them at once.
message = lltype.malloc(rffi.CCHARP.TO, size + 4, flavor='raw')
try:
- rffi.cast(rffi.UINTP, message)[0] = rffi.r_uint(size) # XXX htonl!
+ length = rffi.r_uint(rsocket.htonl(
+ rffi.cast(lltype.Unsigned, size)))
+ rffi.cast(rffi.UINTP, message)[0] = length
i = size - 1
while i >= 0:
message[4 + i] = buffer[offset + i]
@@ -264,7 +266,8 @@
def do_recv_string(self, space, buflength, maxlength):
with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1) as
length_ptr:
self._recvall(space, rffi.cast(rffi.CCHARP, length_ptr), 4)
- length = intmask(length_ptr[0])
+ length = intmask(rsocket.ntohl(
+ rffi.cast(lltype.Unsigned, length_ptr[0])))
if length > maxlength: # bad message, close connection
self.flags &= ~READABLE
if self.flags == 0:
diff --git a/pypy/module/_multiprocessing/test/test_connection.py
b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -37,6 +37,9 @@
def test_connection(self):
rhandle, whandle = self.make_pair()
+ whandle.send_bytes("abc")
+ assert rhandle.recv_bytes(100) == "abc"
+
obj = [1, 2.0, "hello"]
whandle.send(obj)
obj2 = rhandle.recv()
@@ -150,4 +153,20 @@
import _multiprocessing
raises(IOError, _multiprocessing.Connection, -1)
- raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
+ raises(IOError, _multiprocessing.Connection, -15)
+
+ def test_byte_order(self):
+ # The exact format of net strings (length in network byte
+ # order) is important for interoperation with others
+ # implementations.
+ rhandle, whandle = self.make_pair()
+ whandle.send_bytes("abc")
+ whandle.send_bytes("defg")
+ import socket
+ sock = socket.fromfd(rhandle.fileno(),
+ socket.AF_INET, socket.SOCK_STREAM)
+ data1 = sock.recv(7)
+ assert data1 == '\x00\x00\x00\x03abc'
+ data2 = sock.recv(8)
+ assert data2 == '\x00\x00\x00\x04defg'
+
diff --git a/pypy/module/micronumpy/interp_iter.py
b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -102,3 +102,27 @@
class ConstantIterator(BaseIterator):
def next(self, shapelen):
return self
+
+# ------ other iterators that are not part of the computation frame ----------
+
+class AxisIterator(object):
+ """ This object will return offsets of each start of the last stride
+ """
+ def __init__(self, arr):
+ self.arr = arr
+ self.indices = [0] * (len(arr.shape) - 1)
+ self.done = False
+ self.offset = arr.start
+
+ def next(self):
+ for i in range(len(self.arr.shape) - 2, -1, -1):
+ if self.indices[i] < self.arr.shape[i] - 1:
+ self.indices[i] += 1
+ self.offset += self.arr.strides[i]
+ break
+ else:
+ self.indices[i] = 0
+ self.offset -= self.arr.backstrides[i]
+ else:
+ self.done = True
+
diff --git a/pypy/module/micronumpy/interp_numarray.py
b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -9,7 +9,7 @@
from pypy.tool.sourcetools import func_with_new_name
from pypy.rlib.rstring import StringBuilder
from pypy.module.micronumpy.interp_iter import ArrayIterator,\
- view_iter_from_arr, OneDimIterator
+ view_iter_from_arr, OneDimIterator, AxisIterator
numpy_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
@@ -390,10 +390,10 @@
return space.wrap(self.size)
def descr_copy(self, space):
- return self.copy()
+ return self.copy(space)
- def copy(self):
- return self.get_concrete().copy()
+ def copy(self, space):
+ return self.get_concrete().copy(space)
def descr_len(self, space):
if len(self.shape):
@@ -536,7 +536,7 @@
new_shape, self)
else:
# Create copy with contiguous data
- arr = concrete.copy()
+ arr = concrete.copy(space)
arr.setshape(space, new_shape)
return arr
@@ -606,6 +606,9 @@
space.w_False]))
return w_d
+ def supports_fast_slicing(self):
+ return False
+
def convert_to_array(space, w_obj):
if isinstance(w_obj, BaseArray):
return w_obj
@@ -639,7 +642,7 @@
def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
builder.append(self.dtype.itemtype.str_format(self.value))
- def copy(self):
+ def copy(self, space):
return Scalar(self.dtype, self.value)
def create_sig(self, res_shape):
@@ -790,6 +793,9 @@
def get_concrete(self):
return self
+ def supports_fast_slicing(self):
+ return self.order == 'C' and self.strides[-1] == 1
+
def find_dtype(self):
return self.dtype
@@ -929,39 +935,35 @@
item += v * self.strides[i]
return item
-
-class ViewArray(ConcreteArray):
- def copy(self):
- array = W_NDimArray(self.size, self.shape[:], self.find_dtype())
- iter = view_iter_from_arr(self)
- a_iter = ArrayIterator(array.size)
- while not iter.done():
- array.setitem(a_iter.offset, self.getitem(iter.offset))
- iter = iter.next(len(self.shape))
- a_iter = a_iter.next(len(array.shape))
- return array
-
- def create_sig(self, res_shape):
- return signature.ViewSignature(self.dtype)
-
-
-class W_NDimSlice(ViewArray):
- def __init__(self, start, strides, backstrides, shape, parent):
- assert isinstance(parent, ConcreteArray)
- if isinstance(parent, W_NDimSlice):
- parent = parent.parent
- size = 1
- for sh in shape:
- size *= sh
- self.strides = strides
- self.backstrides = backstrides
- ViewArray.__init__(self, size, shape, parent.dtype, parent.order,
- parent)
- self.start = start
-
def setslice(self, space, w_value):
res_shape = shape_agreement(space, self.shape, w_value.shape)
- self._sliceloop(w_value, res_shape)
+ if (res_shape == w_value.shape and self.supports_fast_slicing() and
+ w_value.supports_fast_slicing() and
+ self.dtype is w_value.find_dtype()):
+ self._fast_setslice(space, w_value)
+ else:
+ self._sliceloop(w_value, res_shape)
+
+ def _fast_setslice(self, space, w_value):
+ assert isinstance(w_value, ConcreteArray)
+ itemsize = self.dtype.itemtype.get_element_size()
+ if len(self.shape) == 1:
+ rffi.c_memcpy(
+ rffi.ptradd(self.storage, self.start * itemsize),
+ rffi.ptradd(w_value.storage, w_value.start * itemsize),
+ self.size * itemsize
+ )
+ else:
+ dest = AxisIterator(self)
+ source = AxisIterator(w_value)
+ while not dest.done:
+ rffi.c_memcpy(
+ rffi.ptradd(self.storage, dest.offset * itemsize),
+ rffi.ptradd(w_value.storage, source.offset * itemsize),
+ self.shape[-1] * itemsize
+ )
+ source.next()
+ dest.next()
def _sliceloop(self, source, res_shape):
sig = source.find_sig(res_shape)
@@ -979,6 +981,31 @@
frame.next(shapelen)
res_iter = res_iter.next(shapelen)
+ def copy(self, space):
+ array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
+ array.setslice(space, self)
+ return array
+
+
+class ViewArray(ConcreteArray):
+ def create_sig(self, res_shape):
+ return signature.ViewSignature(self.dtype)
+
+
+class W_NDimSlice(ViewArray):
+ def __init__(self, start, strides, backstrides, shape, parent):
+ assert isinstance(parent, ConcreteArray)
+ if isinstance(parent, W_NDimSlice):
+ parent = parent.parent
+ size = 1
+ for sh in shape:
+ size *= sh
+ self.strides = strides
+ self.backstrides = backstrides
+ ViewArray.__init__(self, size, shape, parent.dtype, parent.order,
+ parent)
+ self.start = start
+
def setshape(self, space, new_shape):
if len(self.shape) < 1:
return
@@ -1017,15 +1044,6 @@
""" A class representing contiguous array. We know that each iteration
by say ufunc will increase the data index by one
"""
- def copy(self):
- array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
- rffi.c_memcpy(
- array.storage,
- self.storage,
- self.size * self.dtype.itemtype.get_element_size()
- )
- return array
-
def setitem(self, item, value):
self.invalidated()
self.dtype.setitem(self.storage, item, value)
diff --git a/pypy/module/micronumpy/test/test_numarray.py
b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1077,6 +1077,17 @@
a = ones((1, 2, 3))
assert a[0, 1, 2] == 1.0
+ def test_multidim_setslice(self):
+ from numpypy import zeros, ones
+ a = zeros((3, 3))
+ b = ones((3, 3))
+ a[:,1:3] = b[:,1:3]
+ assert (a == [[0, 1, 1], [0, 1, 1], [0, 1, 1]]).all()
+ a = zeros((3, 3))
+ b = ones((3, 3))
+ a[:,::2] = b[:,::2]
+ assert (a == [[1, 0, 1], [1, 0, 1], [1, 0, 1]]).all()
+
def test_broadcast_ufunc(self):
from numpypy import array
a = array([[1, 2], [3, 4], [5, 6]])
diff --git a/pypy/objspace/std/stringobject.py
b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -510,6 +510,16 @@
return space.wrap(res)
+def _replace_overflow_check(space, builder, new_piece):
+ # Checks if adding new_piece chars to the builder would overflow, and
+ # converts into an OverflowError.
+ try:
+ ovfcheck(builder.getlength() + new_piece)
+ except OverflowError:
+ raise OperationError(space.w_OverflowError,
+ space.wrap("replace string is too long")
+ )
+
def _string_replace(space, input, sub, by, maxsplit):
if maxsplit == 0:
return space.wrap(input)
@@ -519,7 +529,7 @@
if maxsplit > 0 and maxsplit < upper + 2:
upper = maxsplit - 1
assert upper >= 0
-
+
try:
result_size = ovfcheck(upper * len(by))
result_size = ovfcheck(result_size + upper)
@@ -548,14 +558,18 @@
if next < 0:
break
if not first:
+ _replace_overflow_check(space, builder, len(by))
builder.append(by)
first = False
+ _replace_overflow_check(space, builder, next - start)
builder.append_slice(input, start, next)
start = next + sublen
maxsplit -= 1 # NB. if it's already < 0, it stays < 0
if not first:
+ _replace_overflow_check(space, builder, len(by))
builder.append(by)
+ _replace_overflow_check(space, builder, len(input) - start)
builder.append_slice(input, start, len(input))
return space.wrap(builder.build())
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit
