Author: Carl Friedrich Bolz <cfb...@gmx.de> Branch: optinfo-into-bridges Changeset: r87848:1f9d308fe99b Date: 2016-10-17 17:58 +0200 http://bitbucket.org/pypy/pypy/changeset/1f9d308fe99b/
Log: merge default diff --git a/lib-python/2.7/test/test_cmd_line.py b/lib-python/2.7/test/test_cmd_line.py --- a/lib-python/2.7/test/test_cmd_line.py +++ b/lib-python/2.7/test/test_cmd_line.py @@ -75,7 +75,7 @@ p.stdin.write('Timer\n') p.stdin.write('exit()\n') data = kill_python(p) - self.assertTrue(data.startswith('1 loop')) + self.assertIn('1 loop', data) self.assertIn('__main__.Timer', data) def test_run_code(self): diff --git a/lib-python/2.7/test/test_timeit.py b/lib-python/2.7/test/test_timeit.py --- a/lib-python/2.7/test/test_timeit.py +++ b/lib-python/2.7/test/test_timeit.py @@ -318,9 +318,8 @@ s = self.run_main(seconds_per_increment=2.0, switches=['-n35', '-s', 'print("CustomSetup")']) self.assertIn(dedent("""\ WARNING: timeit is a very unreliable tool. use perf or something else for real measurements - pypy -m pip install perf - pypy -m perf timeit -n35 -s 'print("CustomSetup")' 'import timeit; timeit._fake_timer.inc()' """), s) + self.assertIn("-m pip install perf", s) diff --git a/lib_pypy/_tkinter/tclobj.py b/lib_pypy/_tkinter/tclobj.py --- a/lib_pypy/_tkinter/tclobj.py +++ b/lib_pypy/_tkinter/tclobj.py @@ -171,7 +171,7 @@ encoded = value.encode('utf-16')[2:] buf = tkffi.new("char[]", encoded) inbuf = tkffi.cast("Tcl_UniChar*", buf) - return tklib.Tcl_NewUnicodeObj(buf, len(encoded)/2) + return tklib.Tcl_NewUnicodeObj(inbuf, len(encoded)/2) if isinstance(value, TclObject): tklib.Tcl_IncrRefCount(value._value) return value._value diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py --- a/lib_pypy/_tkinter/tklib_build.py +++ b/lib_pypy/_tkinter/tklib_build.py @@ -107,7 +107,7 @@ int Tcl_Init(Tcl_Interp* interp); int Tk_Init(Tcl_Interp* interp); -void Tcl_Free(char* ptr); +void Tcl_Free(void* ptr); const char *Tcl_SetVar(Tcl_Interp* interp, const char* varName, const char* newValue, int flags); const char *Tcl_SetVar2(Tcl_Interp* interp, const char* name1, const char* name2, const char* newValue, int flags); diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -32,7 +32,7 @@ .. toctree:: - whatsnew-pypy3.3-5.5.0.rst + whatsnew-pypy3-5.5.0.rst CPython 3.2 compatible versions ------------------------------- diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -67,11 +67,16 @@ Make optimiseopt iterative instead of recursive so it can be reasoned about more easily and debugging is faster. -.. branch: stdlib-2.7.11 +.. branch: Tiberiumk/fix-2412-1476011166874 +.. branch: redirect-assembler-jitlog -Update stdlib to version 2.7.11 -.. branch: vendor/stdlib + .. branch: stdlib-2.7.12 Update stdlib to version 2.7.12 + +.. branch: buffer-interface2 + +Improve support for new buffer interface in cpyext, bf_getbuffer on built-in +types still missing diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -207,6 +207,9 @@ def buffer_w(self, space, flags): w_impl = space.lookup(self, '__buffer__') + if w_impl is None: + # cpyext types that may have only old buffer interface + w_impl = space.lookup(self, '__wbuffer__') if w_impl is not None: w_result = space.get_and_call_function(w_impl, self, space.newint(flags)) @@ -215,7 +218,10 @@ raise BufferInterfaceNotFound def readbuf_w(self, space): - w_impl = space.lookup(self, '__buffer__') + # cpyext types that may have old buffer protocol + w_impl = space.lookup(self, '__rbuffer__') + if w_impl is None: + w_impl = space.lookup(self, '__buffer__') if w_impl is not None: w_result = space.get_and_call_function(w_impl, self, space.newint(space.BUF_FULL_RO)) @@ -224,7 +230,10 @@ raise BufferInterfaceNotFound def writebuf_w(self, space): - w_impl = space.lookup(self, '__buffer__') + # cpyext types that may have old buffer protocol + w_impl = space.lookup(self, '__wbuffer__') + if w_impl is None: + w_impl = space.lookup(self, '__buffer__') if w_impl is not None: w_result = space.get_and_call_function(w_impl, self, space.newint(space.BUF_FULL)) diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -165,7 +165,7 @@ "will be forbidden in the future (check that the types " "are as you expect; use an explicit ffi.cast() if they " "are correct)" % (other.name, self.name)) - space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1) + space.warn(space.wrap(msg), space.w_UserWarning) else: raise self._convert_error("compatible pointer", w_ob) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3689,3 +3689,5 @@ assert len(w) == 2 newp(new_pointer_type(BIntP), z3) # fine assert len(w) == 2 + # check that the warnings are associated with lines in this file + assert w[1].lineno == w[0].lineno + 4 diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -144,7 +144,7 @@ /* Py3k buffer interface, adapted for PyPy */ #define Py_MAX_NDIMS 32 -#define Py_MAX_FMT 5 +#define Py_MAX_FMT 128 typedef struct bufferinfo { void *buf; PyObject *obj; /* owned reference */ diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -4,6 +4,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import widen from pypy.objspace.std.memoryobject import W_MemoryView +from pypy.module.cpyext.import_ import PyImport_Import PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview") @@ -33,33 +34,43 @@ view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) except ValueError: raise BufferError("could not create buffer from object") + ret = fill_Py_buffer(space, buf, view) view.c_obj = make_ref(space, w_obj) - return fill_Py_buffer(space, buf, view) + return ret -def fill_Py_buffer(space, buf, view): +def fill_Py_buffer(space, buf, view): # c_buf, c_obj have been filled in ndim = buf.getndim() view.c_len = buf.getlength() view.c_itemsize = buf.getitemsize() rffi.setintfield(view, 'c_ndim', ndim) view.c_format = rffi.cast(rffi.CCHARP, view.c__format) - view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape) - view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides) fmt = buf.getformat() n = Py_MAX_FMT - 1 # NULL terminated buffer if len(fmt) > n: - ### WARN? - pass + w_message = space.newbytes("PyPy specific Py_MAX_FMT is %d which is too " + "small for buffer format, %d needed" % ( + Py_MAX_FMT, len(fmt))) + w_stacklevel = space.newint(1) + w_module = PyImport_Import(space, space.newbytes("warnings")) + w_warn = space.getattr(w_module, space.newbytes("warn")) + space.call_function(w_warn, w_message, space.w_None, w_stacklevel) else: n = len(fmt) for i in range(n): view.c_format[i] = fmt[i] - view.c_format[n] = '\x00' - shape = buf.getshape() - strides = buf.getstrides() - for i in range(ndim): - view.c_shape[i] = shape[i] - view.c_strides[i] = strides[i] + view.c_format[n] = '\x00' + if ndim > 0: + view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape) + view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides) + shape = buf.getshape() + strides = buf.getstrides() + for i in range(ndim): + view.c_shape[i] = shape[i] + view.c_strides[i] = strides[i] + else: + view.c_shape = lltype.nullptr(Py_ssize_tP.TO) + view.c_strides = lltype.nullptr(Py_ssize_tP.TO) view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) view.c_internal = lltype.nullptr(rffi.VOIDP.TO) return 0 @@ -102,12 +113,12 @@ @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL) def PyBuffer_IsContiguous(space, view, fort): - """Return 1 if the memory defined by the view is C-style (fortran is - 'C') or Fortran-style (fortran is 'F') contiguous or either one - (fortran is 'A'). Return 0 otherwise.""" + """Return 1 if the memory defined by the view is C-style (fort is + 'C') or Fortran-style (fort is 'F') contiguous or either one + (fort is 'A'). Return 0 otherwise.""" # traverse the strides, checking for consistent stride increases from # right-to-left (c) or left-to-right (fortran). Copied from cpython - if not view.c_suboffsets: + if view.c_suboffsets: return 0 if (fort == 'C'): return _IsCContiguous(view) @@ -139,6 +150,7 @@ if ndim >= Py_MAX_NDIMS: # XXX warn? return view + fill_Py_buffer(space, w_obj.buf, view) try: view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address()) view.c_obj = make_ref(space, w_obj) @@ -147,8 +159,8 @@ except ValueError: w_s = w_obj.descr_tobytes(space) view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s), track_allocation=False)) rffi.setintfield(view, 'c_readonly', 1) isstr = True - fill_Py_buffer(space, w_obj.buf, view) return view diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -21,7 +21,7 @@ @cpython_api([PyObject, CONST_STRING], lltype.Void) def PyErr_SetString(space, w_type, message_ptr): message = rffi.charp2str(message_ptr) - PyErr_SetObject(space, w_type, space.wrap(message)) + PyErr_SetObject(space, w_type, space.newbytes(message)) @cpython_api([PyObject], lltype.Void, error=CANNOT_FAIL) def PyErr_SetNone(space, w_type): @@ -150,12 +150,12 @@ Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. if llfilename: - w_filename = rffi.charp2str(llfilename) - filename = space.wrap(w_filename) + filename = rffi.charp2str(llfilename) + w_filename = space.newbytes(filename) else: - filename = space.w_None + w_filename = space.w_None - PyErr_SetFromErrnoWithFilenameObject(space, w_type, filename) + PyErr_SetFromErrnoWithFilenameObject(space, w_type, w_filename) @cpython_api([PyObject, PyObject], PyObject) @jit.dont_look_inside # direct use of _get_errno() @@ -170,13 +170,13 @@ msg = os.strerror(errno) if w_value: w_error = space.call_function(w_type, - space.wrap(errno), - space.wrap(msg), + space.newint(errno), + space.newbytes(msg), w_value) else: w_error = space.call_function(w_type, - space.wrap(errno), - space.wrap(msg)) + space.newint(errno), + space.newbytes(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) @@ -252,11 +252,11 @@ documentation. There is no C API for warning control.""" if w_category is None: w_category = space.w_None - w_message = space.wrap(rffi.charp2str(message_ptr)) - w_stacklevel = space.wrap(rffi.cast(lltype.Signed, stacklevel)) + w_message = space.newbytes(rffi.charp2str(message_ptr)) + w_stacklevel = space.newint(rffi.cast(lltype.Signed, stacklevel)) - w_module = PyImport_Import(space, space.wrap("warnings")) - w_warn = space.getattr(w_module, space.wrap("warn")) + w_module = PyImport_Import(space, space.newbytes("warnings")) + w_warn = space.getattr(w_module, space.newbytes("warn")) space.call_function(w_warn, w_message, w_category, w_stacklevel) return 0 @@ -317,10 +317,10 @@ @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) def PyTraceBack_Print(space, w_tb, w_file): - space.call_method(w_file, "write", space.wrap( + space.call_method(w_file, "write", space.newbytes( 'Traceback (most recent call last):\n')) w_traceback = space.call_method(space.builtin, '__import__', - space.wrap("traceback")) + space.newbytes("traceback")) space.call_method(w_traceback, "print_tb", w_tb, space.w_None, w_file) return 0 diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -15,6 +15,7 @@ readbufferproc, getbufferproc, ssizessizeobjargproc) from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef from pypy.module.cpyext.pyerrors import PyErr_Occurred +from pypy.module.cpyext.memoryobject import fill_Py_buffer from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.argument import Arguments @@ -349,6 +350,10 @@ def getndim(self): return self.ndim + def setitem(self, index, char): + # absolutely no safety checks, what could go wrong? + self.ptr[index] = char + def wrap_getreadbuffer(space, w_self, w_args, func): func_target = rffi.cast(readbufferproc, func) with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: @@ -358,6 +363,15 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.newbuffer(CPyBuffer(ptr[0], size, w_self)) +def wrap_getwritebuffer(space, w_self, w_args, func): + func_target = rffi.cast(readbufferproc, func) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbuffer(CPyBuffer(ptr[0], size, w_self, readonly=False)) + def wrap_getbuffer(space, w_self, w_args, func): func_target = rffi.cast(getbufferproc, func) with lltype.scoped_alloc(Py_buffer) as pybuf: @@ -608,13 +622,27 @@ @cpython_api([PyObject, Py_bufferP, rffi.INT_real], rffi.INT_real, header=None, error=-1) @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def buff_w(space, w_self, pybuf, flags): - # XXX this is wrong, needs a test - raise oefmt(space.w_NotImplemented, - "calling bf_getbuffer on a builtin type not supported yet") - #args = Arguments(space, [w_self], - # w_stararg=w_args, w_starstararg=w_kwds) - #return space.call_args(space.get(buff_fn, w_self), args) + def buff_w(space, w_self, view, flags): + args = Arguments(space, [space.newint(flags)]) + w_obj = space.call_args(space.get(buff_fn, w_self), args) + if view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + except ValueError: + w_s = space.newbytes(buf.as_str()) + view.c_obj = make_ref(space, w_s) + view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + space.str_w(w_s), track_allocation=False)) + rffi.setintfield(view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, view) + return ret + return 0 + # XXX remove this when it no longer crashes a translated PyPy + return api_func = buff_w.api_func else: # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce @@ -924,13 +952,13 @@ slotdefs = eval(slotdefs_str) # PyPy addition slotdefs += ( - # XXX that might not be what we want! TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""), ) if not PY3: slotdefs += ( - TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""), + TPSLOT("__rbuffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""), + TPSLOT("__wbuffer__", "tp_as_buffer.c_bf_getwritebuffer", None, "wrap_getwritebuffer", ""), ) diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2394,14 +2394,29 @@ (iternextfunc)arrayiter_next, /* tp_iternext */ 0, /* tp_methods */ }; +static PyObject * +readbuffer_as_string(PyObject *self, PyObject *args) +{ + PyObject *obj; + const void *ptr; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "O", &obj)) { + return NULL; + } + if (PyObject_AsReadBuffer(obj, &ptr, &size) < 0) + return NULL; + return PyString_FromStringAndSize((char*)ptr, size); +} + /*********************** Install Module **************************/ -/* No functions in array module. */ static PyMethodDef a_methods[] = { {"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL}, {"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL}, + {"readbuffer_as_string", (PyCFunction)readbuffer_as_string, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_abstract.py b/pypy/module/cpyext/test/test_abstract.py --- a/pypy/module/cpyext/test/test_abstract.py +++ b/pypy/module/cpyext/test/test_abstract.py @@ -104,3 +104,17 @@ assert raises(TypeError, buffer_support.readbuffer_as_string, 42) assert raises(TypeError, buffer_support.writebuffer_as_string, 42) assert raises(TypeError, buffer_support.charbuffer_as_string, 42) + + def test_user_class(self): + class MyBuf(str): + pass + s = 'a\0x' + buf = MyBuf(s) + buffer_support = self.get_buffer_support() + + assert buffer_support.check_readbuffer(buf) + assert s == buffer_support.readbuffer_as_string(buf) + assert raises(TypeError, buffer_support.writebuffer_as_string, buf) + assert s == buffer_support.charbuffer_as_string(buf) + + diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -1,8 +1,17 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - +from pypy.conftest import option class AppTestArrayModule(AppTestCpythonExtensionBase): - enable_leak_checking = False + enable_leak_checking = True + + def setup_class(cls): + from rpython.tool.udir import udir + AppTestCpythonExtensionBase.setup_class.im_func(cls) + if option.runappdirect: + cls.w_udir = str(udir) + else: + cls.w_udir = cls.space.wrap(str(udir)) + def test_basic(self): module = self.import_module(name='array') @@ -90,6 +99,7 @@ assert res == [2, 4, 6] def test_subclass(self): + import struct module = self.import_module(name='array') class Sub(module.array): pass @@ -98,3 +108,37 @@ res = [1, 2, 3] * arr assert res == [1, 2, 3, 1, 2, 3] + val = module.readbuffer_as_string(arr) + assert val == struct.pack('i', 2) + + def test_unicode_readbuffer(self): + # Not really part of array, refactor + import struct + module = self.import_module(name='array') + val = module.readbuffer_as_string('abcd') + assert val == 'abcd' + val = module.readbuffer_as_string(u'\u03a3') + assert val is not None + + def test_readinto(self): + module = self.import_module(name='array') + a = module.array('c') + a.fromstring('0123456789') + filename = self.udir + "/_test_file" + f = open(filename, 'w+b') + f.write('foobar') + f.seek(0) + n = f.readinto(a) + f.close() + assert n == 6 + assert len(a) == 10 + assert a.tostring() == 'foobar6789' + + def test_iowrite(self): + module = self.import_module(name='array') + from io import BytesIO + a = module.array('c') + a.fromstring('0123456789') + fd = BytesIO() + # only test that it works + fd.write(a) diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -139,6 +139,12 @@ return result; ''') assert 'foo\0bar\0baz' == pybuffer(buffer('foo\0bar\0baz')) + import sys + if '__pypy__' not in sys.builtin_module_names: + class A(object): + def __buffer__(self, flags): + return buffer('123') + assert pybuffer(A()) == '123' def test_pyarg_parse_string_fails(self): diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -1,9 +1,14 @@ +import pytest + from rpython.rtyper.lltypesystem import rffi from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from rpython.rlib.buffer import StringBuffer +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" + class TestMemoryViewObject(BaseApiTest): + skip('needs c_bf_getbuffer wrapper from slotdefs') def test_fromobject(self, space, api): w_hello = space.newbytes("hello") assert api.PyObject_CheckBuffer(w_hello) @@ -23,6 +28,8 @@ assert w_view.c_shape[0] == 5 assert w_view.c_strides[0] == 1 assert w_view.c_len == 5 + o = rffi.charp2str(w_view.c_buf) + assert o == 'hello' class AppTestBufferProtocol(AppTestCpythonExtensionBase): def test_buffer_protocol(self): @@ -39,8 +46,12 @@ viewlen = module.test_buffer(arr) assert viewlen == y.itemsize * len(y) + @pytest.mark.skipif(only_pypy, reason='pypy only test') def test_buffer_info(self): - from _numpypy import multiarray as np + try: + from _numpypy import multiarray as np + except ImportError: + skip('pypy built without _numpypy') module = self.import_module(name='buffer_test') get_buffer_info = module.get_buffer_info raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',)) @@ -50,3 +61,29 @@ arr = np.zeros((10, 1), order='C') shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) assert strides[-1] == 8 + dt1 = np.dtype( + [('a', 'b'), ('b', 'i'), + ('sub0', np.dtype('b,i')), + ('sub1', np.dtype('b,i')), + ('sub2', np.dtype('b,i')), + ('sub3', np.dtype('b,i')), + ('sub4', np.dtype('b,i')), + ('sub5', np.dtype('b,i')), + ('sub6', np.dtype('b,i')), + ('sub7', np.dtype('b,i')), + ('c', 'i')], + ) + x = np.arange(dt1.itemsize, dtype='int8').view(dt1) + # pytest can catch warnings from v2.8 and up, we ship 2.5 + import warnings + warnings.filterwarnings("error") + try: + try: + y = get_buffer_info(x, ['SIMPLE']) + except UserWarning as e: + pass + else: + assert False ,"PyPy-specific UserWarning not raised" \ + " on too long format string" + finally: + warnings.resetwarnings() diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -514,10 +514,13 @@ @cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, header=None, error=-1) def bf_getreadbuffer(space, w_buf, segment, ref): + from rpython.rlib.buffer import StringBuffer if segment != 0: raise oefmt(space.w_SystemError, "accessing non-existent segment") buf = space.readbuf_w(w_buf) + if isinstance(buf, StringBuffer): + return str_getreadbuffer(space, w_buf, segment, ref) address = buf.get_raw_address() ref[0] = address return len(buf) @@ -533,7 +536,6 @@ if segment != 0: raise oefmt(space.w_SystemError, "accessing non-existent segment") - buf = space.writebuf_w(w_buf) ref[0] = buf.get_raw_address() return len(buf) @@ -551,6 +553,20 @@ Py_DecRef(space, pyref) return space.len_w(w_str) +@cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, + header=None, error=-1) +def unicode_getreadbuffer(space, w_str, segment, ref): + from pypy.module.cpyext.unicodeobject import ( + PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE) + if segment != 0: + raise oefmt(space.w_SystemError, + "accessing non-existent unicode segment") + pyref = make_ref(space, w_str) + ref[0] = PyUnicode_AS_UNICODE(space, pyref) + # Stolen reference: the object has better exist somewhere else + Py_DecRef(space, pyref) + return PyUnicode_GET_DATA_SIZE(space, w_str) + @cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, header=None, error=-1) def str_getcharbuffer(space, w_buf, segment, ref): @@ -574,8 +590,8 @@ def setup_buffer_procs(space, w_type, pto): bufspec = w_type.layout.typedef.buffer - if bufspec is None: - # not a buffer + if bufspec is None and not space.is_w(w_type, space.w_unicode): + # not a buffer, but let w_unicode be a read buffer return c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) lltype.render_immortal(c_buf) @@ -591,6 +607,13 @@ c_buf.c_bf_getcharbuffer = llhelper( str_getcharbuffer.api_func.functype, str_getcharbuffer.api_func.get_wrapper(space)) + elif space.is_w(w_type, space.w_unicode): + # Special case: unicode doesn't support get_raw_address(), so we have a + # custom get*buffer that instead gives the address of the char* in the + # PyUnicodeObject*! + c_buf.c_bf_getreadbuffer = llhelper( + unicode_getreadbuffer.api_func.functype, + unicode_getreadbuffer.api_func.get_wrapper(space)) elif space.is_w(w_type, space.w_buffer): # Special case: we store a permanent address on the cpyext wrapper, # so we'll reuse that. @@ -706,7 +729,7 @@ # uninitialized fields: # c_tp_print # XXX implement - # c_tp_compare and the following fields (see http://docs.python.org/c-api/typeobj.html ) + # c_tp_compare and more? w_base = best_base(space, w_type.bases_w) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base)) @@ -764,7 +787,6 @@ return find_best_base(bases_w) def inherit_slots(space, pto, w_base): - # XXX missing: nearly everything base_pyo = make_ref(space, w_base) try: base = rffi.cast(PyTypeObjectPtr, base_pyo) @@ -783,6 +805,25 @@ pto.c_tp_getattro = base.c_tp_getattro if not pto.c_tp_as_buffer: pto.c_tp_as_buffer = base.c_tp_as_buffer + if base.c_tp_as_buffer: + # inherit base.c_tp_as_buffer functions not inherited from w_type + # note: builtin types are handled in setup_buffer_procs + pto_as = pto.c_tp_as_buffer + base_as = base.c_tp_as_buffer + if not pto_as.c_bf_getbuffer: + pto_as.c_bf_getbuffer = base_as.c_bf_getbuffer + if not pto_as.c_bf_getcharbuffer: + pto_as.c_bf_getcharbuffer = base_as.c_bf_getcharbuffer + if not pto_as.c_bf_getwritebuffer: + pto_as.c_bf_getwritebuffer = base_as.c_bf_getwritebuffer + if not pto_as.c_bf_getreadbuffer: + pto_as.c_bf_getreadbuffer = base_as.c_bf_getreadbuffer + if not pto_as.c_bf_getsegcount: + pto_as.c_bf_getsegcount = base_as.c_bf_getsegcount + if not pto_as.c_bf_getcharbuffer: + pto_as.c_bf_getcharbuffer = base_as.c_bf_getcharbuffer + if not pto_as.c_bf_releasebuffer: + pto_as.c_bf_releasebuffer = base_as.c_bf_releasebuffer finally: Py_DecRef(space, base_pyo) @@ -812,13 +853,14 @@ w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype) track_reference(space, py_obj, w_obj) - w_obj.__init__(space, py_type) + # __init__ wraps all slotdefs functions from py_type via add_operators + w_obj.__init__(space, py_type) w_obj.ready() finish_type_2(space, py_type, w_obj) - # inheriting tp_as_* slots base = py_type.c_tp_base if base: + # XXX refactor - parts of this are done in finish_type_2 -> inherit_slots if not py_type.c_tp_as_number: py_type.c_tp_as_number = base.c_tp_as_number py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES @@ -827,7 +869,7 @@ py_type.c_tp_as_sequence = base.c_tp_as_sequence py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping = base.c_tp_as_mapping - if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer + #if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer = base.c_tp_as_buffer return w_obj diff --git a/pypy/module/micronumpy/test/dummy_module.py b/pypy/module/micronumpy/test/dummy_module.py --- a/pypy/module/micronumpy/test/dummy_module.py +++ b/pypy/module/micronumpy/test/dummy_module.py @@ -27,7 +27,8 @@ globals()['uint'] = dtype('uint').type types = ['Generic', 'Number', 'Integer', 'SignedInteger', 'UnsignedInteger', - 'Inexact', 'Floating', 'ComplexFloating', 'Flexible', 'Character'] + 'Inexact', 'Floating', 'ComplexFloating', 'Flexible', 'Character', + ] for t in types: globals()[t.lower()] = typeinfo[t] @@ -40,4 +41,4 @@ return a def isscalar(a): - return type(a) in [typeinfo[t] for t in types] + return any([isinstance(a, typeinfo[t]) for t in types]) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -1486,7 +1486,7 @@ assert d[1] == 12 def test_sum(self): - from numpy import array, zeros, float16, complex64, str_ + from numpy import array, zeros, float16, complex64, str_, isscalar, add a = array(range(5)) assert a.sum() == 10 assert a[:4].sum() == 6 @@ -1515,6 +1515,13 @@ assert list(zeros((0, 2)).sum(axis=1)) == [] + a = array([1, 2, 3, 4]).sum() + s = isscalar(a) + assert s is True + a = add.reduce([1.0, 2, 3, 4]) + s = isscalar(a) + assert s is True,'%r is not a scalar' % type(a) + def test_reduce_nd(self): from numpy import arange, array a = arange(15).reshape(5, 3) diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -288,10 +288,8 @@ _, dtype, _ = self.find_specialization(space, dtype, dtype, out, casting='unsafe') - call__array_wrap__ = True if shapelen == len(axes): if out: - call__array_wrap__ = False if out.ndims() > 0: raise oefmt(space.w_ValueError, "output parameter for reduction operation %s has " @@ -302,15 +300,20 @@ if out: out.set_scalar_value(res) return out + w_NDimArray = space.gettypefor(W_NDimArray) + call__array_wrap__ = False if keepdims: shape = [1] * len(obj_shape) out = W_NDimArray.from_shape(space, shape, dtype, w_instance=obj) out.implementation.setitem(0, res) + call__array_wrap__ = True res = out - elif not space.is_w(space.type(w_obj), space.gettypefor(W_NDimArray)): + elif (space.issubtype_w(space.type(w_obj), w_NDimArray) and + not space.is_w(space.type(w_obj), w_NDimArray)): # subtypes return a ndarray subtype, not a scalar out = W_NDimArray.from_shape(space, [1], dtype, w_instance=obj) out.implementation.setitem(0, res) + call__array_wrap__ = True res = out if call__array_wrap__: res = space.call_method(obj, '__array_wrap__', res, space.w_None) @@ -359,8 +362,7 @@ return out loop.reduce( space, self.func, obj, axis_flags, dtype, out, self.identity) - if call__array_wrap__: - out = space.call_method(obj, '__array_wrap__', out, space.w_None) + out = space.call_method(obj, '__array_wrap__', out, space.w_None) return out def descr_outer(self, space, args_w): diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py --- a/pypy/module/operator/test/test_operator.py +++ b/pypy/module/operator/test/test_operator.py @@ -251,6 +251,13 @@ exc = raises(TypeError, operator.index, "abc") assert str(exc.value) == "'str' object cannot be interpreted as an index" + def test_index_int_subclass(self): + import operator + class myint(int): + def __index__(self): + return 13289 + assert operator.index(myint(7)) == 7 + def test_compare_digest(self): import operator diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -522,6 +522,23 @@ def isinstance(space, w_inst, w_type): return space.wrap(space.isinstance_w(w_inst, w_type)) + def index(space, w_obj): + if (space.isinstance_w(w_obj, space.w_int) or + space.isinstance_w(w_obj, space.w_long)): + return w_obj + w_impl = space.lookup(w_obj, '__index__') + if w_impl is None: + raise oefmt(space.w_TypeError, + "'%T' object cannot be interpreted as an index", + w_obj) + w_result = space.get_and_call_function(w_impl, w_obj) + + if (space.isinstance_w(w_result, space.w_int) or + space.isinstance_w(w_result, space.w_long)): + return w_result + raise oefmt(space.w_TypeError, + "__index__ returned non-(int,long) (type '%T')", w_result) + # helpers @@ -797,17 +814,13 @@ # more of the above manually-coded operations as well) for targetname, specialname, checkerspec in [ - ('index', '__index__', ("space.w_int", "space.w_long")), ('long', '__long__', ("space.w_int", "space.w_long")), ('float', '__float__', ("space.w_float",))]: l = ["space.isinstance_w(w_result, %s)" % x for x in checkerspec] checker = " or ".join(l) - if targetname == 'index': - msg = "'%%T' object cannot be interpreted as an index" - else: - msg = "unsupported operand type for %(targetname)s(): '%%T'" + msg = "unsupported operand type for %(targetname)s(): '%%T'" msg = msg % locals() source = """if 1: def %(targetname)s(space, w_obj): diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py --- a/pypy/objspace/std/bufferobject.py +++ b/pypy/objspace/std/bufferobject.py @@ -32,6 +32,10 @@ def charbuf_w(self, space): return self.buf.as_str() + def descr_getbuffer(self, space, w_flags): + space.check_buf_flags(space.int_w(w_flags), self.buf.readonly) + return self + @staticmethod @unwrap_spec(offset=int, size=int) def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1): @@ -160,6 +164,7 @@ __mul__ = interp2app(W_Buffer.descr_mul), __rmul__ = interp2app(W_Buffer.descr_mul), __repr__ = interp2app(W_Buffer.descr_repr), + __buffer__ = interp2app(W_Buffer.descr_getbuffer), _pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address), ) W_Buffer.typedef.acceptable_as_base_class = False diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -64,8 +64,6 @@ def setitem_str(self, w_dict, key, w_value): cell = self.getdictvalue_no_unwrapping(w_dict, key) - #if (key == '__package__' or key == "__path__") and cell is not None and w_value is not cell: - # print "WARNING", key, w_value, cell, self return self._setitem_str_cell_known(cell, w_dict, key, w_value) def _setitem_str_cell_known(self, cell, w_dict, key, w_value): diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -131,9 +131,13 @@ return space.newbool(bool(self.buf.readonly)) def w_get_shape(self, space): + if self.buf.getndim() == 0: + return space.w_None return space.newtuple([space.wrap(x) for x in self.buf.getshape()]) def w_get_strides(self, space): + if self.buf.getndim() == 0: + return space.w_None return space.newtuple([space.wrap(x) for x in self.buf.getstrides()]) def w_get_suboffsets(self, space): diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -1067,6 +1067,7 @@ def prepare_op_guard_not_forced_2(self, op, fcond): self.rm.before_call(op.getfailargs(), save_all_regs=True) + self.vfprm.before_call(op.getfailargs(), save_all_regs=True) fail_locs = self._prepare_guard(op) self.assembler.store_force_descr(op, fail_locs[1:], fail_locs[0].value) self.possibly_free_vars(op.getfailargs()) diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py --- a/rpython/jit/backend/detect_cpu.py +++ b/rpython/jit/backend/detect_cpu.py @@ -61,6 +61,7 @@ 'i86pc': MODEL_X86, # Solaris/Intel 'x86': MODEL_X86, # Apple 'Power Macintosh': MODEL_PPC_64, + 'powerpc': MODEL_PPC_64, # freebsd 'ppc64': MODEL_PPC_64, 'ppc64le': MODEL_PPC_64, 'x86_64': MODEL_X86, diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -965,6 +965,7 @@ def prepare_guard_not_forced_2(self, op): self.rm.before_call(op.getfailargs(), save_all_regs=True) + self.fprm.before_call(op.getfailargs(), save_all_regs=True) arglocs = self._prepare_guard(op) return arglocs diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2647,6 +2647,32 @@ deadframe2 = self.cpu.force(frame) assert self.cpu.get_int_value(deadframe2, 0) == 30 + def test_guard_not_forced_2_float(self): + cpu = self.cpu + if not cpu.supports_floats: + py.test.skip("requires floats") + faildescr = BasicFailDescr(1) + finaldescr = BasicFinalDescr(0) + loop = parse(""" + [f0] + f1 = float_add(f0, 2.5) + p2 = force_token() + guard_not_forced_2(descr=faildescr) [f1] + finish(p2, descr=finaldescr) + """, namespace=locals()) + looptoken = JitCellToken() + self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + deadframe = self.cpu.execute_token(looptoken, 20.25) + fail = self.cpu.get_latest_descr(deadframe) + assert fail.identifier == 0 + frame = self.cpu.get_ref_value(deadframe, 0) + # actually, we should get the same pointer in 'frame' and 'deadframe' + # but it is not the case on LLGraph + if not getattr(self.cpu, 'is_llgraph', False): + assert frame == deadframe + deadframe2 = self.cpu.force(frame) + assert self.cpu.get_float_value(deadframe2, 0) == 22.75 + def test_call_to_c_function(self): from rpython.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL from rpython.rtyper.lltypesystem.ll2ctypes import libc_name diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1428,6 +1428,7 @@ def consider_guard_not_forced_2(self, op): self.rm.before_call(op.getfailargs(), save_all_regs=True) + self.xrm.before_call(op.getfailargs(), save_all_regs=True) fail_locs = [self.loc(v) for v in op.getfailargs()] self.assembler.store_force_descr(op, fail_locs, self.fm.get_frame_depth()) diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -1138,6 +1138,7 @@ def prepare_guard_not_forced_2(self, op): self.rm.before_call(op.getfailargs(), save_all_regs=True) + self.fprm.before_call(op.getfailargs(), save_all_regs=True) arglocs = self._prepare_guard(op) return arglocs diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -679,7 +679,7 @@ # return newgcdependencies - def get_finish_tables(self): + def enum_type_info_members(self): # We must first make sure that the type_info_group's members # are all followed. Do it repeatedly while new members show up. # Once it is really done, do finish_tables(). @@ -688,6 +688,15 @@ curtotal = len(self.layoutbuilder.type_info_group.members) yield self.layoutbuilder.type_info_group.members[seen:curtotal] seen = curtotal + + def get_finish_helpers(self): + for dep in self.enum_type_info_members(): + yield dep + yield self.finish_helpers() + + def get_finish_tables(self): + for dep in self.enum_type_info_members(): + yield dep yield self.finish_tables() def write_typeid_list(self): diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py --- a/rpython/memory/gctransform/transform.py +++ b/rpython/memory/gctransform/transform.py @@ -286,6 +286,9 @@ newgcdependencies = self.ll_finalizers_ptrs return newgcdependencies + def get_finish_helpers(self): + return self.finish_helpers + def finish_tables(self): pass diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -505,6 +505,8 @@ # ---------- +HASH_ALGORITHM = "rpython" # XXX Is there a better name? + def _hash_string(s): """The algorithm behind compute_hash() for a string or a unicode.""" from rpython.rlib.rarithmetic import intmask diff --git a/rpython/rlib/rlocale.py b/rpython/rlib/rlocale.py --- a/rpython/rlib/rlocale.py +++ b/rpython/rlib/rlocale.py @@ -195,6 +195,7 @@ isalpha = external('isalpha', [rffi.INT], rffi.INT) isupper = external('isupper', [rffi.INT], rffi.INT) +toupper = external('toupper', [rffi.INT], rffi.INT) islower = external('islower', [rffi.INT], rffi.INT) tolower = external('tolower', [rffi.INT], rffi.INT) isalnum = external('isalnum', [rffi.INT], rffi.INT) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -2305,3 +2305,49 @@ self.cached_nosys = -1 _pipe2_syscall = ENoSysCache() + +post_include_bits=['int _cpu_count(void);'] +# cpu count for linux, windows and mac (+ bsds) +# note that the code is copied from cpython and split up here +if sys.platform.startswith('linux'): + cpucount_eci = ExternalCompilationInfo(includes=["unistd.h"], + separate_module_sources=[""" + RPY_EXTERN int _cpu_count(void) { + return sysconf(_SC_NPROCESSORS_ONLN); + } + """], post_include_bits=post_include_bits) +elif sys.platform == "win32": + cpucount_eci = ExternalCompilationInfo(includes=["Windows.h"], + separate_module_sources=[""" + RPY_EXTERN int _cpu_count(void) { + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + return sysinfo.dwNumberOfProcessors; + } + """], post_include_bits=post_include_bits) +else: + cpucount_eci = ExternalCompilationInfo(includes=["sys/types.h", "sys/sysctl.h"], + separate_module_sources=[""" + RPY_EXTERN int _cpu_count(void) { + int ncpu = 0; + #if defined(__DragonFly__) || \ + defined(__OpenBSD__) || \ + defined(__FreeBSD__) || \ + defined(__NetBSD__) || \ + defined(__APPLE__) + int mib[2]; + size_t len = sizeof(ncpu); + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + if (sysctl(mib, 2, &ncpu, &len, NULL, 0) != 0) + ncpu = 0; + #endif + return ncpu; + } + """], post_include_bits=post_include_bits) + +_cpu_count = rffi.llexternal('_cpu_count', [], rffi.INT_real, + compilation_info=cpucount_eci) + +def cpu_count(): + return rffi.cast(lltype.Signed, _cpu_count()) diff --git a/rpython/rlib/rsre/rpy/sre_constants.py b/rpython/rlib/rsre/rpy/sre_constants.py --- a/rpython/rlib/rsre/rpy/sre_constants.py +++ b/rpython/rlib/rsre/rpy/sre_constants.py @@ -58,6 +58,7 @@ REPEAT_ONE = "repeat_one" SUBPATTERN = "subpattern" MIN_REPEAT_ONE = "min_repeat_one" +RANGE_IGNORE = "range_ignore" # positions AT_BEGINNING = "at_beginning" @@ -119,8 +120,8 @@ REPEAT, REPEAT_ONE, SUBPATTERN, - MIN_REPEAT_ONE - + MIN_REPEAT_ONE, + RANGE_IGNORE, ] ATCODES = [ diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py --- a/rpython/rlib/rsre/rsre_char.py +++ b/rpython/rlib/rsre/rsre_char.py @@ -2,7 +2,7 @@ Character categories and charsets. """ import sys -from rpython.rlib.rlocale import tolower, isalnum +from rpython.rlib.rlocale import tolower, toupper, isalnum from rpython.rlib.unroll import unrolling_iterable from rpython.rlib import jit from rpython.rlib.rarithmetic import int_between @@ -67,6 +67,19 @@ char_ord += ord('a') - ord('A') return char_ord +def getupper(char_ord, flags): + if flags & SRE_FLAG_LOCALE: + if char_ord < 256: # cheating! Well, CPython does too. + char_ord = toupper(char_ord) + return char_ord + elif flags & SRE_FLAG_UNICODE: + assert unicodedb is not None + char_ord = unicodedb.toupper(char_ord) + else: + if int_between(ord('a'), char_ord, ord('z') + 1): # ASCII upper + char_ord += ord('A') - ord('a') + return char_ord + #### Category helpers is_a_word = [(chr(i).isalnum() or chr(i) == '_') for i in range(256)] @@ -139,16 +152,17 @@ ##### Charset evaluation @jit.unroll_safe -def check_charset(pattern, ppos, char_code): +def check_charset(ctx, ppos, char_code): """Checks whether a character matches set of arbitrary length. The set starts at pattern[ppos].""" negated = False result = False + pattern = ctx.pattern while True: opcode = pattern[ppos] for i, function in set_dispatch_unroll: if opcode == i: - newresult, ppos = function(pattern, ppos, char_code) + newresult, ppos = function(ctx, ppos, char_code) result |= newresult break else: @@ -163,18 +177,21 @@ return not result return result -def set_literal(pat, index, char_code): +def set_literal(ctx, index, char_code): # <LITERAL> <code> + pat = ctx.pattern match = pat[index+1] == char_code return match, index + 2 -def set_category(pat, index, char_code): +def set_category(ctx, index, char_code): # <CATEGORY> <code> + pat = ctx.pattern match = category_dispatch(pat[index+1], char_code) return match, index + 2 -def set_charset(pat, index, char_code): +def set_charset(ctx, index, char_code): # <CHARSET> <bitmap> (16 bits per code word) + pat = ctx.pattern if CODESIZE == 2: match = char_code < 256 and \ (pat[index+1+(char_code >> 4)] & (1 << (char_code & 15))) @@ -184,13 +201,25 @@ (pat[index+1+(char_code >> 5)] & (1 << (char_code & 31))) return match, index + 9 # skip bitmap -def set_range(pat, index, char_code): +def set_range(ctx, index, char_code): # <RANGE> <lower> <upper> + pat = ctx.pattern match = int_between(pat[index+1], char_code, pat[index+2] + 1) return match, index + 3 -def set_bigcharset(pat, index, char_code): +def set_range_ignore(ctx, index, char_code): + # <RANGE_IGNORE> <lower> <upper> + # the char_code is already lower cased + pat = ctx.pattern + lower = pat[index + 1] + upper = pat[index + 2] + match1 = int_between(lower, char_code, upper + 1) + match2 = int_between(lower, getupper(char_code, ctx.flags), upper + 1) + return match1 | match2, index + 3 + +def set_bigcharset(ctx, index, char_code): # <BIGCHARSET> <blockcount> <256 blockindices> <blocks> + pat = ctx.pattern count = pat[index+1] index += 2 @@ -224,7 +253,7 @@ index += count * (32 / CODESIZE) # skip blocks return match, index -def set_unicode_general_category(pat, index, char_code): +def set_unicode_general_category(ctx, index, char_code): # Unicode "General category property code" (not used by Python). # A general category is two letters. 'pat[index+1]' contains both # the first character, and the second character shifted by 8. @@ -233,6 +262,7 @@ # Negative matches are triggered by bit number 7. assert unicodedb is not None cat = unicodedb.category(char_code) + pat = ctx.pattern category_code = pat[index + 1] first_character = category_code & 0x7F second_character = (category_code >> 8) & 0x7F @@ -260,6 +290,7 @@ 11: set_bigcharset, 19: set_literal, 27: set_range, + 32: set_range_ignore, 70: set_unicode_general_category, } set_dispatch_unroll = unrolling_iterable(sorted(set_dispatch_table.items())) diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -40,6 +40,7 @@ OPCODE_REPEAT_ONE = 29 #OPCODE_SUBPATTERN = 30 OPCODE_MIN_REPEAT_ONE = 31 +OPCODE_RANGE_IGNORE = 32 # not used by Python itself OPCODE_UNICODE_GENERAL_CATEGORY = 70 @@ -640,8 +641,7 @@ elif op == OPCODE_IN: # match set member (or non_member) # <IN> <skip> <set> - if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern, - ppos+1, + if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1, ctx.str(ptr)): return ppos += ctx.pat(ppos) @@ -650,8 +650,7 @@ elif op == OPCODE_IN_IGNORE: # match set member (or non_member), ignoring case # <IN> <skip> <set> - if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern, - ppos+1, + if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1, ctx.lowstr(ptr)): return ppos += ctx.pat(ppos) @@ -871,10 +870,10 @@ return True # match anything (including a newline) @specializectx def match_IN(ctx, ptr, ppos): - return rsre_char.check_charset(ctx.pattern, ppos+2, ctx.str(ptr)) + return rsre_char.check_charset(ctx, ppos+2, ctx.str(ptr)) @specializectx def match_IN_IGNORE(ctx, ptr, ppos): - return rsre_char.check_charset(ctx.pattern, ppos+2, ctx.lowstr(ptr)) + return rsre_char.check_charset(ctx, ppos+2, ctx.lowstr(ptr)) @specializectx def match_LITERAL(ctx, ptr, ppos): return ctx.str(ptr) == ctx.pat(ppos+1) @@ -1134,7 +1133,7 @@ while start < ctx.end: ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start, base=base) - if rsre_char.check_charset(ctx.pattern, 5, ctx.str(start)): + if rsre_char.check_charset(ctx, 5, ctx.str(start)): if sre_match(ctx, base, start, None) is not None: ctx.match_start = start return True diff --git a/rpython/rlib/rsre/test/test_char.py b/rpython/rlib/rsre/test/test_char.py --- a/rpython/rlib/rsre/test/test_char.py +++ b/rpython/rlib/rsre/test/test_char.py @@ -34,6 +34,22 @@ assert rsre_char.getlower(UPPER_PI, SRE_FLAG_LOCALE | SRE_FLAG_UNICODE) \ == UPPER_PI +def test_getupper(): + assert rsre_char.getupper(ord('A'), 0) == ord('A') + assert rsre_char.getupper(ord('b'), 0) == ord('B') + assert rsre_char.getupper(10, 0) == 10 + assert rsre_char.getupper(LOWER_PI, 0) == LOWER_PI + # + assert rsre_char.getupper(ord('a'), SRE_FLAG_UNICODE) == ord('A') + assert rsre_char.getupper(ord('2'), SRE_FLAG_UNICODE) == ord('2') + assert rsre_char.getupper(10, SRE_FLAG_UNICODE) == 10 + assert rsre_char.getupper(LOWER_PI, SRE_FLAG_UNICODE) == UPPER_PI + # + assert rsre_char.getupper(LOWER_PI, SRE_FLAG_LOCALE) == LOWER_PI + assert rsre_char.getupper(LOWER_PI, SRE_FLAG_LOCALE | SRE_FLAG_UNICODE) \ + == LOWER_PI + + def test_is_word(): assert rsre_char.is_word(ord('A')) assert rsre_char.is_word(ord('_')) @@ -128,6 +144,10 @@ assert cat(CHCODES["category_uni_not_digit"], DINGBAT_CIRCLED) +class Ctx: + def __init__(self, pattern): + self.pattern = pattern + def test_general_category(): from rpython.rlib.unicodedata import unicodedb @@ -137,12 +157,12 @@ pat_neg = [70, ord(cat) | 0x80, 0] for c in positive: assert unicodedb.category(ord(c)).startswith(cat) - assert rsre_char.check_charset(pat_pos, 0, ord(c)) - assert not rsre_char.check_charset(pat_neg, 0, ord(c)) + assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) + assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) for c in negative: assert not unicodedb.category(ord(c)).startswith(cat) - assert not rsre_char.check_charset(pat_pos, 0, ord(c)) - assert rsre_char.check_charset(pat_neg, 0, ord(c)) + assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) + assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) def cat2num(cat): return ord(cat[0]) | (ord(cat[1]) << 8) @@ -153,17 +173,17 @@ pat_neg = [70, cat2num(cat) | 0x80, 0] for c in positive: assert unicodedb.category(ord(c)) == cat - assert rsre_char.check_charset(pat_pos, 0, ord(c)) - assert not rsre_char.check_charset(pat_neg, 0, ord(c)) + assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) + assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) for c in negative: assert unicodedb.category(ord(c)) != cat - assert not rsre_char.check_charset(pat_pos, 0, ord(c)) - assert rsre_char.check_charset(pat_neg, 0, ord(c)) + assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) + assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) # test for how the common 'L&' pattern might be compiled pat = [70, cat2num('Lu'), 70, cat2num('Ll'), 70, cat2num('Lt'), 0] - assert rsre_char.check_charset(pat, 0, 65) # Lu - assert rsre_char.check_charset(pat, 0, 99) # Ll - assert rsre_char.check_charset(pat, 0, 453) # Lt - assert not rsre_char.check_charset(pat, 0, 688) # Lm - assert not rsre_char.check_charset(pat, 0, 5870) # Nl + assert rsre_char.check_charset(Ctx(pat), 0, 65) # Lu + assert rsre_char.check_charset(Ctx(pat), 0, 99) # Ll + assert rsre_char.check_charset(Ctx(pat), 0, 453) # Lt + assert not rsre_char.check_charset(Ctx(pat), 0, 688) # Lm + assert not rsre_char.check_charset(Ctx(pat), 0, 5870) # Nl diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py --- a/rpython/rlib/rsre/test/test_match.py +++ b/rpython/rlib/rsre/test/test_match.py @@ -1,5 +1,5 @@ import re, random, py -from rpython.rlib.rsre import rsre_core +from rpython.rlib.rsre import rsre_core, rsre_char from rpython.rlib.rsre.rpy import get_code, VERSION @@ -299,3 +299,12 @@ assert rsre_core.fullmatch(r, "ab") r = get_code(r"(?!a)..") assert not rsre_core.fullmatch(r, "ab") + + def test_range_ignore(self): + from rpython.rlib.unicodedata import unicodedb + rsre_char.set_unicode_db(unicodedb) + # + r = get_code(u"[\U00010428-\U0001044f]", re.I) + assert r.count(27) == 1 # OPCODE_RANGE + r[r.index(27)] = 32 # => OPCODE_RANGE_IGNORE + assert rsre_core.match(r, u"\U00010428") diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -129,6 +129,9 @@ def acquire(self, flag): return True + def is_acquired(self): + return False + def release(self): pass @@ -164,6 +167,15 @@ res = rffi.cast(lltype.Signed, res) return bool(res) + def is_acquired(self): + """ check if the lock is acquired (does not release the GIL) """ + res = c_thread_acquirelock_timed_NOAUTO( + self._lock, + rffi.cast(rffi.LONGLONG, 0), + rffi.cast(rffi.INT, 0)) + res = rffi.cast(lltype.Signed, res) + return not bool(res) + def acquire_timed(self, timeout): """Timeout is in microseconds. Returns 0 in case of failure, 1 in case it works, 2 if interrupted by a signal.""" diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -608,3 +608,7 @@ def test_sync(): if sys.platform != 'win32': rposix.sync() + +def test_cpu_count(): + cc = rposix.cpu_count() + assert cc >= 1 diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -16,6 +16,14 @@ res = ok1 and not ok2 and ok3 assert res == 1 +def test_lock_is_aquired(): + l = allocate_lock() + ok1 = l.acquire(True) + assert l.is_acquired() == True + assert l.is_acquired() == True + l.release() + assert l.is_acquired() == False + def test_thread_error(): l = allocate_lock() try: diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py --- a/rpython/tool/runsubprocess.py +++ b/rpython/tool/runsubprocess.py @@ -8,9 +8,15 @@ import os from subprocess import PIPE, Popen +PY2 = (sys.version_info.major == 2) +if PY2: + text = unicode +else: + text = str + def run_subprocess(executable, args, env=None, cwd=None): if isinstance(args, list): - args = [a.encode('latin1') if isinstance(a, unicode) else a + args = [a.encode('latin1') if isinstance(a, text) else a for a in args] return _run(executable, args, env, cwd) @@ -71,24 +77,38 @@ _source = os.path.join(_source, 'runsubprocess.py') # and not e.g. '.pyc' def spawn_subprocess(): - global _child + global _child, child_stdin, child_stdout _child = Popen([sys.executable, _source], bufsize=0, stdin=PIPE, stdout=PIPE, close_fds=True) + if PY2: + child_stdin = _child.stdin + child_stdout = _child.stdout + else: + # create TextIOWrappers which (hopefully) have the same newline + # behavior as the child's stdin / stdout + from io import TextIOWrapper + child_stdin = TextIOWrapper(_child.stdin, + newline=sys.stdin.newlines, + write_through=True) + child_stdout = TextIOWrapper(_child.stdout, + newline=sys.stdout.newlines) spawn_subprocess() def cleanup_subprocess(): - global _child + global _child, child_stdin, child_stdout _child = None + child_stdin = None + child_stdout = None import atexit; atexit.register(cleanup_subprocess) def _run(*args): try: - _child.stdin.write('%r\n' % (args,)) + child_stdin.write('%r\n' % (args,)) except (OSError, IOError): # lost the child. Try again... spawn_subprocess() - _child.stdin.write('%r\n' % (args,)) - results = _child.stdout.readline() + child_stdin.write('%r\n' % (args,)) + results = child_stdout.readline() assert results.startswith('(') results = eval(results) if results[0] is None: diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -275,7 +275,7 @@ finish_callbacks = [] if self.gctransformer: finish_callbacks.append(('GC transformer: finished helpers', - self.gctransformer.finish_helpers)) + self.gctransformer.get_finish_helpers())) finish_callbacks.append(('GC transformer: finished tables', self.gctransformer.get_finish_tables())) diff --git a/rpython/translator/c/src/asm.c b/rpython/translator/c/src/asm.c --- a/rpython/translator/c/src/asm.c +++ b/rpython/translator/c/src/asm.c @@ -8,10 +8,6 @@ /* # include "src/asm_gcc_x86_64.c" */ #endif -#if defined(__GNUC__) && defined(__ppc__) -# include "src/asm_ppc.c" -#endif - #if defined(_MSC_VER) # include "src/asm_msvc.c" #endif diff --git a/rpython/translator/c/src/asm.h b/rpython/translator/c/src/asm.h --- a/rpython/translator/c/src/asm.h +++ b/rpython/translator/c/src/asm.h @@ -10,10 +10,6 @@ # include "src/asm_gcc_x86_64.h" #endif -#if defined(__GNUC__) && defined(__ppc__) -# include "src/asm_ppc.h" -#endif - #if defined(MS_WINDOWS) && defined(_MSC_VER) # include "src/asm_msvc.h" #endif _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit