Author: Armin Rigo <ar...@tunes.org> Branch: py3.5 Changeset: r87309:7f7862ca4a74 Date: 2016-09-22 11:29 +0200 http://bitbucket.org/pypy/pypy/changeset/7f7862ca4a74/
Log: hg merge py3k diff too long, truncating to 2000 out of 3083 lines diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -13,6 +13,7 @@ import sys import os import shlex +import imp from distutils.errors import DistutilsPlatformError @@ -62,8 +63,7 @@ """Initialize the module as appropriate for POSIX systems.""" g = {} g['EXE'] = "" - g['SO'] = ".so" - g['SOABI'] = g['SO'].rsplit('.')[0] + g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] g['LIBDIR'] = os.path.join(sys.prefix, 'lib') g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check @@ -75,8 +75,7 @@ """Initialize the module as appropriate for NT""" g = {} g['EXE'] = ".exe" - g['SO'] = ".pyd" - g['SOABI'] = g['SO'].rsplit('.')[0] + g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] global _config_vars _config_vars = g diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -529,7 +529,7 @@ for suffix, mode, type_ in imp.get_suffixes(): if type_ == imp.C_EXTENSION: _CONFIG_VARS['SOABI'] = suffix.split('.')[1] - break + break if args: vals = [] diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -83,6 +83,7 @@ BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD); BOOL WINAPI TerminateProcess(HANDLE, UINT); HANDLE WINAPI GetStdHandle(DWORD); +DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD); """) # -------------------- diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -162,18 +162,19 @@ return int(_ffi.cast("intptr_t", res)) def CloseHandle(handle): - res = _CloseHandle(handle) + res = _kernel32.CloseHandle(_ffi.cast("HANDLE", handle)) if not res: raise _WinError() def GetModuleFileName(module): - buf = ctypes.create_unicode_buffer(_MAX_PATH) - res = _GetModuleFileNameW(module, buf, _MAX_PATH) + buf = _ffi.new("wchar_t[]", _MAX_PATH) + res = _kernel32.GetModuleFileNameW(_ffi.cast("HANDLE", module), + buf, _MAX_PATH) if not res: raise _WinError() - return buf.value + return _ffi.string(buf) STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.8.2 +Version: 1.8.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.8.2" -__version_info__ = (1, 8, 2) +__version__ = "1.8.4" +__version_info__ = (1, 8, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.8.2" + "\ncompiled with cffi version: 1.8.4" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -332,7 +332,7 @@ realtype = model.unknown_ptr_type(decl.name) else: realtype, quals = self._get_type_and_quals( - decl.type, name=decl.name) + decl.type, name=decl.name, partial_length_ok=True) self._declare('typedef ' + decl.name, realtype, quals=quals) else: raise api.CDefError("unrecognized construct", decl) @@ -781,11 +781,14 @@ exprnode.name in self._int_constants): return self._int_constants[exprnode.name] # - if partial_length_ok: - if (isinstance(exprnode, pycparser.c_ast.ID) and + if (isinstance(exprnode, pycparser.c_ast.ID) and exprnode.name == '__dotdotdotarray__'): + if partial_length_ok: self._partial_length = True return '...' + raise api.FFIError(":%d: unsupported '[...]' here, cannot derive " + "the actual array length in this context" + % exprnode.coord.line) # raise api.FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -587,8 +587,11 @@ # ---------- # typedefs + def _typedef_type(self, tp, name): + return self._global_type(tp, "(*(%s *)0)" % (name,)) + def _generate_cpy_typedef_collecttype(self, tp, name): - self._do_collect_type(tp) + self._do_collect_type(self._typedef_type(tp, name)) def _generate_cpy_typedef_decl(self, tp, name): pass @@ -598,6 +601,7 @@ self._lsts["typename"].append(TypenameExpr(name, type_index)) def _generate_cpy_typedef_ctx(self, tp, name): + tp = self._typedef_type(tp, name) self._typedef_ctx(tp, name) if getattr(tp, "origin", None) == "unknown_type": self._struct_ctx(tp, tp.name, approxname=None) diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt --- a/pypy/doc/config/translation.profopt.txt +++ b/pypy/doc/config/translation.profopt.txt @@ -3,3 +3,14 @@ RPython program) to gather profile data. Example for pypy-c: "-c 'from richards import main;main(); from test import pystone; pystone.main()'" + +NOTE: be aware of what this does in JIT-enabled executables. What it +does is instrument and later optimize the C code that happens to run in +the example you specify, ignoring any execution of the JIT-generated +assembler. That means that you have to choose the example wisely. If +it is something that will just generate assembler and stay there, there +is little value. If it is something that exercises heavily library +routines that are anyway written in C, then it will optimize that. Most +interesting would be something that causes a lot of JIT-compilation, +like running a medium-sized test suite several times in a row, in order +to optimize the warm-up in general. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -449,6 +449,27 @@ support (see ``multiline_input()``). On the other hand, ``parse_and_bind()`` calls are ignored (issue `#2072`_). +* ``sys.getsizeof()`` always raises ``TypeError``. This is because a + memory profiler using this function is most likely to give results + inconsistent with reality on PyPy. It would be possible to have + ``sys.getsizeof()`` return a number (with enough work), but that may + or may not represent how much memory the object uses. It doesn't even + make really sense to ask how much *one* object uses, in isolation with + the rest of the system. For example, instances have maps, which are + often shared across many instances; in this case the maps would + probably be ignored by an implementation of ``sys.getsizeof()``, but + their overhead is important in some cases if they are many instances + with unique maps. Conversely, equal strings may share their internal + string data even if they are different objects---or empty containers + may share parts of their internals as long as they are empty. Even + stranger, some lists create objects as you read them; if you try to + estimate the size in memory of ``range(10**6)`` as the sum of all + items' size, that operation will by itself create one million integer + objects that never existed in the first place. Note that some of + these concerns also exist on CPython, just less so. For this reason + we explicitly don't implement ``sys.getsizeof()``. + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -16,3 +16,8 @@ Improve merging of virtual states in the JIT in order to avoid jumping to the preamble. Accomplished by allocating virtual objects where non-virtuals are expected. + +.. branch: conditional_call_value_3 +JIT residual calls: if the called function starts with a fast-path +like "if x.foo != 0: return x.foo", then inline the check before +doing the CALL. For now, string hashing is about the only case. diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -246,6 +246,10 @@ raise Exception("Cannot use the --output option with PyPy " "when --shared is on (it is by default). " "See issue #1971.") + if config.translation.profopt is not None: + raise Exception("Cannot use the --profopt option " + "when --shared is on (it is by default). " + "See issue #2398.") if sys.platform == 'win32': libdir = thisdir.join('..', '..', 'libs') libdir.ensure(dir=1) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1961,12 +1961,10 @@ 'NotImplementedError', 'OSError', 'OverflowError', - 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning', 'RecursionError', 'RuntimeError', - 'RuntimeWarning', 'StopIteration', 'SyntaxError', 'SyntaxWarning', @@ -1980,10 +1978,12 @@ 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', - 'UserWarning', 'ValueError', 'Warning', - 'ZeroDivisionError' + 'ZeroDivisionError', + 'RuntimeWarning', + 'PendingDeprecationWarning', + 'UserWarning', ] if sys.platform.startswith("win"): diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py --- a/pypy/interpreter/test/test_generator.py +++ b/pypy/interpreter/test/test_generator.py @@ -57,12 +57,14 @@ def f(): yield 2 g = f() + # two arguments version raises(NameError, g.throw, NameError, "Error") def test_throw2(self): def f(): yield 2 g = f() + # single argument version raises(NameError, g.throw, NameError("Error")) def test_throw3(self): @@ -236,7 +238,8 @@ def f(): yield 1 g = f() - raises(TypeError, g.send, 1) + raises(TypeError, g.send) # one argument required + raises(TypeError, g.send, 1) # not started, must send None def test_generator_explicit_stopiteration(self): def f(): diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -583,3 +583,21 @@ assert a2ref() is None, "stack not cleared" # raises(StopIteration, next, gen) + + def test_throw_trace_bug(self): + import sys + def f(): + yield 5 + gen = f() + assert next(gen) == 5 + seen = [] + def trace_func(frame, event, *args): + seen.append(event) + return trace_func + sys.settrace(trace_func) + try: + gen.throw(ValueError) + except ValueError: + pass + sys.settrace(None) + assert seen == ['call', 'exception', 'return'] diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.8.2" +VERSION = "1.8.4" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -11,7 +11,7 @@ from rpython.rlib.rarithmetic import ovfcheck from pypy.module._cffi_backend import cdataobj -from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray +from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray, W_CTypePointer from pypy.module._cffi_backend import ctypeprim @@ -22,6 +22,7 @@ is_nonfunc_pointer_or_array = True def __init__(self, space, ctptr, length, arraysize, extra): + assert isinstance(ctptr, W_CTypePointer) W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0, ctptr.ctitem) self.length = length diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -35,8 +35,7 @@ assert isinstance(ellipsis, bool) extra, xpos = self._compute_extra_text(fargs, fresult, ellipsis, abi) size = rffi.sizeof(rffi.VOIDP) - W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult, - could_cast_anything=False) + W_CTypePtrBase.__init__(self, space, size, extra, xpos, fresult) self.fargs = fargs self.ellipsis = ellipsis self.abi = abi @@ -59,6 +58,16 @@ lltype.free(self.cif_descr, flavor='raw') self.cif_descr = lltype.nullptr(CIF_DESCRIPTION) + def is_unichar_ptr_or_array(self): + return False + + def is_char_or_unichar_ptr_or_array(self): + return False + + def string(self, cdataobj, maxlen): + # Can't use ffi.string() on a function pointer + return W_CType.string(self, cdataobj, maxlen) + def new_ctypefunc_completing_argtypes(self, args_w): space = self.space nargs_declared = len(self.fargs) diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -19,7 +19,6 @@ # XXX this could be improved with an elidable method get_size() # that raises in case it's still -1... - cast_anything = False is_primitive_integer = False is_nonfunc_pointer_or_array = False is_indirect_arg_for_call_python = False diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -120,7 +120,6 @@ class W_CTypePrimitiveChar(W_CTypePrimitiveCharOrUniChar): _attrs_ = [] - cast_anything = True def cast_to_int(self, cdata): return self.space.wrap(ord(cdata[0])) diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -16,12 +16,11 @@ class W_CTypePtrOrArray(W_CType): - _attrs_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length'] - _immutable_fields_ = ['ctitem', 'can_cast_anything', 'accept_str', 'length'] + _attrs_ = ['ctitem', 'accept_str', 'length'] + _immutable_fields_ = ['ctitem', 'accept_str', 'length'] length = -1 - def __init__(self, space, size, extra, extra_position, ctitem, - could_cast_anything=True): + def __init__(self, space, size, extra, extra_position, ctitem): name, name_position = ctitem.insert_name(extra, extra_position) W_CType.__init__(self, space, size, name, name_position) # this is the "underlying type": @@ -29,10 +28,11 @@ # - for arrays, it is the array item type # - for functions, it is the return type self.ctitem = ctitem - self.can_cast_anything = could_cast_anything and ctitem.cast_anything - self.accept_str = (self.can_cast_anything or - (ctitem.is_primitive_integer and - ctitem.size == rffi.sizeof(lltype.Char))) + self.accept_str = (self.is_nonfunc_pointer_or_array and + (isinstance(ctitem, ctypevoid.W_CTypeVoid) or + isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar) or + (ctitem.is_primitive_integer and + ctitem.size == rffi.sizeof(lltype.Char)))) def is_unichar_ptr_or_array(self): return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar) @@ -139,7 +139,10 @@ class W_CTypePtrBase(W_CTypePtrOrArray): # base class for both pointers and pointers-to-functions - _attrs_ = [] + _attrs_ = ['is_void_ptr', 'is_voidchar_ptr'] + _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr'] + is_void_ptr = False + is_voidchar_ptr = False def convert_to_object(self, cdata): ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0] @@ -156,7 +159,16 @@ else: raise self._convert_error("compatible pointer", w_ob) if self is not other: - if not (self.can_cast_anything or other.can_cast_anything): + if self.is_void_ptr or other.is_void_ptr: + pass # cast from or to 'void *' + elif self.is_voidchar_ptr or other.is_voidchar_ptr: + space = self.space + msg = ("implicit cast from '%s' to '%s' " + "will be forbidden in the future (check that the types " + "are as you expect; use an explicit ffi.cast() if they " + "are correct)" % (other.name, self.name)) + space.warn(space.wrap(msg), space.w_UserWarning, stacklevel=1) + else: raise self._convert_error("compatible pointer", w_ob) rffi.cast(rffi.CCHARPP, cdata)[0] = w_ob.unsafe_escaping_ptr() @@ -167,8 +179,8 @@ class W_CTypePointer(W_CTypePtrBase): - _attrs_ = ['is_file', 'cache_array_type', 'is_void_ptr', '_array_types'] - _immutable_fields_ = ['is_file', 'cache_array_type?', 'is_void_ptr'] + _attrs_ = ['is_file', 'cache_array_type', '_array_types'] + _immutable_fields_ = ['is_file', 'cache_array_type?'] kind = "pointer" cache_array_type = None is_nonfunc_pointer_or_array = True @@ -183,6 +195,8 @@ self.is_file = (ctitem.name == "struct _IO_FILE" or ctitem.name == "FILE") self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid) + self.is_voidchar_ptr = (self.is_void_ptr or + isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)) W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem) def newp(self, w_init, allocator): diff --git a/pypy/module/_cffi_backend/ctypevoid.py b/pypy/module/_cffi_backend/ctypevoid.py --- a/pypy/module/_cffi_backend/ctypevoid.py +++ b/pypy/module/_cffi_backend/ctypevoid.py @@ -7,7 +7,6 @@ class W_CTypeVoid(W_CType): _attrs_ = [] - cast_anything = True kind = "void" def __init__(self, space): diff --git a/pypy/module/_cffi_backend/handle.py b/pypy/module/_cffi_backend/handle.py --- a/pypy/module/_cffi_backend/handle.py +++ b/pypy/module/_cffi_backend/handle.py @@ -32,8 +32,8 @@ @unwrap_spec(w_cdata=cdataobj.W_CData) def from_handle(space, w_cdata): ctype = w_cdata.ctype - if (not isinstance(ctype, ctypeptr.W_CTypePtrOrArray) or - not ctype.can_cast_anything): + if (not isinstance(ctype, ctypeptr.W_CTypePointer) or + not ctype.is_voidchar_ptr): raise oefmt(space.w_TypeError, "expected a 'cdata' object with a 'void *' out of " "new_handle(), got '%s'", ctype.name) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.8.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.8.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -3665,3 +3665,27 @@ check_dir(pp, []) check_dir(pp[0], ['a1', 'a2']) check_dir(pp[0][0], ['a1', 'a2']) + +def test_char_pointer_conversion(): + import warnings + assert __version__.startswith(("1.8", "1.9")), ( + "consider turning the warning into an error") + BCharP = new_pointer_type(new_primitive_type("char")) + BIntP = new_pointer_type(new_primitive_type("int")) + BVoidP = new_pointer_type(new_void_type()) + z1 = cast(BCharP, 0) + z2 = cast(BIntP, 0) + z3 = cast(BVoidP, 0) + with warnings.catch_warnings(record=True) as w: + newp(new_pointer_type(BIntP), z1) # warn + assert len(w) == 1 + newp(new_pointer_type(BVoidP), z1) # fine + assert len(w) == 1 + newp(new_pointer_type(BCharP), z2) # warn + assert len(w) == 2 + newp(new_pointer_type(BVoidP), z2) # fine + assert len(w) == 2 + newp(new_pointer_type(BCharP), z3) # fine + assert len(w) == 2 + newp(new_pointer_type(BIntP), z3) # fine + assert len(w) == 2 diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -503,3 +503,15 @@ assert ffi.unpack(p+1, 7) == b"bc\x00def\x00" p = ffi.new("int[]", [-123456789]) assert ffi.unpack(p, 1) == [-123456789] + + def test_bug_1(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + q = ffi.new("char[]", b"abcd") + p = ffi.cast("char(*)(void)", q) + raises(TypeError, ffi.string, p) + + def test_negative_array_size(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + raises(ffi.error, ffi.cast, "int[-5]", 0) diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -8,7 +8,7 @@ @unwrap_spec(cdef=str, module_name=str, source=str) def prepare(space, cdef, module_name, source, w_includes=None, - w_extra_source=None): + w_extra_source=None, w_min_version=None): try: import cffi from cffi import FFI # <== the system one, which @@ -16,8 +16,13 @@ from cffi import ffiplatform except ImportError: py.test.skip("system cffi module not found or older than 1.0.0") - if cffi.__version_info__ < (1, 4, 0): - py.test.skip("system cffi module needs to be at least 1.4.0") + if w_min_version is None: + min_version = (1, 4, 0) + else: + min_version = tuple(space.unwrap(w_min_version)) + if cffi.__version_info__ < min_version: + py.test.skip("system cffi module needs to be at least %s, got %s" % ( + min_version, cffi.__version_info__)) space.appexec([], """(): import _cffi_backend # force it to be initialized """) @@ -1790,3 +1795,28 @@ "void f(void) { }") assert lib.f.__get__(42) is lib.f assert lib.f.__get__(42, int) is lib.f + + def test_typedef_array_dotdotdot(self): + ffi, lib = self.prepare(""" + typedef int foo_t[...], bar_t[...]; + int gv[...]; + typedef int mat_t[...][...]; + typedef int vmat_t[][...]; + """, + "test_typedef_array_dotdotdot", """ + typedef int foo_t[50], bar_t[50]; + int gv[23]; + typedef int mat_t[6][7]; + typedef int vmat_t[][8]; + """, min_version=(1, 8, 4)) + assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int") + assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int") + assert len(ffi.new("foo_t")) == 50 + assert len(ffi.new("bar_t")) == 50 + assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int") + assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int") + assert len(ffi.new("mat_t")) == 6 + assert len(ffi.new("mat_t")[3]) == 7 + raises(ffi.error, ffi.sizeof, "vmat_t") + p = ffi.new("vmat_t", 4) + assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int") diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -1126,8 +1126,8 @@ a[3] = b'x' b = memoryview(a) assert len(b) == 10 - assert b[3] == b'x' - b[6] = b'y' + assert b[3] == ord(b'x') + b[6] = ord(b'y') assert a[6] == b'y' b[3:5] = b'zt' assert a[3] == b'z' @@ -1135,9 +1135,9 @@ b = memoryview(a) assert len(b) == 10 - assert b[3] == b'z' - b[3] = b'x' - assert b[3] == b'x' + assert b[3] == ord(b'z') + b[3] = ord(b'x') + assert b[3] == ord(b'x') def test_pypy_raw_address(self): import _rawffi diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -33,28 +33,27 @@ if typecode == tc: a = space.allocate_instance(types[tc].w_class, w_cls) a.__init__(space) - - if len(__args__.arguments_w) > 0: - w_initializer = __args__.arguments_w[0] - if isinstance(w_initializer, W_ArrayBase): - a.extend(w_initializer, True) - elif space.type(w_initializer) is space.w_list: - a.descr_fromlist(space, w_initializer) - else: - try: - buf = space.bufferstr_w(w_initializer) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise - a.extend(w_initializer, True) - else: - a.descr_frombytes(space, buf) break else: raise oefmt(space.w_ValueError, "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f " "or d)") + if len(__args__.arguments_w) > 0: + w_initializer = __args__.arguments_w[0] + if isinstance(w_initializer, W_ArrayBase): + a.extend(w_initializer, True) + elif space.type(w_initializer) is space.w_list: + a.descr_fromlist(space, w_initializer) + else: + try: + buf = space.bufferstr_w(w_initializer) + except OperationError as e: + if not e.match(space, space.w_TypeError): + raise + a.extend(w_initializer, True) + else: + a.descr_frombytes(space, buf) return a diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -5,7 +5,6 @@ from pypy.objspace.std.longobject import W_LongObject from pypy.interpreter.error import OperationError from rpython.rlib.rbigint import rbigint -from rpython.rlib.rarithmetic import intmask PyLong_Check, PyLong_CheckExact = build_type_checkers("Long", "w_int") @@ -27,25 +26,25 @@ """Return a new PyLongObject object from a C size_t, or NULL on failure. """ - return space.wrap(val) + return space.newlong_from_rarith_int(val) @cpython_api([rffi.LONGLONG], PyObject) def PyLong_FromLongLong(space, val): """Return a new PyLongObject object from a C long long, or NULL on failure.""" - return space.wrap(val) + return space.newlong_from_rarith_int(val) @cpython_api([rffi.ULONG], PyObject) def PyLong_FromUnsignedLong(space, val): """Return a new PyLongObject object from a C unsigned long, or NULL on failure.""" - return space.wrap(val) + return space.newlong_from_rarith_int(val) @cpython_api([rffi.ULONGLONG], PyObject) def PyLong_FromUnsignedLongLong(space, val): """Return a new PyLongObject object from a C unsigned long long, or NULL on failure.""" - return space.wrap(val) + return space.newlong_from_rarith_int(val) @cpython_api([PyObject], rffi.ULONG, error=-1) def PyLong_AsUnsignedLong(space, w_long): @@ -212,7 +211,10 @@ can be retrieved from the resulting value using PyLong_AsVoidPtr(). If the integer is larger than LONG_MAX, a positive long integer is returned.""" - return space.wrap(rffi.cast(ADDR, p)) + value = rffi.cast(ADDR, p) # signed integer + if value < 0: + return space.newlong_from_rarith_int(rffi.cast(lltype.Unsigned, p)) + return space.wrap(value) @cpython_api([PyObject], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO)) def PyLong_AsVoidPtr(space, w_long): diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -5,7 +5,6 @@ from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) from pypy.module.cpyext.frameobject import PyFrameObject -from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError from pypy.interpreter.pytraceback import PyTraceback from pypy.interpreter import pycode diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -1,5 +1,6 @@ import sys, py from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.rarithmetic import maxint from pypy.objspace.std.longobject import W_LongObject from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -7,18 +8,20 @@ class TestLongObject(BaseApiTest): def test_FromLong(self, space, api): - value = api.PyLong_FromLong(3) - assert isinstance(value, W_LongObject) - assert space.unwrap(value) == 3 + w_value = api.PyLong_FromLong(3) + assert isinstance(w_value, W_LongObject) + assert space.unwrap(w_value) == 3 - value = api.PyLong_FromLong(sys.maxint) - assert isinstance(value, W_LongObject) - assert space.unwrap(value) == sys.maxint + w_value = api.PyLong_FromLong(sys.maxint) + assert isinstance(w_value, W_LongObject) + assert space.unwrap(w_value) == sys.maxint def test_aslong(self, space, api): w_value = api.PyLong_FromLong((sys.maxint - 1) / 2) + assert isinstance(w_value, W_LongObject) w_value = space.mul(w_value, space.wrap(2)) + assert isinstance(w_value, W_LongObject) value = api.PyLong_AsLong(w_value) assert value == (sys.maxint - 1) @@ -34,12 +37,16 @@ def test_as_ssize_t(self, space, api): w_value = space.newlong(2) + assert isinstance(w_value, W_LongObject) value = api.PyLong_AsSsize_t(w_value) assert value == 2 - assert space.eq_w(w_value, api.PyLong_FromSsize_t(2)) + w_val2 = api.PyLong_FromSsize_t(2) + assert isinstance(w_val2, W_LongObject) + assert space.eq_w(w_value, w_val2) def test_fromdouble(self, space, api): w_value = api.PyLong_FromDouble(-12.74) + assert space.isinstance_w(w_value, space.w_int) assert space.unwrap(w_value) == -12 assert api.PyLong_AsDouble(w_value) == -12 @@ -102,9 +109,22 @@ def test_as_voidptr(self, space, api): w_l = api.PyLong_FromVoidPtr(lltype.nullptr(rffi.VOIDP.TO)) - assert space.unwrap(w_l) == 0L + assert space.is_w(space.type(w_l), space.w_int) + assert space.unwrap(w_l) == 0 assert api.PyLong_AsVoidPtr(w_l) == lltype.nullptr(rffi.VOIDP.TO) + p = rffi.cast(rffi.VOIDP, maxint) + w_l = api.PyLong_FromVoidPtr(p) + assert space.is_w(space.type(w_l), space.w_int) + assert space.unwrap(w_l) == maxint + assert api.PyLong_AsVoidPtr(w_l) == p + + p = rffi.cast(rffi.VOIDP, -maxint-1) + w_l = api.PyLong_FromVoidPtr(p) + assert space.is_w(space.type(w_l), space.w_int) + assert space.unwrap(w_l) == maxint+1 + assert api.PyLong_AsVoidPtr(w_l) == p + def test_sign_and_bits(self, space, api): if space.is_true(space.lt(space.sys.get('version_info'), space.wrap((2, 7)))): @@ -133,23 +153,58 @@ module = self.import_extension('foo', [ ("from_unsignedlong", "METH_NOARGS", """ - return PyLong_FromUnsignedLong((unsigned long)-1); + PyObject * obj; + obj = PyLong_FromUnsignedLong((unsigned long)-1); + if (obj->ob_type != &PyLong_Type) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, + "PyLong_FromLongLong did not return PyLongObject"); + return NULL; + } + return obj; """)]) import sys assert module.from_unsignedlong() == 2 * sys.maxsize + 1 def test_fromlonglong(self): module = self.import_extension('foo', [ - ("from_longlong", "METH_NOARGS", + ("from_longlong", "METH_VARARGS", """ - return PyLong_FromLongLong((long long)-1); + int val; + PyObject * obj; + if (!PyArg_ParseTuple(args, "i", &val)) + return NULL; + obj = PyLong_FromLongLong((long long)val); + if (obj->ob_type != &PyLong_Type) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, + "PyLong_FromLongLong did not return PyLongObject"); + return NULL; + } + return obj; """), - ("from_unsignedlonglong", "METH_NOARGS", + ("from_unsignedlonglong", "METH_VARARGS", """ - return PyLong_FromUnsignedLongLong((unsigned long long)-1); + int val; + PyObject * obj; + if (!PyArg_ParseTuple(args, "i", &val)) + return NULL; + obj = PyLong_FromUnsignedLongLong((long long)val); + if (obj->ob_type != &PyLong_Type) + { + Py_DECREF(obj); + PyErr_SetString(PyExc_ValueError, + "PyLong_FromLongLong did not return PyLongObject"); + return NULL; + } + return obj; """)]) - assert module.from_longlong() == -1 - assert module.from_unsignedlonglong() == (1<<64) - 1 + assert module.from_longlong(-1) == -1 + assert module.from_longlong(0) == 0 + assert module.from_unsignedlonglong(0) == 0 + assert module.from_unsignedlonglong(-1) == (1<<64) - 1 def test_from_size_t(self): module = self.import_extension('foo', [ @@ -228,7 +283,7 @@ return PyLong_FromLong(3); if (str + strlen(str) != end) return PyLong_FromLong(4); - return PyLong_FromLong(0); + return PyLong_FromLong(0); """)]) assert module.from_str() == 0 @@ -237,10 +292,15 @@ ("has_sub", "METH_NOARGS", """ PyObject *ret, *obj = PyLong_FromLong(42); - if (obj->ob_type->tp_as_number->nb_subtract) - ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj); + if (obj->ob_type != &PyLong_Type) + ret = PyLong_FromLong(-2); else - ret = PyLong_FromLong(-1); + { + if (obj->ob_type->tp_as_number->nb_subtract) + ret = obj->ob_type->tp_as_number->nb_subtract(obj, obj); + else + ret = PyLong_FromLong(-1); + } Py_DECREF(obj); return ret; """), @@ -279,4 +339,4 @@ assert module.has_pow() == 0 assert module.has_hex() == '0x2aL' assert module.has_oct() == '052L' - + diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py --- a/pypy/module/cpyext/test/test_version.py +++ b/pypy/module/cpyext/test/test_version.py @@ -3,7 +3,7 @@ import py, pytest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" def test_pragma_version(): from pypy.module.sys.version import CPYTHON_VERSION @@ -46,10 +46,18 @@ def test_pypy_versions(self): import sys init = """ + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "foo", /* m_name */ + NULL, /* m_doc */ + -1, /* m_size */ + NULL /* m_methods */ + }; if (Py_IsInitialized()) { - PyObject *m = Py_InitModule("foo", NULL); + PyObject *m = PyModule_Create(&moduledef); PyModule_AddStringConstant(m, "pypy_version", PYPY_VERSION); PyModule_AddIntConstant(m, "pypy_version_num", PYPY_VERSION_NUM); + return m; } Py_RETURN_NONE; """ diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -40,9 +40,6 @@ st_atime = structseqfield(11, "time of last access") st_mtime = structseqfield(12, "time of last modification") st_ctime = structseqfield(13, "time of last change") - st_atime_ns = structseqfield(14, "time of last access in nanoseconds") - st_mtime_ns = structseqfield(15, "time of last modification in nanoseconds") - st_ctime_ns = structseqfield(16, "time of last change in nanoseconds") if "st_blksize" in posix._statfields: st_blksize = structseqfield(20, "blocksize for filesystem I/O") @@ -64,6 +61,21 @@ if self.st_ctime is None: self.__dict__['st_ctime'] = self[9] + @property + def st_atime_ns(self): + "time of last access in nanoseconds" + return int(self[7]) * 1000000000 + self.nsec_atime + + @property + def st_mtime_ns(self): + "time of last modification in nanoseconds" + return int(self[8]) * 1000000000 + self.nsec_mtime + + @property + def st_ctime_ns(self): + "time of last change in nanoseconds" + return int(self[9]) * 1000000000 + self.nsec_ctime + class statvfs_result(metaclass=structseqtype): diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -364,56 +364,50 @@ STAT_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS)) -N_INDEXABLE_FIELDS = 10 - -def _time_ns_from_float(ftime): - "Convert a floating-point time (in seconds) into a (s, ns) pair of ints" - fracpart, intpart = modf(ftime) - if fracpart < 0: - fracpart += 1. - intpart -= 1. - return int(intpart), int(fracpart * 1e9) - -@specialize.arg(4) -def _fill_time(space, lst, index, w_keywords, attrname, ftime): - stat_float_times = space.fromcache(StatState).stat_float_times - seconds, fractional_ns = _time_ns_from_float(ftime) - lst[index] = space.wrap(seconds) - if stat_float_times: - space.setitem(w_keywords, space.wrap(attrname), space.wrap(ftime)) - else: - space.setitem(w_keywords, space.wrap(attrname), space.wrap(seconds)) - w_billion = space.wrap(1000000000) - w_total_ns = space.add(space.mul(space.wrap(seconds), w_billion), - space.wrap(fractional_ns)) - space.setitem(w_keywords, space.wrap(attrname + '_ns'), w_total_ns) - -STANDARD_FIELDS = unrolling_iterable(enumerate(rposix_stat.STAT_FIELDS[:7])) -EXTRA_FIELDS = unrolling_iterable(rposix_stat.STAT_FIELDS[10:]) +STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS)) def build_stat_result(space, st): - lst = [None] * N_INDEXABLE_FIELDS + FIELDS = STAT_FIELDS # also when not translating at all + lst = [None] * rposix_stat.N_INDEXABLE_FIELDS w_keywords = space.newdict() - for (i, (name, TYPE)) in STANDARD_FIELDS: - value = getattr(st, name) - w_value = space.wrap(value) - lst[i] = w_value + stat_float_times = space.fromcache(StatState).stat_float_times + for i, (name, TYPE) in FIELDS: + if i < rposix_stat.N_INDEXABLE_FIELDS: + # get the first 10 items by indexing; this gives us + # 'st_Xtime' as an integer, too + w_value = space.wrap(st[i]) + lst[i] = w_value + else: + try: + value = getattr(st, name) + except AttributeError: + # untranslated, there is no nsec_Xtime attribute + assert name.startswith('nsec_') + value = rposix_stat.get_stat_ns_as_bigint(st, name[5:]) + value = value.tolong() % 1000000000 + w_value = space.wrap(value) + space.setitem(w_keywords, space.wrap(name), w_value) - _fill_time(space, lst, 7, w_keywords, 'st_atime', st.st_atime) - _fill_time(space, lst, 8, w_keywords, 'st_mtime', st.st_mtime) - _fill_time(space, lst, 9, w_keywords, 'st_ctime', st.st_ctime) + # Note: 'w_keywords' contains the three attributes 'nsec_Xtime'. + # We have an app-level property in app_posix.stat_result to + # compute the full 'st_Xtime_ns' value. - for name, TYPE in EXTRA_FIELDS: - value = getattr(st, name) - w_value = space.wrap(value) - space.setitem(w_keywords, space.wrap(name), w_value) + # non-rounded values for name-based access + if stat_float_times: + space.setitem(w_keywords, + space.wrap('st_atime'), space.wrap(st.st_atime)) + space.setitem(w_keywords, + space.wrap('st_mtime'), space.wrap(st.st_mtime)) + space.setitem(w_keywords, + space.wrap('st_ctime'), space.wrap(st.st_ctime)) + #else: + # filled by the __init__ method w_tuple = space.newtuple(lst) w_stat_result = space.getattr(space.getbuiltinmodule(os.name), space.wrap('stat_result')) return space.call_function(w_stat_result, w_tuple, w_keywords) -STATVFS_FIELDS = unrolling_iterable(enumerate(rposix_stat.STATVFS_FIELDS)) def build_statvfs_result(space, st): vals_w = [None] * len(rposix_stat.STATVFS_FIELDS) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -126,9 +126,9 @@ assert st[4] == st.st_uid assert st[5] == st.st_gid assert st[6] == st.st_size - assert st[7] == int(st.st_atime) - assert st[8] == int(st.st_mtime) - assert st[9] == int(st.st_ctime) + assert st[7] == int(st.st_atime) # in complete corner cases, rounding + assert st[8] == int(st.st_mtime) # here could maybe get the wrong + assert st[9] == int(st.st_ctime) # integer... assert stat.S_IMODE(st.st_mode) & stat.S_IRUSR assert stat.S_IMODE(st.st_mode) & stat.S_IWUSR @@ -138,15 +138,17 @@ assert st.st_size == 14 assert st.st_nlink == 1 - #if sys.platform.startswith('linux'): - # # expects non-integer timestamps - it's unlikely that they are - # # all three integers - # assert ((st.st_atime, st.st_mtime, st.st_ctime) != - # (st[7], st[8], st[9])) - # assert st.st_blksize * st.st_blocks >= st.st_size if sys.platform.startswith('linux'): + assert isinstance(st.st_atime, float) + assert isinstance(st.st_mtime, float) + assert isinstance(st.st_ctime, float) assert hasattr(st, 'st_rdev') + assert isinstance(st.st_atime_ns, int) + assert abs(st.st_atime_ns - 1e9*st.st_atime) < 500 + assert abs(st.st_mtime_ns - 1e9*st.st_mtime) < 500 + assert abs(st.st_ctime_ns - 1e9*st.st_ctime) < 500 + def test_stat_float_times(self): path = self.path posix = self.posix diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -242,13 +242,33 @@ from rpython.rtyper.lltypesystem import lltype, rffi return space.wrap(rffi.cast(lltype.Signed, handle)) +getsizeof_missing = """sys.getsizeof() is not implemented on PyPy. + +A memory profiler using this function is most likely to give results +inconsistent with reality on PyPy. It would be possible to have +sys.getsizeof() return a number (with enough work), but that may or +may not represent how much memory the object uses. It doesn't even +make really sense to ask how much *one* object uses, in isolation +with the rest of the system. For example, instances have maps, +which are often shared across many instances; in this case the maps +would probably be ignored by an implementation of sys.getsizeof(), +but their overhead is important in some cases if they are many +instances with unique maps. Conversely, equal strings may share +their internal string data even if they are different objects---or +empty containers may share parts of their internals as long as they +are empty. Even stranger, some lists create objects as you read +them; if you try to estimate the size in memory of range(10**6) as +the sum of all items' size, that operation will by itself create one +million integer objects that never existed in the first place. +""" + def getsizeof(space, w_object, w_default=None): - """Not implemented on PyPy.""" if w_default is None: - raise oefmt(space.w_TypeError, - "sys.getsizeof() not implemented on PyPy") + raise oefmt(space.w_TypeError, getsizeof_missing) return w_default +getsizeof.__doc__ = getsizeof_missing + def intern(space, w_str): """``Intern'' the given string. This enters the string in the (global) table of interned strings whose purpose is to speed up dictionary lookups. diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py @@ -480,3 +480,7 @@ assert ffi.unpack(p+1, 7) == b"bc\x00def\x00" p = ffi.new("int[]", [-123456789]) assert ffi.unpack(p, 1) == [-123456789] + + def test_negative_array_size(self): + ffi = FFI() + py.test.raises(ValueError, ffi.cast, "int[-5]", 0) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_ffi_obj.py @@ -503,3 +503,7 @@ assert ffi.unpack(p+1, 7) == b"bc\x00def\x00" p = ffi.new("int[]", [-123456789]) assert ffi.unpack(p, 1) == [-123456789] + +def test_negative_array_size(): + ffi = _cffi1_backend.FFI() + py.test.raises(ffi.error, ffi.cast, "int[-5]", 0) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -1981,3 +1981,29 @@ static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; } """) assert lib.f1(52).a == 52 + +def test_typedef_array_dotdotdot(): + ffi = FFI() + ffi.cdef(""" + typedef int foo_t[...], bar_t[...]; + int gv[...]; + typedef int mat_t[...][...]; + typedef int vmat_t[][...]; + """) + lib = verify(ffi, "test_typedef_array_dotdotdot", """ + typedef int foo_t[50], bar_t[50]; + int gv[23]; + typedef int mat_t[6][7]; + typedef int vmat_t[][8]; + """) + assert ffi.sizeof("foo_t") == 50 * ffi.sizeof("int") + assert ffi.sizeof("bar_t") == 50 * ffi.sizeof("int") + assert len(ffi.new("foo_t")) == 50 + assert len(ffi.new("bar_t")) == 50 + assert ffi.sizeof(lib.gv) == 23 * ffi.sizeof("int") + assert ffi.sizeof("mat_t") == 6 * 7 * ffi.sizeof("int") + assert len(ffi.new("mat_t")) == 6 + assert len(ffi.new("mat_t")[3]) == 7 + py.test.raises(ffi.error, ffi.sizeof, "vmat_t") + p = ffi.new("vmat_t", 4) + assert ffi.sizeof(p[3]) == 8 * ffi.sizeof("int") diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -3,7 +3,7 @@ from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import lltype -from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat +from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_longfloat, widen from rpython.rlib.rtime import (TIMEB, c_ftime, GETTIMEOFDAY_NO_TZ, TIMEVAL, HAVE_GETTIMEOFDAY, HAVE_FTIME) @@ -290,12 +290,14 @@ if rffi.cast(rffi.LONG, errcode) == 0: if w_info is not None: _setinfo(space, w_info, "gettimeofday()", 1e-6, False, True) - return space.wrap(timeval.c_tv_sec + timeval.c_tv_usec * 1e-6) + return space.wrap( + widen(timeval.c_tv_sec) + + widen(timeval.c_tv_usec) * 1e-6) if HAVE_FTIME: with lltype.scoped_alloc(TIMEB) as t: c_ftime(t) - result = (float(intmask(t.c_time)) + - float(intmask(t.c_millitm)) * 0.001) + result = (widen(t.c_time) + + widen(t.c_millitm) * 0.001) if w_info is not None: _setinfo(space, w_info, "ftime()", 1e-3, False, True) @@ -535,7 +537,7 @@ if not allowNone: raise oefmt(space.w_TypeError, "tuple expected") # default to the current local time - tt = rffi.r_time_t(int(pytime.time())) + tt = rffi.cast(rffi.TIME_T, pytime.time()) t_ref = lltype.malloc(rffi.TIME_TP.TO, 1, flavor='raw') t_ref[0] = tt pbuf = c_localtime(t_ref) @@ -737,7 +739,7 @@ if HAS_CLOCK_GETTIME: def _timespec_to_seconds(timespec): - return int(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9 + return widen(timespec.c_tv_sec) + widen(timespec.c_tv_nsec) * 1e-9 @unwrap_spec(clk_id='c_int') def clock_gettime(space, clk_id): @@ -751,8 +753,9 @@ @unwrap_spec(clk_id='c_int', secs=float) def clock_settime(space, clk_id, secs): with lltype.scoped_alloc(TIMESPEC) as timespec: - frac = math.fmod(secs, 1.0) - rffi.setintfield(timespec, 'c_tv_sec', int(secs)) + integer_secs = rffi.cast(TIMESPEC.c_tv_sec, secs) + frac = secs - widen(integer_secs) + rffi.setintfield(timespec, 'c_tv_sec', integer_secs) rffi.setintfield(timespec, 'c_tv_nsec', int(frac * 1e9)) ret = c_clock_settime(clk_id, timespec) if ret != 0: @@ -1051,16 +1054,15 @@ return win_perf_counter(space, w_info=w_info) except ValueError: pass - value = _clock() - # Is this casting correct? - if intmask(value) == intmask(rffi.cast(rposix.CLOCK_T, -1)): + value = widen(_clock()) + if value == widen(rffi.cast(rposix.CLOCK_T, -1)): raise oefmt(space.w_RuntimeError, "the processor time used is not available or its value" "cannot be represented") if w_info is not None: _setinfo(space, w_info, "clock()", 1.0 / CLOCKS_PER_SEC, True, False) - return space.wrap((1.0 * value) / CLOCKS_PER_SEC) + return space.wrap(float(value) / CLOCKS_PER_SEC) def _setinfo(space, w_info, impl, res, mono, adj): diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -23,7 +23,6 @@ from pypy.objspace.std.util import get_positive_index from pypy.objspace.std.formatting import mod_format, FORMAT_BYTEARRAY -NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d" class W_BytearrayObject(W_Root): diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -288,6 +288,10 @@ return W_SmallLongObject.fromint(val) return W_LongObject.fromint(self, val) + @specialize.argtype(1) + def newlong_from_rarith_int(self, val): # val is an rarithmetic type + return W_LongObject.fromrarith_int(val) + def newlong_from_rbigint(self, val): return newlong(self, val) diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py --- a/pypy/objspace/std/test/test_longobject.py +++ b/pypy/objspace/std/test/test_longobject.py @@ -26,7 +26,6 @@ space.raises_w(space.w_OverflowError, space.float_w, w_big) def test_rint_variants(self): - py.test.skip("XXX broken!") from rpython.rtyper.tool.rfficache import platform space = self.space for r in platform.numbertype_to_rclass.values(): @@ -37,8 +36,8 @@ for x in values: if not r.SIGNED: x &= r.MASK - w_obj = space.wrap(r(x)) - assert space.bigint_w(w_obj).eq(rbigint.fromint(x)) + w_obj = space.newlong_from_rarith_int(r(x)) + assert space.bigint_w(w_obj).eq(rbigint.fromlong(x)) class AppTestLong: diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -805,9 +805,7 @@ class AA(object): __slots__ = ('a',) aa = AA() - # the following line works on CPython >= 2.6 but not on PyPy. - # but see below for more - raises(TypeError, "aa.__class__ = A") + aa.__class__ = A raises(TypeError, "aa.__class__ = object") class Z1(A): pass @@ -869,9 +867,13 @@ __slots__ = ['a', 'b'] class Order2(object): __slots__ = ['b', 'a'] - # the following line works on CPython >= 2.6 but not on PyPy. - # but see below for more - raises(TypeError, "Order1().__class__ = Order2") + Order1().__class__ = Order2 + + # like CPython, the order of slot names doesn't matter + x = Order1() + x.a, x.b = 1, 2 + x.__class__ = Order2 + assert (x.a, x.b) == (1, 2) class U1(object): __slots__ = ['a', 'b'] @@ -881,10 +883,11 @@ __slots__ = ['a', 'b'] class V2(V1): __slots__ = ['c', 'd', 'e'] - # the following line does not work on CPython >= 2.6 either. - # that's just obscure. Really really. So we just ignore - # the whole issue until someone comes complaining. Then we'll - # just kill slots altogether apart from maybe doing a few checks. + # the following line does not work on CPython either: we can't + # change a class if the old and new class have different layouts + # that look compatible but aren't, because they don't have the + # same base-layout class (even if these base classes are + # themselves compatible)... obscure. raises(TypeError, "U2().__class__ = V2") def test_name(self): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -103,9 +103,10 @@ """ _immutable_ = True - def __init__(self, typedef, nslots, base_layout=None): + def __init__(self, typedef, nslots, newslotnames=[], base_layout=None): self.typedef = typedef self.nslots = nslots + self.newslotnames = newslotnames[:] # make a fixed-size list self.base_layout = base_layout def issublayout(self, parent): @@ -115,6 +116,12 @@ return False return True + def expand(self, hasdict, weakrefable): + """Turn this Layout into a tuple. If two classes get equal + tuples, it means their instances have a fully compatible layout.""" + return (self.typedef, self.newslotnames, self.base_layout, + hasdict, weakrefable) + # possible values of compares_by_identity_status UNKNOWN = 0 @@ -287,8 +294,7 @@ # compute a tuple that fully describes the instance layout def get_full_instance_layout(self): - layout = self.layout - return (layout, self.hasdict, self.weakrefable) + return self.layout.expand(self.hasdict, self.weakrefable) def compute_default_mro(self): return compute_C3_mro(self.space, self) @@ -1022,11 +1028,15 @@ w_self.weakrefable = w_self.weakrefable or w_base.weakrefable return hasoldstylebase + def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout): + from pypy.objspace.std.listobject import StringSort + base_layout = w_bestbase.layout index_next_extra_slot = base_layout.nslots space = w_self.space dict_w = w_self.dict_w + newslotnames = [] if '__slots__' not in dict_w: wantdict = True wantweakref = True @@ -1052,9 +1062,22 @@ "__weakref__ slot disallowed: we already got one") wantweakref = True else: - index_next_extra_slot = create_slot(w_self, w_slot_name, - slot_name, - index_next_extra_slot) + newslotnames.append(slot_name) + # Sort the list of names collected so far + sorter = StringSort(newslotnames, len(newslotnames)) + sorter.sort() + # Try to create all slots in order. The creation of some of + # them might silently fail; then we delete the name from the + # list. At the end, 'index_next_extra_slot' has been advanced + # by the final length of 'newslotnames'. + i = 0 + while i < len(newslotnames): + if create_slot(w_self, newslotnames[i], index_next_extra_slot): + index_next_extra_slot += 1 + i += 1 + else: + del newslotnames[i] + # wantdict = wantdict or hasoldstylebase if wantdict: create_dict_slot(w_self) @@ -1063,35 +1086,34 @@ if '__del__' in dict_w: w_self.hasuserdel = True # + assert index_next_extra_slot == base_layout.nslots + len(newslotnames) if index_next_extra_slot == base_layout.nslots and not force_new_layout: return base_layout else: return Layout(base_layout.typedef, index_next_extra_slot, - base_layout=base_layout) + newslotnames, base_layout=base_layout) -def create_slot(w_self, w_slot_name, slot_name, index_next_extra_slot): +def create_slot(w_self, slot_name, index_next_extra_slot): space = w_self.space if not valid_slot_name(slot_name): raise oefmt(space.w_TypeError, "__slots__ must be identifiers") # create member slot_name = mangle(slot_name, w_self.name) - if slot_name in w_self.dict_w: - w_prev = w_self.dict_w[slot_name] - if isinstance(w_prev, Member) and w_prev.w_cls is w_self: - pass # special case: duplicate __slots__ entry, ignored - # (e.g. occurs in datetime.py, fwiw) - else: - raise oefmt(space.w_ValueError, - "%R in __slots__ conflicts with class variable", - w_slot_name) - else: + if slot_name not in w_self.dict_w: # Force interning of slot names. slot_name = space.str_w(space.new_interned_str(slot_name)) # in cpython it is ignored less, but we probably don't care member = Member(index_next_extra_slot, slot_name, w_self) - index_next_extra_slot += 1 w_self.dict_w[slot_name] = space.wrap(member) - return index_next_extra_slot + return True + else: + w_prev = w_self.dict_w[slot_name] + if isinstance(w_prev, Member) and w_prev.w_cls is w_self: + return False # special case: duplicate __slots__ entry, ignored + # (e.g. occurs in datetime.py, fwiw) + raise oefmt(space.w_ValueError, + "'%8' in __slots__ conflicts with class variable", + slot_name) def create_dict_slot(w_self): if not w_self.hasdict: diff --git a/rpython/doc/jit/backend.rst b/rpython/doc/jit/backend.rst new file mode 100644 --- /dev/null +++ b/rpython/doc/jit/backend.rst @@ -0,0 +1,263 @@ +========================= +PyPy's assembler backends +========================= + +Draft notes about the organization of assembler backends in the PyPy JIT, in 2016 +================================================================================= + + +input: linear sequence of instructions, called a "trace". + +A trace is a sequence of instructions in SSA form. Most instructions +correspond to one or a few CPU-level instructions. There are a few +meta-instructions like `label` and debugging stuff. All branching is +done with guards, which are instructions that check that a condition is +true and exit the trace if not. A failing guard can have a new trace +added to it later, called a "bridge". A patched guard becomes a direct +`Jcond` instruction going to the bridge, with no indirection, no +register spilling, etc. + +A trace ends with either a `return` or a `jump to label`. The target +label is either inside the same trace, or in some older one. For +historical reasons we call a "loop" a trace that is not a bridge. The +machine code that we generate is organized as a forest of trees; the +trunk of the tree is a "loop", and the branches are all bridges +(branching off the trunk or off another branch). + +* every trunk or branch that ends in a `jump to label` can target a + label from a different tree, too. + +* the whole process of assembling a loop or a branch is basically + single-threaded, so no synchronization issue there (including to patch + older generated instructions). + +* the generated assembler has got a "frame" in %rbp, which is actually + not on the stack at all, but is a GC object (called a "jitframe"). + Spilling goes there. + +* the guards are `Jcond` to a very small piece of generated code, which + is basically pushing a couple of constants on the stack and then + jumping to the general guard-recovery code. That code will save the + registers into the jitframe and then exit the whole generated + function. The caller of that generated function checks how it + finished: if it finished by hitting a guard, then the caller is + responsible for calling the "blackhole interpreter". This is the part + of the front-end that recovers from failing guards and finishes + running the frame (including, possibly, by jumping again into + generated assembler). + + +Details about the JITting process: + +* front-end and optimization pass + +* rewrite (includes gc related transformation as well as simplifactions) + +* assembler generation + + +Front-end and optimization pass +------------------------------- + +Not discussed here in detail. This produces loops and bridges using an +instruction set that is "high-level" in some sense: it contains +intructions like "new"/"new_array", and +"setfield"/"setarrayitem"/"setinteriorfield" which describe the action +of storing a value in a precise field of the structure or array. For +example, the "setfield" action might require implicitly a GC write +barrier. This is the high-level trace that we send to the following +step. + + +Rewrite +------- + +A mostly but not completely CPU-independent phase: lowers some +instructions. For example, the variants of "new" are lowered to +"malloc" and a few "gc_store": it bumps the pointer of the GC and then +sets a few fields explicitly in the newly allocated structure. The +"setfield" is replaced with a "cond_gc_wb_call" (conditional call to the +write barrier) if needed, followed by a "gc_store". + +The "gc_store" instruction can be encoded in a single MOV assembler +instruction, but is not as flexible as a MOV. The address is always +specified as "some GC pointer + an offset". We don't have the notion of +interior pointer for GC objects. + +A different instruction, "gc_store_indexed", offers additional operands, +which can be mapped to a single MOV instruction using forms like +`[rax+8*rcx+24]`. + +Some other complex instructions pass through to the backend, which must +deal with them: for example, "card marking" in the GC. (Writing an +object pointer inside an array would require walking the whole array +later to find "young" references. Instead of that, we flip a bit for +every range of 128 entries. This is a common GC optimization.) Setting +the card bit of a GC object requires a sequence of assembler +instructions that depends too much on the target CPU to be expressed +explicitly here (moreover, it contains a few branches, which are hard to +express at this level). + + +Assembly +-------- + +No fancy code generation technique, but greedy forward pass that tries +to avoid some pitfalls + + +Handling instructions +~~~~~~~~~~~~~~~~~~~~~ + +* One by one (forward direction). Each instruction asks the register + allocator to ensure that some arguments are in registers (not in the + jitframe); asks for a register to put its result into; and asks for + additional scratch registers that will be freed at the end of the + instruction. There is a special case for boolean variables: they are + stored in the condition code flags instead of being materialized as a + 0/1 value. (They are materialized later, except in the common case + where they are only used by the next `guard_false` or `guard_true` and + then forgotten.) + +* Instruction arguments are loaded into a register on demand. This + makes the backend quite easy to write, but leads do some bad + decisions. + + +Linear scan register allocation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Although it's always a linear trace that we consider, we don't use +advanced techniques for register allocation: we do forward, on-demand +allocation as the backend produces the assembler. When it asks for a +register to put some value into, we give it any free register, without +consideration for what will be done with it later. We compute the +longevity of all variables, but only use it when choosing which register +to spill (we spill the variable with the longest longevity). + +This works to some extend because it is well integrated with the earlier +optimization pass. Loops are unrolled once by the optimization pass to +allow more powerful optimizations---the optimization pass itself is the +place that benefits the most, but it also has benefits here in the +assembly pass. These are: + +* The first peeling initializes the register binding on the first use. + +* This leads to an already allocated register of the trace loop. + +* As well as allocated registers when exiting bridges + +[Try to better allocate registers to match the ABI (minor to non benefit +in the current state)] + + +More complex mappings +~~~~~~~~~~~~~~~~~~~~~ + +Some instructions generate more complex code. These are either or both of: + +* complex instructions generating some local control flow, like + "cond_gc_wb_call" (for write barriers), "call_assembler" (a call + followed by a few checks). + +* instructions that invoke custom assembler helpers, like the slow-path + of write barriers or the slow-path of allocations. These slow-paths + are typically generated too, so that we are not constrained by the + usual calling conventions. + + +GC pointers +~~~~~~~~~~~ + +Around most CALL instructions, we need to record a description of where +the GC pointers are (registers and stack frame). This is needed in case +the CALL invokes a garbage collection. The GC pointers can move; the +pointers in the registers and stack frame are updated by the GC. That's +a reason for why we don't have explicit interior pointers. + +GC pointers can appear as constants in the trace. We are busy changing +that to use a constant table and `MOV REG, (%RIP+offset)`. The +"constant" in the table is actually updated by the GC if the object +move. + + +Vectorization +~~~~~~~~~~~~~ + +Optimization developed to use SIMD instructions for trace loops. Primary +idea was to use it as an optimization of micro numpy. It has several +passes on the already optimized trace. + +Shortly explained: It builds dependencies for an unrolled trace loop, +gathering pairs/packs of operations that could be executed in parallel +and finally schedules the operations. + +What did it add to the code base: + +* Dependencies can be constructed + +* Code motion of guards to relax dependencies + +* Scheduler to reorder trace + +* Array bound check removal (especially for unrolled traces) + +What can it do: + +* Transform vector loops (element wise operations) + +* Accumulation (`reduce([...],operator,0)`). Requires Operation to be + associative and commutative + +* SSE 4.1 as "vector backend" + + +We do not +~~~~~~~~~ + +* Keep tracing data around to reoptimize the trace tree. (Once a trace + is compiled, minimal data is kept.) This is one reason (there are + others in the front-end) for the following result: JIT-compiling a + small loop with two common paths ends up as one "loop" and one bridge + assembled, and the bridge-following path is slightly less efficient. + This is notably because this bridge is assembled with two constraints: + the input registers are fixed (from the guard), and the output + registers are fixed (from the jump target); usually these two sets of + fixed registers are different, and copying around is needed. + +* We don't join trace tails: we only assemble *trees*. + +* We don't do any reordering (neither of trace instructions nor of + individual assembler instructions) + +* We don't do any cross-instruction optimization that makes sense only + for the backend and can't easily be expressed at a higher level. I'm + sure there are tons of examples of that, but e.g. loading a large + constant in a register that will survive for several instructions; + moving out of loops *parts* of some instruction like the address + calculation; etc. etc. + +* Other optimization opportunities I can think about: look at the + function prologue/epilogue; look at the overhead (small but not zero) + at the start of a bridge. Also check if the way guards are + implemented makes sense. Also, we generate large-ish sequences of + assembler instructions with tons of `Jcond` that are almost never + followed; any optimization opportunity there? (They all go forward, + if it changes anything.) In theory we could also replace some of + these with a signal handler on segfault (e.g. `guard_nonnull_class`). + + +a GCC or LLVM backend? +~~~~~~~~~~~~~~~~~~~~~~ + +At least for comparison we'd like a JIT backend that emits its code +using GCC or LLVM (irrespective of the time it would take). But it's +hard to map reasonably well the guards to the C language or to LLVM IR. +The problems are: (1) we have many guards, we would like to avoid having +many paths that each do a full +saving-all-local-variables-that-are-still-alive; (2) it's hard to patch +a guard when a bridge is compiled from it; (3) instructions like a CALL +need to expose the local variables that are GC pointers; CALL_MAY_FORCE +need to expose *all* local variables for optional off-line +reconstruction of the interpreter state. + diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst --- a/rpython/doc/jit/index.rst +++ b/rpython/doc/jit/index.rst @@ -26,6 +26,7 @@ optimizer virtualizable vectorization + backend - :doc:`Overview <overview>`: motivating our approach @@ -34,5 +35,8 @@ - :doc:`Optimizer <optimizer>`: the step between tracing and writing machine code -- :doc:`Virtulizable <virtualizable>` how virtualizables work and what they are - (in other words how to make frames more efficient). +- :doc:`Virtualizable <virtualizable>`: how virtualizables work and what + they are (in other words how to make frames more efficient). + +- :doc:`Assembler backend <backend>`: draft notes about the organization + of the assembler backends diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py --- a/rpython/jit/backend/test/test_ll_random.py +++ b/rpython/jit/backend/test/test_ll_random.py @@ -710,6 +710,12 @@ # 6. a conditional call (for now always with no exception raised) class CondCallOperation(BaseCallOperation): + + def filter(self, builder): + if not builder.cpu.supports_cond_call_value and \ + self.opnum == rop.COND_CALL_VALUE_I: + raise CannotProduceOperation + def produce_into(self, builder, r): fail_subset = builder.subset_of_intvars(r) if self.opnum == rop.COND_CALL: diff --git a/rpython/jit/backend/test/zll_stress.py b/rpython/jit/backend/test/zll_stress.py --- a/rpython/jit/backend/test/zll_stress.py +++ b/rpython/jit/backend/test/zll_stress.py @@ -1,6 +1,7 @@ from rpython.jit.backend.test.test_random import check_random_function, Random from rpython.jit.backend.test.test_ll_random import LLtypeOperationBuilder from rpython.jit.backend.detect_cpu import getcpuclass +from rpython.jit.metainterp.resoperation import rop import platform CPU = getcpuclass() diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -200,8 +200,12 @@ or v.concretetype != lltype.Bool): return False for op in block.operations[::-1]: - if v in op.args: - return False # variable is also used in cur block + # check if variable is used in block + for arg in op.args: + if arg == v: + return False + if isinstance(arg, ListOfKind) and v in arg.content: + return False if v is op.result: if op.opname not in ('int_lt', 'int_le', 'int_eq', 'int_ne', 'int_gt', 'int_ge', diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -243,6 +243,20 @@ assert block.exitswitch == (opname, v1, '-live-before') assert block.exits == exits +def test_optimize_goto_if_not__argument_to_call(): + for opname in ['ptr_iszero', 'ptr_nonzero']: + v1 = Variable() + v3 = Variable(); v3.concretetype = lltype.Bool + v4 = Variable() + block = Block([v1]) + callop = SpaceOperation('residual_call_r_i', + ["fake", ListOfKind('int', [v3])], v4) + block.operations = [SpaceOperation(opname, [v1], v3), callop] + block.exitswitch = v3 + block.exits = exits = [FakeLink(False), FakeLink(True)] + res = Transformer().optimize_goto_if_not(block) + assert not res + def test_symmetric(): ops = {'int_add': 'int_add', 'int_or': 'int_or', diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -90,18 +90,23 @@ assert vs.make_inputargs(args, optimizer) == [] def test_make_inputargs_2(self): - # Ensure that make_inputargs properly errors with VirtualStatesCantMatch - # when the type information for a virtual field conflicts. In practice the - # expected and given field always share a common subclass. - # This check is needed as not all paths to make_inputargs in unroll.py - # are guarded by a call to generate_guards. + # Ensure that make_inputargs does not error when the lengths of the fields + # for the runtime box does not match what the virtual state expected. + # This can occur in unroll.py, as not all paths to make_inputargs are + # guareded with a generalization_of check. The property is validated + # subsequently in all cases, so we just need to ensure that this case does + # not cause segfaults. optimizer = FakeOptimizer(self.cpu) classbox1 = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr)) - innervalue1 = info.InstancePtrInfo(known_class=classbox1, is_virtual=True, descr=self.valuedescr.get_parent_descr()) + innervalue1 = info.InstancePtrInfo( + known_class=classbox1, is_virtual=True, + descr=self.valuedescr.get_parent_descr()) for field in self.valuedescr.get_parent_descr().get_all_fielddescrs(): innervalue1.setfield(field, None, ConstInt(42)) classbox2 = self.cpu.ts.cls_of_box(InputArgRef(self.myptr3)) - innervalue2 = info.InstancePtrInfo(known_class=classbox2, is_virtual=True, descr=self.valuedescr3.get_parent_descr()) + innervalue2 = info.InstancePtrInfo( + known_class=classbox2, is_virtual=True, + descr=self.valuedescr3.get_parent_descr()) for field in self.valuedescr3.get_parent_descr().get_all_fielddescrs(): innervalue2.setfield(field, None, ConstInt(42)) @@ -111,10 +116,14 @@ nodebox2.set_forwarded(innervalue2) constr = VirtualStateConstructor(optimizer) - vs = constr.get_virtual_state([nodebox1]) + vs1 = constr.get_virtual_state([nodebox1]) + constr = VirtualStateConstructor(optimizer) + vs2 = constr.get_virtual_state([nodebox2]) - with py.test.raises(VirtualStatesCantMatch): - args = vs.make_inputargs([nodebox2], optimizer, force_boxes=True) + # This should succeed with no exceptions + vs1.make_inputargs([nodebox2], optimizer, force_boxes=False) + assert not vs1.generalization_of(vs2, optimizer) + assert not vs2.generalization_of(vs1, optimizer) def test_position_generalization(self): def postest(info1, info2): diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -167,7 +167,8 @@ [self.get_box_replacement(x) for x in end_jump.getarglist()], self.optimizer, force_boxes=True) for arg in args: - self.optimizer.force_box(arg) + if arg is not None: + self.optimizer.force_box(arg) except VirtualStatesCantMatch: raise InvalidLoop("Virtual states did not match " "after picking the virtual state, when forcing" diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py @@ -177,14 +177,6 @@ def _generalization_of_structpart(self, other): raise NotImplementedError - @staticmethod - def descr_issubclass(descr1, descr2, optimizer): - if not descr1.is_object() or not descr2.is_object(): - return True - vtable1 = descr1.get_vtable() - vtable2 = descr2.get_vtable() - return optimizer._check_subclass(vtable1, vtable2) - def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False): box = optimizer.get_box_replacement(box) info = optimizer.getptrinfo(box) @@ -193,13 +185,12 @@ else: assert isinstance(info, AbstractStructPtrInfo) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit