Author: Ronan Lamy <[email protected]> Branch: exc-later Changeset: r80820:c61a1f069e26 Date: 2015-11-21 16:50 +0000 http://bitbucket.org/pypy/pypy/changeset/c61a1f069e26/
Log: hg merge default diff too long, truncating to 2000 out of 10259 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -17,3 +17,4 @@ 295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0 f3ad1e1e1d6215e20d34bb65ab85ff9188c9f559 release-2.6.1 850edf14b2c75573720f59e95767335fb1affe55 release-4.0.0 +5f8302b8bf9f53056e40426f10c72151564e5b19 release-4.0.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -56,14 +56,15 @@ Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen + Richard Plangger Richard Emslie Alexander Schremmer Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager Anders Lehmann + Remi Meier Aurelien Campeas - Remi Meier Niklaus Haldimann Camillo Bruni Laura Creighton @@ -87,7 +88,6 @@ Ludovic Aubry Jacob Hallen Jason Creighton - Richard Plangger Alex Martelli Michal Bendowski stian @@ -200,9 +200,12 @@ Alex Perry Vincent Legoll Alan McIntyre + Spenser Bauman Alexander Sedov Attila Gobi Christopher Pope + Devin Jeanpierre + Vaibhav Sood Christian Tismer Marc Abramowitz Dan Stromberg @@ -234,6 +237,7 @@ Lutz Paelike Lucio Torre Lars Wassermann + Philipp Rustemeuer Henrik Vendelbo Dan Buch Miguel de Val Borro @@ -244,6 +248,7 @@ Martin Blais Lene Wagner Tomo Cocoa + Kim Jin Su Toni Mattis Lucas Stadler Julian Berman @@ -253,6 +258,7 @@ Anna Katrina Dominguez William Leslie Bobby Impollonia + Faye Zhao [email protected] Andrew Thompson Yusei Tahara @@ -283,6 +289,7 @@ shoma hosaka Daniel Neuhäuser Ben Mather + Niclas Olofsson halgari Boglarka Vezer Chris Pressey @@ -309,13 +316,16 @@ Stefan Marr jiaaro Mads Kiilerich + Richard Lancaster opassembler.py Antony Lee + Yaroslav Fedevych Jim Hunziker Markus Unterwaditzer Even Wiik Thomassen jbs squeaky + Zearin soareschen Kurt Griffiths Mike Bayer @@ -327,6 +337,7 @@ Anna Ravencroft Andrey Churin Dan Crosta + Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.3.0 +Version: 1.3.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.3.0" -__version_info__ = (1, 3, 0) +__version__ = "1.3.1" +__version_info__ = (1, 3, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py --- a/lib_pypy/cffi/cffi_opcode.py +++ b/lib_pypy/cffi/cffi_opcode.py @@ -110,6 +110,8 @@ _UNKNOWN_FLOAT_PRIM = -2 _UNKNOWN_LONG_DOUBLE = -3 +_IO_FILE_STRUCT = -1 + PRIMITIVE_TO_INDEX = { 'char': PRIM_CHAR, 'short': PRIM_SHORT, diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py --- a/lib_pypy/cffi/commontypes.py +++ b/lib_pypy/cffi/commontypes.py @@ -2,10 +2,17 @@ from . import api, model -COMMON_TYPES = { - 'FILE': model.unknown_type('FILE', '_IO_FILE'), - 'bool': '_Bool', - } +COMMON_TYPES = {} + +try: + # fetch "bool" and all simple Windows types + from _cffi_backend import _get_common_types + _get_common_types(COMMON_TYPES) +except ImportError: + pass + +COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE') +COMMON_TYPES['bool'] = '_Bool' # in case we got ImportError above for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES: if _type.endswith('_t'): @@ -14,212 +21,35 @@ _CACHE = {} -def resolve_common_type(commontype): +def resolve_common_type(parser, commontype): try: return _CACHE[commontype] except KeyError: - result = COMMON_TYPES.get(commontype, commontype) - if not isinstance(result, str): - pass # result is already a BaseType - elif result.endswith(' *'): - if result.startswith('const '): - result = model.ConstPointerType( - resolve_common_type(result[6:-2])) - else: - result = model.PointerType(resolve_common_type(result[:-2])) - elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES: - result = model.PrimitiveType(result) - elif result == 'set-unicode-needed': + cdecl = COMMON_TYPES.get(commontype, commontype) + if not isinstance(cdecl, str): + result, quals = cdecl, 0 # cdecl is already a BaseType + elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + result, quals = model.PrimitiveType(cdecl), 0 + elif cdecl == 'set-unicode-needed': raise api.FFIError("The Windows type %r is only available after " "you call ffi.set_unicode()" % (commontype,)) else: - if commontype == result: + if commontype == cdecl: raise api.FFIError("Unsupported type: %r. Please file a bug " "if you think it should be." % (commontype,)) - result = resolve_common_type(result) # recursively + result, quals = parser.parse_type_and_quals(cdecl) # recursive + assert isinstance(result, model.BaseTypeByIdentity) - _CACHE[commontype] = result - return result + _CACHE[commontype] = result, quals + return result, quals # ____________________________________________________________ -# Windows common types +# extra types for Windows (most of them are in commontypes.c) -def win_common_types(maxsize): - result = {} - if maxsize < (1<<32): - result.update({ # Windows 32-bits - 'HALF_PTR': 'short', - 'INT_PTR': 'int', - 'LONG_PTR': 'long', - 'UHALF_PTR': 'unsigned short', - 'UINT_PTR': 'unsigned int', - 'ULONG_PTR': 'unsigned long', - }) - else: - result.update({ # Windows 64-bits - 'HALF_PTR': 'int', - 'INT_PTR': 'long long', - 'LONG_PTR': 'long long', - 'UHALF_PTR': 'unsigned int', - 'UINT_PTR': 'unsigned long long', - 'ULONG_PTR': 'unsigned long long', - }) - result.update({ - "BYTE": "unsigned char", - "BOOL": "int", - "CCHAR": "char", - "CHAR": "char", - "DWORD": "unsigned long", - "DWORD32": "unsigned int", - "DWORD64": "unsigned long long", - "FLOAT": "float", - "INT": "int", - "INT8": "signed char", - "INT16": "short", - "INT32": "int", - "INT64": "long long", - "LONG": "long", - "LONGLONG": "long long", - "LONG32": "int", - "LONG64": "long long", - "WORD": "unsigned short", - "PVOID": model.voidp_type, - "ULONGLONG": "unsigned long long", - "WCHAR": "wchar_t", - "SHORT": "short", - "UCHAR": "unsigned char", - "UINT": "unsigned int", - "UINT8": "unsigned char", - "UINT16": "unsigned short", - "UINT32": "unsigned int", - "UINT64": "unsigned long long", - "ULONG": "unsigned long", - "ULONG32": "unsigned int", - "ULONG64": "unsigned long long", - "USHORT": "unsigned short", - - "SIZE_T": "ULONG_PTR", - "SSIZE_T": "LONG_PTR", - "ATOM": "WORD", - "BOOLEAN": "BYTE", - "COLORREF": "DWORD", - - "HANDLE": "PVOID", - "DWORDLONG": "ULONGLONG", - "DWORD_PTR": "ULONG_PTR", - "HACCEL": "HANDLE", - - "HBITMAP": "HANDLE", - "HBRUSH": "HANDLE", - "HCOLORSPACE": "HANDLE", - "HCONV": "HANDLE", - "HCONVLIST": "HANDLE", - "HDC": "HANDLE", - "HDDEDATA": "HANDLE", - "HDESK": "HANDLE", - "HDROP": "HANDLE", - "HDWP": "HANDLE", - "HENHMETAFILE": "HANDLE", - "HFILE": "int", - "HFONT": "HANDLE", - "HGDIOBJ": "HANDLE", - "HGLOBAL": "HANDLE", - "HHOOK": "HANDLE", - "HICON": "HANDLE", - "HCURSOR": "HICON", - "HINSTANCE": "HANDLE", - "HKEY": "HANDLE", - "HKL": "HANDLE", - "HLOCAL": "HANDLE", - "HMENU": "HANDLE", - "HMETAFILE": "HANDLE", - "HMODULE": "HINSTANCE", - "HMONITOR": "HANDLE", - "HPALETTE": "HANDLE", - "HPEN": "HANDLE", - "HRESULT": "LONG", - "HRGN": "HANDLE", - "HRSRC": "HANDLE", - "HSZ": "HANDLE", - "WINSTA": "HANDLE", - "HWND": "HANDLE", - - "LANGID": "WORD", - "LCID": "DWORD", - "LCTYPE": "DWORD", - "LGRPID": "DWORD", - "LPARAM": "LONG_PTR", - "LPBOOL": "BOOL *", - "LPBYTE": "BYTE *", - "LPCOLORREF": "DWORD *", - "LPCSTR": "const char *", - - "LPCVOID": model.const_voidp_type, - "LPCWSTR": "const WCHAR *", - "LPDWORD": "DWORD *", - "LPHANDLE": "HANDLE *", - "LPINT": "int *", - "LPLONG": "long *", - "LPSTR": "CHAR *", - "LPWSTR": "WCHAR *", - "LPVOID": model.voidp_type, - "LPWORD": "WORD *", - "LRESULT": "LONG_PTR", - "PBOOL": "BOOL *", - "PBOOLEAN": "BOOLEAN *", - "PBYTE": "BYTE *", - "PCHAR": "CHAR *", - "PCSTR": "const CHAR *", - "PCWSTR": "const WCHAR *", - "PDWORD": "DWORD *", - "PDWORDLONG": "DWORDLONG *", - "PDWORD_PTR": "DWORD_PTR *", - "PDWORD32": "DWORD32 *", - "PDWORD64": "DWORD64 *", - "PFLOAT": "FLOAT *", - "PHALF_PTR": "HALF_PTR *", - "PHANDLE": "HANDLE *", - "PHKEY": "HKEY *", - "PINT": "int *", - "PINT_PTR": "INT_PTR *", - "PINT8": "INT8 *", - "PINT16": "INT16 *", - "PINT32": "INT32 *", - "PINT64": "INT64 *", - "PLCID": "PDWORD", - "PLONG": "LONG *", - "PLONGLONG": "LONGLONG *", - "PLONG_PTR": "LONG_PTR *", - "PLONG32": "LONG32 *", - "PLONG64": "LONG64 *", - "PSHORT": "SHORT *", - "PSIZE_T": "SIZE_T *", - "PSSIZE_T": "SSIZE_T *", - "PSTR": "CHAR *", - "PUCHAR": "UCHAR *", - "PUHALF_PTR": "UHALF_PTR *", - "PUINT": "UINT *", - "PUINT_PTR": "UINT_PTR *", - "PUINT8": "UINT8 *", - "PUINT16": "UINT16 *", - "PUINT32": "UINT32 *", - "PUINT64": "UINT64 *", - "PULONG": "ULONG *", - "PULONGLONG": "ULONGLONG *", - "PULONG_PTR": "ULONG_PTR *", - "PULONG32": "ULONG32 *", - "PULONG64": "ULONG64 *", - "PUSHORT": "USHORT *", - "PWCHAR": "WCHAR *", - "PWORD": "WORD *", - "PWSTR": "WCHAR *", - "QWORD": "unsigned long long", - "SC_HANDLE": "HANDLE", - "SC_LOCK": "LPVOID", - "SERVICE_STATUS_HANDLE": "HANDLE", - +def win_common_types(): + return { "UNICODE_STRING": model.StructType( "_UNICODE_STRING", ["Length", @@ -232,10 +62,6 @@ "PUNICODE_STRING": "UNICODE_STRING *", "PCUNICODE_STRING": "const UNICODE_STRING *", - "USN": "LONGLONG", - "VOID": model.void_type, - "WPARAM": "UINT_PTR", - "TBYTE": "set-unicode-needed", "TCHAR": "set-unicode-needed", "LPCTSTR": "set-unicode-needed", @@ -244,9 +70,7 @@ "PTSTR": "set-unicode-needed", "PTBYTE": "set-unicode-needed", "PTCHAR": "set-unicode-needed", - }) - return result - + } if sys.platform == 'win32': - COMMON_TYPES.update(win_common_types(sys.maxsize)) + COMMON_TYPES.update(win_common_types()) diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -29,6 +29,8 @@ _r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") _r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") _r_cdecl = re.compile(r"\b__cdecl\b") +_r_star_const_space = re.compile( # matches "* const " + r"[*]\s*((const|volatile|restrict)\b\s*)+") def _get_parser(): global _parser_cache @@ -36,6 +38,48 @@ _parser_cache = pycparser.CParser() return _parser_cache +def _workaround_for_old_pycparser(csource): + # Workaround for a pycparser issue (fixed between pycparser 2.10 and + # 2.14): "char*const***" gives us a wrong syntax tree, the same as + # for "char***(*const)". This means we can't tell the difference + # afterwards. But "char(*const(***))" gives us the right syntax + # tree. The issue only occurs if there are several stars in + # sequence with no parenthesis inbetween, just possibly qualifiers. + # Attempt to fix it by adding some parentheses in the source: each + # time we see "* const" or "* const *", we add an opening + # parenthesis before each star---the hard part is figuring out where + # to close them. + parts = [] + while True: + match = _r_star_const_space.search(csource) + if not match: + break + #print repr(''.join(parts)+csource), '=>', + parts.append(csource[:match.start()]) + parts.append('('); closing = ')' + parts.append(match.group()) # e.g. "* const " + endpos = match.end() + if csource.startswith('*', endpos): + parts.append('('); closing += ')' + level = 0 + i = endpos + while i < len(csource): + c = csource[i] + if c == '(': + level += 1 + elif c == ')': + if level == 0: + break + level -= 1 + elif c in ',;=': + if level == 0: + break + i += 1 + csource = csource[endpos:i] + closing + csource[i:] + #print repr(''.join(parts)+csource) + parts.append(csource) + return ''.join(parts) + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -47,6 +91,10 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + # + if pycparser.__version__ < '2.14': + csource = _workaround_for_old_pycparser(csource) + # # BIG HACK: replace WINAPI or __stdcall with "volatile const". # It doesn't make sense for the return type of a function to be # "volatile volatile const", so we abuse it to detect __stdcall... @@ -320,13 +368,15 @@ self._declare('variable ' + decl.name, tp, quals=quals) def parse_type(self, cdecl): + return self.parse_type_and_quals(cdecl)[0] + + def parse_type_and_quals(self, cdecl): ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): raise api.CDefError("unknown identifier '%s'" % (exprnode.name,)) - tp, quals = self._get_type_and_quals(exprnode.type) - return tp + return self._get_type_and_quals(exprnode.type) def _declare(self, name, obj, included=False, quals=0): if name in self._declarations: @@ -348,6 +398,8 @@ pycparser.c_ast.PtrDecl)): if 'const' in type.quals: quals |= model.Q_CONST + if 'volatile' in type.quals: + quals |= model.Q_VOLATILE if 'restrict' in type.quals: quals |= model.Q_RESTRICT return quals @@ -422,7 +474,8 @@ if ident == '__dotdotdot__': raise api.FFIError(':%d: bad usage of "..."' % typenode.coord.line) - return resolve_common_type(ident), quals + tp0, quals0 = resolve_common_type(self, ident) + return tp0, (quals | quals0) # if isinstance(type, pycparser.c_ast.Struct): # 'struct foobar' @@ -456,6 +509,13 @@ def _parse_function_type(self, typenode, funcname=None): params = list(getattr(typenode.args, 'params', [])) + for i, arg in enumerate(params): + if not hasattr(arg, 'type'): + raise api.CDefError("%s arg %d: unknown type '%s'" + " (if you meant to use the old C syntax of giving" + " untyped arguments, it is not supported)" + % (funcname or 'in expression', i + 1, + getattr(arg, 'name', '?'))) ellipsis = ( len(params) > 0 and isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -7,10 +7,13 @@ # type qualifiers Q_CONST = 0x01 Q_RESTRICT = 0x02 +Q_VOLATILE = 0x04 def qualify(quals, replace_with): if quals & Q_CONST: replace_with = ' const ' + replace_with.lstrip() + if quals & Q_VOLATILE: + replace_with = ' volatile ' + replace_with.lstrip() if quals & Q_RESTRICT: # It seems that __restrict is supported by gcc and msvc. # If you hit some different compiler, add a #define in @@ -511,12 +514,17 @@ if self.baseinttype is not None: return self.baseinttype.get_cached_btype(ffi, finishlist) # + from . import api if self.enumvalues: smallest_value = min(self.enumvalues) largest_value = max(self.enumvalues) else: - smallest_value = 0 - largest_value = 0 + import warnings + warnings.warn("%r has no values explicitly defined; next version " + "will refuse to guess which integer type it is " + "meant to be (unsigned/signed, int/long)" + % self._get_c_name()) + smallest_value = largest_value = 0 if smallest_value < 0: # needs a signed type sign = 1 candidate1 = PrimitiveType("int") diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h --- a/lib_pypy/cffi/parse_c_type.h +++ b/lib_pypy/cffi/parse_c_type.h @@ -83,6 +83,8 @@ #define _CFFI__UNKNOWN_FLOAT_PRIM (-2) #define _CFFI__UNKNOWN_LONG_DOUBLE (-3) +#define _CFFI__IO_FILE_STRUCT (-1) + struct _cffi_global_s { const char *name; diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -88,9 +88,19 @@ # try: unbound_method = getattr(_continulet, methodname) + _tls.leaving = current args, kwds = unbound_method(current, *baseargs, to=target) - finally: _tls.current = current + except: + _tls.current = current + if hasattr(_tls, 'trace'): + _run_trace_callback('throw') + _tls.leaving = None + raise + else: + if hasattr(_tls, 'trace'): + _run_trace_callback('switch') + _tls.leaving = None # if kwds: if args: @@ -122,6 +132,34 @@ return f.f_back.f_back.f_back # go past start(), __switch(), switch() # ____________________________________________________________ +# Recent additions + +GREENLET_USE_GC = True +GREENLET_USE_TRACING = True + +def gettrace(): + return getattr(_tls, 'trace', None) + +def settrace(callback): + try: + prev = _tls.trace + del _tls.trace + except AttributeError: + prev = None + if callback is not None: + _tls.trace = callback + return prev + +def _run_trace_callback(event): + try: + _tls.trace(event, (_tls.leaving, _tls.current)) + except: + # In case of exceptions trace function is removed + if hasattr(_tls, 'trace'): + del _tls.trace + raise + +# ____________________________________________________________ # Internal stuff try: @@ -143,22 +181,32 @@ _tls.current = gmain def _greenlet_start(greenlet, args): - args, kwds = args - _tls.current = greenlet try: - res = greenlet.run(*args, **kwds) - except GreenletExit, e: - res = e + args, kwds = args + _tls.current = greenlet + try: + if hasattr(_tls, 'trace'): + _run_trace_callback('switch') + res = greenlet.run(*args, **kwds) + except GreenletExit, e: + res = e + finally: + _continuation.permute(greenlet, greenlet.parent) + return ((res,), None) finally: - _continuation.permute(greenlet, greenlet.parent) - return ((res,), None) + _tls.leaving = greenlet def _greenlet_throw(greenlet, exc, value, tb): - _tls.current = greenlet try: - raise exc, value, tb - except GreenletExit, e: - res = e + _tls.current = greenlet + try: + if hasattr(_tls, 'trace'): + _run_trace_callback('throw') + raise exc, value, tb + except GreenletExit, e: + res = e + finally: + _continuation.permute(greenlet, greenlet.parent) + return ((res,), None) finally: - _continuation.permute(greenlet, greenlet.parent) - return ((res,), None) + _tls.leaving = greenlet diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -76,10 +76,10 @@ if "cppyy" in working_modules: working_modules.remove("cppyy") # depends on ctypes -if sys.platform.startswith("linux"): - _mach = os.popen('uname -m', 'r').read().strip() - if _mach.startswith('ppc'): - working_modules.remove("_continuation") +#if sys.platform.startswith("linux"): +# _mach = os.popen('uname -m', 'r').read().strip() +# if _mach.startswith(...): +# working_modules.remove("_continuation") module_dependencies = { diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -85,13 +85,16 @@ _ssl libssl +gdbm + libgdbm-dev + Make sure to have these libraries (with development headers) installed before building PyPy, otherwise the resulting binary will not contain these modules. On Debian, this is the command to install all build-time dependencies:: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ - libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev + libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev For the optional lzma module on PyPy3 you will also need ``liblzma-dev``. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -26,15 +26,15 @@ Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen + Richard Plangger Richard Emslie Alexander Schremmer Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager Anders Lehmann - Richard Plangger + Remi Meier Aurelien Campeas - Remi Meier Niklaus Haldimann Camillo Bruni Laura Creighton @@ -170,9 +170,12 @@ Alex Perry Vincent Legoll Alan McIntyre + Spenser Bauman Alexander Sedov Attila Gobi Christopher Pope + Devin Jeanpierre + Vaibhav Sood Christian Tismer Marc Abramowitz Dan Stromberg @@ -204,6 +207,7 @@ Lutz Paelike Lucio Torre Lars Wassermann + Philipp Rustemeuer Henrik Vendelbo Dan Buch Miguel de Val Borro @@ -214,6 +218,7 @@ Martin Blais Lene Wagner Tomo Cocoa + Kim Jin Su Toni Mattis Lucas Stadler Julian Berman @@ -223,6 +228,7 @@ Anna Katrina Dominguez William Leslie Bobby Impollonia + Faye Zhao [email protected] Andrew Thompson Yusei Tahara @@ -280,13 +286,16 @@ Stefan Marr jiaaro Mads Kiilerich + Richard Lancaster opassembler.py Antony Lee + Yaroslav Fedevych Jim Hunziker Markus Unterwaditzer Even Wiik Thomassen jbs squeaky + Zearin soareschen Kurt Griffiths Mike Bayer @@ -298,6 +307,7 @@ Anna Ravencroft Andrey Churin Dan Crosta + Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-4.0.1.rst release-4.0.0.rst release-2.6.1.rst release-2.6.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-4.0.1.rst whatsnew-4.0.0.rst whatsnew-2.6.1.rst whatsnew-2.6.0.rst diff --git a/pypy/doc/release-4.0.1.rst b/pypy/doc/release-4.0.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-4.0.1.rst @@ -0,0 +1,106 @@ +========== +PyPy 4.0.1 +========== + +We have released PyPy 4.0.1, three weeks after PyPy 4.0.0. We have fixed +a few critical bugs in the JIT compiled code, reported by users. We therefore +encourage all users of PyPy to update to this version. There are a few minor +enhancements in this version as well. + +You can download the PyPy 4.0.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +CFFI +==== + +While not applicable only to PyPy, `cffi`_ is arguably our most significant +contribution to the python ecosystem. PyPy 4.0.1 ships with +`cffi-1.3.1`_ with the improvements it brings. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`cffi`: https://cffi.readthedocs.org +.. _`cffi-1.3.1`: http://cffi.readthedocs.org/en/latest/whatsnew.html#v1-3-1 +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html +.. _`numpy`: https://bitbucket.org/pypy/numpy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, freebsd), +newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, and the +big- and little-endian variants of **ppc64** running Linux. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 4.0.0 released three weeks ago) +======================================================= + +* Bug Fixes + + * Fix a bug when unrolling double loops in JITted code + + * Fix multiple memory leaks in the ssl module, one of which affected + `cpython` as well (thanks to Alex Gaynor for pointing those out) + + * Use pkg-config to find ssl headers on OS-X + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* New features: + + * Internal cleanup of RPython class handling + + * Support stackless and greenlets on PPC machines + + * Improve debug logging in subprocesses: use PYPYLOG=jit:log.%d + for example to have all subprocesses write the JIT log to a file + called 'log.%d', with '%d' replaced with the subprocess' PID. + + * Support PyOS_double_to_string in our cpyext capi compatibility layer + +* Numpy: + + * Improve support for __array_interface__ + + * Propagate NAN mantissas through float16-float32-float64 conversions + + +* Performance improvements and refactorings: + + * Improvements in slicing byte arrays + + * Improvements in enumerate() + + * Silence some warnings while translating + +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-4.0.1.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -69,7 +69,9 @@ 'Rami Chowdhury': ['necaris'], 'Stanislaw Halik':['w31rd0'], 'Wenzhu Man':['wenzhu man', 'wenzhuman'], - 'Anton Gulenko':['anton gulenko'], + 'Anton Gulenko':['anton gulenko', 'anton_gulenko'], + 'Richard Lancaster':['richardlancaster'], + 'William Leslie':['William ML Leslie'], } alias_map = {} diff --git a/pypy/doc/whatsnew-4.0.1.rst b/pypy/doc/whatsnew-4.0.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-4.0.1.rst @@ -0,0 +1,35 @@ +========================= +What's new in PyPy 4.0.1 +========================= + +.. this is a revision shortly after release-4.0.0 +.. startrev: 57c9a47c70f6 + +.. branch: 2174-fix-osx-10-11-translation + +Use pkg-config to find ssl headers on OS-X + +.. branch: Zearin/minor-whatsnewrst-markup-tweaks-edited-o-1446387512092 + +.. branch: ppc-stacklet + +The PPC machines now support the _continuation module (stackless, greenlets) + +.. branch: int_0/i-need-this-library-to-build-on-ubuntu-1-1446717626227 + +Document that libgdbm-dev is required for translation/packaging + +.. branch: propogate-nans + +Ensure that ndarray conversion from int16->float16->float32->float16->int16 +preserves all int16 values, even across nan conversions. Also fix argmax, argmin +for nan comparisons + +.. branch: array_interface + +Support common use-cases for __array_interface__, passes upstream tests + +.. branch: no-class-specialize + +Some refactoring of class handling in the annotator. +Remove class specialisation and _settled_ flag. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,10 +1,18 @@ ========================= -What's new in PyPy 4.0.+ +What's new in PyPy 4.1.+ ========================= -.. this is a revision shortly after release-4.0.0 -.. startrev: 57c9a47c70f6 +.. this is a revision shortly after release-4.0.1 +.. startrev: 4b5c840d0da2 -.. branch: 2174-fix-osx-10-11-translation +.. branch: numpy-1.10 -Use pkg-config to find ssl headers on OS-X +Fix tests to run cleanly with -A and start to fix micronumpy for upstream numpy +which is now 1.10.2 + +.. branch: osx-flat-namespace + +Fix the cpyext tests on OSX by linking with -flat_namespace + +.. branch: anntype +Refactor and improve exception analysis in the annotator. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -28,7 +28,6 @@ """This is the abstract root class of all wrapped objects that live in a 'normal' object space like StdObjSpace.""" __slots__ = () - _settled_ = True user_overridden_class = False def getdict(self, space): @@ -392,7 +391,7 @@ self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) self._code_of_sys_exc_info = None - + # can be overridden to a subclass self.initialize() diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -50,6 +50,9 @@ kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None return Signature(argnames, varargname, kwargname) +class CodeHookCache(object): + def __init__(self, space): + self._code_hook = None class PyCode(eval.Code): "CPython-style code objects." @@ -86,6 +89,15 @@ self._signature = cpython_code_signature(self) self._initialize() self._init_ready() + self.new_code_hook() + + def new_code_hook(self): + code_hook = self.space.fromcache(CodeHookCache)._code_hook + if code_hook is not None: + try: + self.space.call_function(code_hook, self) + except OperationError, e: + e.write_unraisable(self.space, "new_code_hook()") def _initialize(self): if self.co_cellvars: diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -390,15 +390,20 @@ def test_pickle_enum(self): import pickle - e = enumerate(range(10)) + e = enumerate(range(100, 106)) e.next() e.next() pckl = pickle.dumps(e) result = pickle.loads(pckl) - e.next() - result.next() + res = e.next() + assert res == (2, 102) + res = result.next() + assert res == (2, 102) assert type(e) is type(result) - assert list(e) == list(result) + res = list(e) + assert res == [(3, 103), (4, 104), (5, 105)] + res = list(result) + assert res == [(3, 103), (4, 104), (5, 105)] def test_pickle_xrangeiter(self): import pickle diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -8,7 +8,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.typedef import TypeDef -from rpython.rlib import jit +from rpython.rlib import jit, rarithmetic from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.rbigint import rbigint @@ -229,27 +229,70 @@ return min_max(space, __args__, "min") + class W_Enumerate(W_Root): - def __init__(self, w_iter, w_start): - self.w_iter = w_iter + def __init__(self, w_iter_or_list, start, w_start): + # 'w_index' should never be a wrapped int here; if it would be, + # then it is actually None and the unwrapped int is in 'index'. + self.w_iter_or_list = w_iter_or_list + self.index = start self.w_index = w_start def descr___new__(space, w_subtype, w_iterable, w_start=None): - self = space.allocate_instance(W_Enumerate, w_subtype) + from pypy.objspace.std.listobject import W_ListObject + if w_start is None: - w_start = space.wrap(0) + start = 0 else: w_start = space.index(w_start) - self.__init__(space.iter(w_iterable), w_start) + if space.is_w(space.type(w_start), space.w_int): + start = space.int_w(w_start) + w_start = None + else: + start = -1 + + if start == 0 and type(w_iterable) is W_ListObject: + w_iter = w_iterable + else: + w_iter = space.iter(w_iterable) + + self = space.allocate_instance(W_Enumerate, w_subtype) + self.__init__(w_iter, start, w_start) return space.wrap(self) def descr___iter__(self, space): return space.wrap(self) def descr_next(self, space): - w_item = space.next(self.w_iter) + from pypy.objspace.std.listobject import W_ListObject w_index = self.w_index - self.w_index = space.add(w_index, space.wrap(1)) + w_iter_or_list = self.w_iter_or_list + w_item = None + if w_index is None: + index = self.index + if type(w_iter_or_list) is W_ListObject: + try: + w_item = w_iter_or_list.getitem(index) + except IndexError: + self.w_iter_or_list = None + raise OperationError(space.w_StopIteration, space.w_None) + self.index = index + 1 + elif w_iter_or_list is None: + raise OperationError(space.w_StopIteration, space.w_None) + else: + try: + newval = rarithmetic.ovfcheck(index + 1) + except OverflowError: + w_index = space.wrap(index) + self.w_index = space.add(w_index, space.wrap(1)) + self.index = -1 + else: + self.index = newval + w_index = space.wrap(index) + else: + self.w_index = space.add(w_index, space.wrap(1)) + if w_item is None: + w_item = space.next(self.w_iter_or_list) return space.newtuple([w_index, w_item]) def descr___reduce__(self, space): @@ -257,12 +300,20 @@ w_mod = space.getbuiltinmodule('_pickle_support') mod = space.interp_w(MixedModule, w_mod) w_new_inst = mod.get('enumerate_new') - w_info = space.newtuple([self.w_iter, self.w_index]) + w_index = self.w_index + if w_index is None: + w_index = space.wrap(self.index) + w_info = space.newtuple([self.w_iter_or_list, w_index]) return space.newtuple([w_new_inst, w_info]) # exported through _pickle_support -def _make_enumerate(space, w_iter, w_index): - return space.wrap(W_Enumerate(w_iter, w_index)) +def _make_enumerate(space, w_iter_or_list, w_index): + if space.is_w(space.type(w_index), space.w_int): + index = space.int_w(w_index) + w_index = None + else: + index = -1 + return space.wrap(W_Enumerate(w_iter_or_list, index, w_index)) W_Enumerate.typedef = TypeDef("enumerate", __new__=interp2app(W_Enumerate.descr___new__.im_func), diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -264,6 +264,7 @@ raises(StopIteration,x.next) def test_enumerate(self): + import sys seq = range(2,4) enum = enumerate(seq) assert enum.next() == (0, 2) @@ -274,6 +275,15 @@ enum = enumerate(range(5), 2) assert list(enum) == zip(range(2, 7), range(5)) + enum = enumerate(range(2), 2**100) + assert list(enum) == [(2**100, 0), (2**100+1, 1)] + + enum = enumerate(range(2), sys.maxint) + assert list(enum) == [(sys.maxint, 0), (sys.maxint+1, 1)] + + raises(TypeError, enumerate, range(2), 5.5) + + def test_next(self): x = iter(['a', 'b', 'c']) assert next(x) == 'a' diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -86,6 +86,7 @@ 'specialized_zip_2_lists' : 'interp_magic.specialized_zip_2_lists', 'set_debug' : 'interp_magic.set_debug', 'locals_to_fast' : 'interp_magic.locals_to_fast', + 'set_code_callback' : 'interp_magic.set_code_callback', 'save_module_content_for_future_reload': 'interp_magic.save_module_content_for_future_reload', } diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError, wrap_oserror from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.pycode import CodeHookCache from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib.objectmodel import we_are_translated @@ -151,3 +152,10 @@ def specialized_zip_2_lists(space, w_list1, w_list2): from pypy.objspace.std.specialisedtupleobject import specialized_zip_2_lists return specialized_zip_2_lists(space, w_list1, w_list2) + +def set_code_callback(space, w_callable): + cache = space.fromcache(CodeHookCache) + if space.is_none(w_callable): + cache._code_hook = None + else: + cache._code_hook = w_callable \ No newline at end of file diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py --- a/pypy/module/__pypy__/test/test_magic.py +++ b/pypy/module/__pypy__/test/test_magic.py @@ -13,3 +13,21 @@ # sys.dont_write_bytecode = d __pypy__.save_module_content_for_future_reload(sys) + + def test_new_code_hook(self): + l = [] + + def callable(code): + l.append(code) + + import __pypy__ + __pypy__.set_code_callback(callable) + d = {} + try: + exec """ +def f(): + pass +""" in d + finally: + __pypy__.set_code_callback(None) + assert d['f'].__code__ in l \ No newline at end of file diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload, clibffi -VERSION = "1.3.0" +VERSION = "1.3.1" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: @@ -43,6 +43,7 @@ 'newp_handle': 'handle.newp_handle', 'from_handle': 'handle.from_handle', '_get_types': 'func._get_types', + '_get_common_types': 'func._get_common_types', 'from_buffer': 'func.from_buffer', 'string': 'func.string', diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py --- a/pypy/module/_cffi_backend/cffi_opcode.py +++ b/pypy/module/_cffi_backend/cffi_opcode.py @@ -110,6 +110,9 @@ _UNKNOWN_FLOAT_PRIM = -2 _UNKNOWN_LONG_DOUBLE = -3 +_IO_FILE_STRUCT = -1 + + PRIMITIVE_TO_INDEX = { 'char': PRIM_CHAR, 'short': PRIM_SHORT, diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -448,7 +448,7 @@ 'alloc' is called with the size as argument. If it returns NULL, a MemoryError is raised. 'free' is called with the result of 'alloc' -as argument. Both can be either Python function or directly C +as argument. Both can be either Python functions or directly C functions. If 'free' is None, then no free function is called. If both 'alloc' and 'free' are None, the default is used. diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -84,6 +84,20 @@ # ____________________________________________________________ +def _get_common_types(space, w_dict): + from pypy.module._cffi_backend.parse_c_type import ll_enum_common_types + index = 0 + while True: + p = ll_enum_common_types(rffi.cast(rffi.INT, index)) + if not p: + break + key = rffi.charp2str(p) + value = rffi.charp2str(rffi.ptradd(p, len(key) + 1)) + space.setitem_str(w_dict, key, space.wrap(value)) + index += 1 + +# ____________________________________________________________ + def _fetch_as_read_buffer(space, w_x): # xxx do we really need to implement the same mess as in CPython 2.7 # w.r.t. buffers and memoryviews?? diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py --- a/pypy/module/_cffi_backend/parse_c_type.py +++ b/pypy/module/_cffi_backend/parse_c_type.py @@ -87,6 +87,8 @@ ll_set_cdl_realize_global_int = llexternal('pypy_set_cdl_realize_global_int', [lltype.Ptr(GLOBAL_S)], lltype.Void) +ll_enum_common_types = llexternal('pypy_enum_common_types', + [rffi.INT], rffi.CCHARP) def parse_c_type(info, input): p_input = rffi.str2charp(input) diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py --- a/pypy/module/_cffi_backend/realize_c_type.py +++ b/pypy/module/_cffi_backend/realize_c_type.py @@ -74,7 +74,15 @@ assert len(NAMES) == cffi_opcode._NUM_PRIM def __init__(self, space): + self.space = space self.all_primitives = [None] * cffi_opcode._NUM_PRIM + self.file_struct = None + + def get_file_struct(self): + if self.file_struct is None: + self.file_struct = ctypestruct.W_CTypeStruct(self.space, "FILE") + return self.file_struct + def get_primitive_type(ffi, num): space = ffi.space @@ -266,6 +274,10 @@ def _realize_c_struct_or_union(ffi, sindex): + if sindex == cffi_opcode._IO_FILE_STRUCT: + # returns a single global cached opaque type + return ffi.space.fromcache(RealizeCache).get_file_struct() + s = ffi.ctxobj.ctx.c_struct_unions[sindex] type_index = rffi.getintfield(s, 'c_type_index') if ffi.cached_types[type_index] is not None: @@ -281,7 +293,10 @@ x = ctypestruct.W_CTypeUnion(space, name) else: name = _realize_name("struct ", s.c_name) - x = ctypestruct.W_CTypeStruct(space, name) + if name == "struct _IO_FILE": + x = space.fromcache(RealizeCache).get_file_struct() + else: + x = ctypestruct.W_CTypeStruct(space, name) if (c_flags & cffi_opcode.F_OPAQUE) == 0: assert c_first_field_index >= 0 w_ctype = x diff --git a/pypy/module/_cffi_backend/src/commontypes.c b/pypy/module/_cffi_backend/src/commontypes.c new file mode 100644 --- /dev/null +++ b/pypy/module/_cffi_backend/src/commontypes.c @@ -0,0 +1,207 @@ +/* This file must be kept in alphabetical order. See test_commontypes.py */ + +#define EQ(key, value) key "\0" value /* string concatenation */ +#ifdef _WIN64 +# define W32_64(X,Y) Y +# else +# define W32_64(X,Y) X +# endif + + +static const char *common_simple_types[] = { + +#ifdef MS_WIN32 /* Windows types */ + EQ("ATOM", "WORD"), + EQ("BOOL", "int"), + EQ("BOOLEAN", "BYTE"), + EQ("BYTE", "unsigned char"), + EQ("CCHAR", "char"), + EQ("CHAR", "char"), + EQ("COLORREF", "DWORD"), + EQ("DWORD", "unsigned long"), + EQ("DWORD32", "unsigned int"), + EQ("DWORD64", "unsigned long long"), + EQ("DWORDLONG", "ULONGLONG"), + EQ("DWORD_PTR", "ULONG_PTR"), +#endif + + EQ("FILE", "struct _IO_FILE"), + +#ifdef MS_WIN32 /* more Windows types */ + EQ("FLOAT", "float"), + EQ("HACCEL", "HANDLE"), + EQ("HALF_PTR", W32_64("short","int")), + EQ("HANDLE", "PVOID"), + EQ("HBITMAP", "HANDLE"), + EQ("HBRUSH", "HANDLE"), + EQ("HCOLORSPACE", "HANDLE"), + EQ("HCONV", "HANDLE"), + EQ("HCONVLIST", "HANDLE"), + EQ("HCURSOR", "HICON"), + EQ("HDC", "HANDLE"), + EQ("HDDEDATA", "HANDLE"), + EQ("HDESK", "HANDLE"), + EQ("HDROP", "HANDLE"), + EQ("HDWP", "HANDLE"), + EQ("HENHMETAFILE", "HANDLE"), + EQ("HFILE", "int"), + EQ("HFONT", "HANDLE"), + EQ("HGDIOBJ", "HANDLE"), + EQ("HGLOBAL", "HANDLE"), + EQ("HHOOK", "HANDLE"), + EQ("HICON", "HANDLE"), + EQ("HINSTANCE", "HANDLE"), + EQ("HKEY", "HANDLE"), + EQ("HKL", "HANDLE"), + EQ("HLOCAL", "HANDLE"), + EQ("HMENU", "HANDLE"), + EQ("HMETAFILE", "HANDLE"), + EQ("HMODULE", "HINSTANCE"), + EQ("HMONITOR", "HANDLE"), + EQ("HPALETTE", "HANDLE"), + EQ("HPEN", "HANDLE"), + EQ("HRESULT", "LONG"), + EQ("HRGN", "HANDLE"), + EQ("HRSRC", "HANDLE"), + EQ("HSZ", "HANDLE"), + EQ("HWND", "HANDLE"), + EQ("INT", "int"), + EQ("INT16", "short"), + EQ("INT32", "int"), + EQ("INT64", "long long"), + EQ("INT8", "signed char"), + EQ("INT_PTR", W32_64("int","long long")), + EQ("LANGID", "WORD"), + EQ("LCID", "DWORD"), + EQ("LCTYPE", "DWORD"), + EQ("LGRPID", "DWORD"), + EQ("LONG", "long"), + EQ("LONG32", "int"), + EQ("LONG64", "long long"), + EQ("LONGLONG", "long long"), + EQ("LONG_PTR", W32_64("long","long long")), + EQ("LPARAM", "LONG_PTR"), + EQ("LPBOOL", "BOOL *"), + EQ("LPBYTE", "BYTE *"), + EQ("LPCOLORREF", "DWORD *"), + EQ("LPCSTR", "const char *"), + EQ("LPCVOID", "const void *"), + EQ("LPCWSTR", "const WCHAR *"), + EQ("LPDWORD", "DWORD *"), + EQ("LPHANDLE", "HANDLE *"), + EQ("LPINT", "int *"), + EQ("LPLONG", "long *"), + EQ("LPSTR", "CHAR *"), + EQ("LPVOID", "void *"), + EQ("LPWORD", "WORD *"), + EQ("LPWSTR", "WCHAR *"), + EQ("LRESULT", "LONG_PTR"), + EQ("PBOOL", "BOOL *"), + EQ("PBOOLEAN", "BOOLEAN *"), + EQ("PBYTE", "BYTE *"), + EQ("PCHAR", "CHAR *"), + EQ("PCSTR", "const CHAR *"), + EQ("PCWSTR", "const WCHAR *"), + EQ("PDWORD", "DWORD *"), + EQ("PDWORD32", "DWORD32 *"), + EQ("PDWORD64", "DWORD64 *"), + EQ("PDWORDLONG", "DWORDLONG *"), + EQ("PDWORD_PTR", "DWORD_PTR *"), + EQ("PFLOAT", "FLOAT *"), + EQ("PHALF_PTR", "HALF_PTR *"), + EQ("PHANDLE", "HANDLE *"), + EQ("PHKEY", "HKEY *"), + EQ("PINT", "int *"), + EQ("PINT16", "INT16 *"), + EQ("PINT32", "INT32 *"), + EQ("PINT64", "INT64 *"), + EQ("PINT8", "INT8 *"), + EQ("PINT_PTR", "INT_PTR *"), + EQ("PLCID", "PDWORD"), + EQ("PLONG", "LONG *"), + EQ("PLONG32", "LONG32 *"), + EQ("PLONG64", "LONG64 *"), + EQ("PLONGLONG", "LONGLONG *"), + EQ("PLONG_PTR", "LONG_PTR *"), + EQ("PSHORT", "SHORT *"), + EQ("PSIZE_T", "SIZE_T *"), + EQ("PSSIZE_T", "SSIZE_T *"), + EQ("PSTR", "CHAR *"), + EQ("PUCHAR", "UCHAR *"), + EQ("PUHALF_PTR", "UHALF_PTR *"), + EQ("PUINT", "UINT *"), + EQ("PUINT16", "UINT16 *"), + EQ("PUINT32", "UINT32 *"), + EQ("PUINT64", "UINT64 *"), + EQ("PUINT8", "UINT8 *"), + EQ("PUINT_PTR", "UINT_PTR *"), + EQ("PULONG", "ULONG *"), + EQ("PULONG32", "ULONG32 *"), + EQ("PULONG64", "ULONG64 *"), + EQ("PULONGLONG", "ULONGLONG *"), + EQ("PULONG_PTR", "ULONG_PTR *"), + EQ("PUSHORT", "USHORT *"), + EQ("PVOID", "void *"), + EQ("PWCHAR", "WCHAR *"), + EQ("PWORD", "WORD *"), + EQ("PWSTR", "WCHAR *"), + EQ("QWORD", "unsigned long long"), + EQ("SC_HANDLE", "HANDLE"), + EQ("SC_LOCK", "LPVOID"), + EQ("SERVICE_STATUS_HANDLE", "HANDLE"), + EQ("SHORT", "short"), + EQ("SIZE_T", "ULONG_PTR"), + EQ("SSIZE_T", "LONG_PTR"), + EQ("UCHAR", "unsigned char"), + EQ("UHALF_PTR", W32_64("unsigned short","unsigned int")), + EQ("UINT", "unsigned int"), + EQ("UINT16", "unsigned short"), + EQ("UINT32", "unsigned int"), + EQ("UINT64", "unsigned long long"), + EQ("UINT8", "unsigned char"), + EQ("UINT_PTR", W32_64("unsigned int","unsigned long long")), + EQ("ULONG", "unsigned long"), + EQ("ULONG32", "unsigned int"), + EQ("ULONG64", "unsigned long long"), + EQ("ULONGLONG", "unsigned long long"), + EQ("ULONG_PTR", W32_64("unsigned long","unsigned long long")), + EQ("USHORT", "unsigned short"), + EQ("USN", "LONGLONG"), + EQ("VOID", "void"), + EQ("WCHAR", "wchar_t"), + EQ("WINSTA", "HANDLE"), + EQ("WORD", "unsigned short"), + EQ("WPARAM", "UINT_PTR"), +#endif + + EQ("bool", "_Bool"), +}; + + +#undef EQ +#undef W32_64 + +#define num_common_simple_types \ + (sizeof(common_simple_types) / sizeof(common_simple_types[0])) + + +static const char *get_common_type(const char *search, size_t search_len) +{ + const char *entry; + int index = search_sorted(common_simple_types, sizeof(const char *), + num_common_simple_types, search, search_len); + if (index < 0) + return NULL; + + entry = common_simple_types[index]; + return entry + strlen(entry) + 1; +} + +RPY_EXTERN +char *pypy_enum_common_types(int index) +{ + if (index < num_common_simple_types) + return (char *)(common_simple_types[index]); + else + return NULL; +} diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c --- a/pypy/module/_cffi_backend/src/parse_c_type.c +++ b/pypy/module/_cffi_backend/src/parse_c_type.c @@ -4,6 +4,7 @@ #include <errno.h> #if defined(_MSC_VER) +# define MS_WIN32 typedef size_t uintptr_t; #else # include <stdint.h> @@ -231,6 +232,8 @@ #define MAX_SSIZE_T (((size_t)-1) >> 1) static int parse_complete(token_t *tok); +static const char *get_common_type(const char *search, size_t search_len); +static int parse_common_type_replacement(token_t *tok, const char *replacement); static int parse_sequel(token_t *tok, int outer) { @@ -387,11 +390,18 @@ case TOK_INTEGER: errno = 0; -#ifndef _MSC_VER - if (sizeof(length) > sizeof(unsigned long)) + if (sizeof(length) > sizeof(unsigned long)) { +#ifdef MS_WIN32 +# ifdef _WIN64 + length = _strtoui64(tok->p, &endptr, 0); +# else + abort(); /* unreachable */ +# endif +#else length = strtoull(tok->p, &endptr, 0); +#endif + } else -#endif length = strtoul(tok->p, &endptr, 0); if (endptr != tok->p + tok->size) return parse_error(tok, "invalid number"); @@ -446,26 +456,34 @@ return _CFFI_GETARG(result); } +static int search_sorted(const char *const *base, + size_t item_size, int array_len, + const char *search, size_t search_len) +{ + int left = 0, right = array_len; + const char *baseptr = (const char *)base; -#define MAKE_SEARCH_FUNC(FIELD) \ - RPY_EXTERN int \ - pypy_search_in_##FIELD(const struct _cffi_type_context_s *ctx,\ - const char *search, size_t search_len) \ - { \ - int left = 0, right = ctx->num_##FIELD; \ - \ - while (left < right) { \ - int middle = (left + right) / 2; \ - const char *src = ctx->FIELD[middle].name; \ - int diff = strncmp(src, search, search_len); \ - if (diff == 0 && src[search_len] == '\0') \ - return middle; \ - else if (diff >= 0) \ - right = middle; \ - else \ - left = middle + 1; \ - } \ - return -1; \ + while (left < right) { + int middle = (left + right) / 2; + const char *src = *(const char *const *)(baseptr + middle * item_size); + int diff = strncmp(src, search, search_len); + if (diff == 0 && src[search_len] == '\0') + return middle; + else if (diff >= 0) + right = middle; + else + left = middle + 1; + } + return -1; +} + +#define MAKE_SEARCH_FUNC(FIELD) \ + RPY_EXTERN int \ + pypy_search_in_##FIELD(const struct _cffi_type_context_s *ctx, \ + const char *search, size_t search_len) \ + { \ + return search_sorted(&ctx->FIELD->name, sizeof(*ctx->FIELD), \ + ctx->num_##FIELD, search, search_len); \ } MAKE_SEARCH_FUNC(globals) @@ -719,6 +737,7 @@ break; case TOK_IDENTIFIER: { + const char *replacement; int n = search_in_typenames(tok->info->ctx, tok->p, tok->size); if (n >= 0) { t1 = _CFFI_OP(_CFFI_OP_TYPENAME, n); @@ -729,6 +748,14 @@ t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, n); break; } + replacement = get_common_type(tok->p, tok->size); + if (replacement != NULL) { + n = parse_common_type_replacement(tok, replacement); + if (n < 0) + return parse_error(tok, "internal error, please report!"); + t1 = _CFFI_OP(_CFFI_OP_NOOP, n); + break; + } return parse_error(tok, "undefined type name"); } case TOK_STRUCT: @@ -740,10 +767,15 @@ return parse_error(tok, "struct or union name expected"); n = search_in_struct_unions(tok->info->ctx, tok->p, tok->size); - if (n < 0) - return parse_error(tok, "undefined struct/union name"); - if (((tok->info->ctx->struct_unions[n].flags & _CFFI_F_UNION) != 0) - ^ (kind == TOK_UNION)) + if (n < 0) { + if (kind == TOK_STRUCT && tok->size == 8 && + !memcmp(tok->p, "_IO_FILE", 8)) + n = _CFFI__IO_FILE_STRUCT; + else + return parse_error(tok, "undefined struct/union name"); + } + else if (((tok->info->ctx->struct_unions[n].flags & _CFFI_F_UNION) + != 0) ^ (kind == TOK_UNION)) return parse_error(tok, "wrong kind of tag: struct vs union"); t1 = _CFFI_OP(_CFFI_OP_STRUCT_UNION, n); @@ -773,8 +805,9 @@ } -RPY_EXTERN -int pypy_parse_c_type(struct _cffi_parse_info_s *info, const char *input) +static +int parse_c_type_from(struct _cffi_parse_info_s *info, size_t *output_index, + const char *input) { int result; token_t token; @@ -785,16 +818,33 @@ token.p = input; token.size = 0; token.output = info->output; - token.output_index = 0; + token.output_index = *output_index; next_token(&token); result = parse_complete(&token); + *output_index = token.output_index; if (token.kind != TOK_END) return parse_error(&token, "unexpected symbol"); return result; } +RPY_EXTERN +int pypy_parse_c_type(struct _cffi_parse_info_s *info, const char *input) +{ + size_t output_index = 0; + return parse_c_type_from(info, &output_index, input); +} + +static +int parse_common_type_replacement(token_t *tok, const char *replacement) +{ + return parse_c_type_from(tok->info, &tok->output_index, replacement); +} + + +#include "commontypes.c" /* laziness hack: include this file here */ + /************************************************************/ /* extra from cdlopen.c */ diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h --- a/pypy/module/_cffi_backend/src/parse_c_type.h +++ b/pypy/module/_cffi_backend/src/parse_c_type.h @@ -5,7 +5,7 @@ #define _CFFI_OP(opcode, arg) (_cffi_opcode_t)(opcode | (((uintptr_t)(arg)) << 8)) #define _CFFI_GETOP(cffi_opcode) ((unsigned char)(uintptr_t)cffi_opcode) -#define _CFFI_GETARG(cffi_opcode) (((uintptr_t)cffi_opcode) >> 8) +#define _CFFI_GETARG(cffi_opcode) (((intptr_t)cffi_opcode) >> 8) #define _CFFI_OP_PRIMITIVE 1 #define _CFFI_OP_POINTER 3 @@ -25,6 +25,8 @@ #define _CFFI_OP_CONSTANT_INT 31 #define _CFFI_OP_GLOBAL_VAR 33 #define _CFFI_OP_DLOPEN_FUNC 35 +#define _CFFI_OP_DLOPEN_CONST 37 +#define _CFFI_OP_GLOBAL_VAR_F 39 #define _CFFI_PRIM_VOID 0 #define _CFFI_PRIM_BOOL 1 @@ -77,6 +79,11 @@ #define _CFFI_PRIM_UINTMAX 47 #define _CFFI__NUM_PRIM 48 +#define _CFFI__UNKNOWN_PRIM (-1) +#define _CFFI__UNKNOWN_FLOAT_PRIM (-2) +#define _CFFI__UNKNOWN_LONG_DOUBLE (-3) + +#define _CFFI__IO_FILE_STRUCT (-1) struct _cffi_global_s { @@ -164,4 +171,6 @@ const char *search, size_t search_len); RPY_EXTERN void pypy_set_cdl_realize_global_int(struct _cffi_global_s *target); +RPY_EXTERN +char *pypy_enum_common_types(int index); #endif diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.3.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.3.1", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -3509,3 +3509,8 @@ assert repr(BFunc) == "<ctype 'int(__stdcall *)(int, int)'>" else: assert repr(BFunc) == "<ctype 'int(*)(int, int)'>" + +def test_get_common_types(): + d = {} + _get_common_types(d) + assert d['bool'] == '_Bool' diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -424,3 +424,26 @@ return ffi.NULL alloc5 = ffi.new_allocator(myalloc5) raises(MemoryError, alloc5, "int[5]") + + def test_bool_issue228(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + fntype = ffi.typeof("int(*callback)(bool is_valid)") + assert repr(fntype.args[0]) == "<ctype '_Bool'>" + + def test_FILE_issue228(self): + import _cffi_backend as _cffi1_backend + fntype1 = _cffi1_backend.FFI().typeof("FILE *") + fntype2 = _cffi1_backend.FFI().typeof("FILE *") + assert repr(fntype1) == "<ctype 'FILE *'>" + assert fntype1 is fntype2 + + def test_cast_from_int_type_to_bool(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + for basetype in ['char', 'short', 'int', 'long', 'long long']: + for sign in ['signed', 'unsigned']: + type = '%s %s' % (sign, basetype) + assert int(ffi.cast("_Bool", ffi.cast(type, 42))) == 1 + assert int(ffi.cast("bool", ffi.cast(type, 42))) == 1 + assert int(ffi.cast("_Bool", ffi.cast(type, 0))) == 0 diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -16,8 +16,8 @@ from cffi import ffiplatform except ImportError: py.test.skip("system cffi module not found or older than 1.0.0") - if cffi.__version_info__ < (1, 2, 0): - py.test.skip("system cffi module needs to be at least 1.2.0") + if cffi.__version_info__ < (1, 3, 0): + py.test.skip("system cffi module needs to be at least 1.3.0") space.appexec([], """(): import _cffi_backend # force it to be initialized """) @@ -28,6 +28,7 @@ module_name = '_CFFI_' + module_name rdir = udir.ensure('recompiler', dir=1) rdir.join('Python.h').write( + '#include <stdio.h>\n' '#define PYPY_VERSION XX\n' '#define PyMODINIT_FUNC /*exported*/ void\n' ) @@ -1076,3 +1077,302 @@ raises(ffi.error, getattr, lib, 'my_value') e = raises(ffi.error, setattr, lib, 'my_value', 50) assert str(e.value) == "global variable 'my_value' is at address NULL" + + def test_const_fields(self): + ffi, lib = self.prepare( + """struct foo_s { const int a; void *const b; };""", + 'test_const_fields', + """struct foo_s { const int a; void *const b; };""") + foo_s = ffi.typeof("struct foo_s") + assert foo_s.fields[0][0] == 'a' + assert foo_s.fields[0][1].type is ffi.typeof("int") + assert foo_s.fields[1][0] == 'b' + assert foo_s.fields[1][1].type is ffi.typeof("void *") + + def test_restrict_fields(self): + ffi, lib = self.prepare( + """struct foo_s { void * restrict b; };""", + 'test_restrict_fields', + """struct foo_s { void * __restrict b; };""") + foo_s = ffi.typeof("struct foo_s") + assert foo_s.fields[0][0] == 'b' + assert foo_s.fields[0][1].type is ffi.typeof("void *") + + def test_volatile_fields(self): + ffi, lib = self.prepare( + """struct foo_s { void * volatile b; };""", + 'test_volatile_fields', + """struct foo_s { void * volatile b; };""") + foo_s = ffi.typeof("struct foo_s") + assert foo_s.fields[0][0] == 'b' + assert foo_s.fields[0][1].type is ffi.typeof("void *") + + def test_const_array_fields(self): + ffi, lib = self.prepare( + """struct foo_s { const int a[4]; };""", + 'test_const_array_fields', + """struct foo_s { const int a[4]; };""") + foo_s = ffi.typeof("struct foo_s") + assert foo_s.fields[0][0] == 'a' + assert foo_s.fields[0][1].type is ffi.typeof("int[4]") + + def test_const_array_fields_varlength(self): + ffi, lib = self.prepare( + """struct foo_s { const int a[]; ...; };""", + 'test_const_array_fields_varlength', + """struct foo_s { const int a[4]; };""") + foo_s = ffi.typeof("struct foo_s") + assert foo_s.fields[0][0] == 'a' + assert foo_s.fields[0][1].type is ffi.typeof("int[]") + + def test_const_array_fields_unknownlength(self): + ffi, lb = self.prepare( + """struct foo_s { const int a[...]; ...; };""", + 'test_const_array_fields_unknownlength', + """struct foo_s { const int a[4]; };""") _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
