Author: Remi Meier <[email protected]> Branch: stmgc-c8 Changeset: r80747:6971eb50f3ff Date: 2015-11-18 09:23 +0100 http://bitbucket.org/pypy/pypy/changeset/6971eb50f3ff/
Log: Merge with default diff too long, truncating to 2000 out of 3927 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -56,14 +56,15 @@ Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen + Richard Plangger Richard Emslie Alexander Schremmer Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager Anders Lehmann + Remi Meier Aurelien Campeas - Remi Meier Niklaus Haldimann Camillo Bruni Laura Creighton @@ -87,7 +88,6 @@ Ludovic Aubry Jacob Hallen Jason Creighton - Richard Plangger Alex Martelli Michal Bendowski stian @@ -200,9 +200,12 @@ Alex Perry Vincent Legoll Alan McIntyre + Spenser Bauman Alexander Sedov Attila Gobi Christopher Pope + Devin Jeanpierre + Vaibhav Sood Christian Tismer Marc Abramowitz Dan Stromberg @@ -234,6 +237,7 @@ Lutz Paelike Lucio Torre Lars Wassermann + Philipp Rustemeuer Henrik Vendelbo Dan Buch Miguel de Val Borro @@ -244,6 +248,7 @@ Martin Blais Lene Wagner Tomo Cocoa + Kim Jin Su Toni Mattis Lucas Stadler Julian Berman @@ -253,6 +258,7 @@ Anna Katrina Dominguez William Leslie Bobby Impollonia + Faye Zhao [email protected] Andrew Thompson Yusei Tahara @@ -283,6 +289,7 @@ shoma hosaka Daniel Neuhäuser Ben Mather + Niclas Olofsson halgari Boglarka Vezer Chris Pressey @@ -309,13 +316,16 @@ Stefan Marr jiaaro Mads Kiilerich + Richard Lancaster opassembler.py Antony Lee + Yaroslav Fedevych Jim Hunziker Markus Unterwaditzer Even Wiik Thomassen jbs squeaky + Zearin soareschen Kurt Griffiths Mike Bayer @@ -327,6 +337,7 @@ Anna Ravencroft Andrey Churin Dan Crosta + Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.3.0 +Version: 1.3.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.3.0" -__version_info__ = (1, 3, 0) +__version__ = "1.3.1" +__version_info__ = (1, 3, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -62,7 +62,8 @@ if csource.startswith('*', endpos): parts.append('('); closing += ')' level = 0 - for i in xrange(endpos, len(csource)): + i = endpos + while i < len(csource): c = csource[i] if c == '(': level += 1 @@ -73,6 +74,7 @@ elif c in ',;=': if level == 0: break + i += 1 csource = csource[endpos:i] + closing + csource[i:] #print repr(''.join(parts)+csource) parts.append(csource) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -514,12 +514,17 @@ if self.baseinttype is not None: return self.baseinttype.get_cached_btype(ffi, finishlist) # + from . import api if self.enumvalues: smallest_value = min(self.enumvalues) largest_value = max(self.enumvalues) else: - smallest_value = 0 - largest_value = 0 + import warnings + warnings.warn("%r has no values explicitly defined; next version " + "will refuse to guess which integer type it is " + "meant to be (unsigned/signed, int/long)" + % self._get_c_name()) + smallest_value = largest_value = 0 if smallest_value < 0: # needs a signed type sign = 1 candidate1 = PrimitiveType("int") diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -26,15 +26,15 @@ Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen + Richard Plangger Richard Emslie Alexander Schremmer Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager Anders Lehmann - Richard Plangger + Remi Meier Aurelien Campeas - Remi Meier Niklaus Haldimann Camillo Bruni Laura Creighton @@ -170,9 +170,12 @@ Alex Perry Vincent Legoll Alan McIntyre + Spenser Bauman Alexander Sedov Attila Gobi Christopher Pope + Devin Jeanpierre + Vaibhav Sood Christian Tismer Marc Abramowitz Dan Stromberg @@ -204,6 +207,7 @@ Lutz Paelike Lucio Torre Lars Wassermann + Philipp Rustemeuer Henrik Vendelbo Dan Buch Miguel de Val Borro @@ -214,6 +218,7 @@ Martin Blais Lene Wagner Tomo Cocoa + Kim Jin Su Toni Mattis Lucas Stadler Julian Berman @@ -223,6 +228,7 @@ Anna Katrina Dominguez William Leslie Bobby Impollonia + Faye Zhao [email protected] Andrew Thompson Yusei Tahara @@ -280,13 +286,16 @@ Stefan Marr jiaaro Mads Kiilerich + Richard Lancaster opassembler.py Antony Lee + Yaroslav Fedevych Jim Hunziker Markus Unterwaditzer Even Wiik Thomassen jbs squeaky + Zearin soareschen Kurt Griffiths Mike Bayer @@ -298,6 +307,7 @@ Anna Ravencroft Andrey Churin Dan Crosta + Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-4.0.1.rst release-4.0.0.rst release-2.6.1.rst release-2.6.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-4.0.1.rst whatsnew-4.0.0.rst whatsnew-2.6.1.rst whatsnew-2.6.0.rst diff --git a/pypy/doc/release-4.0.1.rst b/pypy/doc/release-4.0.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-4.0.1.rst @@ -0,0 +1,106 @@ +========== +PyPy 4.0.1 +========== + +We have released PyPy 4.0.1, three weeks after PyPy 4.0.0. We have fixed +a few critical bugs in the JIT compiled code, reported by users. We therefore +encourage all users of PyPy to update to this version. There are a few minor +enhancements in this version as well. + +You can download the PyPy 4.0.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +CFFI +==== + +While not applicable only to PyPy, `cffi`_ is arguably our most significant +contribution to the python ecosystem. PyPy 4.0.1 ships with +`cffi-1.3.1`_ with the improvements it brings. + +.. _`PyPy`: http://doc.pypy.org +.. _`RPython`: https://rpython.readthedocs.org +.. _`cffi`: https://cffi.readthedocs.org +.. _`cffi-1.3.1`: http://cffi.readthedocs.org/en/latest/whatsnew.html#v1-3-1 +.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html +.. _`numpy`: https://bitbucket.org/pypy/numpy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, freebsd), +newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, and the +big- and little-endian variants of **ppc64** running Linux. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Other Highlights (since 4.0.0 released three weeks ago) +======================================================= + +* Bug Fixes + + * Fix a bug when unrolling double loops in JITted code + + * Fix multiple memory leaks in the ssl module, one of which affected + `cpython` as well (thanks to Alex Gaynor for pointing those out) + + * Use pkg-config to find ssl headers on OS-X + + * Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy + +* New features: + + * Internal cleanup of RPython class handling + + * Support stackless and greenlets on PPC machines + + * Improve debug logging in subprocesses: use PYPYLOG=jit:log.%d + for example to have all subprocesses write the JIT log to a file + called 'log.%d', with '%d' replaced with the subprocess' PID. + + * Support PyOS_double_to_string in our cpyext capi compatibility layer + +* Numpy: + + * Improve support for __array_interface__ + + * Propagate NAN mantissas through float16-float32-float64 conversions + + +* Performance improvements and refactorings: + + * Improvements in slicing byte arrays + + * Improvements in enumerate() + + * Silence some warnings while translating + +.. _resolved: http://doc.pypy.org/en/latest/whatsnew-4.0.1.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -69,7 +69,9 @@ 'Rami Chowdhury': ['necaris'], 'Stanislaw Halik':['w31rd0'], 'Wenzhu Man':['wenzhu man', 'wenzhuman'], - 'Anton Gulenko':['anton gulenko'], + 'Anton Gulenko':['anton gulenko', 'anton_gulenko'], + 'Richard Lancaster':['richardlancaster'], + 'William Leslie':['William ML Leslie'], } alias_map = {} diff --git a/pypy/doc/whatsnew-4.0.1.rst b/pypy/doc/whatsnew-4.0.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-4.0.1.rst @@ -0,0 +1,35 @@ +========================= +What's new in PyPy 4.0.1 +========================= + +.. this is a revision shortly after release-4.0.0 +.. startrev: 57c9a47c70f6 + +.. branch: 2174-fix-osx-10-11-translation + +Use pkg-config to find ssl headers on OS-X + +.. branch: Zearin/minor-whatsnewrst-markup-tweaks-edited-o-1446387512092 + +.. branch: ppc-stacklet + +The PPC machines now support the _continuation module (stackless, greenlets) + +.. branch: int_0/i-need-this-library-to-build-on-ubuntu-1-1446717626227 + +Document that libgdbm-dev is required for translation/packaging + +.. branch: propogate-nans + +Ensure that ndarray conversion from int16->float16->float32->float16->int16 +preserves all int16 values, even across nan conversions. Also fix argmax, argmin +for nan comparisons + +.. branch: array_interface + +Support common use-cases for __array_interface__, passes upstream tests + +.. branch: no-class-specialize + +Some refactoring of class handling in the annotator. +Remove class specialisation and _settled_ flag. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,30 +2,6 @@ What's new in PyPy 4.0.+ ========================= -.. this is a revision shortly after release-4.0.0 -.. startrev: 57c9a47c70f6 +.. this is a revision shortly after release-4.0.1 +.. startrev: 4b5c840d0da2 -.. branch: 2174-fix-osx-10-11-translation - -Use pkg-config to find ssl headers on OS-X - -.. branch: Zearin/minor-whatsnewrst-markup-tweaks-edited-o-1446387512092 - -.. branch: ppc-stacklet - -The PPC machines now support the _continuation module (stackless, greenlets) - -.. branch: int_0/i-need-this-library-to-build-on-ubuntu-1-1446717626227 - -Document that libgdbm-dev is required for translation/packaging - -.. branch: propogate-nans - -Ensure that ndarray conversion from int16->float16->float32->float16->int16 -preserves all int16 values, even across nan conversions. Also fix argmax, argmin -for nan comparisons - -.. branch: array_interface - -Support common use-cases for __array_interface__, passes upstream tests - diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -28,7 +28,6 @@ """This is the abstract root class of all wrapped objects that live in a 'normal' object space like StdObjSpace.""" __slots__ = () - _settled_ = True user_overridden_class = False def getdict(self, space): diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload, clibffi -VERSION = "1.3.0" +VERSION = "1.3.1" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -448,7 +448,7 @@ 'alloc' is called with the size as argument. If it returns NULL, a MemoryError is raised. 'free' is called with the result of 'alloc' -as argument. Both can be either Python function or directly C +as argument. Both can be either Python functions or directly C functions. If 'free' is None, then no free function is called. If both 'alloc' and 'free' are None, the default is used. diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c --- a/pypy/module/_cffi_backend/src/parse_c_type.c +++ b/pypy/module/_cffi_backend/src/parse_c_type.c @@ -4,6 +4,7 @@ #include <errno.h> #if defined(_MSC_VER) +# define MS_WIN32 typedef size_t uintptr_t; #else # include <stdint.h> diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.3.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.3.1", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -16,8 +16,8 @@ from cffi import ffiplatform except ImportError: py.test.skip("system cffi module not found or older than 1.0.0") - if cffi.__version_info__ < (1, 2, 0): - py.test.skip("system cffi module needs to be at least 1.2.0") + if cffi.__version_info__ < (1, 3, 0): + py.test.skip("system cffi module needs to be at least 1.3.0") space.appexec([], """(): import _cffi_backend # force it to be initialized """) diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -13,7 +13,7 @@ def __init__(self, msg): self.msg = msg -from rpython.annotator.description import FORCE_ATTRIBUTES_INTO_CLASSES +from rpython.annotator.classdesc import FORCE_ATTRIBUTES_INTO_CLASSES from rpython.annotator.model import SomeString # this is necessary due to annmixlevel diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py --- a/pypy/module/_multiprocessing/interp_win32.py +++ b/pypy/module/_multiprocessing/interp_win32.py @@ -17,7 +17,7 @@ NMPWAIT_WAIT_FOREVER ERROR_PIPE_CONNECTED ERROR_SEM_TIMEOUT ERROR_PIPE_BUSY ERROR_NO_SYSTEM_RESOURCES ERROR_BROKEN_PIPE ERROR_MORE_DATA - ERROR_ALREADY_EXISTS + ERROR_ALREADY_EXISTS ERROR_NO_DATA """.split() class CConfig: diff --git a/pypy/module/cpyext/pystrtod.py b/pypy/module/cpyext/pystrtod.py --- a/pypy/module/cpyext/pystrtod.py +++ b/pypy/module/cpyext/pystrtod.py @@ -5,10 +5,23 @@ from rpython.rlib import rdtoa from rpython.rlib import rfloat from rpython.rlib import rposix, jit +from rpython.rlib.rarithmetic import intmask from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem import rffi +# PyOS_double_to_string's "type", if non-NULL, will be set to one of: +Py_DTST_FINITE = 0 +Py_DTST_INFINITE = 1 +Py_DTST_NAN = 2 + +# Match the "type" back to values in CPython +DOUBLE_TO_STRING_TYPES_MAP = { + rfloat.DIST_FINITE: Py_DTST_FINITE, + rfloat.DIST_INFINITY: Py_DTST_INFINITE, + rfloat.DIST_NAN: Py_DTST_NAN +} + @cpython_api([rffi.CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) @jit.dont_look_inside # direct use of _get_errno() def PyOS_string_to_double(space, s, endptr, w_overflow_exception): @@ -68,3 +81,42 @@ finally: if not user_endptr: lltype.free(endptr, flavor='raw') + +@cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) +def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): + """Convert a double val to a string using supplied + format_code, precision, and flags. + + format_code must be one of 'e', 'E', 'f', 'F', + 'g', 'G' or 'r'. For 'r', the supplied precision + must be 0 and is ignored. The 'r' format code specifies the + standard repr() format. + + flags can be zero or more of the values Py_DTSF_SIGN, + Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: + + Py_DTSF_SIGN means to always precede the returned string with a sign + character, even if val is non-negative. + + Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look + like an integer. + + Py_DTSF_ALT means to apply "alternate" formatting rules. See the + documentation for the PyOS_snprintf() '#' specifier for + details. + + If ptype is non-NULL, then the value it points to will be set to one of + Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that + val is a finite number, an infinite number, or not a number, respectively. + + The return value is a pointer to buffer with the converted string or + NULL if the conversion failed. The caller is responsible for freeing the + returned string by calling PyMem_Free(). + """ + buffer, rtype = rfloat.double_to_string(val, format_code, + intmask(precision), + intmask(flags)) + if ptype != lltype.nullptr(rffi.INTP.TO): + ptype[0] = rffi.cast(rffi.INT, DOUBLE_TO_STRING_TYPES_MAP[rtype]) + bufp = rffi.str2charp(buffer) + return bufp diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py --- a/pypy/module/cpyext/test/test_pystrtod.py +++ b/pypy/module/cpyext/test/test_pystrtod.py @@ -1,5 +1,6 @@ import math +from pypy.module.cpyext import pystrtod from pypy.module.cpyext.test.test_api import BaseApiTest from rpython.rtyper.lltypesystem import rffi from rpython.rtyper.lltypesystem import lltype @@ -91,3 +92,76 @@ api.PyErr_Clear() rffi.free_charp(s) lltype.free(endp, flavor='raw') + + +class TestPyOS_double_to_string(BaseApiTest): + + def test_format_code(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(150.0, 'e', 1, 0, ptype) + assert '1.5e+02' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_precision(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(3.14159269397, 'g', 5, 0, ptype) + assert '3.1416' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_flags_sign(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(-3.14, 'g', 3, 1, ptype) + assert '-3.14' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_flags_add_dot_0(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(3, 'g', 5, 2, ptype) + assert '3.0' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_flags_alt(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(314., 'g', 3, 4, ptype) + assert '314.' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_FINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_ptype_nan(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(float('nan'), 'g', 3, 4, ptype) + assert 'nan' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_NAN == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_ptype_infinity(self, api): + ptype = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + r = api.PyOS_double_to_string(1e200 * 1e200, 'g', 0, 0, ptype) + assert 'inf' == rffi.charp2str(r) + type_value = rffi.cast(lltype.Signed, ptype[0]) + assert pystrtod.Py_DTST_INFINITE == type_value + rffi.free_charp(r) + lltype.free(ptype, flavor='raw') + + def test_ptype_null(self, api): + ptype = lltype.nullptr(rffi.INTP.TO) + r = api.PyOS_double_to_string(3.14, 'g', 3, 0, ptype) + assert '3.14' == rffi.charp2str(r) + assert ptype == lltype.nullptr(rffi.INTP.TO) + rffi.free_charp(r) \ No newline at end of file diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -156,9 +156,6 @@ put_tuple_w(TYPE, tuple_w) puts tuple_w, an unwrapped list of wrapped objects """ - # _annspecialcase_ = "specialize:ctr_location" # polymorphic - # does not work with subclassing - def __init__(self, space, writer, version): self.space = space ## self.put = putfunc diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -684,8 +684,9 @@ arr_iter, arr_state = arr.create_iter() arr_dtype = arr.get_dtype() index_dtype = index.get_dtype() - # XXX length of shape of index as well? - while not index_iter.done(index_state): + # support the deprecated form where arr([True]) will return arr[0, ...] + # by iterating over res_iter, not index_iter + while not res_iter.done(res_state): getitem_filter_driver.jit_merge_point(shapelen=shapelen, index_dtype=index_dtype, arr_dtype=arr_dtype, diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2238,6 +2238,9 @@ c = array([True,False,True],bool) b = a[c] assert (a[c] == [[1, 2, 3], [7, 8, 9]]).all() + c = array([True]) + b = a[c] + assert b.shape == (1, 3) def test_bool_array_index_setitem(self): from numpy import arange, array diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1336,7 +1336,8 @@ # these depend on user-defined data, so should not be shared assert ffi1.typeof("struct foo") is not ffi2.typeof("struct foo") assert ffi1.typeof("union foo *") is not ffi2.typeof("union foo*") - assert ffi1.typeof("enum foo") is not ffi2.typeof("enum foo") + # the following test is an opaque enum, which we no longer support + #assert ffi1.typeof("enum foo") is not ffi2.typeof("enum foo") # sanity check: twice 'ffi1' assert ffi1.typeof("struct foo*") is ffi1.typeof("struct foo *") @@ -1348,6 +1349,17 @@ assert ffi.getctype("pe") == 'e *' assert ffi.getctype("e1*") == 'e1 *' + def test_opaque_enum(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef("enum foo;") + from cffi import __version_info__ + if __version_info__ < (1, 4): + py.test.skip("re-enable me in version 1.4") + e = py.test.raises(CDefError, ffi.cast, "enum foo", -1) + assert str(e.value) == ( + "'enum foo' has no values explicitly defined: refusing to guess " + "which integer type it is meant to be (unsigned/signed, int/long)") + def test_new_ctype(self): ffi = FFI(backend=self.Backend()) p = ffi.new("int *") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py new file mode 100644 --- /dev/null +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_commontypes.py @@ -0,0 +1,35 @@ +# Generated by pypy/tool/import_cffi.py +import py, os, cffi, re +import _cffi_backend + + +def getlines(): + try: + f = open(os.path.join(os.path.dirname(cffi.__file__), + '..', 'c', 'commontypes.c')) + except IOError: + py.test.skip("cannot find ../c/commontypes.c") + lines = [line for line in f.readlines() if line.strip().startswith('EQ(')] + f.close() + return lines + +def test_alphabetical_order(): + lines = getlines() + assert lines == sorted(lines) + +def test_dependencies(): + r = re.compile(r'EQ[(]"([^"]+)",(?:\s*"([A-Z0-9_]+)\s*[*]*"[)])?') + lines = getlines() + d = {} + for line in lines: + match = r.search(line) + if match is not None: + d[match.group(1)] = match.group(2) + for value in d.values(): + if value: + assert value in d + +def test_get_common_types(): + d = {} + _cffi_backend._get_common_types(d) + assert d["bool"] == "_Bool" diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -1231,6 +1231,21 @@ def setitem(self, index, char): self.data[index] = char + def getslice(self, start, stop, step, size): + if size == 0: + return "" + if step == 1: + assert 0 <= start <= stop + if start == 0 and stop == len(self.data): + return "".join(self.data) + return "".join(self.data[start:stop]) + return Buffer.getslice(self, start, stop, step, size) + + def setslice(self, start, string): + # No bounds checks. + for i in range(len(string)): + self.data[start + i] = string[i] + @specialize.argtype(1) def _memcmp(selfvalue, buffer, length): diff --git a/pypy/objspace/std/test/test_tupleobject.py b/pypy/objspace/std/test/test_tupleobject.py --- a/pypy/objspace/std/test/test_tupleobject.py +++ b/pypy/objspace/std/test/test_tupleobject.py @@ -413,8 +413,9 @@ from __pypy__ import specialized_zip_2_lists except ImportError: specialized_zip_2_lists = zip - raises(TypeError, specialized_zip_2_lists, [], ()) - raises(TypeError, specialized_zip_2_lists, (), []) + else: + raises(TypeError, specialized_zip_2_lists, [], ()) + raises(TypeError, specialized_zip_2_lists, (), []) assert specialized_zip_2_lists([], []) == [ ] assert specialized_zip_2_lists([2, 3], []) == [ diff --git a/pypy/tool/ann_override.py b/pypy/tool/ann_override.py --- a/pypy/tool/ann_override.py +++ b/pypy/tool/ann_override.py @@ -2,6 +2,7 @@ from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Constant from rpython.annotator import specialize +from rpython.annotator.classdesc import InstanceSource, ClassDef @@ -20,7 +21,6 @@ def specialize__wrap(self, funcdesc, args_s): from pypy.interpreter.baseobjspace import W_Root - from rpython.annotator.classdef import ClassDef W_Root_def = funcdesc.bookkeeper.getuniqueclassdef(W_Root) typ = args_s[1].knowntype if isinstance(typ, ClassDef): @@ -50,54 +50,34 @@ typ = (None, str) return funcdesc.cachedgraph(typ) - def _remember_immutable(self, t, cached): - # for jit benefit - if cached not in t._immutable_fields_: # accessed this way just - # for convenience - t._immutable_fields_.append(cached) - - def attach_lookup(self, t, attr): - cached = "cached_%s" % attr - if not t.is_heaptype() and not t.is_cpytype(): - self._remember_immutable(t, cached) - setattr(t, cached, t._lookup(attr)) - return True - return False - - def attach_lookup_in_type_where(self, t, attr): - cached = "cached_where_%s" % attr - if not t.is_heaptype() and not t.is_cpytype(): - self._remember_immutable(t, cached) - setattr(t, cached, t._lookup_where(attr)) - return True - return False - def consider_lookup(self, bookkeeper, attr): - from rpython.annotator.classdef import InstanceSource assert attr not in self.lookups from pypy.objspace.std import typeobject cached = "cached_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc + classdesc.immutable_fields.add(cached) classdesc.classdict[cached] = Constant(None) clsdef.add_source_for_attribute(cached, classdesc) for t in self.pypytypes: - if self.attach_lookup(t, attr): + if not (t.is_heaptype() or t.is_cpytype()): + setattr(t, cached, t._lookup(attr)) source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) self.lookups[attr] = True def consider_lookup_in_type_where(self, bookkeeper, attr): - from rpython.annotator.classdef import InstanceSource assert attr not in self.lookups_where from pypy.objspace.std import typeobject cached = "cached_where_%s" % attr clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) classdesc = clsdef.classdesc + classdesc.immutable_fields.add(cached) classdesc.classdict[cached] = Constant((None, None)) clsdef.add_source_for_attribute(cached, classdesc) for t in self.pypytypes: - if self.attach_lookup_in_type_where(t, attr): + if not (t.is_heaptype() or t.is_cpytype()): + setattr(t, cached, t._lookup_where(attr)) source = InstanceSource(bookkeeper, t) clsdef.add_source_for_attribute(cached, source) self.lookups_where[attr] = True @@ -135,18 +115,19 @@ def event(self, bookkeeper, what, x): from pypy.objspace.std import typeobject if isinstance(x, typeobject.W_TypeObject): - from rpython.annotator.classdef import InstanceSource clsdef = bookkeeper.getuniqueclassdef(typeobject.W_TypeObject) self.pypytypes[x] = True #print "TYPE", x for attr in self.lookups: - if attr and self.attach_lookup(x, attr): + if attr and not (x.is_heaptype() or x.is_cpytype()): cached = "cached_%s" % attr + setattr(x, cached, x._lookup(attr)) source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) for attr in self.lookups_where: - if attr and self.attach_lookup_in_type_where(x, attr): + if attr and not (x.is_heaptype() or x.is_cpytype()): cached = "cached_where_%s" % attr + setattr(x, cached, x._lookup_where(attr)) source = InstanceSource(bookkeeper, x) clsdef.add_source_for_attribute(cached, source) return diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -14,8 +14,8 @@ SomeBuiltin, SomePBC, SomeInteger, TLS, TlsClass, SomeUnicodeCodePoint, s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked, - SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty, AnnotatorError) -from rpython.annotator.classdef import InstanceSource, ClassDef + SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty) +from rpython.annotator.classdesc import ClassDef, ClassDesc from rpython.annotator.listdef import ListDef, ListItem from rpython.annotator.dictdef import DictDef from rpython.annotator import description @@ -23,7 +23,6 @@ from rpython.annotator.argument import simple_args from rpython.rlib.objectmodel import r_dict, r_ordereddict, Symbolic from rpython.tool.algo.unionfind import UnionFind -from rpython.tool.flattenrec import FlattenRecursion from rpython.rtyper import extregistry try: @@ -215,9 +214,7 @@ s_callable.consider_call_site(args, s_result, call_op) def getuniqueclassdef(self, cls): - """Get the ClassDef associated with the given user cls. - Avoid using this! It breaks for classes that must be specialized. - """ + """Get the ClassDef associated with the given user cls.""" assert cls is not object desc = self.getdesc(cls) return desc.getuniqueclassdef() @@ -386,8 +383,9 @@ and x.__class__.__module__ != '__builtin__': if hasattr(x, '_cleanup_'): x._cleanup_() - self.see_mutable(x) - result = SomeInstance(self.getuniqueclassdef(x.__class__)) + classdef = self.getuniqueclassdef(x.__class__) + classdef.see_instance(x) + result = SomeInstance(classdef) elif x is None: return s_None else: @@ -414,7 +412,7 @@ if pyobj.__module__ == '__builtin__': # avoid making classdefs for builtin types result = self.getfrozen(pyobj) else: - result = description.ClassDesc(self, pyobj) + result = ClassDesc(self, pyobj) elif isinstance(pyobj, types.MethodType): if pyobj.im_self is None: # unbound return self.getdesc(pyobj.im_func) @@ -427,11 +425,11 @@ self.getdesc(pyobj.im_self)) # frozendesc else: # regular method origincls, name = origin_of_meth(pyobj) - self.see_mutable(pyobj.im_self) + classdef = self.getuniqueclassdef(pyobj.im_class) + classdef.see_instance(pyobj.im_self) assert pyobj == getattr(pyobj.im_self, name), ( "%r is not %s.%s ??" % (pyobj, pyobj.im_self, name)) # emulate a getattr to make sure it's on the classdef - classdef = self.getuniqueclassdef(pyobj.im_class) classdef.find_attribute(name) result = self.getmethoddesc( self.getdesc(pyobj.im_func), # funcdesc @@ -452,15 +450,6 @@ self.descs[pyobj] = result return result - def have_seen(self, x): - # this might need to expand some more. - if x in self.descs: - return True - elif (x.__class__, x) in self.seen_mutable: - return True - else: - return False - def getfrozen(self, pyobj): return description.FrozenDesc(self, pyobj) @@ -477,22 +466,6 @@ self.methoddescs[key] = result return result - _see_mutable_flattenrec = FlattenRecursion() - - def see_mutable(self, x): - key = (x.__class__, x) - if key in self.seen_mutable: - return - clsdef = self.getuniqueclassdef(x.__class__) - self.seen_mutable[key] = True - self.event('mutable', x) - source = InstanceSource(self, x) - def delayed(): - for attr in source.all_instance_attributes(): - clsdef.add_source_for_attribute(attr, source) - # ^^^ can trigger reflowing - self._see_mutable_flattenrec(delayed) - def valueoftype(self, t): return annotationoftype(t, self) @@ -547,6 +520,20 @@ return s_result + def getattr_locations(self, clsdesc, attrname): + attrdef = clsdesc.classdef.find_attribute(attrname) + return attrdef.read_locations + + def record_getattr(self, clsdesc, attrname): + locations = self.getattr_locations(clsdesc, attrname) + locations.add(self.position_key) + + def update_attr(self, clsdef, attrdef): + locations = self.getattr_locations(clsdef.classdesc, attrdef.name) + for position in locations: + self.annotator.reflowfromposition(position) + attrdef.validate(homedef=clsdef) + def pbc_call(self, pbc, args, emulated=None): """Analyse a call to a SomePBC() with the given args (list of annotations). diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -5,13 +5,14 @@ from collections import OrderedDict from rpython.annotator.model import ( - SomeInteger, SomeObject, SomeChar, SomeBool, SomeString, SomeTuple, + SomeInteger, SomeChar, SomeBool, SomeString, SomeTuple, SomeUnicodeCodePoint, SomeFloat, unionof, SomeUnicodeString, SomePBC, SomeInstance, SomeDict, SomeList, SomeWeakRef, SomeIterator, SomeOrderedDict, SomeByteArray, add_knowntypedata, s_ImpossibleValue,) from rpython.annotator.bookkeeper import ( getbookkeeper, immutablevalue, BUILTIN_ANALYZERS, analyzer_for) from rpython.annotator import description +from rpython.annotator.classdesc import ClassDef from rpython.flowspace.model import Constant import rpython.rlib.rarithmetic import rpython.rlib.objectmodel @@ -124,7 +125,6 @@ def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" - from rpython.annotator.classdef import ClassDef if cls2 is object: return True def classify(cls): diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdesc.py rename from rpython/annotator/classdef.py rename to rpython/annotator/classdesc.py --- a/rpython/annotator/classdef.py +++ b/rpython/annotator/classdesc.py @@ -1,9 +1,18 @@ """ Type inference for user-defined classes. """ +from __future__ import absolute_import +import types + +from rpython.flowspace.model import Constant +from rpython.tool.flattenrec import FlattenRecursion +from rpython.tool.sourcetools import func_with_new_name +from rpython.tool.uid import Hashable from rpython.annotator.model import ( - SomePBC, s_ImpossibleValue, unionof, s_None, AnnotatorError) -from rpython.annotator import description + SomePBC, s_ImpossibleValue, unionof, s_None, AnnotatorError, SomeInteger, + SomeString, SomeImpossibleValue, SomeList, HarmlesslyBlocked) +from rpython.annotator.description import ( + Desc, FunctionDesc, MethodDesc, NODEFAULT) # The main purpose of a ClassDef is to collect information about class/instance @@ -67,56 +76,39 @@ # Both writing to the instance attribute and discovering prebuilt # instances that have the attribute set will turn off readonly-ness. - def __init__(self, name, bookkeeper): + def __init__(self, name): assert name != '__class__' self.name = name - self.bookkeeper = bookkeeper self.s_value = s_ImpossibleValue self.readonly = True self.attr_allowed = True - self.read_locations = {} + self.read_locations = set() def add_constant_source(self, classdef, source): s_value = source.s_get_value(classdef, self.name) if source.instance_level: # a prebuilt instance source forces readonly=False, see above self.modified(classdef) - s_new_value = unionof(self.s_value, s_value) # XXX "source %r attr %s" % (source, self.name), + s_new_value = unionof(self.s_value, s_value) self.s_value = s_new_value - def getvalue(self): - # Same as 'self.s_value' for historical reasons. - return self.s_value - - def merge(self, other, classdef='?'): + def merge(self, other, classdef): assert self.name == other.name - s_new_value = unionof(self.s_value, other.s_value) # XXX "%s attr %s" % (classdef, self.name) + s_new_value = unionof(self.s_value, other.s_value) self.s_value = s_new_value if not other.readonly: self.modified(classdef) self.read_locations.update(other.read_locations) - def mutated(self, homedef): # reflow from attr read positions - s_newvalue = self.getvalue() - - for position in self.read_locations: - self.bookkeeper.annotator.reflowfromposition(position) - - # check for method demotion and after-the-fact method additions + def validate(self, homedef): + s_newvalue = self.s_value + # check for after-the-fact method additions if isinstance(s_newvalue, SomePBC): attr = self.name - if s_newvalue.getKind() == description.MethodDesc: + if s_newvalue.getKind() == MethodDesc: # is method if homedef.classdesc.read_attribute(attr, None) is None: - if not homedef.check_missing_attribute_update(attr): - for desc in s_newvalue.descriptions: - if desc.selfclassdef is None: - if homedef.classdesc.settled: - raise AnnotatorError( - "demoting method %s to settled class " - "%s not allowed" % (self.name, homedef) - ) - break + homedef.check_missing_attribute_update(attr) # check for attributes forbidden by slots or _attrs_ if homedef.classdesc.all_enforced_attrs is not None: @@ -124,19 +116,22 @@ self.attr_allowed = False if not self.readonly: raise NoSuchAttrError( - "the attribute %r goes here to %r, " - "but it is forbidden here" % ( - self.name, homedef)) + "the attribute %r goes here to %r, but it is " + "forbidden here" % (self.name, homedef)) def modified(self, classdef='?'): self.readonly = False if not self.attr_allowed: + from rpython.annotator.bookkeeper import getbookkeeper + bk = getbookkeeper() + classdesc = classdef.classdesc + locations = bk.getattr_locations(classdesc, self.name) raise NoSuchAttrError( "Attribute %r on %r should be read-only.\n" % (self.name, classdef) + "This error can be caused by another 'getattr' that promoted\n" "the attribute here; the list of read locations is:\n" + - '\n'.join([str(loc[0]) for loc in self.read_locations])) + '\n'.join([str(loc[0]) for loc in locations])) class ClassDef(object): "Wraps a user class." @@ -152,6 +147,7 @@ self.read_locations_of__class__ = {} self.repr = None self.extra_access_sets = {} + self.instances_seen = set() if classdesc.basedesc: self.basedef = classdesc.basedesc.getuniqueclassdef() @@ -169,6 +165,27 @@ if self.bookkeeper: self.bookkeeper.event('classdef_setup', self) + def s_getattr(self, attrname, flags): + attrdef = self.find_attribute(attrname) + s_result = attrdef.s_value + # hack: if s_result is a set of methods, discard the ones + # that can't possibly apply to an instance of self. + # XXX do it more nicely + if isinstance(s_result, SomePBC): + s_result = self.lookup_filter(s_result, attrname, flags) + elif isinstance(s_result, SomeImpossibleValue): + self.check_missing_attribute_update(attrname) + # blocking is harmless if the attribute is explicitly listed + # in the class or a parent class. + for basedef in self.getmro(): + if basedef.classdesc.all_enforced_attrs is not None: + if attrname in basedef.classdesc.all_enforced_attrs: + raise HarmlesslyBlocked("get enforced attr") + elif isinstance(s_result, SomeList): + s_result = self.classdesc.maybe_return_immutable_list( + attrname, s_result) + return s_result + def add_source_for_attribute(self, attr, source): """Adds information about a constant source for an attribute. """ @@ -182,7 +199,7 @@ # but as an optimization we try to see if the attribute # has really been generalized if attrdef.s_value != s_prev_value: - attrdef.mutated(cdef) # reflow from all read positions + self.bookkeeper.update_attr(cdef, attrdef) return else: # remember the source in self.attr_sources @@ -200,16 +217,24 @@ s_prev_value = attrdef.s_value attrdef.add_constant_source(self, source) if attrdef.s_value != s_prev_value: - attrdef.mutated(subdef) # reflow from all read positions + self.bookkeeper.update_attr(subdef, attrdef) + + def get_owner(self, attrname): + """Return the classdef owning the attribute `attrname`.""" + for cdef in self.getmro(): + if attrname in cdef.attrs: + return cdef + else: + return None + def locate_attribute(self, attr): - while True: - for cdef in self.getmro(): - if attr in cdef.attrs: - return cdef - self.generalize_attr(attr) - # the return value will likely be 'self' now, but not always -- see - # test_annrpython.test_attr_moving_from_subclass_to_class_to_parent + cdef = self.get_owner(attr) + if cdef: + return cdef + else: + self._generalize_attr(attr, s_value=None) + return self def find_attribute(self, attr): return self.locate_attribute(attr).attrs[attr] @@ -247,13 +272,18 @@ seen[sub] = True def _generalize_attr(self, attr, s_value): - # first remove the attribute from subclasses -- including us! + # create the Attribute and do the generalization asked for + newattr = Attribute(attr) + if s_value: + newattr.s_value = s_value + + # remove the attribute from subclasses -- including us! # invariant (I) - subclass_attrs = [] constant_sources = [] # [(classdef-of-origin, source)] for subdef in self.getallsubdefs(): if attr in subdef.attrs: - subclass_attrs.append(subdef.attrs[attr]) + subattr = subdef.attrs[attr] + newattr.merge(subattr, classdef=self) del subdef.attrs[attr] if attr in subdef.attr_sources: # accumulate attr_sources for this attribute from all subclasses @@ -270,17 +300,6 @@ if not source.instance_level: constant_sources.append((superdef, source)) - # create the Attribute and do the generalization asked for - newattr = Attribute(attr, self.bookkeeper) - if s_value: - #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False): - # import pdb; pdb.set_trace() - newattr.s_value = s_value - - # keep all subattributes' values - for subattr in subclass_attrs: - newattr.merge(subattr, classdef=self) - # store this new Attribute, generalizing the previous ones from # subclasses -- invariant (A) self.attrs[attr] = newattr @@ -291,15 +310,14 @@ newattr.add_constant_source(origin_classdef, source) # reflow from all read positions - newattr.mutated(self) + self.bookkeeper.update_attr(self, newattr) def generalize_attr(self, attr, s_value=None): # if the attribute exists in a superclass, generalize there, # as imposed by invariant (I) - for clsdef in self.getmro(): - if attr in clsdef.attrs: - clsdef._generalize_attr(attr, s_value) - break + clsdef = self.get_owner(attr) + if clsdef: + clsdef._generalize_attr(attr, s_value) else: self._generalize_attr(attr, s_value) @@ -326,11 +344,10 @@ for desc in pbc.descriptions: # pick methods but ignore already-bound methods, which can come # from an instance attribute - if (isinstance(desc, description.MethodDesc) - and desc.selfclassdef is None): + if (isinstance(desc, MethodDesc) and desc.selfclassdef is None): methclassdef = desc.originclassdef if methclassdef is not self and methclassdef.issubclass(self): - pass # subclasses methods are always candidates + pass # subclasses methods are always candidates elif self.issubclass(methclassdef): # upward consider only the best match if uplookup is None or methclassdef.issubclass(uplookup): @@ -341,7 +358,7 @@ # clsdef1.lookup_filter(pbc) includes # clsdef2.lookup_filter(pbc) (see formal proof...) else: - continue # not matching + continue # not matching # bind the method by giving it a selfclassdef. Use the # more precise subclass that it's coming from. desc = desc.bind_self(methclassdef, flags) @@ -382,6 +399,22 @@ else: return False + _see_instance_flattenrec = FlattenRecursion() + + def see_instance(self, x): + assert isinstance(x, self.classdesc.pyobj) + key = Hashable(x) + if key in self.instances_seen: + return + self.instances_seen.add(key) + self.bookkeeper.event('mutable', x) + source = InstanceSource(self.bookkeeper, x) + def delayed(): + for attr in source.all_instance_attributes(): + self.add_source_for_attribute(attr, source) + # ^^^ can trigger reflowing + self._see_instance_flattenrec(delayed) + def see_new_subclass(self, classdef): for position in self.read_locations_of__class__: self.bookkeeper.annotator.reflowfromposition(position) @@ -394,7 +427,8 @@ return SomePBC([subdef.classdesc for subdef in self.getallsubdefs()]) def _freeze_(self): - raise Exception("ClassDefs are used as knowntype for instances but cannot be used as immutablevalue arguments directly") + raise Exception("ClassDefs are used as knowntype for instances but " + "cannot be used as immutablevalue arguments directly") # ____________________________________________________________ @@ -432,3 +466,479 @@ class NoSuchAttrError(AnnotatorError): """Raised when an attribute is found on a class where __slots__ or _attrs_ forbits it.""" + + +def is_mixin(cls): + return cls.__dict__.get('_mixin_', False) + + +class ClassDesc(Desc): + knowntype = type + instance_level = False + all_enforced_attrs = None # or a set + _detect_invalid_attrs = None + + def __init__(self, bookkeeper, cls, + name=None, basedesc=None, classdict=None): + super(ClassDesc, self).__init__(bookkeeper, cls) + if '__NOT_RPYTHON__' in cls.__dict__: + raise AnnotatorError('Bad class') + + if name is None: + name = cls.__module__ + '.' + cls.__name__ + self.name = name + self.basedesc = basedesc + if classdict is None: + classdict = {} # populated below + self.classdict = classdict # {attr: Constant-or-Desc} + if cls.__dict__.get('_annspecialcase_', ''): + raise AnnotatorError( + "Class specialization has been removed. The " + "'_annspecialcase_' class tag is now unsupported.") + self.classdef = None + + if is_mixin(cls): + raise AnnotatorError("cannot use directly the class %r because " + "it is a _mixin_" % (cls,)) + + assert cls.__module__ != '__builtin__' + baselist = list(cls.__bases__) + + # special case: skip BaseException, and pretend + # that all exceptions ultimately inherit from Exception instead + # of BaseException (XXX hack) + if cls is Exception: + baselist = [] + elif baselist == [BaseException]: + baselist = [Exception] + + immutable_fields = cls.__dict__.get('_immutable_fields_', []) + # To prevent confusion, we forbid strings. Any other bona fide sequence + # of strings is OK. + if isinstance(immutable_fields, basestring): + raise AnnotatorError( + "In class %s, '_immutable_fields_' must be a sequence of " + "attribute names, not a string." % cls) + self.immutable_fields = set(immutable_fields) + + mixins_before = [] + mixins_after = [] + base = object + for b1 in baselist: + if b1 is object: + continue + if is_mixin(b1): + if base is object: + mixins_before.append(b1) + else: + mixins_after.append(b1) + else: + assert base is object, ("multiple inheritance only supported " + "with _mixin_: %r" % (cls,)) + base = b1 + if mixins_before and mixins_after: + raise Exception("unsupported: class %r has mixin bases both" + " before and after the regular base" % (self,)) + self.add_mixins(mixins_after, check_not_in=base) + self.add_mixins(mixins_before) + self.add_sources_for_class(cls) + + if base is not object: + self.basedesc = bookkeeper.getdesc(base) + + if '__slots__' in cls.__dict__ or '_attrs_' in cls.__dict__: + attrs = {} + for decl in ('__slots__', '_attrs_'): + decl = cls.__dict__.get(decl, []) + if isinstance(decl, str): + decl = (decl,) + decl = dict.fromkeys(decl) + attrs.update(decl) + if self.basedesc is not None: + if self.basedesc.all_enforced_attrs is None: + raise Exception("%r has slots or _attrs_, " + "but not its base class" % (cls,)) + attrs.update(self.basedesc.all_enforced_attrs) + self.all_enforced_attrs = attrs + + if (self.is_builtin_exception_class() and + self.all_enforced_attrs is None): + if cls not in FORCE_ATTRIBUTES_INTO_CLASSES: + self.all_enforced_attrs = [] # no attribute allowed + + def add_source_attribute(self, name, value, mixin=False): + if isinstance(value, property): + # special case for property object + if value.fget is not None: + newname = name + '__getter__' + func = func_with_new_name(value.fget, newname) + self.add_source_attribute(newname, func, mixin) + if value.fset is not None: + newname = name + '__setter__' + func = func_with_new_name(value.fset, newname) + self.add_source_attribute(newname, func, mixin) + self.classdict[name] = Constant(value) + return + + if isinstance(value, types.FunctionType): + # for debugging + if not hasattr(value, 'class_'): + value.class_ = self.pyobj + if mixin: + # make a new copy of the FunctionDesc for this class, + # but don't specialize further for all subclasses + funcdesc = FunctionDesc(self.bookkeeper, value) + self.classdict[name] = funcdesc + return + # NB. if value is, say, AssertionError.__init__, then we + # should not use getdesc() on it. Never. The problem is + # that the py lib has its own AssertionError.__init__ which + # is of type FunctionType. But bookkeeper.immutablevalue() + # will do the right thing in s_get_value(). + if isinstance(value, staticmethod) and mixin: + # make a new copy of staticmethod + func = value.__get__(42) + value = staticmethod(func_with_new_name(func, func.__name__)) + + if type(value) in MemberDescriptorTypes: + # skip __slots__, showing up in the class as 'member' objects + return + if name == '__init__' and self.is_builtin_exception_class(): + # pretend that built-in exceptions have no __init__, + # unless explicitly specified in builtin.py + from rpython.annotator.builtin import BUILTIN_ANALYZERS + value = getattr(value, 'im_func', value) + if value not in BUILTIN_ANALYZERS: + return + self.classdict[name] = Constant(value) + + def add_mixins(self, mixins, check_not_in=object): + if not mixins: + return + A = type('tmp', tuple(mixins) + (object,), {}) + mro = A.__mro__ + assert mro[0] is A and mro[-1] is object + mro = mro[1:-1] + # + skip = set() + def add(cls): + if cls is not object: + for base in cls.__bases__: + add(base) + for name in cls.__dict__: + skip.add(name) + add(check_not_in) + # + for base in reversed(mro): + assert is_mixin(base), ( + "Mixin class %r has non mixin base class %r" % (mixins, base)) + for name, value in base.__dict__.items(): + if name in skip: + continue + self.add_source_attribute(name, value, mixin=True) + if '_immutable_fields_' in base.__dict__: + self.immutable_fields.update( + set(base.__dict__['_immutable_fields_'])) + + + def add_sources_for_class(self, cls): + for name, value in cls.__dict__.items(): + self.add_source_attribute(name, value) + + def getclassdef(self, key): + return self.getuniqueclassdef() + + def _init_classdef(self): + classdef = ClassDef(self.bookkeeper, self) + self.bookkeeper.classdefs.append(classdef) + self.classdef = classdef + + # forced attributes + cls = self.pyobj + if cls in FORCE_ATTRIBUTES_INTO_CLASSES: + for name, s_value in FORCE_ATTRIBUTES_INTO_CLASSES[cls].items(): + classdef.generalize_attr(name, s_value) + classdef.find_attribute(name).modified(classdef) + + # register all class attributes as coming from this ClassDesc + # (as opposed to prebuilt instances) + classsources = {} + for attr in self.classdict: + classsources[attr] = self # comes from this ClassDesc + classdef.setup(classsources) + # look for a __del__ method and annotate it if it's there + if '__del__' in self.classdict: + from rpython.annotator.model import s_None, SomeInstance + s_func = self.s_read_attribute('__del__') + args_s = [SomeInstance(classdef)] + s = self.bookkeeper.emulate_pbc_call(classdef, s_func, args_s) + assert s_None.contains(s) + return classdef + + def getuniqueclassdef(self): + if self.classdef is None: + self._init_classdef() + return self.classdef + + def pycall(self, whence, args, s_previous_result, op=None): + from rpython.annotator.model import SomeInstance, SomeImpossibleValue + classdef = self.getuniqueclassdef() + s_instance = SomeInstance(classdef) + # look up __init__ directly on the class, bypassing the normal + # lookup mechanisms ClassDef (to avoid influencing Attribute placement) + s_init = self.s_read_attribute('__init__') + if isinstance(s_init, SomeImpossibleValue): + # no __init__: check that there are no constructor args + if not self.is_exception_class(): + try: + args.fixedunpack(0) + except ValueError: + raise Exception("default __init__ takes no argument" + " (class %s)" % (self.name,)) + elif self.pyobj is Exception: + # check explicitly against "raise Exception, x" where x + # is a low-level exception pointer + try: + [s_arg] = args.fixedunpack(1) + except ValueError: + pass + else: + from rpython.rtyper.llannotation import SomePtr + assert not isinstance(s_arg, SomePtr) + else: + # call the constructor + args = args.prepend(s_instance) + s_init.call(args) + return s_instance + + def is_exception_class(self): + return issubclass(self.pyobj, BaseException) + + def is_builtin_exception_class(self): + if self.is_exception_class(): + if self.pyobj.__module__ == 'exceptions': + return True + if issubclass(self.pyobj, AssertionError): + return True + return False + + def lookup(self, name): + cdesc = self + while name not in cdesc.classdict: + cdesc = cdesc.basedesc + if cdesc is None: + return None + else: + return cdesc + + def get_param(self, name, default=None, inherit=True): + cls = self.pyobj + if inherit: + return getattr(cls, name, default) + else: + return cls.__dict__.get(name, default) + + def read_attribute(self, name, default=NODEFAULT): + cdesc = self.lookup(name) + if cdesc is None: + if default is NODEFAULT: + raise AttributeError + else: + return default + else: + return cdesc.classdict[name] + + def s_read_attribute(self, name): + # look up an attribute in the class + cdesc = self.lookup(name) + if cdesc is None: + return s_ImpossibleValue + else: + # delegate to s_get_value to turn it into an annotation + return cdesc.s_get_value(None, name) + + def s_get_value(self, classdef, name): + obj = self.classdict[name] + if isinstance(obj, Constant): + value = obj.value + if isinstance(value, staticmethod): # special case + value = value.__get__(42) + classdef = None # don't bind + elif isinstance(value, classmethod): + raise AnnotatorError("classmethods are not supported") + s_value = self.bookkeeper.immutablevalue(value) + if classdef is not None: + s_value = s_value.bind_callables_under(classdef, name) + elif isinstance(obj, Desc): + if classdef is not None: + obj = obj.bind_under(classdef, name) + s_value = SomePBC([obj]) + else: + raise TypeError("classdict should not contain %r" % (obj,)) + return s_value + + def create_new_attribute(self, name, value): + assert name not in self.classdict, "name clash: %r" % (name,) + self.classdict[name] = Constant(value) + + def find_source_for(self, name): + if name in self.classdict: + return self + # check whether there is a new attribute + cls = self.pyobj + if name in cls.__dict__: + self.add_source_attribute(name, cls.__dict__[name]) + if name in self.classdict: + return self + return None + + def maybe_return_immutable_list(self, attr, s_result): + # hack: 'x.lst' where lst is listed in _immutable_fields_ as + # either 'lst[*]' or 'lst?[*]' + # should really return an immutable list as a result. Implemented + # by changing the result's annotation (but not, of course, doing an + # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist, + # test_immutable_list_out_of_instance. + if self._detect_invalid_attrs and attr in self._detect_invalid_attrs: + raise Exception("field %r was migrated to %r from a subclass in " + "which it was declared as _immutable_fields_" % + (attr, self.pyobj)) + search1 = '%s[*]' % (attr,) + search2 = '%s?[*]' % (attr,) + cdesc = self + while cdesc is not None: + immutable_fields = cdesc.immutable_fields + if immutable_fields: + if (search1 in immutable_fields or search2 in immutable_fields): + s_result.listdef.never_resize() + s_copy = s_result.listdef.offspring() + s_copy.listdef.mark_as_immutable() + # + cdesc = cdesc.basedesc + while cdesc is not None: + if cdesc._detect_invalid_attrs is None: + cdesc._detect_invalid_attrs = set() + cdesc._detect_invalid_attrs.add(attr) + cdesc = cdesc.basedesc + # + return s_copy + cdesc = cdesc.basedesc + return s_result # common case + + @staticmethod + def consider_call_site(descs, args, s_result, op): + descs[0].getcallfamily() + descs[0].mergecallfamilies(*descs[1:]) + from rpython.annotator.model import SomeInstance, SomePBC, s_None + if len(descs) == 1: + # call to a single class, look at the result annotation + # in case it was specialized + if not isinstance(s_result, SomeInstance): + raise Exception("calling a class didn't return an instance??") + classdefs = [s_result.classdef] + else: + # call to multiple classes: specialization not supported + classdefs = [desc.getuniqueclassdef() for desc in descs] + # If some of the classes have an __init__ and others not, then + # we complain, even though in theory it could work if all the + # __init__s take no argument. But it's messy to implement, so + # let's just say it is not RPython and you have to add an empty + # __init__ to your base class. + has_init = False + for desc in descs: + s_init = desc.s_read_attribute('__init__') + has_init |= isinstance(s_init, SomePBC) + basedesc = ClassDesc.getcommonbase(descs) + s_init = basedesc.s_read_attribute('__init__') + parent_has_init = isinstance(s_init, SomePBC) + if has_init and not parent_has_init: + raise AnnotatorError( + "some subclasses among %r declare __init__()," + " but not the common parent class" % (descs,)) + # make a PBC of MethodDescs, one for the __init__ of each class + initdescs = [] + for desc, classdef in zip(descs, classdefs): + s_init = desc.s_read_attribute('__init__') + if isinstance(s_init, SomePBC): + assert len(s_init.descriptions) == 1, ( + "unexpected dynamic __init__?") + initfuncdesc, = s_init.descriptions + if isinstance(initfuncdesc, FunctionDesc): + from rpython.annotator.bookkeeper import getbookkeeper + initmethdesc = getbookkeeper().getmethoddesc( + initfuncdesc, classdef, classdef, '__init__') + initdescs.append(initmethdesc) + # register a call to exactly these __init__ methods + if initdescs: + initdescs[0].mergecallfamilies(*initdescs[1:]) + MethodDesc.consider_call_site(initdescs, args, s_None, op) + + def getallbases(self): + desc = self + while desc is not None: + yield desc + desc = desc.basedesc + + @staticmethod + def getcommonbase(descs): + commondesc = descs[0] + for desc in descs[1:]: + allbases = set(commondesc.getallbases()) + while desc not in allbases: + assert desc is not None, "no common base for %r" % (descs,) + desc = desc.basedesc + commondesc = desc + return commondesc + + def rowkey(self): + return self + + def getattrfamily(self, attrname): + "Get the ClassAttrFamily object for attrname. Possibly creates one." + access_sets = self.bookkeeper.get_classpbc_attr_families(attrname) + _, _, attrfamily = access_sets.find(self) + return attrfamily + + def queryattrfamily(self, attrname): + """Retrieve the ClassAttrFamily object for attrname if there is one, + otherwise return None.""" + access_sets = self.bookkeeper.get_classpbc_attr_families(attrname) + try: + return access_sets[self] + except KeyError: + return None + + def mergeattrfamilies(self, others, attrname): + """Merge the attr families of the given Descs into one.""" + access_sets = self.bookkeeper.get_classpbc_attr_families(attrname) + changed, rep, attrfamily = access_sets.find(self) + for desc in others: + changed1, rep, attrfamily = access_sets.union(rep, desc) + changed = changed or changed1 + return changed + +# ____________________________________________________________ + +class Sample(object): + __slots__ = 'x' +MemberDescriptorTypes = [type(Sample.x)] +del Sample +try: + MemberDescriptorTypes.append(type(OSError.errno)) +except AttributeError: # on CPython <= 2.4 + pass + +# ____________________________________________________________ + +FORCE_ATTRIBUTES_INTO_CLASSES = { + EnvironmentError: {'errno': SomeInteger(), + 'strerror': SomeString(can_be_None=True), + 'filename': SomeString(can_be_None=True)}, +} + +try: + WindowsError +except NameError: + pass +else: + FORCE_ATTRIBUTES_INTO_CLASSES[WindowsError] = {'winerror': SomeInteger()} diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -2,13 +2,12 @@ import types from rpython.annotator.signature import ( enforce_signature_args, enforce_signature_return, finish_type) -from rpython.flowspace.model import Constant, FunctionGraph +from rpython.flowspace.model import FunctionGraph from rpython.flowspace.bytecode import cpython_code_signature from rpython.annotator.argument import rawshape, ArgErr, simple_args -from rpython.tool.sourcetools import valid_identifier, func_with_new_name +from rpython.tool.sourcetools import valid_identifier from rpython.tool.pairtype import extendabletype -from rpython.annotator.model import ( - AnnotatorError, SomeInteger, SomeString, s_ImpossibleValue) +from rpython.annotator.model import AnnotatorError, s_ImpossibleValue class CallFamily(object): """A family of Desc objects that could be called from common call sites. @@ -188,6 +187,8 @@ class NoStandardGraph(Exception): """The function doesn't have a single standard non-specialized graph.""" +NODEFAULT = object() + class FunctionDesc(Desc): knowntype = types.FunctionType @@ -404,469 +405,6 @@ return s_sigs -def is_mixin(cls): - return cls.__dict__.get('_mixin_', False) - -NODEFAULT = object() - -class ClassDesc(Desc): - knowntype = type - instance_level = False - all_enforced_attrs = None # or a set - settled = False - _detect_invalid_attrs = None - - def __init__(self, bookkeeper, cls, - name=None, basedesc=None, classdict=None, - specialize=None): - super(ClassDesc, self).__init__(bookkeeper, cls) - if '__NOT_RPYTHON__' in cls.__dict__: - raise AnnotatorError('Bad class') - - if name is None: - name = cls.__module__ + '.' + cls.__name__ - self.name = name - self.basedesc = basedesc - if classdict is None: - classdict = {} # populated below - self.classdict = classdict # {attr: Constant-or-Desc} - if specialize is None: - specialize = cls.__dict__.get('_annspecialcase_', '') - self.specialize = specialize - self._classdefs = {} - - if is_mixin(cls): - raise AnnotatorError("cannot use directly the class %r because " - "it is a _mixin_" % (cls,)) - - assert cls.__module__ != '__builtin__' _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
