Author: Carl Friedrich Bolz-Tereick <[email protected]> Branch: regalloc-playground Changeset: r92477:ad2fa98841d9 Date: 2017-09-27 16:05 +0200 http://bitbucket.org/pypy/pypy/changeset/ad2fa98841d9/
Log: merge default diff too long, truncating to 2000 out of 6770 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -60,8 +60,8 @@ Wim Lavrijsen Eric van Riet Paap Richard Emslie + Remi Meier Alexander Schremmer - Remi Meier Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager @@ -102,6 +102,7 @@ Michael Foord Stephan Diehl Stefano Rivera + Jean-Paul Calderone Stefan Schwarzer Tomek Meka Valentino Volonghi @@ -110,14 +111,13 @@ Bob Ippolito Bruno Gola David Malcolm - Jean-Paul Calderone Squeaky Edd Barrett Timo Paulssen Marius Gedminas + Nicolas Truessel Alexandre Fayolle Simon Burton - Nicolas Truessel Martin Matusiak Laurence Tratt Wenzhu Man @@ -156,6 +156,7 @@ Stefan H. Muller Tim Felgentreff Eugene Oden + Dodan Mihai Jeff Terrace Henry Mason Vasily Kuznetsov @@ -182,11 +183,13 @@ Rocco Moretti Gintautas Miliauskas Lucian Branescu Mihaila + Mariano Anaya anatoly techtonik - Dodan Mihai Karl Bartel + Stefan Beyer Gabriel Lavoie Jared Grubb + Alecsandru Patrascu Olivier Dormond Wouter van Heyst Sebastian Pawluś @@ -194,6 +197,7 @@ Victor Stinner Andrews Medina Aaron Iles + [email protected] Toby Watson Daniel Patrick Stuart Williams @@ -204,6 +208,7 @@ Michael Cheng Mikael Schönenberg Stanislaw Halik + Mihnea Saracin Berkin Ilbeyi Gasper Zejn Faye Zhao @@ -214,14 +219,12 @@ Jonathan David Riehl Beatrice During Alex Perry - [email protected] Robert Zaremba Alan McIntyre Alexander Sedov Vaibhav Sood Reuben Cummings Attila Gobi - Alecsandru Patrascu Christopher Pope Tristan Arthur Christian Tismer @@ -243,7 +246,6 @@ Jacek Generowicz Sylvain Thenault Jakub Stasiak - Stefan Beyer Andrew Dalke Alejandro J. Cura Vladimir Kryachko @@ -275,6 +277,7 @@ Christoph Gerum Miguel de Val Borro Artur Lisiecki + afteryu Toni Mattis Laurens Van Houtven Bobby Impollonia @@ -305,6 +308,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Anthony Sottile Nate Bragg Ben Darnell Juan Francisco Cantero Hurtado @@ -325,12 +329,14 @@ Mike Bayer Rodrigo Araújo Daniil Yarancev + Min RK OlivierBlanvillain Jonas Pfannschmidt Zearin Andrey Churin Dan Crosta [email protected] + Stanisław Halik Julien Phalip Roman Podoliaka Eli Stevens diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -364,7 +364,7 @@ pypy_dll = _ffi.CDLL(name, mode) else: pypy_dll = _ffi.WinDLL(name, mode) - self._pypy_dll = pypy_dll + self.__pypy_dll__ = pypy_dll handle = int(pypy_dll) if _sys.maxint > 2 ** 32: handle = int(handle) # long -> int diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py --- a/lib-python/2.7/ctypes/test/test_byteswap.py +++ b/lib-python/2.7/ctypes/test/test_byteswap.py @@ -23,7 +23,6 @@ setattr(bits, "i%s" % i, 1) dump(bits) - @xfail def test_endian_short(self): if sys.byteorder == "little": self.assertIs(c_short.__ctype_le__, c_short) @@ -51,7 +50,6 @@ self.assertEqual(bin(s), "3412") self.assertEqual(s.value, 0x1234) - @xfail def test_endian_int(self): if sys.byteorder == "little": self.assertIs(c_int.__ctype_le__, c_int) @@ -80,7 +78,6 @@ self.assertEqual(bin(s), "78563412") self.assertEqual(s.value, 0x12345678) - @xfail def test_endian_longlong(self): if sys.byteorder == "little": self.assertIs(c_longlong.__ctype_le__, c_longlong) @@ -109,7 +106,6 @@ self.assertEqual(bin(s), "EFCDAB9078563412") self.assertEqual(s.value, 0x1234567890ABCDEF) - @xfail def test_endian_float(self): if sys.byteorder == "little": self.assertIs(c_float.__ctype_le__, c_float) @@ -128,7 +124,6 @@ self.assertAlmostEqual(s.value, math.pi, 6) self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s)) - @xfail def test_endian_double(self): if sys.byteorder == "little": self.assertIs(c_double.__ctype_le__, c_double) @@ -156,7 +151,6 @@ self.assertIs(c_char.__ctype_le__, c_char) self.assertIs(c_char.__ctype_be__, c_char) - @xfail def test_struct_fields_1(self): if sys.byteorder == "little": base = BigEndianStructure @@ -192,7 +186,6 @@ pass self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)]) - @xfail def test_struct_struct(self): # nested structures with different byteorders @@ -221,7 +214,6 @@ self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) - @xfail def test_struct_fields_2(self): # standard packing in struct uses no alignment. # So, we have to align using pad bytes. @@ -245,7 +237,6 @@ s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) self.assertEqual(bin(s1), bin(s2)) - @xfail def test_unaligned_nonnative_struct_fields(self): if sys.byteorder == "little": base = BigEndianStructure diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py --- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py +++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py @@ -37,10 +37,7 @@ for typ in byteswapped_structures: ## print >> sys.stderr, typ.value self.assertEqual(typ.value.offset, 1) - try: - o = typ() - except NotImplementedError as e: - self.skipTest(str(e)) # for PyPy + o = typ() o.value = 4 self.assertEqual(o.value, 4) diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -203,7 +203,7 @@ f_locals local namespace seen by this frame f_restricted 0 or 1 if frame is in restricted execution mode f_trace tracing function for this frame, or None""" - return isinstance(object, types.FrameType) + return isinstance(object, (types.FrameType, types.FakeFrameType)) def iscode(object): """Return true if the object is a code object. diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py --- a/lib-python/2.7/multiprocessing/heap.py +++ b/lib-python/2.7/multiprocessing/heap.py @@ -62,7 +62,7 @@ self.size = size self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next()) self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - assert win32.GetLastError() == 0, 'tagname already in use' + #assert win32.GetLastError() == 0, 'tagname already in use' self._state = (self.size, self.name) def __getstate__(self): @@ -72,7 +72,7 @@ def __setstate__(self, state): self.size, self.name = self._state = state self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS + #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS else: diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -75,7 +75,7 @@ for i in range(256): buf[i] = i for i in range(n): - buf[ord(fromstr[i])] = tostr[i] + buf[ord(fromstr[i])] = ord(tostr[i]) return str(buf) diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py --- a/lib-python/2.7/types.py +++ b/lib-python/2.7/types.py @@ -71,6 +71,12 @@ FrameType = type(tb.tb_frame) del tb +# PyPy extension +try: + FakeFrameType = type(next(sys._current_frames().itervalues())) +except (AttributeError, StopIteration): + FakeFrameType = FrameType + SliceType = slice EllipsisType = type(Ellipsis) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -82,7 +82,7 @@ return False def in_dll(self, dll, name): - return self.from_address(dll._pypy_dll.getaddressindll(name)) + return self.from_address(dll.__pypy_dll__.getaddressindll(name)) def from_buffer(self, obj, offset=0): size = self._sizeofinstances() diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -430,7 +430,7 @@ ffires = restype.get_ffi_argtype() return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_) - cdll = self.dll._pypy_dll + cdll = self.dll.__pypy_dll__ try: ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes] ffi_restype = restype.get_ffi_argtype() diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -142,6 +142,10 @@ ptr._buffer = tp._ffiarray(1, autofree=True) ptr._buffer[0] = obj._buffer result = ptr + elif isinstance(obj, bytes): + result = tp() + result._buffer[0] = buffer(obj)._pypy_raw_address() + return result elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -61,6 +61,54 @@ pyobj_container = GlobalPyobjContainer() +def swap_bytes(value, sizeof, typeof, get_or_set): + def swap_2(): + return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00) + + def swap_4(): + return ((value & 0x000000FF) << 24) | \ + ((value & 0x0000FF00) << 8) | \ + ((value & 0x00FF0000) >> 8) | \ + ((value >> 24) & 0xFF) + + def swap_8(): + return ((value & 0x00000000000000FFL) << 56) | \ + ((value & 0x000000000000FF00L) << 40) | \ + ((value & 0x0000000000FF0000L) << 24) | \ + ((value & 0x00000000FF000000L) << 8) | \ + ((value & 0x000000FF00000000L) >> 8) | \ + ((value & 0x0000FF0000000000L) >> 24) | \ + ((value & 0x00FF000000000000L) >> 40) | \ + ((value >> 56) & 0xFF) + + def swap_double_float(typ): + from struct import pack, unpack + if get_or_set == 'set': + if sys.byteorder == 'little': + st = pack(''.join(['>', typ]), value) + else: + st = pack(''.join(['<', typ]), value) + return unpack(typ, st)[0] + else: + packed = pack(typ, value) + if sys.byteorder == 'little': + st = unpack(''.join(['>', typ]), packed) + else: + st = unpack(''.join(['<', typ]), packed) + return st[0] + + if typeof in ('c_float', 'c_float_le', 'c_float_be'): + return swap_double_float('f') + elif typeof in ('c_double', 'c_double_le', 'c_double_be'): + return swap_double_float('d') + else: + if sizeof == 2: + return swap_2() + elif sizeof == 4: + return swap_4() + elif sizeof == 8: + return swap_8() + def generic_xxx_p_from_param(cls, value): if value is None: return cls(None) @@ -271,6 +319,31 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) result._as_ffi_pointer_ = _as_ffi_pointer_ + if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \ + and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'): + from sys import byteorder + if byteorder == 'big': + name += '_le' + swapped = self.__new__(self, name, bases, dct) + result.__ctype_le__ = swapped + result.__ctype_be__ = result + swapped.__ctype_be__ = result + swapped.__ctype_le__ = swapped + else: + name += '_be' + swapped = self.__new__(self, name, bases, dct) + result.__ctype_be__ = swapped + result.__ctype_le__ = result + swapped.__ctype_le__ = result + swapped.__ctype_be__ = swapped + from _ctypes import sizeof + def _getval(self): + return swap_bytes(self._buffer[0], sizeof(self), name, 'get') + def _setval(self, value): + d = result() + d.value = value + self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set') + swapped.value = property(_getval, _setval) return result diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -146,6 +146,7 @@ obj._buffer.__setattr__(self.name, arg) + def _set_shape(tp, rawfields, is_union=False): tp._ffistruct_ = _rawffi.Structure(rawfields, is_union, getattr(tp, '_pack_', 0)) @@ -240,19 +241,27 @@ res.__dict__['_index'] = -1 return res - class StructOrUnion(_CData): __metaclass__ = StructOrUnionMeta def __new__(cls, *args, **kwds): from _ctypes import union - self = super(_CData, cls).__new__(cls) - if ('_abstract_' in cls.__dict__ or cls is Structure + if ('_abstract_' in cls.__dict__ or cls is Structure or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - raise NotImplementedError("missing in PyPy: structure/union with " - "swapped (non-native) byte ordering") + fields = [None] * len(cls._fields_) + for i in range(len(cls._fields_)): + if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): + swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) + else: + swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) + if len(cls._fields_[i]) < 3: + fields[i] = (cls._fields_[i][0], swapped) + else: + fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) + names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) return self diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py --- a/lib_pypy/cPickle.py +++ b/lib_pypy/cPickle.py @@ -116,10 +116,20 @@ @builtinify def dump(obj, file, protocol=None): + if protocol > HIGHEST_PROTOCOL: + # use cPickle error message, not pickle.py one + raise ValueError("pickle protocol %d asked for; " + "the highest available protocol is %d" % ( + protocol, HIGHEST_PROTOCOL)) Pickler(file, protocol).dump(obj) @builtinify def dumps(obj, protocol=None): + if protocol > HIGHEST_PROTOCOL: + # use cPickle error message, not pickle.py one + raise ValueError("pickle protocol %d asked for; " + "the highest available protocol is %d" % ( + protocol, HIGHEST_PROTOCOL)) file = StringIO() Pickler(file, protocol).dump(obj) return file.getvalue() diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.0 +Version: 1.11.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.0" -__version_info__ = (1, 11, 0) +__version__ = "1.11.1" +__version_info__ = (1, 11, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.0" + "\ncompiled with cffi version: 1.11.1" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py --- a/lib_pypy/pyrepl/historical_reader.py +++ b/lib_pypy/pyrepl/historical_reader.py @@ -17,7 +17,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from pyrepl import reader, commands +from pyrepl import reader, commands, input from pyrepl.reader import Reader as R isearch_keymap = tuple( @@ -214,7 +214,6 @@ isearch_forwards, isearch_backwards, operate_and_get_next]: self.commands[c.__name__] = c self.commands[c.__name__.replace('_', '-')] = c - from pyrepl import input self.isearch_trans = input.KeymapTranslator( isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -119,8 +119,15 @@ To run untranslated tests, you need the Boehm garbage collector libgc. -On Debian and Ubuntu, this is the command to install all build-time -dependencies:: +On recent Debian and Ubuntu (like 17.04), this is the command to install +all build-time dependencies:: + + apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \ + libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \ + tk-dev libgc-dev python-cffi \ + liblzma-dev libncursesw5-dev # these two only needed on PyPy3 + +On older Debian and Ubuntu (12.04 to 16.04):: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -27,8 +27,8 @@ Wim Lavrijsen Eric van Riet Paap Richard Emslie + Remi Meier Alexander Schremmer - Remi Meier Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager @@ -69,6 +69,7 @@ Michael Foord Stephan Diehl Stefano Rivera + Jean-Paul Calderone Stefan Schwarzer Tomek Meka Valentino Volonghi @@ -77,14 +78,13 @@ Bob Ippolito Bruno Gola David Malcolm - Jean-Paul Calderone Squeaky Edd Barrett Timo Paulssen Marius Gedminas + Nicolas Truessel Alexandre Fayolle Simon Burton - Nicolas Truessel Martin Matusiak Laurence Tratt Wenzhu Man @@ -123,6 +123,7 @@ Stefan H. Muller Tim Felgentreff Eugene Oden + Dodan Mihai Jeff Terrace Henry Mason Vasily Kuznetsov @@ -149,11 +150,13 @@ Rocco Moretti Gintautas Miliauskas Lucian Branescu Mihaila + Mariano Anaya anatoly techtonik - Dodan Mihai Karl Bartel + Stefan Beyer Gabriel Lavoie Jared Grubb + Alecsandru Patrascu Olivier Dormond Wouter van Heyst Sebastian Pawluś @@ -161,6 +164,7 @@ Victor Stinner Andrews Medina Aaron Iles + [email protected] Toby Watson Daniel Patrick Stuart Williams @@ -171,6 +175,7 @@ Michael Cheng Mikael Schönenberg Stanislaw Halik + Mihnea Saracin Berkin Ilbeyi Gasper Zejn Faye Zhao @@ -181,14 +186,12 @@ Jonathan David Riehl Beatrice During Alex Perry - [email protected] Robert Zaremba Alan McIntyre Alexander Sedov Vaibhav Sood Reuben Cummings Attila Gobi - Alecsandru Patrascu Christopher Pope Tristan Arthur Christian Tismer @@ -210,7 +213,6 @@ Jacek Generowicz Sylvain Thenault Jakub Stasiak - Stefan Beyer Andrew Dalke Alejandro J. Cura Vladimir Kryachko @@ -242,6 +244,7 @@ Christoph Gerum Miguel de Val Borro Artur Lisiecki + afteryu Toni Mattis Laurens Van Houtven Bobby Impollonia @@ -272,6 +275,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Anthony Sottile Nate Bragg Ben Darnell Juan Francisco Cantero Hurtado @@ -292,12 +296,14 @@ Mike Bayer Rodrigo Araújo Daniil Yarancev + Min RK OlivierBlanvillain Jonas Pfannschmidt Zearin Andrey Churin Dan Crosta [email protected] + Stanisław Halik Julien Phalip Roman Podoliaka Eli Stevens diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -429,7 +429,8 @@ * the ``__builtins__`` name is always referencing the ``__builtin__`` module, never a dictionary as it sometimes is in CPython. Assigning to - ``__builtins__`` has no effect. + ``__builtins__`` has no effect. (For usages of tools like + RestrictedPython, see `issue #2653`_.) * directly calling the internal magic methods of a few built-in types with invalid arguments may have a slightly different result. For @@ -535,7 +536,12 @@ or ``float`` subtypes. Currently PyPy does not support the ``__class__`` attribute assignment for any non heaptype subtype. +* In PyPy, module and class dictionaries are optimized under the assumption + that deleting attributes from them are rare. Because of this, e.g. + ``del foo.bar`` where ``foo`` is a module (or class) that contains the + function ``bar``, is significantly slower than CPython. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ - +.. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst whatsnew-pypy2-5.6.0.rst @@ -36,6 +37,7 @@ .. toctree:: whatsnew-pypy3-head.rst + whatsnew-pypy3-5.9.0.rst whatsnew-pypy3-5.8.0.rst whatsnew-pypy3-5.7.0.rst diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.9.0.rst @@ -0,0 +1,209 @@ +===================================== +PyPy2.7 and PyPy3.5 v5.9 dual release +===================================== + +The PyPy team is proud to release both PyPy2.7 v5.9 (an interpreter supporting +Python 2.7 syntax), and a beta-quality PyPy3.5 v5.9 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. Note that PyPy3.5 supports Linux 64bit only for now. + +This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and +PyPy3.5 includes the upstream stdlib version 3.5.3. + +Only a handful of failing tests remain in NumPy and Pandas on PyPy2.7, issues +that appeared as excessive memory use were cleared up and other incompatibilities +were resolved. + +Cython 0.27 (released last week) should support more projects with PyPy, both +on PyPy2.7 and PyPy3.5 beta. + +We optimized the JSON parser for recurring string keys, which should decrease +memory use to 50% and increase parsing speed by up to 15% for large JSON files +with many repeating dictionary keys (which is quite common). + +CFFI_, which is part of the PyPy release, has been updated to 1.11, +improving an already great package for interfacing with C. CFFI now supports +complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has +improved support for callbacks. + +Please let us know if your use case is slow, we have ideas how to make things +faster but need real-world examples (not micro-benchmarks) of problematic code. + +Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from +CPython were ported to PyPy. Of course the bug fixes and performance enhancements +mentioned above are part of both PyPy2.7 and PyPy3.5 beta. + +As always, this release fixed many other issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the v5.9 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy 2.7 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.8 released June, 2017) +====================================================================================== + +See also issues that were resolved_ + +Note that these are also merged into PyPy 3.5 + +* New features and cleanups + + * Add support for ``PyFrozenSet_New``, ``PyObject_HashNotImplemented``, + ``PyObject_Print(NULL, ...)``, ``PyObject_RichCompareBool(a, a, ...)``, + ``PyType_IS_GC`` (does nothing), ``PyUnicode_FromFormat`` + * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible + * ``gcdump`` now reports largest object + * More complete support in the ``_curses`` CFFI module + * Add cPickle.Unpickler.find_global (issue 1853_) + * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set + * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()`` + * Update RevDB to version pypy2.7-v5.6.2 + * Previously, ``instance.method`` would return always the same bound method + object, when gotten from the same instance (as far as ``is`` and ``id()`` + can tell). CPython doesn't do that. Now PyPy, like CPython, returns a + different bound method object every time. For ``type.method``, PyPy2 still + returns always the same *unbound* method object; CPython does it for built-in + types but not for user-defined types + * Link to disable PaX protection for the JIT when needed + * Update build instructions and an rarely used Makefile + * Recreate support for using leakfinder in cpyext tests which had suffered + bit-rot, disable due to many false positives + * Add more functionality to ``sysconfig`` + * Added ``_swappedbytes_`` support for ``ctypes.Structure`` + * Better support the ``inspect`` module on ``frames`` + +* Bug Fixes + + * Fix issue 2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value + * Implement ``PyListOjbect.getstorage_copy`` + * Fix for ``reversed(dictproxy)`` issue 2601_ + * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_ + * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16 + * ``gc.get_objects`` now handles objects with finalizers more consistently + * Fixed memory leak in ``SSLContext.getpeercert`` returning validated + certificates and ``SSLContext.get_ca_certs(binary_mode=True)`` + (_get_crl_dp) `CPython issue 29738`_ + +* Performance improvements: + + * Improve performance of ``bytearray.extend`` by rewriting portions in app-level + * Optimize list accesses with constant indexes better by retaining more + information about them + * Add a jit driver for ``array.count`` and ``array.index`` + * Improve information retained in a bridge wrt ``array`` + * Move some dummy CAPI functions and ``Py*_Check`` functions from RPython into + pure C macros + * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap + all the ints + * Cache string keys that occur in JSON dicts, as they are likely to repeat + +* RPython improvements + + * Do not preallocate a RPython list if we only know an upper bound on its size + * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers + * Replace magical NOT RPYTHON comment with a decorator + * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5 + +* Degredations + + * Disable vmprof on win32, due to upstream changes that break the internal ``_vmprof`` module + +.. _here: cpython_differences.html +.. _1853: https://bitbucket.org/pypy/pypy/issues/1853 +.. _2592: https://bitbucket.org/pypy/pypy/issues/2592 +.. _2590: https://bitbucket.org/pypy/pypy/issues/2590 +.. _2621: https://bitbucket.org/pypy/pypy/issues/2621 + +Highlights of the PyPy3.5 release (since 5.8 beta released June 2017) +====================================================================== + +* New features + + * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, + ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``, + ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject`` + * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support + * Support the new version of ``os.stat()`` on win32 + * Use ``stat3()`` on Posix + * Accept buffer objects as filenames, except for `oslistdir`` + * Make slices of array ``memoryview`` s usable as writable buffers if contiguous + * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded + * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version + * Ensure that ``mappingproxy`` is recognised as a mapping, not a sequence + * Enable PGO for CLang + * Rework ``cppyy`` packaging and rename the backend to ``_cppyy`` + * Support for libressl 2.5.4 + * Mirror CPython ``classmethod __reduce__`` which fixes pickling test + * Use utf-8 for ``readline`` history file + * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses + * Add async slot functions in cpyext + +* Bug Fixes + + * Try to make ``openssl`` CFFI bindings more general and future-proof + * Better support ``importlib`` by only listing built-in modules in ``sys.builtin`` + * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue 2579_ + * Fix for ``reversed(mapping object)`` issue 2601_ + * Fixing regression with non-started generator receiving non-``None``, should + always raise ``TypeError`` + * ``itertools.islice``: use same logic as CPython, fixes 2643_ + +* Performance improvements: + + * + +* The following features of Python 3.5 are not implemented yet in PyPy: + + * PEP 442: Safe object finalization + +.. _resolved: whatsnew-pypy2-5.9.0.html +.. _2579: https://bitbucket.org/pypy/pypy/issues/2579 +.. _2601: https://bitbucket.org/pypy/pypy/issues/2601 +.. _2643: https://bitbucket.org/pypy/pypy/issues/2643 +.. _CPython issue 29738: https://bugs.python.org/issue29738 + +Please update, and continue to help us make PyPy better. + +Cheers diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,75 +1,6 @@ -========================== -What's new in PyPy2.7 5.9+ -========================== - -.. this is a revision shortly after release-pypy2.7-v5.8.0 -.. startrev: 558bd00b3dd8 - -In previous versions of PyPy, ``instance.method`` would return always -the same bound method object, when gotten out of the same instance (as -far as ``is`` and ``id()`` can tell). CPython doesn't do that. Now -PyPy, like CPython, returns a different bound method object every time. -For ``type.method``, PyPy2 still returns always the same *unbound* -method object; CPython does it for built-in types but not for -user-defined types. - -.. branch: cffi-complex -.. branch: cffi-char16-char32 - -The two ``cffi-*`` branches are part of the upgrade to cffi 1.11. - -.. branch: ctypes_char_indexing - -Indexing into char* behaves differently than CPython - -.. branch: vmprof-0.4.8 - -Improve and fix issues with vmprof - -.. branch: issue-2592 - -CPyext PyListObject.pop must return the value - -.. branch: cpyext-hash_notimpl - -If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None - -.. branch: cppyy-packaging - -Renaming of ``cppyy`` to ``_cppyy``. -The former is now an external package installable with ``pip install cppyy``. - -.. branch: Enable_PGO_for_clang - -.. branch: nopax - -At the end of translation, run ``attr -q -s pax.flags -V m`` on -PAX-enabled systems on the produced binary. This seems necessary -because PyPy uses a JIT. - -.. branch: pypy_bytearray - -Improve ``bytearray`` performance (backported from py3.5) - -.. branch: gc-del-limit-growth - -Fix the bounds in the GC when allocating a lot of objects with finalizers, -fixes issue #2590 - -.. branch: arrays-force-less - -Small improvement to optimize list accesses with constant indexes better by -throwing away information about them less eagerly. - - -.. branch: getarrayitem-into-bridges - -More information is retained into a bridge: knowledge about the content of -arrays (at fixed indices) is stored in guards (and thus available at the -beginning of bridges). Also, some better feeding of information about known -fields of constant objects into bridges. - -.. branch: cpyext-leakchecking - -Add support for leakfinder in cpyext tests (disabled for now, due to too many -failures). +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:899e5245de1e diff --git a/pypy/doc/whatsnew-pypy2-5.9.0.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.9.0.rst @@ -0,0 +1,87 @@ +========================= +What's new in PyPy2.7 5.9 +========================= + +.. this is a revision shortly after release-pypy2.7-v5.8.0 +.. startrev: 558bd00b3dd8 + +In previous versions of PyPy, ``instance.method`` would return always +the same bound method object, when gotten out of the same instance (as +far as ``is`` and ``id()`` can tell). CPython doesn't do that. Now +PyPy, like CPython, returns a different bound method object every time. +For ``type.method``, PyPy2 still returns always the same *unbound* +method object; CPython does it for built-in types but not for +user-defined types. + +.. branch: cffi-complex +.. branch: cffi-char16-char32 + +The two ``cffi-*`` branches are part of the upgrade to cffi 1.11. + +.. branch: ctypes_char_indexing + +Indexing into char* behaves differently than CPython + +.. branch: vmprof-0.4.8 + +Improve and fix issues with vmprof + +.. branch: issue-2592 + +CPyext PyListObject.pop must return the value + +.. branch: cpyext-hash_notimpl + +If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None + +.. branch: cppyy-packaging + +Renaming of ``cppyy`` to ``_cppyy``. +The former is now an external package installable with ``pip install cppyy``. + +.. branch: Enable_PGO_for_clang + +.. branch: nopax + +At the end of translation, run ``attr -q -s pax.flags -V m`` on +PAX-enabled systems on the produced binary. This seems necessary +because PyPy uses a JIT. + +.. branch: pypy_bytearray + +Improve ``bytearray`` performance (backported from py3.5) + +.. branch: gc-del-limit-growth + +Fix the bounds in the GC when allocating a lot of objects with finalizers, +fixes issue #2590 + +.. branch: arrays-force-less + +Small improvement to optimize list accesses with constant indexes better by +throwing away information about them less eagerly. + + +.. branch: getarrayitem-into-bridges + +More information is retained into a bridge: knowledge about the content of +arrays (at fixed indices) is stored in guards (and thus available at the +beginning of bridges). Also, some better feeding of information about known +fields of constant objects into bridges. + +.. branch: cpyext-leakchecking + +Add support for leakfinder in cpyext tests (disabled for now, due to too many +failures). + +.. branch: pypy_swappedbytes + +Added ``_swappedbytes_`` support for ``ctypes.Structure`` + +.. branch: pycheck-macros + +Convert many Py*_Check cpyext functions into macros, like CPython. + +.. branch: py_ssize_t + +Explicitly use Py_ssize_t as the Signed type in pypy c-api diff --git a/pypy/doc/whatsnew-pypy3-5.9.0.rst b/pypy/doc/whatsnew-pypy3-5.9.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.9.0.rst @@ -0,0 +1,7 @@ +======================= +What's new in PyPy3 5.9 +======================= + +.. this is the revision after release-pypy3.5-5.8 +.. startrev: afbf09453369 + diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,7 +1,7 @@ ========================= -What's new in PyPy3 5.7+ +What's new in PyPy3 5.9+ ========================= -.. this is the revision after release-pypy3.3-5.7.x was branched -.. startrev: afbf09453369 +.. this is the revision after release-pypy3.5-5.9 +.. startrev: be41e3ac0a29 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -114,12 +114,15 @@ INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. -Abridged method (for -Ojit builds using Visual Studio 2008) ------------------------------------------------------------ +Abridged method (using Visual Studio 2008) +------------------------------------------ Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_59.zip +(for post-5.8 builds) with sha256 checksum +``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(for post-5.7.1 builds) with sha256 checksum +(to reproduce 5.8 builds) with sha256 checksum ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip (for 2.4 release and later) or @@ -135,8 +138,8 @@ Now you should be good to go. If you choose this method, you do not need to download/build anything else. -Nonabrided method (building from scratch) ------------------------------------------ +Nonabridged method (building from scratch) +------------------------------------------ If you want to, you can rebuild everything from scratch by continuing. @@ -209,17 +212,85 @@ The expat XML parser ~~~~~~~~~~~~~~~~~~~~ -Download the source code of expat on sourceforge: -https://github.com/libexpat/libexpat/releases and extract it in the base directory. -Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw`` -with Visual Studio; follow the instruction for converting the project files, -switch to the "Release" configuration, use the ``expat_static`` project, -reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for -the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes) +CPython compiles expat from source as part of the build. PyPy uses the same +code base, but expects to link to a static lib of expat. Here are instructions +to reproduce the static lib in version 2.2.4. -Then, copy the file ``win32\bin\release\libexpat.lib`` into -LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in -INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH. +Download the source code of expat: https://github.com/libexpat/libexpat. +``git checkout`` the proper tag, in this case ``R_2_2_4``. Run +``vcvars.bat`` to set up the visual compiler tools, and CD into the source +directory. Create a file ``stdbool.h`` with the content + +.. code-block:: c + + #pragma once + + #define false 0 + #define true 1 + + #define bool int + +and put it in a place on the ``INCLUDE`` path, or create it in the local +directory and add ``.`` to the ``INCLUDE`` path:: + + SET INCLUDE=%INCLUDE%;. + +Then compile all the ``*.c`` file into ``*.obj``:: + + cl.exe /nologo /MD /O2 *c /c + rem for debug + cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c + +You may need to move some variable declarations to the beginning of the +function, to be compliant with C89 standard. Here is the diff for version 2.2.4 + +.. code-block:: diff + + diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c + index 007aed0..a2dcaad 100644 + --- a/expat/lib/xmltok.c + +++ b/expat/lib/xmltok.c + @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), + /* Avoid copying partial characters (due to limited space). */ + const ptrdiff_t bytesAvailable = fromLim - *fromP; + const ptrdiff_t bytesStorable = toLim - *toP; + + const char * fromLimBefore; + + ptrdiff_t bytesToCopy; + if (bytesAvailable > bytesStorable) { + fromLim = *fromP + bytesStorable; + output_exhausted = true; + } + + /* Avoid copying partial characters (from incomplete input). */ + - const char * const fromLimBefore = fromLim; + + fromLimBefore = fromLim; + align_limit_to_full_utf8_characters(*fromP, &fromLim); + if (fromLim < fromLimBefore) { + input_incomplete = true; + } + + - const ptrdiff_t bytesToCopy = fromLim - *fromP; + + bytesToCopy = fromLim - *fromP; + memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); + *fromP += bytesToCopy; + *toP += bytesToCopy; + + +Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: + + cl /LD *.obj libexpat.def /Felibexpat.dll + rem for debug + rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll + + rem this will override the export library created in the step above + rem but tests do not need the export library, they load the dll dynamically + lib *.obj /out:libexpat.lib + +Then, copy + +- ``libexpat.lib`` into LIB +- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE +- ``libexpat.dll`` into PATH The OpenSSL library @@ -363,7 +434,7 @@ It is probably not too much work if the goal is only to get a translated PyPy executable, and to run all tests before translation. But you need to start somewhere, and you should start with some tests in -rpython/translator/c/test/, like ``test_standalone.py`` and +``rpython/translator/c/test/``, like ``test_standalone.py`` and ``test_newgc.py``: try to have them pass on top of CPython64/64. Keep in mind that this runs small translations, and some details may go @@ -373,7 +444,7 @@ should be something like ``long long``. What is more generally needed is to review all the C files in -rpython/translator/c/src for the word ``long``, because this means a +``rpython/translator/c/src`` for the word ``long``, because this means a 32-bit integer even on Win64. Replace it with ``Signed`` most of the times. You can replace one with the other without breaking anything on any other platform, so feel free to. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1442,7 +1442,7 @@ length = 1 return start, stop, step, length - def getindex_w(self, w_obj, w_exception, objdescr=None): + def getindex_w(self, w_obj, w_exception, objdescr=None, errmsg=None): """Return w_obj.__index__() as an RPython int. If w_exception is None, silently clamp in case of overflow; else raise w_exception. @@ -1452,8 +1452,10 @@ except OperationError as err: if objdescr is None or not err.match(self, self.w_TypeError): raise - raise oefmt(self.w_TypeError, "%s must be an integer, not %T", - objdescr, w_obj) + if errmsg is None: + errmsg = " must be an integer" + raise oefmt(self.w_TypeError, "%s%s, not %T", + objdescr, errmsg, w_obj) try: # allow_conversion=False it's not really necessary because the # return type of __index__ is already checked by space.index(), @@ -1629,12 +1631,15 @@ # return text_w(w_obj) or None return None if self.is_none(w_obj) else self.text_w(w_obj) + @specialize.argtype(1) def bytes_w(self, w_obj): """ Takes an application level :py:class:`bytes` (on PyPy2 this equals `str`) and returns a rpython byte string. """ + assert w_obj is not None return w_obj.str_w(self) + @specialize.argtype(1) def text_w(self, w_obj): """ PyPy2 takes either a :py:class:`str` and returns a rpython byte string, or it takes an :py:class:`unicode` @@ -1644,6 +1649,7 @@ On PyPy3 it takes a :py:class:`str` and it will return an utf-8 encoded rpython string. """ + assert w_obj is not None return w_obj.str_w(self) @not_rpython # tests only; should be replaced with bytes_w or text_w @@ -1686,12 +1692,14 @@ if len(string) != 1: raise oefmt(self.w_ValueError, "string must be of size 1") return string[0] - value = self.getindex_w(w_obj, None) + value = self.getindex_w(w_obj, None, "", + "an integer or string of size 1 is required") if not 0 <= value < 256: # this includes the OverflowError in case the long is too large raise oefmt(self.w_ValueError, "byte must be in range(0, 256)") return chr(value) + @specialize.argtype(1) def int_w(self, w_obj, allow_conversion=True): """ Unwrap an app-level int object into an interpret-level int. @@ -1704,26 +1712,35 @@ If allow_conversion=False, w_obj needs to be an app-level int or a subclass. """ + assert w_obj is not None return w_obj.int_w(self, allow_conversion) + @specialize.argtype(1) def int(self, w_obj): + assert w_obj is not None return w_obj.int(self) + @specialize.argtype(1) def uint_w(self, w_obj): + assert w_obj is not None return w_obj.uint_w(self) + @specialize.argtype(1) def bigint_w(self, w_obj, allow_conversion=True): """ Like int_w, but return a rlib.rbigint object and call __long__ if allow_conversion is True. """ + assert w_obj is not None return w_obj.bigint_w(self, allow_conversion) + @specialize.argtype(1) def float_w(self, w_obj, allow_conversion=True): """ Like int_w, but return an interp-level float and call __float__ if allow_conversion is True. """ + assert w_obj is not None return w_obj.float_w(self, allow_conversion) def realtext_w(self, w_obj): @@ -1733,7 +1750,9 @@ raise oefmt(self.w_TypeError, "argument must be a string") return self.bytes_w(w_obj) + @specialize.argtype(1) def unicode_w(self, w_obj): + assert w_obj is not None return w_obj.unicode_w(self) def unicode0_w(self, w_obj): @@ -1758,7 +1777,9 @@ # This is here mostly just for gateway.int_unwrapping_space_method(). return bool(self.int_w(w_obj)) + @specialize.argtype(1) def ord(self, w_obj): + assert w_obj is not None return w_obj.ord(self) # This is all interface for gateway.py. diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -419,3 +419,7 @@ def f(): return x assert f.__closure__[0].cell_contents is x + + def test_get_with_none_arg(self): + raises(TypeError, type.__dict__['__mro__'].__get__, None) + raises(TypeError, type.__dict__['__mro__'].__get__, None, None) diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -297,6 +297,8 @@ if (space.is_w(w_obj, space.w_None) and not space.is_w(w_cls, space.type(space.w_None))): #print self, w_obj, w_cls + if space.is_w(w_cls, space.w_None): + raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") return self else: try: diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.0" +VERSION = "1.11.1" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.1", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py --- a/pypy/module/_csv/interp_csv.py +++ b/pypy/module/_csv/interp_csv.py @@ -29,10 +29,15 @@ return default return space.is_true(w_src) -def _get_int(space, w_src, default): +def _get_int(space, w_src, default, attrname): if w_src is None: return default - return space.int_w(w_src) + try: + return space.int_w(w_src) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, '"%s" must be an int', attrname) + raise def _get_str(space, w_src, default, attrname): if w_src is None: @@ -100,7 +105,7 @@ dialect.escapechar = _get_char(space, w_escapechar, '\0', 'escapechar') dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator') dialect.quotechar = _get_char(space, w_quotechar, '"', 'quotechar') - tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL) + tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting') dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False) dialect.strict = _get_bool(space, w_strict, False) diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py --- a/pypy/module/_csv/test/test_dialect.py +++ b/pypy/module/_csv/test/test_dialect.py @@ -65,7 +65,8 @@ name = attempt[0] for value in attempt[1:]: kwargs = {name: value} - raises(TypeError, _csv.register_dialect, 'foo1', **kwargs) + exc_info = raises(TypeError, _csv.register_dialect, 'foo1', **kwargs) + assert name in exc_info.value.args[0] exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4) assert exc_info.value.args[0] == '"lineterminator" must be a string' diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py --- a/pypy/module/_multiprocessing/interp_win32.py +++ b/pypy/module/_multiprocessing/interp_win32.py @@ -109,6 +109,7 @@ raise wrap_windowserror(space, rwin32.lastSavedWindowsError()) def GetLastError(space): + """NOTE: don't use this. See issue #2658""" return space.newint(rwin32.GetLastError_saved()) # __________________________________________________________ diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -12,6 +12,7 @@ 'itertools', 'select', 'struct', 'binascii']} if sys.platform == 'win32': spaceconfig['usemodules'].append('_rawffi') + spaceconfig['usemodules'].append('_cffi_backend') else: spaceconfig['usemodules'].append('fcntl') @@ -39,6 +40,10 @@ class BaseConnectionTest(object): def test_connection(self): + import sys + # if not translated, for win32 + if not hasattr(sys, 'executable'): + sys.executable = 'from test_connection.py' rhandle, whandle = self.make_pair() whandle.send_bytes("abc") @@ -50,6 +55,10 @@ assert obj == obj2 def test_poll(self): + import sys + # if not translated, for win32 + if not hasattr(sys, 'executable'): + sys.executable = 'from test_connection.py' rhandle, whandle = self.make_pair() assert rhandle.poll() == False @@ -64,6 +73,10 @@ def test_read_into(self): import array, multiprocessing + import sys + # if not translated, for win32 + if not hasattr(sys, 'executable'): + sys.executable = 'from test_connection.py' rhandle, whandle = self.make_pair() obj = [1, 2.0, "hello"] @@ -81,6 +94,7 @@ } if sys.platform == 'win32': spaceconfig['usemodules'].append('_rawffi') + spaceconfig['usemodules'].append('_cffi_backend') def setup_class(cls): if sys.platform != "win32": @@ -109,6 +123,7 @@ } if sys.platform == 'win32': spaceconfig['usemodules'].append('_rawffi') + spaceconfig['usemodules'].append('_cffi_backend') else: spaceconfig['usemodules'].append('fcntl') diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,6 +1,6 @@ import sys from rpython.rlib.rstring import StringBuilder -from rpython.rlib.objectmodel import specialize, always_inline +from rpython.rlib.objectmodel import specialize, always_inline, r_dict from rpython.rlib import rfloat, runicode from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import oefmt @@ -42,6 +42,22 @@ ll_res.chars[i] = cast_primitive(UniChar, ch) return hlunicode(ll_res) +def slice_eq(a, b): + (ll_chars1, start1, length1, _) = a + (ll_chars2, start2, length2, _) = b + if length1 != length2: + return False + j = start2 + for i in range(start1, start1 + length1): + if ll_chars1[i] != ll_chars2[j]: + return False + j += 1 + return True + +def slice_hash(a): + (ll_chars, start, length, h) = a + return h + TYPE_UNKNOWN = 0 TYPE_STRING = 1 class JSONDecoder(object): @@ -55,7 +71,7 @@ self.ll_chars = rffi.str2charp(s) self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 - self.last_type = TYPE_UNKNOWN + self.cache = r_dict(slice_eq, slice_hash) def close(self): rffi.free_charp(self.ll_chars) @@ -248,19 +264,16 @@ def decode_object(self, i): start = i - w_dict = self.space.newdict() - # + i = self.skip_whitespace(i) if self.ll_chars[i] == '}': self.pos = i+1 - return w_dict - # + return self.space.newdict() + + d = {} while True: # parse a key: value - self.last_type = TYPE_UNKNOWN - w_name = self.decode_any(i) - if self.last_type != TYPE_STRING: - self._raise("Key name must be string for object starting at char %d", start) + name = self.decode_key(i) i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': @@ -269,13 +282,13 @@ i = self.skip_whitespace(i) # w_value = self.decode_any(i) - self.space.setitem(w_dict, w_name, w_value) + d[name] = w_value i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 if ch == '}': self.pos = i - return w_dict + return self._create_dict(d) elif ch == ',': pass elif ch == '\0': @@ -284,6 +297,9 @@ self._raise("Unexpected '%s' when decoding object (char %d)", ch, i-1) + def _create_dict(self, d): + from pypy.objspace.std.dictmultiobject import from_unicode_key_dict + return from_unicode_key_dict(self.space, d) def decode_string(self, i): start = i @@ -295,22 +311,23 @@ i += 1 bits |= ord(ch) if ch == '"': - if bits & 0x80: - # the 8th bit is set, it's an utf8 strnig - content_utf8 = self.getslice(start, i-1) - content_unicode = unicodehelper.decode_utf8(self.space, content_utf8) - else: - # ascii only, fast path (ascii is a strict subset of - # latin1, and we already checked that all the chars are < - # 128) - content_unicode = strslice2unicode_latin1(self.s, start, i-1) - self.last_type = TYPE_STRING self.pos = i - return self.space.newunicode(content_unicode) + return self.space.newunicode( + self._create_string(start, i - 1, bits)) elif ch == '\\' or ch < '\x20': self.pos = i-1 return self.decode_string_escaped(start) + def _create_string(self, start, end, bits): + if bits & 0x80: + # the 8th bit is set, it's an utf8 string + content_utf8 = self.getslice(start, end) + return unicodehelper.decode_utf8(self.space, content_utf8) + else: + # ascii only, fast path (ascii is a strict subset of + # latin1, and we already checked that all the chars are < + # 128) + return strslice2unicode_latin1(self.s, start, end) def decode_string_escaped(self, start): i = self.pos @@ -324,7 +341,6 @@ if ch == '"': content_utf8 = builder.build() content_unicode = unicodehelper.decode_utf8(self.space, content_utf8) - self.last_type = TYPE_STRING self.pos = i return self.space.newunicode(content_unicode) elif ch == '\\': @@ -387,6 +403,48 @@ lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00)) + def decode_key(self, i): + """ returns an unwrapped unicode """ + from rpython.rlib.rarithmetic import intmask + + i = self.skip_whitespace(i) + ll_chars = self.ll_chars + ch = ll_chars[i] + if ch != '"': + self._raise("Key name must be string at char %d", i) + i += 1 + + start = i + bits = 0 + strhash = ord(ll_chars[i]) << 7 + while True: + ch = ll_chars[i] + i += 1 + if ch == '"': + break + elif ch == '\\' or ch < '\x20': + self.pos = i-1 + return self.space.unicode_w(self.decode_string_escaped(start)) + strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i])) + bits |= ord(ch) + length = i - start - 1 + if length == 0: + strhash = -1 + else: + strhash ^= length + strhash = intmask(strhash) + self.pos = i + # check cache first: + key = (ll_chars, start, length, strhash) + try: + return self.cache[key] + except KeyError: + pass + res = self._create_string(start, i - 1, bits) + self.cache[key] = res + return res + + def loads(space, w_s): if space.isinstance_w(w_s, space.w_unicode): raise oefmt(space.w_TypeError, diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py --- a/pypy/module/_pypyjson/targetjson.py +++ b/pypy/module/_pypyjson/targetjson.py @@ -5,9 +5,15 @@ import time from pypy.interpreter.error import OperationError -from pypy.module._pypyjson.interp_decoder import loads +from pypy.module._pypyjson.interp_decoder import loads, JSONDecoder from rpython.rlib.objectmodel import specialize, dont_inline +def _create_dict(self, d): + w_res = W_Dict() + w_res.dictval = d + return w_res + +JSONDecoder._create_dict = _create_dict ## MSG = open('msg.json').read() @@ -65,10 +71,14 @@ def isinstance_w(self, w_x, w_type): return isinstance(w_x, w_type) - def str_w(self, w_x): + def bytes_w(self, w_x): assert isinstance(w_x, W_String) return w_x.strval + def unicode_w(self, w_x): + assert isinstance(w_x, W_Unicode) + return w_x.unival + @dont_inline def call_method(self, obj, name, arg): assert name == 'append' @@ -83,13 +93,17 @@ assert isinstance(key, W_Unicode) d.dictval[key.unival] = value - def wrapunicode(self, x): + def newunicode(self, x): return W_Unicode(x) - def wrapint(self, x): + def newtext(self, x): + return W_String(x) + newbytes = newtext + + def newint(self, x): return W_Int(x) - def wrapfloat(self, x): + def newfloat(self, x): return W_Float(x) @specialize.argtype(1) diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -10,7 +10,18 @@ assert dec.skip_whitespace(8) == len(s) dec.close() - +def test_decode_key(): + s1 = "123" * 100 + s = ' "%s" "%s" ' % (s1, s1) + dec = JSONDecoder('fake space', s) + assert dec.pos == 0 + x = dec.decode_key(0) + assert x == s1 + # check caching + y = dec.decode_key(dec.pos) + assert y == s1 + assert y is x + dec.close() class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} @@ -190,6 +201,12 @@ res = _pypyjson.loads(json) assert res == {u'a': u'\ud83d'} + def test_cache_keys(self): + import _pypyjson + json = '[{"a": 1}, {"a": 2}]' + res = _pypyjson.loads(json) + assert res == [{u'a': 1}, {u'a': 2}] + def test_tab_in_string_should_fail(self): import _pypyjson # http://json.org/JSON_checker/test/fail25.json @@ -226,7 +243,7 @@ ('{"spam":[42}', "Unexpected '}' when decoding array (char 11)"), ('["]', 'Unterminated string starting at char 1'), ('["spam":', "Unexpected ':' when decoding array (char 7)"), - ('[{]', "No JSON object could be decoded: unexpected ']' at char 2"), + ('[{]', "Key name must be string at char 2"), ] for inputtext, errmsg in test_cases: exc = raises(ValueError, _pypyjson.loads, inputtext) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -589,7 +589,12 @@ import _socket s = _socket.socket() assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0 - assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00' + ret = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) + if len(ret) == 1: + # win32 returns a byte-as-bool + assert ret == b'\x00' + else: + assert ret == b'\x00\x00' s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True) assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1 s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) @@ -599,7 +604,11 @@ import _socket s = _socket.socket() buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024) - assert buf == b'\x00' * 4 + if len(buf) == 1: + # win32 returns a byte-as-bool + assert buf == b'\x00' + else: + assert buf == b'\x00' * 4 raises(_socket.error, s.getsockopt, _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025) raises(_socket.error, s.getsockopt, diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -996,9 +996,6 @@ libssl_AUTHORITY_INFO_ACCESS_free(info) def _get_crl_dp(space, certificate): - if OPENSSL_VERSION_NUMBER >= 0x10001000: - # Calls x509v3_cache_extensions and sets up crldp - libssl_X509_check_ca(certificate) dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i( certificate, NID_crl_distribution_points, None, None)) if not dps: @@ -1020,8 +1017,7 @@ s_uri = rffi.charpsize2str(uri.c_data, length) cdp_w.append(space.newtext(s_uri)) finally: - if OPENSSL_VERSION_NUMBER < 0x10001000: - libssl_sk_DIST_POINT_free(dps) + libssl_CRL_DIST_POINTS_free(dps) return space.newtuple(cdp_w[:]) def checkwait(space, w_sock, writing): @@ -1319,7 +1315,7 @@ if not ctx: raise ssl_error(space, "failed to allocate SSL context") - rgc.add_memory_pressure(10 * 1024 * 1024) + rgc.add_memory_pressure(10 * 1024) self = space.allocate_instance(_SSLContext, w_subtype) self.ctx = ctx self.check_hostname = False diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -1,5 +1,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib.rvmprof import VMProfPlatformUnsupported +from rpython.translator.platform import CompilationError + class Module(MixedModule): """ @@ -29,3 +31,9 @@ import pypy.module._vmprof.interp_vmprof except VMProfPlatformUnsupported as e: pass +except CompilationError as e: + import sys + if sys.platform == 'win32': + pass + else: + raise diff --git a/pypy/module/_vmprof/conftest.py b/pypy/module/_vmprof/conftest.py --- a/pypy/module/_vmprof/conftest.py +++ b/pypy/module/_vmprof/conftest.py @@ -1,6 +1,8 @@ -import py, platform +import py, platform, sys def pytest_collect_directory(path, parent): if platform.machine() == 's390x': - py.test.skip("zarch tests skipped") + py.test.skip("_vmprof tests skipped") + if sys.platform == 'win32': + py.test.skip("_vmprof tests skipped") pytest_collect_file = pytest_collect_directory diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -40,6 +40,7 @@ from rpython.rlib import rawrefcount from rpython.rlib import rthread from rpython.rlib.debug import fatalerror_notb +from rpython.rlib import rstackovf from pypy.objspace.std.typeobject import W_TypeObject, find_best_base from pypy.module.cpyext.cparser import CTypeSpace @@ -128,6 +129,11 @@ Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE """.split() _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
