Author: Armin Rigo <[email protected]> Branch: unicode-utf8 Changeset: r92741:194830b5fc5a Date: 2017-10-12 18:37 +0200 http://bitbucket.org/pypy/pypy/changeset/194830b5fc5a/
Log: hg merge default diff too long, truncating to 2000 out of 16196 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -40,3 +40,7 @@ 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0 a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0 +03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 +d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 +03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 +84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -60,8 +60,8 @@ Wim Lavrijsen Eric van Riet Paap Richard Emslie + Remi Meier Alexander Schremmer - Remi Meier Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager @@ -102,6 +102,7 @@ Michael Foord Stephan Diehl Stefano Rivera + Jean-Paul Calderone Stefan Schwarzer Tomek Meka Valentino Volonghi @@ -110,14 +111,13 @@ Bob Ippolito Bruno Gola David Malcolm - Jean-Paul Calderone Squeaky Edd Barrett Timo Paulssen Marius Gedminas + Nicolas Truessel Alexandre Fayolle Simon Burton - Nicolas Truessel Martin Matusiak Laurence Tratt Wenzhu Man @@ -156,6 +156,7 @@ Stefan H. Muller Tim Felgentreff Eugene Oden + Dodan Mihai Jeff Terrace Henry Mason Vasily Kuznetsov @@ -182,11 +183,13 @@ Rocco Moretti Gintautas Miliauskas Lucian Branescu Mihaila + Mariano Anaya anatoly techtonik - Dodan Mihai Karl Bartel + Stefan Beyer Gabriel Lavoie Jared Grubb + Alecsandru Patrascu Olivier Dormond Wouter van Heyst Sebastian Pawluś @@ -194,6 +197,7 @@ Victor Stinner Andrews Medina Aaron Iles + [email protected] Toby Watson Daniel Patrick Stuart Williams @@ -204,6 +208,7 @@ Michael Cheng Mikael Schönenberg Stanislaw Halik + Mihnea Saracin Berkin Ilbeyi Gasper Zejn Faye Zhao @@ -214,14 +219,12 @@ Jonathan David Riehl Beatrice During Alex Perry - [email protected] Robert Zaremba Alan McIntyre Alexander Sedov Vaibhav Sood Reuben Cummings Attila Gobi - Alecsandru Patrascu Christopher Pope Tristan Arthur Christian Tismer @@ -243,7 +246,6 @@ Jacek Generowicz Sylvain Thenault Jakub Stasiak - Stefan Beyer Andrew Dalke Alejandro J. Cura Vladimir Kryachko @@ -275,6 +277,7 @@ Christoph Gerum Miguel de Val Borro Artur Lisiecki + afteryu Toni Mattis Laurens Van Houtven Bobby Impollonia @@ -305,6 +308,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Anthony Sottile Nate Bragg Ben Darnell Juan Francisco Cantero Hurtado @@ -325,12 +329,14 @@ Mike Bayer Rodrigo Araújo Daniil Yarancev + Min RK OlivierBlanvillain Jonas Pfannschmidt Zearin Andrey Churin Dan Crosta [email protected] + Stanisław Halik Julien Phalip Roman Podoliaka Eli Stevens diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py --- a/lib-python/2.7/ctypes/test/test_byteswap.py +++ b/lib-python/2.7/ctypes/test/test_byteswap.py @@ -23,7 +23,6 @@ setattr(bits, "i%s" % i, 1) dump(bits) - @xfail def test_endian_short(self): if sys.byteorder == "little": self.assertIs(c_short.__ctype_le__, c_short) @@ -51,7 +50,6 @@ self.assertEqual(bin(s), "3412") self.assertEqual(s.value, 0x1234) - @xfail def test_endian_int(self): if sys.byteorder == "little": self.assertIs(c_int.__ctype_le__, c_int) @@ -80,7 +78,6 @@ self.assertEqual(bin(s), "78563412") self.assertEqual(s.value, 0x12345678) - @xfail def test_endian_longlong(self): if sys.byteorder == "little": self.assertIs(c_longlong.__ctype_le__, c_longlong) @@ -109,7 +106,6 @@ self.assertEqual(bin(s), "EFCDAB9078563412") self.assertEqual(s.value, 0x1234567890ABCDEF) - @xfail def test_endian_float(self): if sys.byteorder == "little": self.assertIs(c_float.__ctype_le__, c_float) @@ -128,7 +124,6 @@ self.assertAlmostEqual(s.value, math.pi, 6) self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s)) - @xfail def test_endian_double(self): if sys.byteorder == "little": self.assertIs(c_double.__ctype_le__, c_double) @@ -156,7 +151,6 @@ self.assertIs(c_char.__ctype_le__, c_char) self.assertIs(c_char.__ctype_be__, c_char) - @xfail def test_struct_fields_1(self): if sys.byteorder == "little": base = BigEndianStructure @@ -192,7 +186,6 @@ pass self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)]) - @xfail def test_struct_struct(self): # nested structures with different byteorders @@ -221,7 +214,6 @@ self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) - @xfail def test_struct_fields_2(self): # standard packing in struct uses no alignment. # So, we have to align using pad bytes. @@ -245,7 +237,6 @@ s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) self.assertEqual(bin(s1), bin(s2)) - @xfail def test_unaligned_nonnative_struct_fields(self): if sys.byteorder == "little": base = BigEndianStructure diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py --- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py +++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py @@ -37,10 +37,7 @@ for typ in byteswapped_structures: ## print >> sys.stderr, typ.value self.assertEqual(typ.value.offset, 1) - try: - o = typ() - except NotImplementedError as e: - self.skipTest(str(e)) # for PyPy + o = typ() o.value = 4 self.assertEqual(o.value, 4) diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -203,7 +203,7 @@ f_locals local namespace seen by this frame f_restricted 0 or 1 if frame is in restricted execution mode f_trace tracing function for this frame, or None""" - return isinstance(object, types.FrameType) + return isinstance(object, (types.FrameType, types.FakeFrameType)) def iscode(object): """Return true if the object is a code object. diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py --- a/lib-python/2.7/multiprocessing/heap.py +++ b/lib-python/2.7/multiprocessing/heap.py @@ -62,7 +62,7 @@ self.size = size self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next()) self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - assert win32.GetLastError() == 0, 'tagname already in use' + #assert win32.GetLastError() == 0, 'tagname already in use' self._state = (self.size, self.name) def __getstate__(self): @@ -72,7 +72,7 @@ def __setstate__(self, state): self.size, self.name = self._state = state self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS + #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS else: diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -75,7 +75,7 @@ for i in range(256): buf[i] = i for i in range(n): - buf[ord(fromstr[i])] = tostr[i] + buf[ord(fromstr[i])] = ord(tostr[i]) return str(buf) diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py --- a/lib-python/2.7/types.py +++ b/lib-python/2.7/types.py @@ -71,6 +71,12 @@ FrameType = type(tb.tb_frame) del tb +# PyPy extension +try: + FakeFrameType = type(next(sys._current_frames().itervalues())) +except (AttributeError, StopIteration): + FakeFrameType = FrameType + SliceType = slice EllipsisType = type(Ellipsis) diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -142,6 +142,10 @@ ptr._buffer = tp._ffiarray(1, autofree=True) ptr._buffer[0] = obj._buffer result = ptr + elif isinstance(obj, bytes): + result = tp() + result._buffer[0] = buffer(obj)._pypy_raw_address() + return result elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -61,6 +61,54 @@ pyobj_container = GlobalPyobjContainer() +def swap_bytes(value, sizeof, typeof, get_or_set): + def swap_2(): + return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00) + + def swap_4(): + return ((value & 0x000000FF) << 24) | \ + ((value & 0x0000FF00) << 8) | \ + ((value & 0x00FF0000) >> 8) | \ + ((value >> 24) & 0xFF) + + def swap_8(): + return ((value & 0x00000000000000FFL) << 56) | \ + ((value & 0x000000000000FF00L) << 40) | \ + ((value & 0x0000000000FF0000L) << 24) | \ + ((value & 0x00000000FF000000L) << 8) | \ + ((value & 0x000000FF00000000L) >> 8) | \ + ((value & 0x0000FF0000000000L) >> 24) | \ + ((value & 0x00FF000000000000L) >> 40) | \ + ((value >> 56) & 0xFF) + + def swap_double_float(typ): + from struct import pack, unpack + if get_or_set == 'set': + if sys.byteorder == 'little': + st = pack(''.join(['>', typ]), value) + else: + st = pack(''.join(['<', typ]), value) + return unpack(typ, st)[0] + else: + packed = pack(typ, value) + if sys.byteorder == 'little': + st = unpack(''.join(['>', typ]), packed) + else: + st = unpack(''.join(['<', typ]), packed) + return st[0] + + if typeof in ('c_float', 'c_float_le', 'c_float_be'): + return swap_double_float('f') + elif typeof in ('c_double', 'c_double_le', 'c_double_be'): + return swap_double_float('d') + else: + if sizeof == 2: + return swap_2() + elif sizeof == 4: + return swap_4() + elif sizeof == 8: + return swap_8() + def generic_xxx_p_from_param(cls, value): if value is None: return cls(None) @@ -271,6 +319,31 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) result._as_ffi_pointer_ = _as_ffi_pointer_ + if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \ + and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'): + from sys import byteorder + if byteorder == 'big': + name += '_le' + swapped = self.__new__(self, name, bases, dct) + result.__ctype_le__ = swapped + result.__ctype_be__ = result + swapped.__ctype_be__ = result + swapped.__ctype_le__ = swapped + else: + name += '_be' + swapped = self.__new__(self, name, bases, dct) + result.__ctype_be__ = swapped + result.__ctype_le__ = result + swapped.__ctype_le__ = result + swapped.__ctype_be__ = swapped + from _ctypes import sizeof + def _getval(self): + return swap_bytes(self._buffer[0], sizeof(self), name, 'get') + def _setval(self, value): + d = result() + d.value = value + self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set') + swapped.value = property(_getval, _setval) return result diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -146,6 +146,7 @@ obj._buffer.__setattr__(self.name, arg) + def _set_shape(tp, rawfields, is_union=False): tp._ffistruct_ = _rawffi.Structure(rawfields, is_union, getattr(tp, '_pack_', 0)) @@ -240,19 +241,27 @@ res.__dict__['_index'] = -1 return res - class StructOrUnion(_CData): __metaclass__ = StructOrUnionMeta def __new__(cls, *args, **kwds): from _ctypes import union - self = super(_CData, cls).__new__(cls) - if ('_abstract_' in cls.__dict__ or cls is Structure + if ('_abstract_' in cls.__dict__ or cls is Structure or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - raise NotImplementedError("missing in PyPy: structure/union with " - "swapped (non-native) byte ordering") + fields = [None] * len(cls._fields_) + for i in range(len(cls._fields_)): + if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): + swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) + else: + swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) + if len(cls._fields_[i]) < 3: + fields[i] = (cls._fields_[i][0], swapped) + else: + fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) + names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) return self diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.0 +Version: 1.11.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.0" -__version_info__ = (1, 11, 0) +__version__ = "1.11.2" +__version_info__ = (1, 11, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -238,9 +238,9 @@ _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x) { if (sizeof(_cffi_wchar_t) == 2) - return _cffi_from_c_wchar_t(x); + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); else - return _cffi_from_c_wchar3216_t(x); + return _cffi_from_c_wchar3216_t((int)x); } _CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o) @@ -254,7 +254,7 @@ _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x) { if (sizeof(_cffi_wchar_t) == 4) - return _cffi_from_c_wchar_t(x); + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); else return _cffi_from_c_wchar3216_t(x); } diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -247,7 +247,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.0" + "\ncompiled with cffi version: 1.11.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py --- a/lib_pypy/pyrepl/historical_reader.py +++ b/lib_pypy/pyrepl/historical_reader.py @@ -17,7 +17,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from pyrepl import reader, commands +from pyrepl import reader, commands, input from pyrepl.reader import Reader as R isearch_keymap = tuple( @@ -214,7 +214,6 @@ isearch_forwards, isearch_backwards, operate_and_get_next]: self.commands[c.__name__] = c self.commands[c.__name__.replace('_', '-')] = c - from pyrepl import input self.isearch_trans = input.KeymapTranslator( isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -119,8 +119,15 @@ To run untranslated tests, you need the Boehm garbage collector libgc. -On Debian and Ubuntu, this is the command to install all build-time -dependencies:: +On recent Debian and Ubuntu (16.04 onwards), this is the command to install +all build-time dependencies:: + + apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \ + libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \ + tk-dev libgc-dev python-cffi \ + liblzma-dev libncursesw5-dev # these two only needed on PyPy3 + +On older Debian and Ubuntu (12.04-14.04):: apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \ libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \ @@ -142,12 +149,23 @@ xz-devel # For lzma on PyPy3. (XXX plus the SLES11 version of libgdbm-dev and tk-dev) -On Mac OS X, most of these build-time dependencies are installed alongside +On Mac OS X:: + +Most of these build-time dependencies are installed alongside the Developer Tools. However, note that in order for the installation to find them you may need to run:: xcode-select --install +An exception is OpenSSL, which is no longer provided with the operating +system. It can be obtained via Homebrew (with ``$ brew install openssl``), +but it will not be available on the system path by default. The easiest +way to enable it for building pypy is to set an environment variable:: + + export PKG_CONFIG_PATH=$(brew --prefix)/opt/openssl/lib/pkgconfig + +After setting this, translation (described next) will find the OpenSSL libs +as expected. Run the translation ------------------- @@ -180,18 +198,18 @@ entire pypy interpreter. This step is currently singe threaded, and RAM hungry. As part of this step, the chain creates a large number of C code files and a Makefile to compile them in a - directory controlled by the ``PYPY_USESSION_DIR`` environment variable. + directory controlled by the ``PYPY_USESSION_DIR`` environment variable. 2. Create an executable ``pypy-c`` by running the Makefile. This step can - utilize all possible cores on the machine. -3. Copy the needed binaries to the current directory. -4. Generate c-extension modules for any cffi-based stdlib modules. + utilize all possible cores on the machine. +3. Copy the needed binaries to the current directory. +4. Generate c-extension modules for any cffi-based stdlib modules. The resulting executable behaves mostly like a normal Python interpreter (see :doc:`cpython_differences`), and is ready for testing, for use as a base interpreter for a new virtualenv, or for packaging into a binary suitable for installation on another machine running the same OS as the build -machine. +machine. Note that step 4 is merely done as a convenience, any of the steps may be rerun without rerunning the previous steps. @@ -248,7 +266,7 @@ * PyPy 2.5.1 or earlier: normal users would see permission errors. Installers need to run ``pypy -c "import gdbm"`` and other similar - commands at install time; the exact list is in + commands at install time; the exact list is in :source:`pypy/tool/release/package.py <package.py>`. Users seeing a broken installation of PyPy can fix it after-the-fact if they have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -27,8 +27,8 @@ Wim Lavrijsen Eric van Riet Paap Richard Emslie + Remi Meier Alexander Schremmer - Remi Meier Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager @@ -69,6 +69,7 @@ Michael Foord Stephan Diehl Stefano Rivera + Jean-Paul Calderone Stefan Schwarzer Tomek Meka Valentino Volonghi @@ -77,14 +78,13 @@ Bob Ippolito Bruno Gola David Malcolm - Jean-Paul Calderone Squeaky Edd Barrett Timo Paulssen Marius Gedminas + Nicolas Truessel Alexandre Fayolle Simon Burton - Nicolas Truessel Martin Matusiak Laurence Tratt Wenzhu Man @@ -123,6 +123,7 @@ Stefan H. Muller Tim Felgentreff Eugene Oden + Dodan Mihai Jeff Terrace Henry Mason Vasily Kuznetsov @@ -149,11 +150,13 @@ Rocco Moretti Gintautas Miliauskas Lucian Branescu Mihaila + Mariano Anaya anatoly techtonik - Dodan Mihai Karl Bartel + Stefan Beyer Gabriel Lavoie Jared Grubb + Alecsandru Patrascu Olivier Dormond Wouter van Heyst Sebastian Pawluś @@ -161,6 +164,7 @@ Victor Stinner Andrews Medina Aaron Iles + [email protected] Toby Watson Daniel Patrick Stuart Williams @@ -171,6 +175,7 @@ Michael Cheng Mikael Schönenberg Stanislaw Halik + Mihnea Saracin Berkin Ilbeyi Gasper Zejn Faye Zhao @@ -181,14 +186,12 @@ Jonathan David Riehl Beatrice During Alex Perry - [email protected] Robert Zaremba Alan McIntyre Alexander Sedov Vaibhav Sood Reuben Cummings Attila Gobi - Alecsandru Patrascu Christopher Pope Tristan Arthur Christian Tismer @@ -210,7 +213,6 @@ Jacek Generowicz Sylvain Thenault Jakub Stasiak - Stefan Beyer Andrew Dalke Alejandro J. Cura Vladimir Kryachko @@ -242,6 +244,7 @@ Christoph Gerum Miguel de Val Borro Artur Lisiecki + afteryu Toni Mattis Laurens Van Houtven Bobby Impollonia @@ -272,6 +275,7 @@ Anna Katrina Dominguez Kim Jin Su Amber Brown + Anthony Sottile Nate Bragg Ben Darnell Juan Francisco Cantero Hurtado @@ -292,12 +296,14 @@ Mike Bayer Rodrigo Araújo Daniil Yarancev + Min RK OlivierBlanvillain Jonas Pfannschmidt Zearin Andrey Churin Dan Crosta [email protected] + Stanisław Halik Julien Phalip Roman Podoliaka Eli Stevens diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -429,7 +429,8 @@ * the ``__builtins__`` name is always referencing the ``__builtin__`` module, never a dictionary as it sometimes is in CPython. Assigning to - ``__builtins__`` has no effect. + ``__builtins__`` has no effect. (For usages of tools like + RestrictedPython, see `issue #2653`_.) * directly calling the internal magic methods of a few built-in types with invalid arguments may have a slightly different result. For @@ -535,7 +536,12 @@ or ``float`` subtypes. Currently PyPy does not support the ``__class__`` attribute assignment for any non heaptype subtype. +* In PyPy, module and class dictionaries are optimized under the assumption + that deleting attributes from them are rare. Because of this, e.g. + ``del foo.bar`` where ``foo`` is a module (or class) that contains the + function ``bar``, is significantly slower than CPython. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ - +.. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/ diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst release-v5.7.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst whatsnew-pypy2-5.6.0.rst @@ -36,6 +37,7 @@ .. toctree:: whatsnew-pypy3-head.rst + whatsnew-pypy3-5.9.0.rst whatsnew-pypy3-5.8.0.rst whatsnew-pypy3-5.7.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -240,9 +240,12 @@ **matplotlib** https://github.com/matplotlib/matplotlib - TODO: the tkagg backend does not work, which makes tests fail on downstream - projects like Pandas, SciPy. It uses id(obj) as a c-pointer to obj in - tkagg.py, which requires refactoring + Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of + matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the + tkagg backend can function. + + TODO: the matplotlib branch passes numpy arrays by value (copying all the + data), this proof-of-concept needs help to become completely compliant **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.9.0.rst @@ -0,0 +1,217 @@ +===================================== +PyPy2.7 and PyPy3.5 v5.9 dual release +===================================== + +The PyPy team is proud to release both PyPy2.7 v5.9 (an interpreter supporting +Python 2.7 syntax), and a beta-quality PyPy3.5 v5.9 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. Note that PyPy3.5 supports Linux 64bit only for now. + +This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and +PyPy3.5 includes the upstream stdlib version 3.5.3. + +NumPy and Pandas now work on PyPy2.7 (together with Cython 0.27.1). Issues +that appeared as excessive memory +use were cleared up and other incompatibilities were resolved. The C-API +compatibility layer does slow down code which crosses the python-c interface +often, we have ideas on how it could be improved, and still recommend +using pure python on PyPy or interfacing via CFFI_. Many other modules +based on C-API exentions now work on PyPy as well. + +Cython 0.27.1 (released very recently) supports more projects with PyPy, both +on PyPy2.7 and PyPy3.5 beta. Note version **0.27.1** is now the minimum +version that supports this version of PyPy, due to some interactions with +updated C-API interface code. + +We optimized the JSON parser for recurring string keys, which should decrease +memory use to 50% and increase parsing speed by up to 15% for large JSON files +with many repeating dictionary keys (which is quite common). + +CFFI_, which is part of the PyPy release, has been updated to 1.11.1, +improving an already great package for interfacing with C. CFFI now supports +complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has +improved support for callbacks. + +Please let us know if your use case is slow, we have ideas how to make things +faster but need real-world examples (not micro-benchmarks) of problematic code. + +Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from +CPython were ported to PyPy. Of course the bug fixes and performance enhancements +mentioned above are part of both PyPy2.7 and PyPy3.5 beta. + +As always, this release fixed many other issues and bugs raised by the +growing community of PyPy users. We strongly recommend updating. + +You can download the v5.9 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy 2.7 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.8 released June, 2017) +====================================================================================== + +See also issues that were resolved_ + +Note that these are also merged into PyPy 3.5 + +* New features and cleanups + + * Add support for ``PyFrozenSet_New``, ``PyObject_HashNotImplemented``, + ``PyObject_Print(NULL, ...)``, ``PyObject_RichCompareBool(a, a, ...)``, + ``PyType_IS_GC`` (does nothing), ``PyUnicode_FromFormat`` + * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible + * ``gcdump`` now reports largest object + * More complete support in the ``_curses`` CFFI module + * Add cPickle.Unpickler.find_global (issue 1853_) + * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set + * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()`` + * Update RevDB to version pypy2.7-v5.6.2 + * Previously, ``instance.method`` would return always the same bound method + object, when gotten from the same instance (as far as ``is`` and ``id()`` + can tell). CPython doesn't do that. Now PyPy, like CPython, returns a + different bound method object every time. For ``type.method``, PyPy2 still + returns always the same *unbound* method object; CPython does it for built-in + types but not for user-defined types + * Link to disable PaX protection for the JIT when needed + * Update build instructions and an rarely used Makefile + * Recreate support for using leakfinder in cpyext tests which had suffered + bit-rot, disable due to many false positives + * Add more functionality to ``sysconfig`` + * Added ``_swappedbytes_`` support for ``ctypes.Structure`` + * Better support the ``inspect`` module on ``frames`` + +* Bug Fixes + + * Fix issue 2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value + * Implement ``PyListOjbect.getstorage_copy`` + * Fix for ``reversed(dictproxy)`` issue 2601_ + * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_ + * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16 + * ``gc.get_objects`` now handles objects with finalizers more consistently + * Fixed memory leak in ``SSLContext.getpeercert`` returning validated + certificates and ``SSLContext.get_ca_certs(binary_mode=True)`` + (_get_crl_dp) `CPython issue 29738`_ + +* Performance improvements: + + * Improve performance of ``bytearray.extend`` by rewriting portions in app-level + * Optimize list accesses with constant indexes better by retaining more + information about them + * Add a jit driver for ``array.count`` and ``array.index`` + * Improve information retained in a bridge wrt ``array`` + * Move some dummy CAPI functions and ``Py*_Check`` functions from RPython into + pure C macros + * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap + all the ints + * Cache string keys that occur in JSON dicts, as they are likely to repeat + +* RPython improvements + + * Do not preallocate a RPython list if we only know an upper bound on its size + * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers + * Replace magical NOT RPYTHON comment with a decorator + * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5 + * Add ``memory_pressure`` for ``_SSLSocket`` objects + +* Degredations + + * Disable vmprof on win32, due to upstream changes that break the internal ``_vmprof`` module + +.. _here: cpython_differences.html +.. _1853: https://bitbucket.org/pypy/pypy/issues/1853 +.. _2592: https://bitbucket.org/pypy/pypy/issues/2592 +.. _2590: https://bitbucket.org/pypy/pypy/issues/2590 +.. _2621: https://bitbucket.org/pypy/pypy/issues/2621 + +Highlights of the PyPy3.5 release (since 5.8 beta released June 2017) +====================================================================== + +* New features + + * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, + ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``, + ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``, + ``PyGenObject``, ``PyUnicode_Substring``, ``PyLong_FromUnicodeObject`` + * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support + * Support the new version of ``os.stat()`` on win32 + * Use ``stat3()`` on Posix + * Accept buffer objects as filenames, except for `oslistdir`` + * Make slices of array ``memoryview`` s usable as writable buffers if contiguous + * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded + * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version + * Ensure that ``mappingproxy`` is recognised as a mapping, not a sequence + * Enable PGO for CLang + * Rework ``cppyy`` packaging and rename the backend to ``_cppyy`` + * Support for libressl 2.5.4 + * Mirror CPython ``classmethod __reduce__`` which fixes pickling test + * Use utf-8 for ``readline`` history file + * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses + * Add async slot functions in cpyext + +* Bug Fixes + + * Try to make ``openssl`` CFFI bindings more general and future-proof + * Better support ``importlib`` by only listing built-in modules in ``sys.builtin`` + * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue 2579_ + * Fix for ``reversed(mapping object)`` issue 2601_ + * Fixing regression with non-started generator receiving non-``None``, should + always raise ``TypeError`` + * ``itertools.islice``: use same logic as CPython, fixes 2643_ + +* Performance improvements: + + * + +* The following features of Python 3.5 are not implemented yet in PyPy: + + * PEP 442: Safe object finalization + +.. _resolved: whatsnew-pypy2-5.9.0.html +.. _2579: https://bitbucket.org/pypy/pypy/issues/2579 +.. _2601: https://bitbucket.org/pypy/pypy/issues/2601 +.. _2643: https://bitbucket.org/pypy/pypy/issues/2643 +.. _CPython issue 29738: https://bugs.python.org/issue29738 + +Please update, and continue to help us make PyPy better. + +Cheers diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,75 +1,12 @@ -========================== -What's new in PyPy2.7 5.9+ -========================== - -.. this is a revision shortly after release-pypy2.7-v5.8.0 -.. startrev: 558bd00b3dd8 - -In previous versions of PyPy, ``instance.method`` would return always -the same bound method object, when gotten out of the same instance (as -far as ``is`` and ``id()`` can tell). CPython doesn't do that. Now -PyPy, like CPython, returns a different bound method object every time. -For ``type.method``, PyPy2 still returns always the same *unbound* -method object; CPython does it for built-in types but not for -user-defined types. - -.. branch: cffi-complex -.. branch: cffi-char16-char32 - -The two ``cffi-*`` branches are part of the upgrade to cffi 1.11. - -.. branch: ctypes_char_indexing - -Indexing into char* behaves differently than CPython - -.. branch: vmprof-0.4.8 - -Improve and fix issues with vmprof - -.. branch: issue-2592 - -CPyext PyListObject.pop must return the value - -.. branch: cpyext-hash_notimpl - -If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None - -.. branch: cppyy-packaging - -Renaming of ``cppyy`` to ``_cppyy``. -The former is now an external package installable with ``pip install cppyy``. - -.. branch: Enable_PGO_for_clang - -.. branch: nopax - -At the end of translation, run ``attr -q -s pax.flags -V m`` on -PAX-enabled systems on the produced binary. This seems necessary -because PyPy uses a JIT. - -.. branch: pypy_bytearray - -Improve ``bytearray`` performance (backported from py3.5) - -.. branch: gc-del-limit-growth - -Fix the bounds in the GC when allocating a lot of objects with finalizers, -fixes issue #2590 - -.. branch: arrays-force-less - -Small improvement to optimize list accesses with constant indexes better by -throwing away information about them less eagerly. - - -.. branch: getarrayitem-into-bridges - -More information is retained into a bridge: knowledge about the content of -arrays (at fixed indices) is stored in guards (and thus available at the -beginning of bridges). Also, some better feeding of information about known -fields of constant objects into bridges. - -.. branch: cpyext-leakchecking - -Add support for leakfinder in cpyext tests (disabled for now, due to too many -failures). +=========================== +What's new in PyPy2.7 5.10+ +=========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + +.. branch: cppyy-packaging +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + diff --git a/pypy/doc/whatsnew-pypy2-5.9.0.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.9.0.rst @@ -0,0 +1,96 @@ +========================= +What's new in PyPy2.7 5.9 +========================= + +.. this is a revision shortly after release-pypy2.7-v5.8.0 +.. startrev: 558bd00b3dd8 + +In previous versions of PyPy, ``instance.method`` would return always +the same bound method object, when gotten out of the same instance (as +far as ``is`` and ``id()`` can tell). CPython doesn't do that. Now +PyPy, like CPython, returns a different bound method object every time. +For ``type.method``, PyPy2 still returns always the same *unbound* +method object; CPython does it for built-in types but not for +user-defined types. + +.. branch: cffi-complex +.. branch: cffi-char16-char32 + +The two ``cffi-*`` branches are part of the upgrade to cffi 1.11. + +.. branch: ctypes_char_indexing + +Indexing into char* behaves differently than CPython + +.. branch: vmprof-0.4.8 + +Improve and fix issues with vmprof + +.. branch: issue-2592 + +CPyext PyListObject.pop must return the value + +.. branch: cpyext-hash_notimpl + +If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None + +.. branch: cppyy-packaging + +Renaming of ``cppyy`` to ``_cppyy``. +The former is now an external package installable with ``pip install cppyy``. + +.. branch: Enable_PGO_for_clang + +.. branch: nopax + +At the end of translation, run ``attr -q -s pax.flags -V m`` on +PAX-enabled systems on the produced binary. This seems necessary +because PyPy uses a JIT. + +.. branch: pypy_bytearray + +Improve ``bytearray`` performance (backported from py3.5) + +.. branch: gc-del-limit-growth + +Fix the bounds in the GC when allocating a lot of objects with finalizers, +fixes issue #2590 + +.. branch: arrays-force-less + +Small improvement to optimize list accesses with constant indexes better by +throwing away information about them less eagerly. + + +.. branch: getarrayitem-into-bridges + +More information is retained into a bridge: knowledge about the content of +arrays (at fixed indices) is stored in guards (and thus available at the +beginning of bridges). Also, some better feeding of information about known +fields of constant objects into bridges. + +.. branch: cpyext-leakchecking + +Add support for leakfinder in cpyext tests (disabled for now, due to too many +failures). + +.. branch: pypy_swappedbytes + +Added ``_swappedbytes_`` support for ``ctypes.Structure`` + +.. branch: pycheck-macros + +Convert many Py*_Check cpyext functions into macros, like CPython. + +.. branch: py_ssize_t + +Explicitly use Py_ssize_t as the Signed type in pypy c-api + +.. branch: cpyext-jit + +Differentiate the code to call METH_NOARGS, METH_O and METH_VARARGS in cpyext: +this allows to write specialized code which is much faster than previous +completely generic version. Moreover, let the JIT to look inside the cpyext +module: the net result is that cpyext calls are up to 7x faster. However, this +is true only for very simple situations: in all real life code, we are still +much slower than CPython (more optimizations to come) diff --git a/pypy/doc/whatsnew-pypy3-5.9.0.rst b/pypy/doc/whatsnew-pypy3-5.9.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-5.9.0.rst @@ -0,0 +1,7 @@ +======================= +What's new in PyPy3 5.9 +======================= + +.. this is the revision after release-pypy3.5-5.8 +.. startrev: afbf09453369 + diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,7 +1,7 @@ ========================= -What's new in PyPy3 5.7+ +What's new in PyPy3 5.9+ ========================= -.. this is the revision after release-pypy3.3-5.7.x was branched -.. startrev: afbf09453369 +.. this is the revision after release-pypy3.5-5.9 +.. startrev: be41e3ac0a29 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -114,12 +114,15 @@ INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. -Abridged method (for -Ojit builds using Visual Studio 2008) ------------------------------------------------------------ +Abridged method (using Visual Studio 2008) +------------------------------------------ Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_59.zip +(for post-5.8 builds) with sha256 checksum +``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(for post-5.7.1 builds) with sha256 checksum +(to reproduce 5.8 builds) with sha256 checksum ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip (for 2.4 release and later) or @@ -135,8 +138,8 @@ Now you should be good to go. If you choose this method, you do not need to download/build anything else. -Nonabrided method (building from scratch) ------------------------------------------ +Nonabridged method (building from scratch) +------------------------------------------ If you want to, you can rebuild everything from scratch by continuing. @@ -209,17 +212,85 @@ The expat XML parser ~~~~~~~~~~~~~~~~~~~~ -Download the source code of expat on sourceforge: -https://github.com/libexpat/libexpat/releases and extract it in the base directory. -Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw`` -with Visual Studio; follow the instruction for converting the project files, -switch to the "Release" configuration, use the ``expat_static`` project, -reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for -the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes) +CPython compiles expat from source as part of the build. PyPy uses the same +code base, but expects to link to a static lib of expat. Here are instructions +to reproduce the static lib in version 2.2.4. -Then, copy the file ``win32\bin\release\libexpat.lib`` into -LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in -INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH. +Download the source code of expat: https://github.com/libexpat/libexpat. +``git checkout`` the proper tag, in this case ``R_2_2_4``. Run +``vcvars.bat`` to set up the visual compiler tools, and CD into the source +directory. Create a file ``stdbool.h`` with the content + +.. code-block:: c + + #pragma once + + #define false 0 + #define true 1 + + #define bool int + +and put it in a place on the ``INCLUDE`` path, or create it in the local +directory and add ``.`` to the ``INCLUDE`` path:: + + SET INCLUDE=%INCLUDE%;. + +Then compile all the ``*.c`` file into ``*.obj``:: + + cl.exe /nologo /MD /O2 *c /c + rem for debug + cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c + +You may need to move some variable declarations to the beginning of the +function, to be compliant with C89 standard. Here is the diff for version 2.2.4 + +.. code-block:: diff + + diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c + index 007aed0..a2dcaad 100644 + --- a/expat/lib/xmltok.c + +++ b/expat/lib/xmltok.c + @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), + /* Avoid copying partial characters (due to limited space). */ + const ptrdiff_t bytesAvailable = fromLim - *fromP; + const ptrdiff_t bytesStorable = toLim - *toP; + + const char * fromLimBefore; + + ptrdiff_t bytesToCopy; + if (bytesAvailable > bytesStorable) { + fromLim = *fromP + bytesStorable; + output_exhausted = true; + } + + /* Avoid copying partial characters (from incomplete input). */ + - const char * const fromLimBefore = fromLim; + + fromLimBefore = fromLim; + align_limit_to_full_utf8_characters(*fromP, &fromLim); + if (fromLim < fromLimBefore) { + input_incomplete = true; + } + + - const ptrdiff_t bytesToCopy = fromLim - *fromP; + + bytesToCopy = fromLim - *fromP; + memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); + *fromP += bytesToCopy; + *toP += bytesToCopy; + + +Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: + + cl /LD *.obj libexpat.def /Felibexpat.dll + rem for debug + rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll + + rem this will override the export library created in the step above + rem but tests do not need the export library, they load the dll dynamically + lib *.obj /out:libexpat.lib + +Then, copy + +- ``libexpat.lib`` into LIB +- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE +- ``libexpat.dll`` into PATH The OpenSSL library @@ -363,7 +434,7 @@ It is probably not too much work if the goal is only to get a translated PyPy executable, and to run all tests before translation. But you need to start somewhere, and you should start with some tests in -rpython/translator/c/test/, like ``test_standalone.py`` and +``rpython/translator/c/test/``, like ``test_standalone.py`` and ``test_newgc.py``: try to have them pass on top of CPython64/64. Keep in mind that this runs small translations, and some details may go @@ -373,7 +444,7 @@ should be something like ``long long``. What is more generally needed is to review all the C files in -rpython/translator/c/src for the word ``long``, because this means a +``rpython/translator/c/src`` for the word ``long``, because this means a 32-bit integer even on Win64. Replace it with ``Signed`` most of the times. You can replace one with the other without breaking anything on any other platform, so feel free to. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1445,7 +1445,7 @@ length = 1 return start, stop, step, length - def getindex_w(self, w_obj, w_exception, objdescr=None): + def getindex_w(self, w_obj, w_exception, objdescr=None, errmsg=None): """Return w_obj.__index__() as an RPython int. If w_exception is None, silently clamp in case of overflow; else raise w_exception. @@ -1455,8 +1455,10 @@ except OperationError as err: if objdescr is None or not err.match(self, self.w_TypeError): raise - raise oefmt(self.w_TypeError, "%s must be an integer, not %T", - objdescr, w_obj) + if errmsg is None: + errmsg = " must be an integer" + raise oefmt(self.w_TypeError, "%s%s, not %T", + objdescr, errmsg, w_obj) try: # allow_conversion=False it's not really necessary because the # return type of __index__ is already checked by space.index(), @@ -1632,12 +1634,15 @@ # return text_w(w_obj) or None return None if self.is_none(w_obj) else self.text_w(w_obj) + @specialize.argtype(1) def bytes_w(self, w_obj): """ Takes an application level :py:class:`bytes` (on PyPy2 this equals `str`) and returns a rpython byte string. """ + assert w_obj is not None return w_obj.str_w(self) + @specialize.argtype(1) def text_w(self, w_obj): """ PyPy2 takes either a :py:class:`str` and returns a rpython byte string, or it takes an :py:class:`unicode` @@ -1647,6 +1652,7 @@ On PyPy3 it takes a :py:class:`str` and it will return an utf-8 encoded rpython string. """ + assert w_obj is not None return w_obj.str_w(self) @not_rpython # tests only; should be replaced with bytes_w or text_w @@ -1689,12 +1695,14 @@ if len(string) != 1: raise oefmt(self.w_ValueError, "string must be of size 1") return string[0] - value = self.getindex_w(w_obj, None) + value = self.getindex_w(w_obj, None, "", + "an integer or string of size 1 is required") if not 0 <= value < 256: # this includes the OverflowError in case the long is too large raise oefmt(self.w_ValueError, "byte must be in range(0, 256)") return chr(value) + @specialize.argtype(1) def int_w(self, w_obj, allow_conversion=True): """ Unwrap an app-level int object into an interpret-level int. @@ -1707,26 +1715,35 @@ If allow_conversion=False, w_obj needs to be an app-level int or a subclass. """ + assert w_obj is not None return w_obj.int_w(self, allow_conversion) + @specialize.argtype(1) def int(self, w_obj): + assert w_obj is not None return w_obj.int(self) + @specialize.argtype(1) def uint_w(self, w_obj): + assert w_obj is not None return w_obj.uint_w(self) + @specialize.argtype(1) def bigint_w(self, w_obj, allow_conversion=True): """ Like int_w, but return a rlib.rbigint object and call __long__ if allow_conversion is True. """ + assert w_obj is not None return w_obj.bigint_w(self, allow_conversion) + @specialize.argtype(1) def float_w(self, w_obj, allow_conversion=True): """ Like int_w, but return an interp-level float and call __float__ if allow_conversion is True. """ + assert w_obj is not None return w_obj.float_w(self, allow_conversion) def realtext_w(self, w_obj): @@ -1739,6 +1756,7 @@ def utf8_w(self, w_obj): return w_obj.utf8_w(self) + @specialize.argtype(1) def unicode_w(self, w_obj): return self.utf8_w(w_obj).decode('utf8') @@ -1775,7 +1793,9 @@ # This is here mostly just for gateway.int_unwrapping_space_method(). return bool(self.int_w(w_obj)) + @specialize.argtype(1) def ord(self, w_obj): + assert w_obj is not None return w_obj.ord(self) # This is all interface for gateway.py. diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -419,3 +419,7 @@ def f(): return x assert f.__closure__[0].cell_contents is x + + def test_get_with_none_arg(self): + raises(TypeError, type.__dict__['__mro__'].__get__, None) + raises(TypeError, type.__dict__['__mro__'].__get__, None, None) diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -297,6 +297,8 @@ if (space.is_w(w_obj, space.w_None) and not space.is_w(w_cls, space.type(space.w_None))): #print self, w_obj, w_cls + if space.is_w(w_cls, space.w_None): + raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") return self else: try: diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.0" +VERSION = "1.11.2" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py --- a/pypy/module/_cffi_backend/cffi1_module.py +++ b/pypy/module/_cffi_backend/cffi1_module.py @@ -1,4 +1,5 @@ from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import jit from pypy.interpreter.error import oefmt from pypy.interpreter.module import Module @@ -15,7 +16,7 @@ INITFUNCPTR = lltype.Ptr(lltype.FuncType([rffi.VOIDPP], lltype.Void)) - [email protected]_look_inside def load_cffi1_module(space, name, path, initptr): # This is called from pypy.module.cpyext.api.load_extension_module() from pypy.module._cffi_backend.call_python import get_ll_cffi_call_python diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -156,10 +156,11 @@ class W_CTypePtrBase(W_CTypePtrOrArray): # base class for both pointers and pointers-to-functions - _attrs_ = ['is_void_ptr', 'is_voidchar_ptr'] - _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr'] + _attrs_ = ['is_void_ptr', 'is_voidchar_ptr', 'is_onebyte_ptr'] + _immutable_fields_ = ['is_void_ptr', 'is_voidchar_ptr', 'is_onebyte_ptr'] is_void_ptr = False is_voidchar_ptr = False + is_onebyte_ptr = False def convert_to_object(self, cdata): ptrdata = rffi.cast(rffi.CCHARPP, cdata)[0] @@ -179,12 +180,20 @@ if self.is_void_ptr or other.is_void_ptr: pass # cast from or to 'void *' elif self.is_voidchar_ptr or other.is_voidchar_ptr: - space = self.space - msg = ("implicit cast from '%s' to '%s' " - "will be forbidden in the future (check that the types " - "are as you expect; use an explicit ffi.cast() if they " - "are correct)" % (other.name, self.name)) - space.warn(space.newtext(msg), space.w_UserWarning) + # for backward compatibility, accept "char *" as either + # source of target. This is not what C does, though, + # so emit a warning that will eventually turn into an + # error. The warning is turned off if both types are + # pointers to single bytes. + if self.is_onebyte_ptr and other.is_onebyte_ptr: + pass # no warning + else: + space = self.space + msg = ("implicit cast from '%s' to '%s' " + "will be forbidden in the future (check that the types " + "are as you expect; use an explicit ffi.cast() if they " + "are correct)" % (other.name, self.name)) + space.warn(space.newtext(msg), space.w_UserWarning) else: raise self._convert_error("compatible pointer", w_ob) @@ -214,6 +223,7 @@ self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid) self.is_voidchar_ptr = (self.is_void_ptr or isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)) + self.is_onebyte_ptr = (ctitem.size == 1) W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem) def newp(self, w_init, allocator): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -2099,7 +2099,8 @@ if sys.platform.startswith("linux"): BWChar = new_primitive_type("wchar_t") assert sizeof(BWChar) == 4 - assert int(cast(BWChar, -1)) == -1 # signed, on linux + # wchar_t is often signed on Linux, but not always (e.g. on ARM) + assert int(cast(BWChar, -1)) in (-1, 4294967295) def test_char16(): BChar16 = new_primitive_type("char16_t") @@ -3903,9 +3904,11 @@ BCharP = new_pointer_type(new_primitive_type("char")) BIntP = new_pointer_type(new_primitive_type("int")) BVoidP = new_pointer_type(new_void_type()) + BUCharP = new_pointer_type(new_primitive_type("unsigned char")) z1 = cast(BCharP, 0) z2 = cast(BIntP, 0) z3 = cast(BVoidP, 0) + z4 = cast(BUCharP, 0) with warnings.catch_warnings(record=True) as w: newp(new_pointer_type(BIntP), z1) # warn assert len(w) == 1 @@ -3919,6 +3922,12 @@ assert len(w) == 2 newp(new_pointer_type(BIntP), z3) # fine assert len(w) == 2 + newp(new_pointer_type(BCharP), z4) # fine (ignore signedness here) + assert len(w) == 2 + newp(new_pointer_type(BUCharP), z1) # fine (ignore signedness here) + assert len(w) == 2 + newp(new_pointer_type(BUCharP), z3) # fine + assert len(w) == 2 # check that the warnings are associated with lines in this file assert w[1].lineno == w[0].lineno + 4 diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -66,20 +66,18 @@ "position %d from error handler out of bounds", newpos) w_replace = space.convert_to_w_unicode(w_replace) - return w_replace._utf8.decode('utf8'), newpos + replace = w_replace._utf8.decode('utf8') + if decode: + return replace, newpos + else: + return replace, None, newpos return call_errorhandler def make_decode_errorhandler(self, space): return self._make_errorhandler(space, True) def make_encode_errorhandler(self, space): - errorhandler = self._make_errorhandler(space, False) - def encode_call_errorhandler(errors, encoding, reason, input, startpos, - endpos): - replace, newpos = errorhandler(errors, encoding, reason, input, - startpos, endpos) - return replace, None, newpos - return encode_call_errorhandler + return self._make_errorhandler(space, False) def get_unicodedata_handler(self, space): if self.unicodedata_handler: diff --git a/pypy/module/_cppyy/__init__.py b/pypy/module/_cppyy/__init__.py --- a/pypy/module/_cppyy/__init__.py +++ b/pypy/module/_cppyy/__init__.py @@ -1,12 +1,10 @@ from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): - "This module provides runtime bindings to C++ code for which reflection\n\ - info has been generated. Current supported back-ends are Reflex and CINT.\n\ - See http://doc.pypy.org/en/latest/cppyy.html for full details." + "This module brigdes the cppyy frontend with its backend, through PyPy.\n\ + See http://cppyy.readthedocs.io/en/latest for full details." interpleveldefs = { - '_load_dictionary' : 'interp_cppyy.load_dictionary', '_resolve_name' : 'interp_cppyy.resolve_name', '_scope_byname' : 'interp_cppyy.scope_byname', '_template_byname' : 'interp_cppyy.template_byname', @@ -15,14 +13,13 @@ '_set_function_generator': 'interp_cppyy.set_function_generator', '_register_class' : 'interp_cppyy.register_class', '_get_nullptr' : 'interp_cppyy.get_nullptr', - 'CPPInstanceBase' : 'interp_cppyy.W_CPPInstance', + 'CPPClassBase' : 'interp_cppyy.W_CPPClass', 'addressof' : 'interp_cppyy.addressof', 'bind_object' : 'interp_cppyy.bind_object', } appleveldefs = { '_init_pythonify' : 'pythonify._init_pythonify', - 'load_reflection_info' : 'pythonify.load_reflection_info', 'add_pythonization' : 'pythonify.add_pythonization', 'Template' : 'pythonify.CPPTemplate', } diff --git a/pypy/module/_cppyy/backend/create_cppyy_package.py b/pypy/module/_cppyy/backend/create_cppyy_package.py deleted file mode 100755 --- a/pypy/module/_cppyy/backend/create_cppyy_package.py +++ /dev/null @@ -1,649 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function - -import os, sys -import argparse, re, shutil, tarfile, urllib2 - - -DEBUG_TESTBUILD = False - -TARBALL_CACHE_DIR = 'releases' - -ROOT_KEEP = ['build', 'cmake', 'config', 'core', 'etc', 'interpreter', - 'io', 'LICENSE', 'net', 'Makefile', 'CMakeLists.txt', 'math', - 'main'] # main only needed in more recent root b/c of rootcling -ROOT_CORE_KEEP = ['CMakeLists.txt', 'base', 'clib', 'clingutils', 'cont', - 'dictgen', 'foundation', 'lzma', 'macosx', 'meta', - 'metacling', 'metautils', 'rootcling_stage1', 'textinput', - 'thread', 'unix', 'utils', 'winnt', 'zip'] -ROOT_IO_KEEP = ['CMakeLists.txt', 'io', 'rootpcm'] -ROOT_NET_KEEP = ['CMakeLists.txt', 'net'] -ROOT_MATH_KEEP = ['CMakeLists.txt', 'mathcore'] -ROOT_ETC_KEEP = ['Makefile.arch', 'class.rules', 'cmake', 'dictpch', - 'gdb-backtrace.sh', 'gitinfo.txt', 'helgrind-root.supp', - 'hostcert.conf', 'system.plugins-ios', - 'valgrind-root-python.supp', 'valgrind-root.supp', 'vmc'] - -ROOT_EXPLICIT_REMOVE = ['core/base/v7', 'math/mathcore/v7', 'io/io/v7'] - - -ERR_RELEASE_NOT_FOUND = 2 - - -# -## CLI arguments -# -class ReleaseValidation(argparse.Action): - def __call__(self, parser, namespace, value, option_string=None): - if not re.match(r'6\.\d\d\.\d\d', value): - raise argparse.ArgumentTypeError( - "release number should of the form '6.dd.dd'") - setattr(namespace, self.dest, value) - return value - -parser = argparse.ArgumentParser( - description='Build PyPi package for cppyy containing the minimum of ROOT') -parser.add_argument('-r', '--release', type=str, nargs='?', - action=ReleaseValidation, help='ROOT release to use') - -args = parser.parse_args() - - -# -## ROOT source pull and cleansing -# -def clean_directory(directory, keeplist, trim_cmake=True): - removed_entries = [] - for entry in os.listdir(directory): - if entry[0] == '.' or entry in keeplist: - continue - removed_entries.append(entry) - entry = os.path.join(directory, entry) - print('now removing', entry) - if os.path.isdir(entry): - shutil.rmtree(entry) - else: - os.remove(entry) - - if not trim_cmake: - return - - # now take the removed entries out of the CMakeLists.txt - if removed_entries: - inp = os.path.join(directory, 'CMakeLists.txt') - print('trimming', inp) - outp = inp+'.new' - new_cml = open(outp, 'w') - for line in open(inp).readlines(): - if ('add_subdirectory' in line) or\ - ('COMMAND' in line and 'copy' in line) or\ - ('ROOT_ADD_TEST_SUBDIRECTORY' in line) or\ - ('install(DIRECTORY' in line): - for sub in removed_entries: - if sub in line: - line = '#'+line - break - new_cml.write(line) - new_cml.close() - os.rename(outp, inp) - else: - print('reusing existing %s/CMakeLists.txt' % (directory,)) - - -class ReleaseValidation(argparse.Action): - def __call__(self, parser, namespace, value, option_string=None): - if not re.match(r'6\.\d\d\.\d\d', value): - raise argparse.ArgumentTypeError( - "release number should of the form '6.dd.dd'") - setattr(namespace, self.dest, value) - return value - -parser = argparse.ArgumentParser( - description='Build PyPi package for cppyy containing the minimum of ROOT') -parser.add_argument('-r', '--release', type=str, nargs='?', - action=ReleaseValidation, help='ROOT release to use') - -args = parser.parse_args() - -if not os.path.exists(TARBALL_CACHE_DIR): - os.mkdir(TARBALL_CACHE_DIR) - -if args.release: - # use provided release - fn = 'root_v%s.source.tar.gz' % args.release - addr = 'https://root.cern.ch/download/'+fn - if not os.path.exists(os.path.join(TARBALL_CACHE_DIR, fn)): - try: - print('retrieving', fn) - resp = urllib2.urlopen(addr, fn) - out = open(os.path.join(TARBALL_CACHE_DIR, fn), 'wb') - out.write(resp.read()) - out.close() - except urllib2.HTTPError: - print('release %s not found' % args.release) - sys.exit(ERR_RELEASE_NOT_FOUND) - else: - print('reusing', fn, 'from local directory') -else: - print('provide release ... getting latest release is not yet implemented ...') - sys.exit(1) - # get latest and set fn, args.release, etc. - -# construct version for package -args.version = '' -testnext = False -for c in args.release: - if testnext: - testnext = False - if c == '0': - continue - if c == '.': - testnext = True - args.version += c -args.version += '.0' - -fn = os.path.join(TARBALL_CACHE_DIR, fn) -pkgdir = os.path.join('root-'+args.release) -if not os.path.exists(pkgdir): - print('now extracting', args.release) - tf = tarfile.TarFile.gzopen(fn) - tf.extractall() - tf.close() -else: - print('reusing existing directory', pkgdir) - -# remove everything except for the listed set of libraries -os.chdir(pkgdir) - -clean_directory(os.path.curdir, ROOT_KEEP) -clean_directory('core', ROOT_CORE_KEEP) -clean_directory('etc', ROOT_ETC_KEEP, trim_cmake=False) -clean_directory('io', ROOT_IO_KEEP) -clean_directory('math', ROOT_MATH_KEEP) -clean_directory('net', ROOT_NET_KEEP) - - -# trim main (only need rootcling) -print('trimming main') -for entry in os.listdir('main/src'): - if entry != 'rootcling.cxx': - os.remove('main/src/'+entry) -inp = 'main/CMakeLists.txt' -outp = inp+'.new' -new_cml = open(outp, 'w') -for line in open(inp).readlines(): - if ('ROOT_EXECUTABLE' in line or\ - 'SET_TARGET_PROPERTIES' in line) and\ - not 'rootcling' in line: - line = '#'+line - new_cml.write(line) -new_cml.close() -os.rename(outp, inp) - - -# remove afterimage and ftgl explicitly -print('trimming externals') -for cmf in ['AfterImage', 'FTGL']: - os.remove('cmake/modules/Find%s.cmake' % (cmf,)) -inp = 'cmake/modules/SearchInstalledSoftware.cmake' -outp = inp+'.new' -now_stripping = False -new_cml = open(outp, 'w') -for line in open(inp).readlines(): - if '#---Check for ftgl if needed' == line[0:28] or\ - '#---Check for AfterImage' == line[0:24]: - now_stripping = True - elif '#---Check' == line[0:9]: - now_stripping = False - if now_stripping: - line = '#'+line - new_cml.write(line) -new_cml.close() -os.rename(outp, inp) - -inp = 'cmake/modules/RootBuildOptions.cmake' -outp = inp+'.new' -new_cml = open(outp, 'w') -for line in open(inp).readlines(): - if 'ROOT_BUILD_OPTION(builtin_ftgl' in line or\ - 'ROOT_BUILD_OPTION(builtin_afterimage' in line: - line = '#'+line - new_cml.write(line) -new_cml.close() -os.rename(outp, inp) - - -# remove testing and examples -print('trimming testing') -inp = 'CMakeLists.txt' -outp = inp+'.new' -now_stripping = False -new_cml = open(outp, 'w') -for line in open(inp).readlines(): - if '#---Configure Testing using CTest' == line[0:33] or\ - '#---hsimple.root' == line[0:16]: - now_stripping = True - elif '#---Packaging' == line[0:13] or\ - '#---version' == line[0:11]: - now_stripping = False - if now_stripping: - line = '#'+line - new_cml.write(line) -new_cml.close() -os.rename(outp, inp) - -print('trimming RootCPack') -inp = 'cmake/modules/RootCPack.cmake' -outp = inp+'.new' -new_cml = open(outp, 'w') -for line in open(inp): - if 'README.txt' in line: - line = '#'+line - new_cml.write(line) -new_cml.close() -os.rename(outp, inp) - -# some more explicit removes: -for dir_to_remove in ROOT_EXPLICIT_REMOVE: - try: - shutil.rmtree(dir_to_remove) - except OSError: - pass - -# special fixes -inp = 'core/base/src/TVirtualPad.cxx' -outp = inp+'.new' -new_cml = open(outp, 'w') -for line in open(inp): - if '#include "X3DBuffer.h"' == line[0:22]: - line = """//#include "X3DBuffer.h" -typedef struct _x3d_sizeof_ { - int numPoints; - int numSegs; - int numPolys; -} Size3D; -""" - new_cml.write(line) -new_cml.close() _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
