Author: Armin Rigo <[email protected]> Branch: reverse-debugger Changeset: r86927:b13abdc2e1b3 Date: 2016-09-07 14:43 +0200 http://bitbucket.org/pypy/pypy/changeset/b13abdc2e1b3/
Log: hg merge default diff too long, truncating to 2000 out of 212749 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -27,3 +27,9 @@ 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 +68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 +68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 +77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0 +050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 +050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 +0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -74,6 +74,7 @@ Seo Sanghyeon Ronny Pfannschmidt Justin Peel + Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak @@ -117,7 +118,6 @@ Wenzhu Man John Witulski Laurence Tratt - Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -141,6 +141,7 @@ tav Taavi Burns Georg Brandl + Nicolas Truessel Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -211,6 +212,7 @@ Vaibhav Sood Alan McIntyre Alexander Sedov + [email protected] Attila Gobi Jasper.Schulz Christopher Pope @@ -221,6 +223,7 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + touilleMan Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -229,12 +232,14 @@ Gabriel Lukas Vacek Kunal Grover + Aaron Gallagher Andrew Dalke Sylvain Thenault Jakub Stasiak Nathan Taylor Vladimir Kryachko Omer Katz + Mark Williams Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -355,12 +360,15 @@ yasirs Michael Chermside Anna Ravencroft + pizi Andrey Churin Dan Crosta + Eli Stevens Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz + werat Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden diff --git a/_pytest/python.py b/_pytest/python.py --- a/_pytest/python.py +++ b/_pytest/python.py @@ -498,7 +498,10 @@ """ Collector for test methods. """ def collect(self): if hasinit(self.obj): - pytest.skip("class %s.%s with __init__ won't get collected" % ( + # XXX used to be skip(), but silently skipping classes + # XXX just because they have been written long ago is + # XXX imho a very, very, very bad idea + pytest.fail("class %s.%s with __init__ won't get collected" % ( self.obj.__module__, self.obj.__name__, )) diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -122,22 +122,24 @@ """Dummy method to let some easy_install packages that have optional C speedup components. """ + def customize(executable, flags): + command = compiler.executables[executable] + flags + setattr(compiler, executable, command) + if compiler.compiler_type == "unix": compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit']) compiler.shared_lib_extension = get_config_var('SO') if "CPPFLAGS" in os.environ: cppflags = shlex.split(os.environ["CPPFLAGS"]) - compiler.compiler.extend(cppflags) - compiler.compiler_so.extend(cppflags) - compiler.linker_so.extend(cppflags) + for executable in ('compiler', 'compiler_so', 'linker_so'): + customize(executable, cppflags) if "CFLAGS" in os.environ: cflags = shlex.split(os.environ["CFLAGS"]) - compiler.compiler.extend(cflags) - compiler.compiler_so.extend(cflags) - compiler.linker_so.extend(cflags) + for executable in ('compiler', 'compiler_so', 'linker_so'): + customize(executable, cflags) if "LDFLAGS" in os.environ: ldflags = shlex.split(os.environ["LDFLAGS"]) - compiler.linker_so.extend(ldflags) + customize('linker_so', ldflags) from sysconfig_cpython import ( diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -342,7 +342,7 @@ thisarg = cast(thisvalue, POINTER(POINTER(c_void_p))) keepalives, newargs, argtypes, outargs, errcheckargs = ( self._convert_args(argtypes, args[1:], kwargs)) - newargs.insert(0, thisvalue.value) + newargs.insert(0, thisarg) argtypes.insert(0, c_void_p) else: thisarg = None diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.8.0 +Version: 1.8.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.8.0" -__version_info__ = (1, 8, 0) +__version__ = "1.8.2" +__version_info__ = (1, 8, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -1,4 +1,20 @@ #define _CFFI_ + +/* We try to define Py_LIMITED_API before including Python.h. + + Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and + Py_REF_DEBUG are not defined. This is a best-effort approximation: + we can learn about Py_DEBUG from pyconfig.h, but it is unclear if + the same works for the other two macros. Py_DEBUG implies them, + but not the other way around. +*/ +#ifndef _CFFI_USE_EMBEDDING +# include <pyconfig.h> +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +#endif + #include <Python.h> #ifdef __cplusplus extern "C" { diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.8.0" + "\ncompiled with cffi version: 1.8.2" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -652,7 +652,7 @@ recompile(self, module_name, source, c_file=filename, call_c_compiler=False, **kwds) - def compile(self, tmpdir='.', verbose=0, target=None): + def compile(self, tmpdir='.', verbose=0, target=None, debug=None): """The 'target' argument gives the final file name of the compiled DLL. Use '*' to force distutils' choice, suitable for regular CPython C API modules. Use a file name ending in '.*' @@ -669,7 +669,7 @@ module_name, source, source_extension, kwds = self._assigned_source return recompile(self, module_name, source, tmpdir=tmpdir, target=target, source_extension=source_extension, - compiler_verbose=verbose, **kwds) + compiler_verbose=verbose, debug=debug, **kwds) def init_once(self, func, tag): # Read _init_once_cache[tag], which is either (False, lock) if diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -997,29 +997,43 @@ assert onerror is None # XXX not implemented return BType(source, error) + _weakref_cache_ref = None + def gcp(self, cdata, destructor): - BType = self.typeof(cdata) + if self._weakref_cache_ref is None: + import weakref + class MyRef(weakref.ref): + def __eq__(self, other): + myref = self() + return self is other or ( + myref is not None and myref is other()) + def __ne__(self, other): + return not (self == other) + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self()) + return self._hash + self._weakref_cache_ref = {}, MyRef + weak_cache, MyRef = self._weakref_cache_ref if destructor is None: - if not (hasattr(BType, '_gcp_type') and - BType._gcp_type is BType): + try: + del weak_cache[MyRef(cdata)] + except KeyError: raise TypeError("Can remove destructor only on a object " "previously returned by ffi.gc()") - cdata._destructor = None return None - try: - gcp_type = BType._gcp_type - except AttributeError: - class CTypesDataGcp(BType): - __slots__ = ['_orig', '_destructor'] - def __del__(self): - if self._destructor is not None: - self._destructor(self._orig) - gcp_type = BType._gcp_type = CTypesDataGcp - new_cdata = self.cast(gcp_type, cdata) - new_cdata._orig = cdata - new_cdata._destructor = destructor + def remove(k): + cdata, destructor = weak_cache.pop(k, (None, None)) + if destructor is not None: + destructor(cdata) + + new_cdata = self.cast(self.typeof(cdata), cdata) + assert new_cdata is not cdata + weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor) return new_cdata typeof = type diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -21,12 +21,12 @@ allsources.append(os.path.normpath(src)) return Extension(name=modname, sources=allsources, **kwds) -def compile(tmpdir, ext, compiler_verbose=0): +def compile(tmpdir, ext, compiler_verbose=0, debug=None): """Compile a C extension module using distutils.""" saved_environ = os.environ.copy() try: - outputfilename = _build(tmpdir, ext, compiler_verbose) + outputfilename = _build(tmpdir, ext, compiler_verbose, debug) outputfilename = os.path.abspath(outputfilename) finally: # workaround for a distutils bugs where some env vars can @@ -36,7 +36,7 @@ os.environ[key] = value return outputfilename -def _build(tmpdir, ext, compiler_verbose=0): +def _build(tmpdir, ext, compiler_verbose=0, debug=None): # XXX compact but horrible :-( from distutils.core import Distribution import distutils.errors, distutils.log @@ -44,6 +44,9 @@ dist = Distribution({'ext_modules': [ext]}) dist.parse_config_files() options = dist.get_option_dict('build_ext') + if debug is None: + debug = sys.flags.debug + options['debug'] = ('ffiplatform', debug) options['force'] = ('ffiplatform', True) options['build_lib'] = ('ffiplatform', tmpdir) options['build_temp'] = ('ffiplatform', tmpdir) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -275,8 +275,8 @@ def write_c_source_to_f(self, f, preamble): self._f = f prnt = self._prnt - if self.ffi._embedding is None: - prnt('#define Py_LIMITED_API') + if self.ffi._embedding is not None: + prnt('#define _CFFI_USE_EMBEDDING') # # first the '#include' (actually done by inlining the file's content) lines = self._rel_readlines('_cffi_include.h') @@ -1431,7 +1431,7 @@ def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, - compiler_verbose=1, target=None, **kwds): + compiler_verbose=1, target=None, debug=None, **kwds): if not isinstance(module_name, str): module_name = module_name.encode('ascii') if ffi._windows_unicode: @@ -1467,7 +1467,8 @@ if target != '*': _patch_for_target(patchlist, target) os.chdir(tmpdir) - outputfilename = ffiplatform.compile('.', ext, compiler_verbose) + outputfilename = ffiplatform.compile('.', ext, + compiler_verbose, debug) finally: os.chdir(cwd) _unpatch_meths(patchlist) diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -69,16 +69,36 @@ else: _add_c_module(dist, ffi, module_name, source, source_extension, kwds) +def _set_py_limited_api(Extension, kwds): + """ + Add py_limited_api to kwds if setuptools >= 26 is in use. + Do not alter the setting if it already exists. + Setuptools takes care of ignoring the flag on Python 2 and PyPy. + """ + if 'py_limited_api' not in kwds: + import setuptools + try: + setuptools_major_version = int(setuptools.__version__.partition('.')[0]) + if setuptools_major_version >= 26: + kwds['py_limited_api'] = True + except ValueError: # certain development versions of setuptools + # If we don't know the version number of setuptools, we + # try to set 'py_limited_api' anyway. At worst, we get a + # warning. + kwds['py_limited_api'] = True + return kwds def _add_c_module(dist, ffi, module_name, source, source_extension, kwds): from distutils.core import Extension - from distutils.command.build_ext import build_ext + # We are a setuptools extension. Need this build_ext for py_limited_api. + from setuptools.command.build_ext import build_ext from distutils.dir_util import mkpath from distutils import log from cffi import recompiler allsources = ['$PLACEHOLDER'] allsources.extend(kwds.pop('sources', [])) + kwds = _set_py_limited_api(Extension, kwds) ext = Extension(name=module_name, sources=allsources, **kwds) def make_mod(tmpdir, pre_run=None): diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -44,6 +44,7 @@ Seo Sanghyeon Ronny Pfannschmidt Justin Peel + Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak @@ -87,7 +88,6 @@ Wenzhu Man John Witulski Laurence Tratt - Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -111,6 +111,7 @@ tav Taavi Burns Georg Brandl + Nicolas Truessel Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -181,6 +182,7 @@ Vaibhav Sood Alan McIntyre Alexander Sedov + [email protected] Attila Gobi Jasper.Schulz Christopher Pope @@ -191,6 +193,7 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + touilleMan Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -199,12 +202,14 @@ Gabriel Lukas Vacek Kunal Grover + Aaron Gallagher Andrew Dalke Sylvain Thenault Jakub Stasiak Nathan Taylor Vladimir Kryachko Omer Katz + Mark Williams Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -325,9 +330,12 @@ yasirs Michael Chermside Anna Ravencroft + pizi Andrey Churin Dan Crosta + Eli Stevens Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz + werat diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-pypy2.7-v5.4.1.rst release-pypy2.7-v5.4.0.rst release-pypy2.7-v5.3.1.rst release-pypy2.7-v5.3.0.rst diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst --- a/pypy/doc/release-pypy2.7-v5.4.0.rst +++ b/pypy/doc/release-pypy2.7-v5.4.0.rst @@ -3,7 +3,8 @@ ============ We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3. -This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream +This new PyPy2.7 release includes incremental improvements to our C-API +compatability layer (cpyext), enabling us to pass over 99% of the upstream numpy `test suite`_. We updated built-in cffi_ support to version 1.8, which now supports the "limited API" mode for c-extensions on CPython >=3.2. @@ -12,9 +13,7 @@ support to OpenBSD and Dragon Fly BSD As always, this release fixed many issues and bugs raised by the -growing community of PyPy users. - -XXXXX MORE ??? +growing community of PyPy users. We strongly recommend updating. You can download the PyPy2.7 v5.4 release here: @@ -110,8 +109,8 @@ * (RPython) add `rposix_scandir` portably, needed for Python 3.5 - * Support for memoryview attributes (format, itemsize, ...) which also - adds support for `PyMemoryView_FromObject` + * Increased but incomplete support for memoryview attributes (format, + itemsize, ...) which also adds support for `PyMemoryView_FromObject` * Bug Fixes @@ -153,10 +152,6 @@ * Make `hash(-1)` return -2, as CPython does, and fix all the ancilary places this matters - * Issues reported with our previous release were resolved_ after - reports from users on our issue tracker at - https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy - * Fix `PyNumber_Check()` to behave more like CPython * (VMProf) Try hard to not miss any Python-level frame in the @@ -169,6 +164,10 @@ * Fix the mapdict cache for subclasses of builtin types that provide a dict + * Issues reported with our previous release were resolved_ after + reports from users on our issue tracker at + https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy + * Performance improvements: * Add a before_call()-like equivalent before a few operations like diff --git a/pypy/doc/release-pypy2.7-v5.4.1.rst b/pypy/doc/release-pypy2.7-v5.4.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy2.7-v5.4.1.rst @@ -0,0 +1,64 @@ +========== +PyPy 5.4.1 +========== + +We have released a bugfix for PyPy2.7-v5.4.0, released last week, +due to the following issues: + + * Update list of contributors in documentation and LICENSE file, + this was unfortunately left out of 5.4.0. My apologies to the new + contributors + + * Allow tests run with `-A` to find `libm.so` even if it is a script not a + dynamically loadable file + + * Bump `sys.setrecursionlimit()` when translating PyPy, for translating with CPython + + * Tweak a float comparison with 0 in `backendopt.inline` to avoid rounding errors + + * Fix for an issue where os.access() accepted a float for mode + + * Fix for and issue where `unicode.decode('utf8', 'custom_replace')` messed up + the last byte of a unicode string sometimes + + * Update built-in cffi_ to the soon-to-be-released 1.8.2 version + + * Explicitly detect that we found as-yet-unsupported OpenSSL 1.1, and crash + translation with a message asking for help porting it + + * Fix a regression where a PyBytesObject was forced (converted to a RPython + object) when not required, reported as issue #2395 + +Thanks to those who reported the issues. + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other +`dynamic languages`_ to see what RPython can do for them. + +This release supports: + + * **x86** machines on most common operating systems + (Linux 32/64, Mac OS X 64, Windows 32, OpenBSD, FreeBSD), + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _cffi: https://cffi.readthedocs.io +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://pypyjs.org + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,4 +5,11 @@ .. this is a revision shortly after release-pypy2.7-v5.4 .. startrev: 522736f816dc +.. branch: rpython-resync +Backport rpython changes made directly on the py3k and py3.5 branches. +.. branch: buffer-interface +Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews +in numpypy + + diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1454,6 +1454,9 @@ BUF_FORMAT = 0x0004 BUF_ND = 0x0008 BUF_STRIDES = 0x0010 | BUF_ND + BUF_C_CONTIGUOUS = 0x0020 | BUF_STRIDES + BUF_F_CONTIGUOUS = 0x0040 | BUF_STRIDES + BUF_ANY_CONTIGUOUS = 0x0080 | BUF_STRIDES BUF_INDIRECT = 0x0100 | BUF_STRIDES BUF_CONTIG_RO = BUF_ND diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py --- a/pypy/module/__builtin__/descriptor.py +++ b/pypy/module/__builtin__/descriptor.py @@ -23,6 +23,14 @@ self.w_objtype = w_type self.w_self = w_obj_or_type + def descr_repr(self, space): + if self.w_objtype is not None: + objtype_name = "<%s object>" % self.w_objtype.getname(space) + else: + objtype_name = 'NULL' + return space.wrap("<super: <class '%s'>, %s>" % ( + self.w_starttype.getname(space), objtype_name)) + def get(self, space, w_obj, w_type=None): if self.w_self is None or space.is_w(w_obj, space.w_None): return self @@ -84,7 +92,10 @@ 'super', __new__ = generic_new_descr(W_Super), __init__ = interp2app(W_Super.descr_init), + __repr__ = interp2app(W_Super.descr_repr), __thisclass__ = interp_attrproperty_w("w_starttype", W_Super), + __self__ = interp_attrproperty_w("w_self", W_Super), + __self_class__ = interp_attrproperty_w("w_objtype", W_Super), __getattribute__ = interp2app(W_Super.getattribute), __get__ = interp2app(W_Super.get), __doc__ = """\ diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py --- a/pypy/module/__builtin__/interp_classobj.py +++ b/pypy/module/__builtin__/interp_classobj.py @@ -38,6 +38,8 @@ class W_ClassObject(W_Root): + _immutable_fields_ = ['bases_w?[*]', 'w_dict?'] + def __init__(self, space, w_name, bases, w_dict): self.name = space.str_w(w_name) make_sure_not_resized(bases) @@ -75,6 +77,7 @@ "__bases__ items must be classes") self.bases_w = bases_w + @jit.unroll_safe def is_subclass_of(self, other): assert isinstance(other, W_ClassObject) if self is other: @@ -313,7 +316,7 @@ # This method ignores the instance dict and the __getattr__. # Returns None if not found. assert isinstance(name, str) - w_value = self.w_class.lookup(space, name) + w_value = jit.promote(self.w_class).lookup(space, name) if w_value is None: return None w_descr_get = space.lookup(w_value, '__get__') diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py --- a/pypy/module/__builtin__/test/test_descriptor.py +++ b/pypy/module/__builtin__/test/test_descriptor.py @@ -250,6 +250,24 @@ assert super(B, B()).__thisclass__ is B assert super(A, B()).__thisclass__ is A + def test_super_self_selfclass(self): + class A(object): + pass + class B(A): + pass + b = B() + assert super(A, b).__self__ is b + assert super(A).__self__ is None + assert super(A, b).__self_class__ is B + assert super(A).__self_class__ is None + + def test_super_repr(self): + class A(object): + def __repr__(self): + return super(A, self).__repr__() + '!' + assert repr(A()).endswith('>!') + assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>" + def test_property_docstring(self): assert property.__doc__.startswith('property') diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.8.0" +VERSION = "1.8.2" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -105,9 +105,6 @@ return True return False - def _check_only_one_argument_for_union(self, w_ob): - pass - def convert_from_object(self, cdata, w_ob): if not self._copy_from_same(cdata, w_ob): self.convert_struct_from_object(cdata, w_ob, optvarsize=-1) @@ -117,19 +114,24 @@ ) def convert_struct_from_object(self, cdata, w_ob, optvarsize): self.force_lazy_struct() - self._check_only_one_argument_for_union(w_ob) space = self.space if (space.isinstance_w(w_ob, space.w_list) or space.isinstance_w(w_ob, space.w_tuple)): lst_w = space.listview(w_ob) - if len(lst_w) > len(self._fields_list): - raise oefmt(space.w_ValueError, - "too many initializers for '%s' (got %d)", - self.name, len(lst_w)) - for i in range(len(lst_w)): - optvarsize = self._fields_list[i].write_v(cdata, lst_w[i], + j = 0 + for w_obj in lst_w: + try: + while (self._fields_list[j].flags & + W_CField.BF_IGNORE_IN_CTOR): + j += 1 + except IndexError: + raise oefmt(space.w_ValueError, + "too many initializers for '%s' (got %d)", + self.name, len(lst_w)) + optvarsize = self._fields_list[j].write_v(cdata, w_obj, optvarsize) + j += 1 return optvarsize elif space.isinstance_w(w_ob, space.w_dict): @@ -185,14 +187,6 @@ class W_CTypeUnion(W_CTypeStructOrUnion): kind = "union" - def _check_only_one_argument_for_union(self, w_ob): - space = self.space - n = space.int_w(space.len(w_ob)) - if n > 1: - raise oefmt(space.w_ValueError, - "initializer for '%s': %d items given, but only one " - "supported (use a dict if needed)", self.name, n) - class W_CField(W_Root): _immutable_ = True @@ -200,18 +194,21 @@ BS_REGULAR = -1 BS_EMPTY_ARRAY = -2 - def __init__(self, ctype, offset, bitshift, bitsize): + BF_IGNORE_IN_CTOR = 0x01 + + def __init__(self, ctype, offset, bitshift, bitsize, flags): self.ctype = ctype self.offset = offset self.bitshift = bitshift # >= 0: bitshift; or BS_REGULAR/BS_EMPTY_ARRAY self.bitsize = bitsize + self.flags = flags # BF_xxx def is_bitfield(self): return self.bitshift >= 0 - def make_shifted(self, offset): + def make_shifted(self, offset, fflags): return W_CField(self.ctype, offset + self.offset, - self.bitshift, self.bitsize) + self.bitshift, self.bitsize, self.flags | fflags) def read(self, cdata): cdata = rffi.ptradd(cdata, self.offset) @@ -341,5 +338,6 @@ offset = interp_attrproperty('offset', W_CField), bitshift = interp_attrproperty('bitshift', W_CField), bitsize = interp_attrproperty('bitsize', W_CField), + flags = interp_attrproperty('flags', W_CField), ) W_CField.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -345,6 +345,11 @@ if alignment < falign and do_align: alignment = falign # + if is_union and i > 0: + fflags = ctypestruct.W_CField.BF_IGNORE_IN_CTOR + else: + fflags = 0 + # if fbitsize < 0: # not a bitfield: common case @@ -372,7 +377,7 @@ for name, srcfld in ftype._fields_dict.items(): srcfield2names[srcfld] = name for srcfld in ftype._fields_list: - fld = srcfld.make_shifted(boffset // 8) + fld = srcfld.make_shifted(boffset // 8, fflags) fields_list.append(fld) try: fields_dict[srcfield2names[srcfld]] = fld @@ -382,7 +387,8 @@ w_ctype._custom_field_pos = True else: # a regular field - fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1) + fld = ctypestruct.W_CField(ftype, boffset // 8, bs_flag, -1, + fflags) fields_list.append(fld) fields_dict[fname] = fld @@ -489,7 +495,7 @@ bitshift = 8 * ftype.size - fbitsize- bitshift fld = ctypestruct.W_CField(ftype, field_offset_bytes, - bitshift, fbitsize) + bitshift, fbitsize, fflags) fields_list.append(fld) fields_dict[fname] = fld diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.8.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.8.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -2525,6 +2525,25 @@ assert d[2][1].bitshift == -1 assert d[2][1].bitsize == -1 +def test_nested_anonymous_struct_2(): + BInt = new_primitive_type("int") + BStruct = new_struct_type("struct foo") + BInnerUnion = new_union_type("union bar") + complete_struct_or_union(BInnerUnion, [('a1', BInt, -1), + ('a2', BInt, -1)]) + complete_struct_or_union(BStruct, [('b1', BInt, -1), + ('', BInnerUnion, -1), + ('b2', BInt, -1)]) + assert sizeof(BInnerUnion) == sizeof(BInt) + assert sizeof(BStruct) == sizeof(BInt) * 3 + fields = [(name, fld.offset, fld.flags) for (name, fld) in BStruct.fields] + assert fields == [ + ('b1', 0 * sizeof(BInt), 0), + ('a1', 1 * sizeof(BInt), 0), + ('a2', 1 * sizeof(BInt), 1), + ('b2', 2 * sizeof(BInt), 0), + ] + def test_sizeof_union(): # a union has the largest alignment of its members, and a total size # that is the largest of its items *possibly further aligned* if diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py --- a/pypy/module/_sre/__init__.py +++ b/pypy/module/_sre/__init__.py @@ -1,4 +1,4 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): @@ -7,7 +7,7 @@ interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', - 'MAGIC': 'space.wrap(interp_sre.MAGIC)', + 'MAGIC': 'space.newint(20031017)', 'MAXREPEAT': 'space.wrap(interp_sre.MAXREPEAT)', 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -14,7 +14,7 @@ # Constants and exposed functions from rpython.rlib.rsre import rsre_core -from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db +from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db @unwrap_spec(char_ord=int, flags=int) diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -597,6 +597,18 @@ def getlength(self): return self.array.len * self.array.itemsize + def getformat(self): + return self.array.typecode + + def getitemsize(self): + return self.array.itemsize + + def getndim(self): + return 1 + + def getstrides(self): + return [self.getitemsize()] + def getitem(self, index): array = self.array data = array._charbuf_start() diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -122,7 +122,7 @@ METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER -Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES +Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS """.split() for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) @@ -645,6 +645,9 @@ ('format', rffi.CCHARP), ('shape', Py_ssize_tP), ('strides', Py_ssize_tP), + ('_format', rffi.UCHAR), + ('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)), + ('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)), ('suboffsets', Py_ssize_tP), #('smalltable', rffi.CFixedArray(Py_ssize_t, 2)), ('internal', rffi.VOIDP) @@ -977,8 +980,10 @@ py_type_ready(space, get_capsule_type()) INIT_FUNCTIONS.append(init_types) from pypy.module.posix.interp_posix import add_fork_hook - reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], lltype.Void, - compilation_info=eci) + _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [], + lltype.Void, compilation_info=eci) + def reinit_tls(space): + _reinit_tls() add_fork_hook('child', reinit_tls) def init_function(func): diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -1,8 +1,9 @@ from pypy.interpreter.error import oefmt from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER) -from pypy.module.cpyext.pyobject import PyObject + cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER, Py_ssize_tP) +from pypy.module.cpyext.pyobject import PyObject, make_ref, incref @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyObject_CheckBuffer(space, pyobj): @@ -33,13 +34,82 @@ raise an error if the object can't support a simpler view of its memory. 0 is returned on success and -1 on error.""" - raise oefmt(space.w_TypeError, - "PyPy does not yet implement the new buffer interface") + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + except ValueError: + raise BufferError("could not create buffer from object") + view.c_len = buf.getlength() + view.c_obj = make_ref(space, w_obj) + ndim = buf.getndim() + view.c_itemsize = buf.getitemsize() + rffi.setintfield(view, 'c_readonly', int(buf.readonly)) + rffi.setintfield(view, 'c_ndim', ndim) + view.c_format = rffi.str2charp(buf.getformat()) + view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw') + view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw') + shape = buf.getshape() + strides = buf.getstrides() + for i in range(ndim): + view.c_shape[i] = shape[i] + view.c_strides[i] = strides[i] + view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) + view.c_internal = lltype.nullptr(rffi.VOIDP.TO) + return 0 + +def _IsFortranContiguous(view): + ndim = widen(view.c_ndim) + if ndim == 0: + return 1 + if not view.c_strides: + return ndim == 1 + sd = view.c_itemsize + if ndim == 1: + return view.c_shape[0] == 1 or sd == view.c_strides[0] + for i in range(view.c_ndim): + dim = view.c_shape[i] + if dim == 0: + return 1 + if view.c_strides[i] != sd: + return 0 + sd *= dim + return 1 + +def _IsCContiguous(view): + ndim = widen(view.c_ndim) + if ndim == 0: + return 1 + if not view.c_strides: + return ndim == 1 + sd = view.c_itemsize + if ndim == 1: + return view.c_shape[0] == 1 or sd == view.c_strides[0] + for i in range(ndim - 1, -1, -1): + dim = view.c_shape[i] + if dim == 0: + return 1 + if view.c_strides[i] != sd: + return 0 + sd *= dim + return 1 + @cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real, error=CANNOT_FAIL) -def PyBuffer_IsContiguous(space, view, fortran): +def PyBuffer_IsContiguous(space, view, fort): """Return 1 if the memory defined by the view is C-style (fortran is 'C') or Fortran-style (fortran is 'F') contiguous or either one (fortran is 'A'). Return 0 otherwise.""" - # PyPy only supports contiguous Py_buffers for now. - return 1 + # traverse the strides, checking for consistent stride increases from + # right-to-left (c) or left-to-right (fortran). Copied from cpython + if not view.c_suboffsets: + return 0 + if (fort == 'C'): + return _IsCContiguous(view) + elif (fort == 'F'): + return _IsFortranContiguous(view) + elif (fort == 'A'): + return (_IsCContiguous(view) or _IsFortranContiguous(view)) + return 0 + + diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -29,19 +29,17 @@ ## Solution ## -------- ## -## PyBytesObject contains two additional members: the ob_size and a pointer to a -## char ob_sval; it may be NULL. +## PyBytesObject contains two additional members: the ob_size and an array +## char ob_sval which holds a \x0 terminated string. ## ## - A string allocated by pypy will be converted into a PyBytesObject with a -## NULL buffer. The first time PyString_AsString() is called, memory is -## allocated (with flavor='raw') and content is copied. +## buffer holding \x0. The first time PyString_AsString() is called, the +## PyStringObject is reallocated, and the string copied into the buffer. The +## ob_size reflects the length of the string. ## ## - A string allocated with PyString_FromStringAndSize(NULL, size) will ## allocate a PyBytesObject structure, and a buffer with the specified -## size+1, but the reference won't be stored in the global map; there is no -## corresponding object in pypy. When from_ref() or Py_INCREF() is called, -## the pypy string is created, and added to the global map of tracked -## objects. The buffer is then supposed to be immutable. +## size+1, as part of the object. The buffer is then supposed to be immutable. ## ##- A buffer obtained from PyString_AS_STRING() could be mutable iff ## there is no corresponding pypy object for the string @@ -156,9 +154,6 @@ "expected string or Unicode object, %T found", from_ref(space, ref)) ref_str = rffi.cast(PyBytesObject, ref) - if not pyobj_has_w_obj(ref): - # XXX Force the ref? - bytes_realize(space, ref) return ref_str.c_ob_sval @cpython_api([rffi.VOIDP], rffi.CCHARP, error=0) diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py --- a/pypy/module/cpyext/import_.py +++ b/pypy/module/cpyext/import_.py @@ -123,5 +123,4 @@ pathname = code.co_filename w_mod = importing.add_module(space, w_name) space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname)) - importing.exec_code_module(space, w_mod, code) - return w_mod + return importing.exec_code_module(space, w_mod, code, w_name) diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -142,7 +142,8 @@ typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *); typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); -/* Py3k buffer interface */ +/* Py3k buffer interface, adapted for PyPy */ +#define Py_MAX_NDIMS 32 typedef struct bufferinfo { void *buf; PyObject *obj; /* owned reference */ @@ -156,12 +157,14 @@ char *format; Py_ssize_t *shape; Py_ssize_t *strides; - Py_ssize_t *suboffsets; - + Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/ + unsigned char _format; + Py_ssize_t _strides[Py_MAX_NDIMS]; + Py_ssize_t _shape[Py_MAX_NDIMS]; /* static store for shape and strides of mono-dimensional buffers. */ /* Py_ssize_t smalltable[2]; */ - void *internal; + void *internal; /* always NULL for app-level objects */ } Py_buffer; diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,8 +29,8 @@ #define PY_VERSION "2.7.10" /* PyPy version as a string */ -#define PYPY_VERSION "5.4.0" -#define PYPY_VERSION_NUM 0x05040000 +#define PYPY_VERSION "5.5.0-alpha0" +#define PYPY_VERSION_NUM 0x05050000 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,7 +1,8 @@ from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL, - build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject -from rpython.rtyper.lltypesystem import lltype + Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP) +from pypy.module.cpyext.pyobject import PyObject, make_ref, incref +from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.objspace.std.memoryobject import W_MemoryView PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView", "w_memoryview") @@ -12,6 +13,7 @@ @cpython_api([PyObject], PyObject) def PyMemoryView_GET_BASE(space, w_obj): # return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER + # XXX needed for numpy on py3k raise NotImplementedError('PyMemoryView_GET_BUFFER') @cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL) @@ -20,21 +22,35 @@ object. The object must be a memoryview instance; this macro doesn't check its type, you must do it yourself or you will risk crashes.""" view = lltype.malloc(Py_buffer, flavor='raw', zero=True) - # TODO - fill in fields - ''' - view.c_buf = buf - view.c_len = length - view.c_obj = obj - Py_IncRef(space, obj) - view.c_itemsize = 1 - rffi.setintfield(view, 'c_readonly', readonly) - rffi.setintfield(view, 'c_ndim', 0) - view.c_format = lltype.nullptr(rffi.CCHARP.TO) - view.c_shape = lltype.nullptr(Py_ssize_tP.TO) - view.c_strides = lltype.nullptr(Py_ssize_tP.TO) + if not isinstance(w_obj, W_MemoryView): + return view + ndim = w_obj.buf.getndim() + if ndim >= Py_MAX_NDIMS: + # XXX warn? + return view + try: + view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address()) + view.c_obj = make_ref(space, w_obj) + rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly) + isstr = False + except ValueError: + w_s = w_obj.descr_tobytes(space) + view.c_obj = make_ref(space, w_s) + rffi.setintfield(view, 'c_readonly', 1) + isstr = True + view.c_len = w_obj.getlength() + view.c_itemsize = w_obj.buf.getitemsize() + rffi.setintfield(view, 'c_ndim', ndim) + view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat()) + view.c_format = rffi.cast(rffi.CCHARP, view.c__format) + view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape) + view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides) + shape = w_obj.buf.getshape() + strides = w_obj.buf.getstrides() + for i in range(ndim): + view.c_shape[i] = shape[i] + view.c_strides[i] = strides[i] view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) view.c_internal = lltype.nullptr(rffi.VOIDP.TO) - ''' return view - diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -508,10 +508,9 @@ @cpython_api([lltype.Ptr(Py_buffer)], lltype.Void, error=CANNOT_FAIL) def PyBuffer_Release(space, view): """ - Releases a Py_buffer obtained from getbuffer ParseTuple's s*. - - This is not a complete re-implementation of the CPython API; it only - provides a subset of CPython's behavior. + Release the buffer view. This should be called when the buffer is + no longer being used as it may free memory from it """ Py_DecRef(space, view.c_obj) view.c_obj = lltype.nullptr(PyObject.TO) + # XXX do other fields leak memory? diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -335,9 +335,15 @@ def getshape(self): return self.shape + def getstrides(self): + return self.strides + def getitemsize(self): return self.itemsize + def getndim(self): + return self.ndim + def wrap_getreadbuffer(space, w_self, w_args, func): func_target = rffi.cast(readbufferproc, func) with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c --- a/pypy/module/cpyext/test/buffer_test.c +++ b/pypy/module/cpyext/test/buffer_test.c @@ -107,14 +107,11 @@ PyMyArray_getbuffer(PyObject *obj, Py_buffer *view, int flags) { PyMyArray* self = (PyMyArray*)obj; - fprintf(stdout, "in PyMyArray_getbuffer\n"); if (view == NULL) { - fprintf(stdout, "view is NULL\n"); PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer"); return -1; } if (flags == 0) { - fprintf(stdout, "flags is 0\n"); PyErr_SetString(PyExc_ValueError, "flags == 0 in getbuffer"); return -1; } @@ -188,7 +185,131 @@ (initproc)PyMyArray_init, /* tp_init */ }; +static PyObject* +test_buffer(PyObject* self, PyObject* args) +{ + Py_buffer* view = NULL; + PyObject* obj = PyTuple_GetItem(args, 0); + PyObject* memoryview = PyMemoryView_FromObject(obj); + if (memoryview == NULL) + return PyInt_FromLong(-1); + view = PyMemoryView_GET_BUFFER(memoryview); + Py_DECREF(memoryview); + return PyInt_FromLong(view->len); +} + +/* Copied from numpy tests */ +/* + * Create python string from a FLAG and or the corresponding PyBuf flag + * for the use in get_buffer_info. + */ +#define GET_PYBUF_FLAG(FLAG) \ + buf_flag = PyUnicode_FromString(#FLAG); \ + flag_matches = PyObject_RichCompareBool(buf_flag, tmp, Py_EQ); \ + Py_DECREF(buf_flag); \ + if (flag_matches == 1) { \ + Py_DECREF(tmp); \ + flags |= PyBUF_##FLAG; \ + continue; \ + } \ + else if (flag_matches == -1) { \ + Py_DECREF(tmp); \ + return NULL; \ + } + + +/* + * Get information for a buffer through PyBuf_GetBuffer with the + * corresponding flags or'ed. Note that the python caller has to + * make sure that or'ing those flags actually makes sense. + * More information should probably be returned for future tests. + */ +static PyObject * +get_buffer_info(PyObject *self, PyObject *args) +{ + PyObject *buffer_obj, *pyflags; + PyObject *tmp, *buf_flag; + Py_buffer buffer; + PyObject *shape, *strides; + Py_ssize_t i, n; + int flag_matches; + int flags = 0; + + if (!PyArg_ParseTuple(args, "OO", &buffer_obj, &pyflags)) { + return NULL; + } + + n = PySequence_Length(pyflags); + if (n < 0) { + return NULL; + } + + for (i=0; i < n; i++) { + tmp = PySequence_GetItem(pyflags, i); + if (tmp == NULL) { + return NULL; + } + + GET_PYBUF_FLAG(SIMPLE); + GET_PYBUF_FLAG(WRITABLE); + GET_PYBUF_FLAG(STRIDES); + GET_PYBUF_FLAG(ND); + GET_PYBUF_FLAG(C_CONTIGUOUS); + GET_PYBUF_FLAG(F_CONTIGUOUS); + GET_PYBUF_FLAG(ANY_CONTIGUOUS); + GET_PYBUF_FLAG(INDIRECT); + GET_PYBUF_FLAG(FORMAT); + GET_PYBUF_FLAG(STRIDED); + GET_PYBUF_FLAG(STRIDED_RO); + GET_PYBUF_FLAG(RECORDS); + GET_PYBUF_FLAG(RECORDS_RO); + GET_PYBUF_FLAG(FULL); + GET_PYBUF_FLAG(FULL_RO); + GET_PYBUF_FLAG(CONTIG); + GET_PYBUF_FLAG(CONTIG_RO); + + Py_DECREF(tmp); + + /* One of the flags must match */ + PyErr_SetString(PyExc_ValueError, "invalid flag used."); + return NULL; + } + + if (PyObject_GetBuffer(buffer_obj, &buffer, flags) < 0) { + return NULL; + } + + if (buffer.shape == NULL) { + Py_INCREF(Py_None); + shape = Py_None; + } + else { + shape = PyTuple_New(buffer.ndim); + for (i=0; i < buffer.ndim; i++) { + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(buffer.shape[i])); + } + } + + if (buffer.strides == NULL) { + Py_INCREF(Py_None); + strides = Py_None; + } + else { + strides = PyTuple_New(buffer.ndim); + for (i=0; i < buffer.ndim; i++) { + PyTuple_SET_ITEM(strides, i, PyLong_FromSsize_t(buffer.strides[i])); + } + } + + PyBuffer_Release(&buffer); + return Py_BuildValue("(NN)", shape, strides); +} + + + static PyMethodDef buffer_functions[] = { + {"test_buffer", (PyCFunction)test_buffer, METH_VARARGS, NULL}, + {"get_buffer_info", (PyCFunction)get_buffer_info, METH_VARARGS, NULL}, {NULL, NULL} /* Sentinel */ }; @@ -198,7 +319,7 @@ "buffer_test", "Module Doc", -1, - buffer_functions; + buffer_functions, NULL, NULL, NULL, diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -183,8 +183,27 @@ Py_INCREF(Py_None); return Py_None; """), + ("c_only", "METH_NOARGS", + """ + int ret; + char * buf2; + PyObject * obj = PyBytes_FromStringAndSize(NULL, 1024); + if (!obj) + return NULL; + buf2 = PyBytes_AsString(obj); + if (!buf2) + return NULL; + /* buf should not have been forced, issue #2395 */ + ret = _PyBytes_Resize(&obj, 512); + if (ret < 0) + return NULL; + Py_DECREF(obj); + Py_INCREF(Py_None); + return Py_None; + """), ]) module.getbytes() + module.c_only() def test_py_string_as_string_Unicode(self): module = self.import_extension('foo', [ diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -92,10 +92,20 @@ link_extra=link_extra, libraries=libraries) from pypy.module.imp.importing import get_so_extension - pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) + ext = get_so_extension(space) + pydname = soname.new(purebasename=modname, ext=ext) soname.rename(pydname) return str(pydname) +def get_so_suffix(): + from imp import get_suffixes, C_EXTENSION + for suffix, mode, typ in get_suffixes(): + if typ == C_EXTENSION: + return suffix + else: + raise RuntimeError("This interpreter does not define a filename " + "suffix for C extensions!") + def compile_extension_module_applevel(space, modname, include_dirs=[], source_files=None, source_strings=None): """ @@ -126,13 +136,9 @@ source_strings=source_strings, compile_extra=compile_extra, link_extra=link_extra) - from imp import get_suffixes, C_EXTENSION - pydname = soname - for suffix, mode, typ in get_suffixes(): - if typ == C_EXTENSION: - pydname = soname.new(purebasename=modname, ext=suffix) - soname.rename(pydname) - break + ext = get_so_suffix() + pydname = soname.new(purebasename=modname, ext=ext) + soname.rename(pydname) return str(pydname) def freeze_refcnts(self): @@ -145,6 +151,24 @@ #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) +class FakeSpace(object): + """Like TinyObjSpace, but different""" + def __init__(self, config): + from distutils.sysconfig import get_python_inc + self.config = config + self.include_dir = get_python_inc() + + def passthrough(self, arg): + return arg + listview = passthrough + str_w = passthrough + + def unwrap(self, args): + try: + return args.str_w(None) + except: + return args + class LeakCheckingTest(object): """Base class for all cpyext tests.""" spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array', @@ -433,21 +457,8 @@ self.imported_module_names = [] if self.runappdirect: + fake = FakeSpace(self.space.config) def interp2app(func): - from distutils.sysconfig import get_python_inc - class FakeSpace(object): - def passthrough(self, arg): - return arg - listview = passthrough - str_w = passthrough - def unwrap(self, args): - try: - return args.str_w(None) - except: - return args - fake = FakeSpace() - fake.include_dir = get_python_inc() - fake.config = self.space.config def run(*args, **kwargs): for k in kwargs.keys(): if k not in func.unwrap_spec and not k.startswith('w_'): diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -1,14 +1,9 @@ -import pytest from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - +from rpython.rlib.buffer import StringBuffer class TestMemoryViewObject(BaseApiTest): def test_fromobject(self, space, api): - if space.is_true(space.lt(space.sys.get('version_info'), - space.wrap((2, 7)))): - py.test.skip("unsupported before Python 2.7") - w_hello = space.newbytes("hello") assert api.PyObject_CheckBuffer(w_hello) w_view = api.PyMemoryView_FromObject(w_hello) @@ -17,6 +12,12 @@ w_bytes = space.call_method(w_view, "tobytes") assert space.unwrap(w_bytes) == "hello" + def test_frombuffer(self, space, api): + w_buf = space.newbuffer(StringBuffer("hello")) + w_memoryview = api.PyMemoryView_FromObject(w_buf) + w_view = api.PyMemoryView_GET_BUFFER(w_memoryview) + ndim = w_view.c_ndim + assert ndim == 1 class AppTestBufferProtocol(AppTestCpythonExtensionBase): def test_buffer_protocol(self): @@ -26,7 +27,25 @@ y = memoryview(arr) assert y.format == 'i' assert y.shape == (10,) + assert len(y) == 10 s = y[3] assert len(s) == struct.calcsize('i') assert s == struct.pack('i', 3) + viewlen = module.test_buffer(arr) + assert viewlen == y.itemsize * len(y) + def test_buffer_info(self): + from _numpypy import multiarray as np + module = self.import_module(name='buffer_test') + get_buffer_info = module.get_buffer_info + # test_export_flags from numpy test_multiarray + raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',)) + # test_relaxed_strides from numpy test_multiarray + arr = np.zeros((1, 10)) + if arr.flags.f_contiguous: + shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS']) + assert strides[0] == 8 + arr = np.ones((10, 1), order='F') + shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS']) + assert strides[-1] == 8 + diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py --- a/pypy/module/cpyext/test/test_version.py +++ b/pypy/module/cpyext/test/test_version.py @@ -32,9 +32,11 @@ assert module.py_minor_version == sys.version_info.minor assert module.py_micro_version == sys.version_info.micro - @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') + #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names, reason='pypy only test') def test_pypy_versions(self): import sys + if '__pypy__' not in sys.builtin_module_names: + py.test.skip("pypy only test") init = """ if (Py_IsInitialized()) { PyObject *m = Py_InitModule("foo", NULL); diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -293,6 +293,8 @@ STRUCT_TYPE = PyNumberMethods elif slot_names[0] == 'c_tp_as_sequence': STRUCT_TYPE = PySequenceMethods + elif slot_names[0] == 'c_tp_as_buffer': + STRUCT_TYPE = PyBufferProcs else: raise AssertionError( "Structure not allocated: %s" % (slot_names[0],)) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -597,6 +597,11 @@ @jit.dont_look_inside def load_module(space, w_modulename, find_info, reuse=False): + """Like load_module() in CPython's import.c, this will normally + make a module object, store it in sys.modules, execute code in it, + and then fetch it again from sys.modules. But this logic is not + used if we're calling a PEP302 loader. + """ if find_info is None: return @@ -625,17 +630,15 @@ try: if find_info.modtype == PY_SOURCE: - load_source_module( + return load_source_module( space, w_modulename, w_mod, find_info.filename, find_info.stream.readall(), find_info.stream.try_to_find_file_descriptor()) - return w_mod elif find_info.modtype == PY_COMPILED: magic = _r_long(find_info.stream) timestamp = _r_long(find_info.stream) - load_compiled_module(space, w_modulename, w_mod, find_info.filename, + return load_compiled_module(space, w_modulename, w_mod, find_info.filename, magic, timestamp, find_info.stream.readall()) - return w_mod elif find_info.modtype == PKG_DIRECTORY: w_path = space.newlist([space.wrap(find_info.filename)]) space.setattr(w_mod, space.wrap('__path__'), w_path) @@ -644,14 +647,13 @@ if find_info is None: return w_mod try: - load_module(space, w_modulename, find_info, reuse=True) + w_mod = load_module(space, w_modulename, find_info, + reuse=True) finally: try: find_info.stream.close() except StreamErrors: pass - # fetch the module again, in case of "substitution" - w_mod = check_sys_modules(space, w_modulename) return w_mod elif find_info.modtype == C_EXTENSION and has_so_extension(space): load_c_extension(space, find_info.filename, space.str_w(w_modulename)) @@ -677,13 +679,6 @@ try: if find_info: w_mod = load_module(space, w_modulename, find_info) - try: - w_mod = space.getitem(space.sys.get("modules"), - w_modulename) - except OperationError as oe: - if not oe.match(space, space.w_KeyError): - raise - raise OperationError(space.w_ImportError, w_modulename) if w_parent is not None: space.setattr(w_parent, space.wrap(partname), w_mod) return w_mod @@ -875,20 +870,32 @@ pycode = ec.compiler.compile(source, pathname, 'exec', 0) return pycode -def exec_code_module(space, w_mod, code_w): +def exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=True): + """ + Execute a code object in the module's dict. Returns + 'sys.modules[modulename]', which must exist. + """ w_dict = space.getattr(w_mod, space.wrap('__dict__')) space.call_method(w_dict, 'setdefault', space.wrap('__builtins__'), space.wrap(space.builtin)) code_w.exec_code(space, w_dict, w_dict) + if check_afterwards: + w_mod = check_sys_modules(space, w_modulename) + if w_mod is None: + raise oefmt(space.w_ImportError, + "Loaded module %R not found in sys.modules", + w_modulename) + return w_mod + @jit.dont_look_inside def load_source_module(space, w_modulename, w_mod, pathname, source, fd, - write_pyc=True): + write_pyc=True, check_afterwards=True): """ - Load a source module from a given file and return its module - object. + Load a source module from a given file. Returns the result + of sys.modules[modulename], which must exist. """ w = space.wrap @@ -927,9 +934,8 @@ code_w.remove_docstrings(space) update_code_filenames(space, code_w, pathname) - exec_code_module(space, w_mod, code_w) - - return w_mod + return exec_code_module(space, w_mod, code_w, w_modulename, + check_afterwards=check_afterwards) def update_code_filenames(space, code_w, pathname, oldname=None): assert isinstance(code_w, PyCode) @@ -1012,10 +1018,10 @@ @jit.dont_look_inside def load_compiled_module(space, w_modulename, w_mod, cpathname, magic, - timestamp, source): + timestamp, source, check_afterwards=True): """ - Load a module from a compiled file, execute it, and return its - module object. + Load a module from a compiled file and execute it. Returns + 'sys.modules[modulename]', which must exist. """ log_pyverbose(space, 1, "import %s # compiled from %s\n" % (space.str_w(w_modulename), cpathname)) @@ -1032,9 +1038,8 @@ if optimize >= 2: code_w.remove_docstrings(space) - exec_code_module(space, w_mod, code_w) - - return w_mod + return exec_code_module(space, w_mod, code_w, w_modulename, + check_afterwards=check_afterwards) def open_exclusive(space, cpathname, mode): try: diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -98,33 +98,35 @@ w_mod = space.wrap(Module(space, w_modulename)) importing._prepare_module(space, w_mod, filename, None) - importing.load_source_module( + w_mod = importing.load_source_module( space, w_modulename, w_mod, filename, stream.readall(), stream.try_to_find_file_descriptor()) if space.is_none(w_file): stream.close() return w_mod -@unwrap_spec(filename='str0') -def _run_compiled_module(space, w_modulename, filename, w_file, w_module): +@unwrap_spec(filename='str0', check_afterwards=int) +def _run_compiled_module(space, w_modulename, filename, w_file, w_module, + check_afterwards=False): # the function 'imp._run_compiled_module' is a pypy-only extension stream = get_file(space, w_file, filename, 'rb') magic = importing._r_long(stream) timestamp = importing._r_long(stream) - importing.load_compiled_module( + w_mod = importing.load_compiled_module( space, w_modulename, w_module, filename, magic, timestamp, - stream.readall()) + stream.readall(), check_afterwards=check_afterwards) if space.is_none(w_file): stream.close() + return w_mod @unwrap_spec(filename='str0') def load_compiled(space, w_modulename, filename, w_file=None): w_mod = space.wrap(Module(space, w_modulename)) importing._prepare_module(space, w_mod, filename, None) - _run_compiled_module(space, w_modulename, filename, w_file, w_mod) - return w_mod + return _run_compiled_module(space, w_modulename, filename, w_file, w_mod, + check_afterwards=True) @unwrap_spec(filename=str) def load_dynamic(space, w_modulename, filename, w_file=None): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -118,7 +118,7 @@ filename = str(p.join("x.py")) stream = streamio.open_file_as_stream(filename, "r") try: - importing.load_source_module( + _load_source_module( space, w_modname, w(importing.Module(space, w_modname)), filename, stream.readall(), stream.try_to_find_file_descriptor()) @@ -139,6 +139,15 @@ return str(root) +def _load_source_module(space, w_modname, w_mod, *args, **kwds): + kwds.setdefault('check_afterwards', False) + return importing.load_source_module(space, w_modname, w_mod, *args, **kwds) + +def _load_compiled_module(space, w_modname, w_mod, *args, **kwds): + kwds.setdefault('check_afterwards', False) + return importing.load_compiled_module(space, w_modname, w_mod, + *args, **kwds) + def _setup(space): dn = setup_directory_structure(space) @@ -887,8 +896,7 @@ w_mod = space.wrap(Module(space, w_modulename)) magic = importing._r_long(stream) timestamp = importing._r_long(stream) - w_ret = importing.load_compiled_module(space, - w_modulename, + w_ret = _load_compiled_module(space, w_modulename, w_mod, cpathname, magic, @@ -946,7 +954,7 @@ pathname = _testfilesource() stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -968,7 +976,7 @@ pathname = _testfilesource() stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor(), @@ -987,7 +995,7 @@ try: space.setattr(space.sys, space.wrap('dont_write_bytecode'), space.w_True) - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -1006,7 +1014,7 @@ pathname = _testfilesource(source="<Syntax Error>") stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -1026,7 +1034,7 @@ pathname = _testfilesource(source="a = unknown_name") stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -1114,7 +1122,7 @@ magic = importing._r_long(stream) timestamp = importing._r_long(stream) space2.raises_w(space2.w_ImportError, - importing.load_compiled_module, + _load_compiled_module, space2, w_modulename, w_mod, @@ -1326,10 +1334,7 @@ # use an import hook that doesn't update sys.modules, then the # import succeeds; but at the same time, you can have the same # result without an import hook (see test_del_from_sys_modules) - # and then the import fails. This looks like even more mess - # to replicate, so we ignore it until someone really hits this - # case... - skip("looks like an inconsistency in CPython") + # and then the import fails. Mess mess mess. class ImportHook(object): def find_module(self, fullname, path=None): diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py --- a/pypy/module/marshal/interp_marshal.py +++ b/pypy/module/marshal/interp_marshal.py @@ -225,15 +225,6 @@ def dump_w_obj(self, w_obj): space = self.space - if space.type(w_obj).is_heaptype(): - try: - buf = space.readbuf_w(w_obj) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise - self.raise_exc("unmarshallable object") - else: - w_obj = space.newbuffer(buf) try: self.put_w_obj(w_obj) except rstackovf.StackOverflow: diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -186,6 +186,8 @@ assert str(exc.value) == 'unmarshallable object' exc = raises(ValueError, marshal.dumps, subtype()) assert str(exc.value) == 'unmarshallable object' + exc = raises(ValueError, marshal.dumps, (subtype(),)) + assert str(exc.value) == 'unmarshallable object' def test_valid_subtypes(self): import marshal diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -460,6 +460,9 @@ def getdictvalue(self, space, key): return self.items[key] _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
