Author: Matti Picus <[email protected]> Branch: buffer-interface Changeset: r86836:802bf4e38a0d Date: 2016-09-02 09:48 +0300 http://bitbucket.org/pypy/pypy/changeset/802bf4e38a0d/
Log: merge default into branch diff too long, truncating to 2000 out of 210666 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -27,3 +27,6 @@ 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 +68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 +68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 +77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -74,6 +74,7 @@ Seo Sanghyeon Ronny Pfannschmidt Justin Peel + Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak @@ -117,7 +118,6 @@ Wenzhu Man John Witulski Laurence Tratt - Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -141,6 +141,7 @@ tav Taavi Burns Georg Brandl + Nicolas Truessel Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -211,6 +212,7 @@ Vaibhav Sood Alan McIntyre Alexander Sedov + [email protected] Attila Gobi Jasper.Schulz Christopher Pope @@ -221,6 +223,7 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + touilleMan Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -229,12 +232,14 @@ Gabriel Lukas Vacek Kunal Grover + Aaron Gallagher Andrew Dalke Sylvain Thenault Jakub Stasiak Nathan Taylor Vladimir Kryachko Omer Katz + Mark Williams Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -355,12 +360,15 @@ yasirs Michael Chermside Anna Ravencroft + pizi Andrey Churin Dan Crosta + Eli Stevens Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz + werat Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden diff --git a/_pytest/python.py b/_pytest/python.py --- a/_pytest/python.py +++ b/_pytest/python.py @@ -498,7 +498,10 @@ """ Collector for test methods. """ def collect(self): if hasinit(self.obj): - pytest.skip("class %s.%s with __init__ won't get collected" % ( + # XXX used to be skip(), but silently skipping classes + # XXX just because they have been written long ago is + # XXX imho a very, very, very bad idea + pytest.fail("class %s.%s with __init__ won't get collected" % ( self.obj.__module__, self.obj.__name__, )) diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -122,22 +122,24 @@ """Dummy method to let some easy_install packages that have optional C speedup components. """ + def customize(executable, flags): + command = compiler.executables[executable] + flags + setattr(compiler, executable, command) + if compiler.compiler_type == "unix": compiler.compiler_so.extend(['-O2', '-fPIC', '-Wimplicit']) compiler.shared_lib_extension = get_config_var('SO') if "CPPFLAGS" in os.environ: cppflags = shlex.split(os.environ["CPPFLAGS"]) - compiler.compiler.extend(cppflags) - compiler.compiler_so.extend(cppflags) - compiler.linker_so.extend(cppflags) + for executable in ('compiler', 'compiler_so', 'linker_so'): + customize(executable, cppflags) if "CFLAGS" in os.environ: cflags = shlex.split(os.environ["CFLAGS"]) - compiler.compiler.extend(cflags) - compiler.compiler_so.extend(cflags) - compiler.linker_so.extend(cflags) + for executable in ('compiler', 'compiler_so', 'linker_so'): + customize(executable, cflags) if "LDFLAGS" in os.environ: ldflags = shlex.split(os.environ["LDFLAGS"]) - compiler.linker_so.extend(ldflags) + customize('linker_so', ldflags) from sysconfig_cpython import ( diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -342,7 +342,7 @@ thisarg = cast(thisvalue, POINTER(POINTER(c_void_p))) keepalives, newargs, argtypes, outargs, errcheckargs = ( self._convert_args(argtypes, args[1:], kwargs)) - newargs.insert(0, thisvalue.value) + newargs.insert(0, thisarg) argtypes.insert(0, c_void_p) else: thisarg = None diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -44,6 +44,7 @@ Seo Sanghyeon Ronny Pfannschmidt Justin Peel + Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak @@ -87,7 +88,6 @@ Wenzhu Man John Witulski Laurence Tratt - Raffael Tfirst Ivan Sichmann Freitas Greg Price Dario Bertini @@ -111,6 +111,7 @@ tav Taavi Burns Georg Brandl + Nicolas Truessel Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -181,6 +182,7 @@ Vaibhav Sood Alan McIntyre Alexander Sedov + [email protected] Attila Gobi Jasper.Schulz Christopher Pope @@ -191,6 +193,7 @@ Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + touilleMan Alexis Daboville Jens-Uwe Mager Carl Meyer @@ -199,12 +202,14 @@ Gabriel Lukas Vacek Kunal Grover + Aaron Gallagher Andrew Dalke Sylvain Thenault Jakub Stasiak Nathan Taylor Vladimir Kryachko Omer Katz + Mark Williams Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -325,9 +330,12 @@ yasirs Michael Chermside Anna Ravencroft + pizi Andrey Churin Dan Crosta + Eli Stevens Tobias Diaz Julien Phalip Roman Podoliaka Dan Loewenherz + werat diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst --- a/pypy/doc/release-pypy2.7-v5.4.0.rst +++ b/pypy/doc/release-pypy2.7-v5.4.0.rst @@ -3,7 +3,8 @@ ============ We have released PyPy2.7 v5.4, a little under two months after PyPy2.7 v5.3. -This new PyPy2.7 release includes further improvements to our C-API compatability layer (cpyext), enabling us to pass over 99% of the upstream +This new PyPy2.7 release includes incremental improvements to our C-API +compatability layer (cpyext), enabling us to pass over 99% of the upstream numpy `test suite`_. We updated built-in cffi_ support to version 1.8, which now supports the "limited API" mode for c-extensions on CPython >=3.2. @@ -12,9 +13,7 @@ support to OpenBSD and Dragon Fly BSD As always, this release fixed many issues and bugs raised by the -growing community of PyPy users. - -XXXXX MORE ??? +growing community of PyPy users. We strongly recommend updating. You can download the PyPy2.7 v5.4 release here: @@ -110,8 +109,8 @@ * (RPython) add `rposix_scandir` portably, needed for Python 3.5 - * Support for memoryview attributes (format, itemsize, ...) which also - adds support for `PyMemoryView_FromObject` + * Increased but incomplete support for memoryview attributes (format, + itemsize, ...) which also adds support for `PyMemoryView_FromObject` * Bug Fixes @@ -153,10 +152,6 @@ * Make `hash(-1)` return -2, as CPython does, and fix all the ancilary places this matters - * Issues reported with our previous release were resolved_ after - reports from users on our issue tracker at - https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy - * Fix `PyNumber_Check()` to behave more like CPython * (VMProf) Try hard to not miss any Python-level frame in the @@ -169,6 +164,10 @@ * Fix the mapdict cache for subclasses of builtin types that provide a dict + * Issues reported with our previous release were resolved_ after + reports from users on our issue tracker at + https://bitbucket.org/pypy/pypy/issues or on IRC at #pypy + * Performance improvements: * Add a before_call()-like equivalent before a few operations like diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,4 +5,5 @@ .. this is a revision shortly after release-pypy2.7-v5.4 .. startrev: 522736f816dc - +.. branch: rpython-resync +Backport rpython changes made directly on the py3k and py3.5 branches. diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py --- a/pypy/module/__builtin__/descriptor.py +++ b/pypy/module/__builtin__/descriptor.py @@ -23,6 +23,14 @@ self.w_objtype = w_type self.w_self = w_obj_or_type + def descr_repr(self, space): + if self.w_objtype is not None: + objtype_name = "<%s object>" % self.w_objtype.getname(space) + else: + objtype_name = 'NULL' + return space.wrap("<super: <class '%s'>, %s>" % ( + self.w_starttype.getname(space), objtype_name)) + def get(self, space, w_obj, w_type=None): if self.w_self is None or space.is_w(w_obj, space.w_None): return self @@ -84,7 +92,10 @@ 'super', __new__ = generic_new_descr(W_Super), __init__ = interp2app(W_Super.descr_init), + __repr__ = interp2app(W_Super.descr_repr), __thisclass__ = interp_attrproperty_w("w_starttype", W_Super), + __self__ = interp_attrproperty_w("w_self", W_Super), + __self_class__ = interp_attrproperty_w("w_objtype", W_Super), __getattribute__ = interp2app(W_Super.getattribute), __get__ = interp2app(W_Super.get), __doc__ = """\ diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py --- a/pypy/module/__builtin__/interp_classobj.py +++ b/pypy/module/__builtin__/interp_classobj.py @@ -38,6 +38,8 @@ class W_ClassObject(W_Root): + _immutable_fields_ = ['bases_w?[*]', 'w_dict?'] + def __init__(self, space, w_name, bases, w_dict): self.name = space.str_w(w_name) make_sure_not_resized(bases) @@ -75,6 +77,7 @@ "__bases__ items must be classes") self.bases_w = bases_w + @jit.unroll_safe def is_subclass_of(self, other): assert isinstance(other, W_ClassObject) if self is other: @@ -313,7 +316,7 @@ # This method ignores the instance dict and the __getattr__. # Returns None if not found. assert isinstance(name, str) - w_value = self.w_class.lookup(space, name) + w_value = jit.promote(self.w_class).lookup(space, name) if w_value is None: return None w_descr_get = space.lookup(w_value, '__get__') diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py --- a/pypy/module/__builtin__/test/test_descriptor.py +++ b/pypy/module/__builtin__/test/test_descriptor.py @@ -250,6 +250,24 @@ assert super(B, B()).__thisclass__ is B assert super(A, B()).__thisclass__ is A + def test_super_self_selfclass(self): + class A(object): + pass + class B(A): + pass + b = B() + assert super(A, b).__self__ is b + assert super(A).__self__ is None + assert super(A, b).__self_class__ is B + assert super(A).__self_class__ is None + + def test_super_repr(self): + class A(object): + def __repr__(self): + return super(A, self).__repr__() + '!' + assert repr(A()).endswith('>!') + assert repr(super(A, A())) == "<super: <class 'A'>, <A object>>" + def test_property_docstring(self): assert property.__doc__.startswith('property') diff --git a/pypy/module/_sre/__init__.py b/pypy/module/_sre/__init__.py --- a/pypy/module/_sre/__init__.py +++ b/pypy/module/_sre/__init__.py @@ -1,4 +1,4 @@ -from pypy.interpreter.mixedmodule import MixedModule +from pypy.interpreter.mixedmodule import MixedModule class Module(MixedModule): @@ -7,7 +7,7 @@ interpleveldefs = { 'CODESIZE': 'space.wrap(interp_sre.CODESIZE)', - 'MAGIC': 'space.wrap(interp_sre.MAGIC)', + 'MAGIC': 'space.newint(20031017)', 'MAXREPEAT': 'space.wrap(interp_sre.MAXREPEAT)', 'compile': 'interp_sre.W_SRE_Pattern', 'getlower': 'interp_sre.w_getlower', diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -14,7 +14,7 @@ # Constants and exposed functions from rpython.rlib.rsre import rsre_core -from rpython.rlib.rsre.rsre_char import MAGIC, CODESIZE, MAXREPEAT, getlower, set_unicode_db +from rpython.rlib.rsre.rsre_char import CODESIZE, MAXREPEAT, getlower, set_unicode_db @unwrap_spec(char_ord=int, flags=int) diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py --- a/pypy/module/cpyext/import_.py +++ b/pypy/module/cpyext/import_.py @@ -123,5 +123,4 @@ pathname = code.co_filename w_mod = importing.add_module(space, w_name) space.setattr(w_mod, space.wrap('__file__'), space.wrap(pathname)) - importing.exec_code_module(space, w_mod, code) - return w_mod + return importing.exec_code_module(space, w_mod, code, w_name) diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -92,10 +92,20 @@ link_extra=link_extra, libraries=libraries) from pypy.module.imp.importing import get_so_extension - pydname = soname.new(purebasename=modname, ext=get_so_extension(space)) + ext = get_so_extension(space) + pydname = soname.new(purebasename=modname, ext=ext) soname.rename(pydname) return str(pydname) +def get_so_suffix(): + from imp import get_suffixes, C_EXTENSION + for suffix, mode, typ in get_suffixes(): + if typ == C_EXTENSION: + return suffix + else: + raise RuntimeError("This interpreter does not define a filename " + "suffix for C extensions!") + def compile_extension_module_applevel(space, modname, include_dirs=[], source_files=None, source_strings=None): """ @@ -126,13 +136,9 @@ source_strings=source_strings, compile_extra=compile_extra, link_extra=link_extra) - from imp import get_suffixes, C_EXTENSION - pydname = soname - for suffix, mode, typ in get_suffixes(): - if typ == C_EXTENSION: - pydname = soname.new(purebasename=modname, ext=suffix) - soname.rename(pydname) - break + ext = get_so_suffix() + pydname = soname.new(purebasename=modname, ext=ext) + soname.rename(pydname) return str(pydname) def freeze_refcnts(self): @@ -145,6 +151,24 @@ #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) +class FakeSpace(object): + """Like TinyObjSpace, but different""" + def __init__(self, config): + from distutils.sysconfig import get_python_inc + self.config = config + self.include_dir = get_python_inc() + + def passthrough(self, arg): + return arg + listview = passthrough + str_w = passthrough + + def unwrap(self, args): + try: + return args.str_w(None) + except: + return args + class LeakCheckingTest(object): """Base class for all cpyext tests.""" spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array', @@ -433,21 +457,8 @@ self.imported_module_names = [] if self.runappdirect: + fake = FakeSpace(self.space.config) def interp2app(func): - from distutils.sysconfig import get_python_inc - class FakeSpace(object): - def passthrough(self, arg): - return arg - listview = passthrough - str_w = passthrough - def unwrap(self, args): - try: - return args.str_w(None) - except: - return args - fake = FakeSpace() - fake.include_dir = get_python_inc() - fake.config = self.space.config def run(*args, **kwargs): for k in kwargs.keys(): if k not in func.unwrap_spec and not k.startswith('w_'): diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -597,6 +597,11 @@ @jit.dont_look_inside def load_module(space, w_modulename, find_info, reuse=False): + """Like load_module() in CPython's import.c, this will normally + make a module object, store it in sys.modules, execute code in it, + and then fetch it again from sys.modules. But this logic is not + used if we're calling a PEP302 loader. + """ if find_info is None: return @@ -625,17 +630,15 @@ try: if find_info.modtype == PY_SOURCE: - load_source_module( + return load_source_module( space, w_modulename, w_mod, find_info.filename, find_info.stream.readall(), find_info.stream.try_to_find_file_descriptor()) - return w_mod elif find_info.modtype == PY_COMPILED: magic = _r_long(find_info.stream) timestamp = _r_long(find_info.stream) - load_compiled_module(space, w_modulename, w_mod, find_info.filename, + return load_compiled_module(space, w_modulename, w_mod, find_info.filename, magic, timestamp, find_info.stream.readall()) - return w_mod elif find_info.modtype == PKG_DIRECTORY: w_path = space.newlist([space.wrap(find_info.filename)]) space.setattr(w_mod, space.wrap('__path__'), w_path) @@ -644,14 +647,13 @@ if find_info is None: return w_mod try: - load_module(space, w_modulename, find_info, reuse=True) + w_mod = load_module(space, w_modulename, find_info, + reuse=True) finally: try: find_info.stream.close() except StreamErrors: pass - # fetch the module again, in case of "substitution" - w_mod = check_sys_modules(space, w_modulename) return w_mod elif find_info.modtype == C_EXTENSION and has_so_extension(space): load_c_extension(space, find_info.filename, space.str_w(w_modulename)) @@ -677,13 +679,6 @@ try: if find_info: w_mod = load_module(space, w_modulename, find_info) - try: - w_mod = space.getitem(space.sys.get("modules"), - w_modulename) - except OperationError as oe: - if not oe.match(space, space.w_KeyError): - raise - raise OperationError(space.w_ImportError, w_modulename) if w_parent is not None: space.setattr(w_parent, space.wrap(partname), w_mod) return w_mod @@ -875,20 +870,32 @@ pycode = ec.compiler.compile(source, pathname, 'exec', 0) return pycode -def exec_code_module(space, w_mod, code_w): +def exec_code_module(space, w_mod, code_w, w_modulename, check_afterwards=True): + """ + Execute a code object in the module's dict. Returns + 'sys.modules[modulename]', which must exist. + """ w_dict = space.getattr(w_mod, space.wrap('__dict__')) space.call_method(w_dict, 'setdefault', space.wrap('__builtins__'), space.wrap(space.builtin)) code_w.exec_code(space, w_dict, w_dict) + if check_afterwards: + w_mod = check_sys_modules(space, w_modulename) + if w_mod is None: + raise oefmt(space.w_ImportError, + "Loaded module %R not found in sys.modules", + w_modulename) + return w_mod + @jit.dont_look_inside def load_source_module(space, w_modulename, w_mod, pathname, source, fd, - write_pyc=True): + write_pyc=True, check_afterwards=True): """ - Load a source module from a given file and return its module - object. + Load a source module from a given file. Returns the result + of sys.modules[modulename], which must exist. """ w = space.wrap @@ -927,9 +934,8 @@ code_w.remove_docstrings(space) update_code_filenames(space, code_w, pathname) - exec_code_module(space, w_mod, code_w) - - return w_mod + return exec_code_module(space, w_mod, code_w, w_modulename, + check_afterwards=check_afterwards) def update_code_filenames(space, code_w, pathname, oldname=None): assert isinstance(code_w, PyCode) @@ -1012,10 +1018,10 @@ @jit.dont_look_inside def load_compiled_module(space, w_modulename, w_mod, cpathname, magic, - timestamp, source): + timestamp, source, check_afterwards=True): """ - Load a module from a compiled file, execute it, and return its - module object. + Load a module from a compiled file and execute it. Returns + 'sys.modules[modulename]', which must exist. """ log_pyverbose(space, 1, "import %s # compiled from %s\n" % (space.str_w(w_modulename), cpathname)) @@ -1032,9 +1038,8 @@ if optimize >= 2: code_w.remove_docstrings(space) - exec_code_module(space, w_mod, code_w) - - return w_mod + return exec_code_module(space, w_mod, code_w, w_modulename, + check_afterwards=check_afterwards) def open_exclusive(space, cpathname, mode): try: diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -98,33 +98,35 @@ w_mod = space.wrap(Module(space, w_modulename)) importing._prepare_module(space, w_mod, filename, None) - importing.load_source_module( + w_mod = importing.load_source_module( space, w_modulename, w_mod, filename, stream.readall(), stream.try_to_find_file_descriptor()) if space.is_none(w_file): stream.close() return w_mod -@unwrap_spec(filename='str0') -def _run_compiled_module(space, w_modulename, filename, w_file, w_module): +@unwrap_spec(filename='str0', check_afterwards=int) +def _run_compiled_module(space, w_modulename, filename, w_file, w_module, + check_afterwards=False): # the function 'imp._run_compiled_module' is a pypy-only extension stream = get_file(space, w_file, filename, 'rb') magic = importing._r_long(stream) timestamp = importing._r_long(stream) - importing.load_compiled_module( + w_mod = importing.load_compiled_module( space, w_modulename, w_module, filename, magic, timestamp, - stream.readall()) + stream.readall(), check_afterwards=check_afterwards) if space.is_none(w_file): stream.close() + return w_mod @unwrap_spec(filename='str0') def load_compiled(space, w_modulename, filename, w_file=None): w_mod = space.wrap(Module(space, w_modulename)) importing._prepare_module(space, w_mod, filename, None) - _run_compiled_module(space, w_modulename, filename, w_file, w_mod) - return w_mod + return _run_compiled_module(space, w_modulename, filename, w_file, w_mod, + check_afterwards=True) @unwrap_spec(filename=str) def load_dynamic(space, w_modulename, filename, w_file=None): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -118,7 +118,7 @@ filename = str(p.join("x.py")) stream = streamio.open_file_as_stream(filename, "r") try: - importing.load_source_module( + _load_source_module( space, w_modname, w(importing.Module(space, w_modname)), filename, stream.readall(), stream.try_to_find_file_descriptor()) @@ -139,6 +139,15 @@ return str(root) +def _load_source_module(space, w_modname, w_mod, *args, **kwds): + kwds.setdefault('check_afterwards', False) + return importing.load_source_module(space, w_modname, w_mod, *args, **kwds) + +def _load_compiled_module(space, w_modname, w_mod, *args, **kwds): + kwds.setdefault('check_afterwards', False) + return importing.load_compiled_module(space, w_modname, w_mod, + *args, **kwds) + def _setup(space): dn = setup_directory_structure(space) @@ -887,8 +896,7 @@ w_mod = space.wrap(Module(space, w_modulename)) magic = importing._r_long(stream) timestamp = importing._r_long(stream) - w_ret = importing.load_compiled_module(space, - w_modulename, + w_ret = _load_compiled_module(space, w_modulename, w_mod, cpathname, magic, @@ -946,7 +954,7 @@ pathname = _testfilesource() stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -968,7 +976,7 @@ pathname = _testfilesource() stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor(), @@ -987,7 +995,7 @@ try: space.setattr(space.sys, space.wrap('dont_write_bytecode'), space.w_True) - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -1006,7 +1014,7 @@ pathname = _testfilesource(source="<Syntax Error>") stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -1026,7 +1034,7 @@ pathname = _testfilesource(source="a = unknown_name") stream = streamio.open_file_as_stream(pathname, "r") try: - w_ret = importing.load_source_module( + w_ret = _load_source_module( space, w_modulename, w_mod, pathname, stream.readall(), stream.try_to_find_file_descriptor()) @@ -1114,7 +1122,7 @@ magic = importing._r_long(stream) timestamp = importing._r_long(stream) space2.raises_w(space2.w_ImportError, - importing.load_compiled_module, + _load_compiled_module, space2, w_modulename, w_mod, @@ -1326,10 +1334,7 @@ # use an import hook that doesn't update sys.modules, then the # import succeeds; but at the same time, you can have the same # result without an import hook (see test_del_from_sys_modules) - # and then the import fails. This looks like even more mess - # to replicate, so we ignore it until someone really hits this - # case... - skip("looks like an inconsistency in CPython") + # and then the import fails. Mess mess mess. class ImportHook(object): def find_module(self, fullname, path=None): diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py --- a/pypy/module/pypyjit/test_pypy_c/test_instance.py +++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py @@ -187,6 +187,43 @@ """) + def test_oldstyle_methcall(self): + def main(): + def g(): pass + class A: + def f(self): + return self.x + 1 + class I(A): + pass + class J(I): + pass + + + class B(J): + def __init__(self, x): + self.x = x + + i = 0 + b = B(1) + while i < 1000: + g() + v = b.f() # ID: meth + i += v + return i + + log = self.run(main, [], threshold=80) + loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True) + assert loop.match_by_id('meth', + ''' + guard_nonnull_class(p18, ..., descr=...) + p52 = getfield_gc_r(p18, descr=...) # read map + guard_value(p52, ConstPtr(ptr53), descr=...) + p54 = getfield_gc_r(p18, descr=...) # read class + guard_value(p54, ConstPtr(ptr55), descr=...) + p56 = force_token() # done + ''') + + def test_oldstyle_newstyle_mix(self): def main(): class A: diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -64,7 +64,7 @@ AsyncAction.__init__(self, space) self.pending_signal = -1 self.fire_in_another_thread = False - # + @rgc.no_collect def _after_thread_switch(): if self.fire_in_another_thread: @@ -251,7 +251,7 @@ except OSError as e: if e.errno == errno.EBADF: raise oefmt(space.w_ValueError, "invalid fd") - old_fd = pypysig_set_wakeup_fd(fd) + old_fd = pypysig_set_wakeup_fd(fd, True) return space.wrap(intmask(old_fd)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py @@ -195,6 +195,29 @@ _fields_ = [('t', enum)] assert isinstance(S().t, enum) + def test_no_missing_shape_to_ffi_type(self): + # whitebox test + import sys + if '__pypy__' not in sys.builtin_module_names: + skip("only for pypy's ctypes") + skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, " + "which I think needs fighting all the way up from " + "rpython.rlib.libffi") + from _ctypes.basics import _shape_to_ffi_type + from _rawffi import Array + for i in range(1, 256): + try: + Array(chr(i)) + except ValueError: + pass + else: + assert chr(i) in _shape_to_ffi_type.typemap + + @py.test.mark.xfail + def test_pointer_to_long_double(self): + import ctypes + ctypes.POINTER(ctypes.c_longdouble) + ## def test_perf(self): ## check_perf() diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py --- a/pypy/module/zipimport/interp_zipimport.py +++ b/pypy/module/zipimport/interp_zipimport.py @@ -152,8 +152,7 @@ importing._prepare_module(space, w_mod, real_name, pkgpath) co_filename = self.make_co_filename(filename) code_w = importing.parse_source_module(space, co_filename, buf) - importing.exec_code_module(space, w_mod, code_w) - return w_mod + return importing.exec_code_module(space, w_mod, code_w, w(modname)) def _parse_mtime(self, space, filename): w = space.wrap @@ -205,10 +204,10 @@ real_name = self.filename + os.path.sep + self.corr_zname(filename) space.setattr(w_mod, w('__loader__'), space.wrap(self)) importing._prepare_module(space, w_mod, real_name, pkgpath) - result = importing.load_compiled_module(space, w(modname), w_mod, + w_result = importing.load_compiled_module(space, w(modname), w_mod, filename, magic, timestamp, buf) - return result + return w_result def have_modulefile(self, space, filename): if ZIPSEP != os.path.sep: diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -58,6 +58,20 @@ return w_iter tuple_iter._annspecialcase_ = 'specialize:memo' +def str_getitem(space): + "Utility that returns the app-level descriptor str.__getitem__." + w_src, w_iter = space.lookup_in_type_where(space.w_str, + '__getitem__') + return w_iter +str_getitem._annspecialcase_ = 'specialize:memo' + +def unicode_getitem(space): + "Utility that returns the app-level descriptor unicode.__getitem__." + w_src, w_iter = space.lookup_in_type_where(space.w_unicode, + '__getitem__') + return w_iter +unicode_getitem._annspecialcase_ = 'specialize:memo' + def raiseattrerror(space, w_obj, name, w_descr=None): if w_descr is None: raise oefmt(space.w_AttributeError, diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -8,6 +8,7 @@ from rpython.rlib.objectmodel import specialize from rpython.rlib.rfloat import copysign, formatd from rpython.rlib.rarithmetic import r_uint, intmask +from pypy.interpreter.signature import Signature @specialize.argtype(1) @@ -40,19 +41,32 @@ ANS_MANUAL = 3 -def make_template_formatting_class(): +format_signature = Signature([], 'args', 'kwargs') + + +def make_template_formatting_class(for_unicode): class TemplateFormatter(object): + is_unicode = for_unicode parser_list_w = None - def __init__(self, space, is_unicode, template): + def __init__(self, space, template): self.space = space - self.is_unicode = is_unicode - self.empty = u"" if is_unicode else "" + self.empty = u"" if self.is_unicode else "" self.template = template def build(self, args): - self.args, self.kwargs = args.unpack() + if self.is_unicode: + # for unicode, use the slower parse_obj() to get self.w_kwargs + # as a wrapped dictionary that may contain full-range unicode + # keys. See test_non_latin1_key + space = self.space + w_args, w_kwds = args.parse_obj(None, 'format', + format_signature) + self.args = space.listview(w_args) + self.w_kwargs = w_kwds + else: + self.args, self.kwargs = args.unpack() self.auto_numbering = 0 self.auto_numbering_state = ANS_INIT return self._build_string(0, len(self.template), 2) @@ -197,17 +211,13 @@ if index == -1: kwarg = name[:i] if self.is_unicode: + w_arg = space.getitem(self.w_kwargs, space.wrap(kwarg)) + else: try: - arg_key = kwarg.encode("latin-1") - except UnicodeEncodeError: - # Not going to be found in a dict of strings. - raise OperationError(space.w_KeyError, space.wrap(kwarg)) - else: - arg_key = kwarg - try: - w_arg = self.kwargs[arg_key] - except KeyError: - raise OperationError(space.w_KeyError, space.wrap(arg_key)) + w_arg = self.kwargs[kwarg] + except KeyError: + raise OperationError(space.w_KeyError, + space.wrap(kwarg)) else: try: w_arg = self.args[index] @@ -351,14 +361,8 @@ return space.iter(space.newlist(self.parser_list_w)) return TemplateFormatter -StrTemplateFormatter = make_template_formatting_class() -UnicodeTemplateFormatter = make_template_formatting_class() - -def str_template_formatter(space, template): - return StrTemplateFormatter(space, False, template) - -def unicode_template_formatter(space, template): - return UnicodeTemplateFormatter(space, True, template) +str_template_formatter = make_template_formatting_class(for_unicode=False) +unicode_template_formatter = make_template_formatting_class(for_unicode=True) def format_method(space, w_string, args, is_unicode): @@ -395,16 +399,16 @@ LONG_DIGITS = string.digits + string.ascii_lowercase -def make_formatting_class(): +def make_formatting_class(for_unicode): class Formatter(BaseFormatter): """__format__ implementation for builtin types.""" + is_unicode = for_unicode _grouped_digits = None - def __init__(self, space, is_unicode, spec): + def __init__(self, space, spec): self.space = space - self.is_unicode = is_unicode - self.empty = u"" if is_unicode else "" + self.empty = u"" if self.is_unicode else "" self.spec = spec def _is_alignment(self, c): @@ -1138,15 +1142,8 @@ self._unknown_presentation("complex") return Formatter -StrFormatter = make_formatting_class() -UnicodeFormatter = make_formatting_class() - - -def unicode_formatter(space, spec): - return StrFormatter(space, True, spec) - -def str_formatter(space, spec): - return UnicodeFormatter(space, False, spec) +str_formatter = make_formatting_class(for_unicode=False) +unicode_formatter = make_formatting_class(for_unicode=True) @specialize.arg(2) diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -445,7 +445,7 @@ return w_obj.listview_bytes() if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject: return w_obj.listview_bytes() - if isinstance(w_obj, W_BytesObject) and self._uses_no_iter(w_obj): + if isinstance(w_obj, W_BytesObject) and self._str_uses_no_iter(w_obj): return w_obj.listview_bytes() if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj): return w_obj.getitems_bytes() @@ -460,7 +460,7 @@ return w_obj.listview_unicode() if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject: return w_obj.listview_unicode() - if isinstance(w_obj, W_UnicodeObject) and self._uses_no_iter(w_obj): + if isinstance(w_obj, W_UnicodeObject) and self._uni_uses_no_iter(w_obj): return w_obj.listview_unicode() if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj): return w_obj.getitems_unicode() @@ -504,8 +504,15 @@ from pypy.objspace.descroperation import tuple_iter return self.lookup(w_obj, '__iter__') is tuple_iter(self) - def _uses_no_iter(self, w_obj): - return self.lookup(w_obj, '__iter__') is None + def _str_uses_no_iter(self, w_obj): + from pypy.objspace.descroperation import str_getitem + return (self.lookup(w_obj, '__iter__') is None and + self.lookup(w_obj, '__getitem__') is str_getitem(self)) + + def _uni_uses_no_iter(self, w_obj): + from pypy.objspace.descroperation import unicode_getitem + return (self.lookup(w_obj, '__iter__') is None and + self.lookup(w_obj, '__getitem__') is unicode_getitem(self)) def sliceindices(self, w_slice, w_length): if isinstance(w_slice, W_SliceObject): diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -432,7 +432,7 @@ class AppTestListObject(object): - spaceconfig = {"objspace.std.withliststrategies": True} # it's the default + #spaceconfig = {"objspace.std.withliststrategies": True} # it's the default def setup_class(cls): import platform @@ -1518,6 +1518,16 @@ def __iter__(self): yield "ok" assert list(U(u"don't see me")) == ["ok"] + # + class S(str): + def __getitem__(self, index): + return str.__getitem__(self, index).upper() + assert list(S("abc")) == list("ABC") + # + class U(unicode): + def __getitem__(self, index): + return unicode.__getitem__(self, index).upper() + assert list(U(u"abc")) == list(u"ABC") def test_extend_from_nonempty_list_with_subclasses(self): l = ["hi!"] @@ -1543,6 +1553,20 @@ l.extend(U(u"don't see me")) # assert l == ["hi!", "okT", "okL", "okL", "okS", "okU"] + # + class S(str): + def __getitem__(self, index): + return str.__getitem__(self, index).upper() + l = [] + l.extend(S("abc")) + assert l == list("ABC") + # + class U(unicode): + def __getitem__(self, index): + return unicode.__getitem__(self, index).upper() + l = [] + l.extend(U(u"abc")) + assert l == list(u"ABC") def test_no_len_on_range_iter(self): iterable = range(10) diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py --- a/pypy/objspace/std/test/test_newformat.py +++ b/pypy/objspace/std/test/test_newformat.py @@ -215,7 +215,9 @@ assert self.s("{!r}").format(x()) == self.s("32") def test_non_latin1_key(self): - raises(KeyError, self.s("{\u1000}").format) + raises(KeyError, u"{\u1000}".format) + d = {u"\u1000": u"foo"} + assert u"{\u1000}".format(**d) == u"foo" class AppTestStringFormat(BaseStringFormatTests): diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,7 +1,7 @@ # Edit these appropriately before running this script maj=5 -min=3 -rev=1 +min=4 +rev=0 branchname=release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy2.7-v$maj.$min.$rev # ==OR== release-$maj.$min diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -164,8 +164,15 @@ # annotations that are passed in, and don't annotate the old # graph -- it's already low-level operations! for a, s_newarg in zip(block.inputargs, cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg + s_oldarg = a.annotation + # XXX: Should use s_oldarg.contains(s_newarg) but that breaks + # PyPy translation + if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg: + raise annmodel.AnnotatorError( + "Late-stage annotation is not allowed to modify the " + "existing annotation for variable %s: %s" % + (a, s_oldarg)) + else: assert not self.frozen if block not in self.annotated: diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -17,7 +17,7 @@ from rpython.flowspace.model import Variable, Constant, const from rpython.flowspace.operation import op from rpython.rlib import rarithmetic -from rpython.annotator.model import AnnotatorError +from rpython.annotator.model import AnnotatorError, TLS BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values() if oper.dispatch == 2]) @@ -436,6 +436,11 @@ class __extend__(pairtype(SomeFloat, SomeFloat)): def union((flt1, flt2)): + if not TLS.allow_int_to_float: + # in this mode, if one of the two is actually the + # subclass SomeInteger, complain + if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger): + raise UnionError(flt1, flt2) return SomeFloat() add = sub = mul = union diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -44,6 +44,7 @@ # A global attribute :-( Patch it with 'True' to enable checking of # the no_nul attribute... check_str_without_nul = False + allow_int_to_float = True TLS = State() class SomeObject(object): @@ -749,6 +750,7 @@ s1 = pair(s1, s2).union() else: # this is just a performance shortcut + # XXX: This is a lie! Grep for no_side_effects_in_union and weep. if s1 != s2: s1 = pair(s1, s2).union() return s1 diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -176,6 +176,7 @@ SOCK_DGRAM SOCK_RAW SOCK_RDM SOCK_SEQPACKET SOCK_STREAM +SOCK_CLOEXEC SOL_SOCKET SOL_IPX SOL_AX25 SOL_ATALK SOL_NETROM SOL_ROSE @@ -319,6 +320,8 @@ [('p_proto', rffi.INT), ]) +CConfig.HAVE_ACCEPT4 = platform.Has('accept4') + if _POSIX: CConfig.nfds_t = platform.SimpleType('nfds_t') CConfig.pollfd = platform.Struct('struct pollfd', @@ -541,6 +544,12 @@ socketaccept = external('accept', [socketfd_type, sockaddr_ptr, socklen_t_ptr], socketfd_type, save_err=SAVE_ERR) +HAVE_ACCEPT4 = cConfig.HAVE_ACCEPT4 +if HAVE_ACCEPT4: + socketaccept4 = external('accept4', [socketfd_type, sockaddr_ptr, + socklen_t_ptr, rffi.INT], + socketfd_type, + save_err=SAVE_ERR) socketbind = external('bind', [socketfd_type, sockaddr_ptr, socklen_t], rffi.INT, save_err=SAVE_ERR) socketlisten = external('listen', [socketfd_type, rffi.INT], rffi.INT, diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py --- a/rpython/rlib/buffer.py +++ b/rpython/rlib/buffer.py @@ -10,6 +10,7 @@ _immutable_ = True def getlength(self): + """Returns the size in bytes (even if getitemsize() > 1).""" raise NotImplementedError def __len__(self): diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py --- a/rpython/rlib/libffi.py +++ b/rpython/rlib/libffi.py @@ -47,6 +47,8 @@ cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG) cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED) cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar) + # XXX long double support: clibffi.ffi_type_longdouble, but then + # XXX fix the whole rest of this file to add a case for long double del cls._import @staticmethod diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py --- a/rpython/rlib/rbigint.py +++ b/rpython/rlib/rbigint.py @@ -1222,6 +1222,9 @@ # base is supposed to be positive or 0.0, which means we use e if base == 10.0: return _loghelper(math.log10, self) + if base == 2.0: + from rpython.rlib import rfloat + return _loghelper(rfloat.log2, self) ret = _loghelper(math.log, self) if base != 0.0: ret /= math.log(base) diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -98,8 +98,15 @@ try: ctypes.CDLL(name) except OSError as e: + # common case: ctypes fails too, with the real dlerror() + # message in str(e). Return that error message. return str(e) else: + # uncommon case: may happen if 'name' is a linker script + # (which the C-level dlopen() can't handle) and we are + # directly running on pypy (whose implementation of ctypes + # or cffi will resolve linker scripts). In that case, + # unsure what we can do. return ("opening %r with ctypes.CDLL() works, " "but not with c_dlopen()??" % (name,)) @@ -160,7 +167,18 @@ mode = _dlopen_default_mode() elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0: mode |= RTLD_NOW + # + # haaaack for 'pypy py.test -A' if libm.so is a linker script + # (see reason in _dlerror_on_dlopen_untranslated()) + must_free = False + if not we_are_translated() and platform.name == "linux": + if name and rffi.charp2str(name) == 'libm.so': + name = rffi.str2charp('libm.so.6') + must_free = True + # res = c_dlopen(name, rffi.cast(rffi.INT, mode)) + if must_free: + rffi.free_charp(name) if not res: if not we_are_translated(): err = _dlerror_on_dlopen_untranslated(name) diff --git a/rpython/rlib/rjitlog/rjitlog.py b/rpython/rlib/rjitlog/rjitlog.py --- a/rpython/rlib/rjitlog/rjitlog.py +++ b/rpython/rlib/rjitlog/rjitlog.py @@ -430,9 +430,8 @@ def encode_merge_point(log, compressor, values): line = [] - unrolled = unrolling_iterable(values) i = 0 - for value in unrolled: + for value in values: line.append(value.encode(log,i,compressor)) i += 1 return ''.join(line) diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py --- a/rpython/rlib/rmarshal.py +++ b/rpython/rlib/rmarshal.py @@ -346,11 +346,15 @@ # on s_bigger. It relies on the fact that s_bigger was created with # an expression like 'annotation([s_item])' which returns a ListDef with # no bookkeeper, on which side-effects are not allowed. + saved = annmodel.TLS.allow_int_to_float try: + annmodel.TLS.allow_int_to_float = False s_union = annmodel.unionof(s_bigger, s_smaller) return s_bigger.contains(s_union) except (annmodel.UnionError, TooLateForChange): return False + finally: + annmodel.TLS.allow_int_to_float = saved class __extend__(pairtype(MTag, annmodel.SomeObject)): diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -126,6 +126,10 @@ "SSL_OP_NO_COMPRESSION") SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") + SSL_OP_CIPHER_SERVER_PREFERENCE = rffi_platform.ConstantInteger( + "SSL_OP_CIPHER_SERVER_PREFERENCE") + SSL_OP_SINGLE_DH_USE = rffi_platform.ConstantInteger( + "SSL_OP_SINGLE_DH_USE") HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME") HAS_NPN = rffi_platform.Defined("OPENSSL_NPN_NEGOTIATED") SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE") @@ -307,6 +311,8 @@ if HAVE_OPENSSL_RAND: ssl_external('RAND_add', [rffi.CCHARP, rffi.INT, rffi.DOUBLE], lltype.Void) + ssl_external('RAND_bytes', [rffi.UCHARP, rffi.INT], rffi.INT) + ssl_external('RAND_pseudo_bytes', [rffi.UCHARP, rffi.INT], rffi.INT) ssl_external('RAND_status', [], rffi.INT) if HAVE_OPENSSL_RAND_EGD: ssl_external('RAND_egd', [rffi.CCHARP], rffi.INT) @@ -465,6 +471,7 @@ ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT) ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME, macro=True) + ssl_external('pypy_GENERAL_NAME_uri', [GENERAL_NAME], ASN1_IA5STRING, macro=True) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -223,7 +223,7 @@ pass if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): @@ -254,10 +254,11 @@ UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32, [('actime', rffi.INT), ('modtime', rffi.INT)]) + CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT) if not _WIN32: UID_T = rffi_platform.SimpleType('uid_t', rffi.UINT) GID_T = rffi_platform.SimpleType('gid_t', rffi.UINT) - CLOCK_T = rffi_platform.SimpleType('clock_t', rffi.INT) + TIOCGWINSZ = rffi_platform.DefinedConstantInteger('TIOCGWINSZ') TMS = rffi_platform.Struct( 'struct tms', [('tms_utime', rffi.INT), @@ -265,6 +266,12 @@ ('tms_cutime', rffi.INT), ('tms_cstime', rffi.INT)]) + WINSIZE = rffi_platform.Struct( + 'struct winsize', [('ws_row', rffi.USHORT), + ('ws_col', rffi.USHORT), + ('ws_xpixel', rffi.USHORT), + ('ws_ypixel', rffi.USHORT)]) + GETPGRP_HAVE_ARG = rffi_platform.Has("getpgrp(0)") SETPGRP_HAVE_ARG = rffi_platform.Has("setpgrp(0, 0)") @@ -365,15 +372,27 @@ raise OSError(get_saved_errno(), '%s failed' % name) return result +def _dup(fd, inheritable=True): + validate_fd(fd) + if inheritable: + res = c_dup(fd) + else: + res = c_dup_noninheritable(fd) + return res + @replace_os_function('dup') -def dup(fd): - validate_fd(fd) - return handle_posix_error('dup', c_dup(fd)) +def dup(fd, inheritable=True): + res = _dup(fd, inheritable) + return handle_posix_error('dup', res) @replace_os_function('dup2') -def dup2(fd, newfd): +def dup2(fd, newfd, inheritable=True): validate_fd(fd) - handle_posix_error('dup2', c_dup2(fd, newfd)) + if inheritable: + res = c_dup2(fd, newfd) + else: + res = c_dup2_noninheritable(fd, newfd) + handle_posix_error('dup2', res) #___________________________________________________________________ @@ -604,7 +623,8 @@ class CConfig: _compilation_info_ = eci DIRENT = rffi_platform.Struct('struct dirent', - [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))] + [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1)), + ('d_ino', lltype.Signed)] + [('d_type', rffi.INT)] if HAVE_D_TYPE else []) if HAVE_D_TYPE: DT_UNKNOWN = rffi_platform.ConstantInteger('DT_UNKNOWN') @@ -628,6 +648,8 @@ macro=True, save_err=rffi.RFFI_FULL_ERRNO_ZERO) c_closedir = external('closedir', [DIRP], rffi.INT, releasegil=False) c_dirfd = external('dirfd', [DIRP], rffi.INT, releasegil=False) + c_ioctl_voidp = external('ioctl', [rffi.INT, rffi.UINT, rffi.VOIDP], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) else: dirent_config = {} @@ -1113,37 +1135,69 @@ c_open_osfhandle = external('_open_osfhandle', [rffi.INTPTR_T, rffi.INT], rffi.INT) + HAVE_PIPE2 = False + HAVE_DUP3 = False + O_CLOEXEC = None else: INT_ARRAY_P = rffi.CArrayPtr(rffi.INT) c_pipe = external('pipe', [INT_ARRAY_P], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) + class CConfig: + _compilation_info_ = eci + HAVE_PIPE2 = rffi_platform.Has('pipe2') + HAVE_DUP3 = rffi_platform.Has('dup3') + O_CLOEXEC = rffi_platform.DefinedConstantInteger('O_CLOEXEC') + config = rffi_platform.configure(CConfig) + HAVE_PIPE2 = config['HAVE_PIPE2'] + HAVE_DUP3 = config['HAVE_DUP3'] + O_CLOEXEC = config['O_CLOEXEC'] + if HAVE_PIPE2: + c_pipe2 = external('pipe2', [INT_ARRAY_P, rffi.INT], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) @replace_os_function('pipe') -def pipe(): +def pipe(flags=0): + # 'flags' might be ignored. Check the result. if _WIN32: - pread = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw') - pwrite = lltype.malloc(rwin32.LPHANDLE.TO, 1, flavor='raw') - try: - if not CreatePipe( - pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0): - raise WindowsError(rwin32.GetLastError_saved(), - "CreatePipe failed") - hread = rffi.cast(rffi.INTPTR_T, pread[0]) - hwrite = rffi.cast(rffi.INTPTR_T, pwrite[0]) - finally: - lltype.free(pwrite, flavor='raw') - lltype.free(pread, flavor='raw') - fdread = c_open_osfhandle(hread, 0) - fdwrite = c_open_osfhandle(hwrite, 1) - return (fdread, fdwrite) + # 'flags' ignored + ralloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1) + walloc = lltype.scoped_alloc(rwin32.LPHANDLE.TO, 1) + with ralloc as pread, walloc as pwrite: + if CreatePipe(pread, pwrite, lltype.nullptr(rffi.VOIDP.TO), 0): + hread = pread[0] + hwrite = pwrite[0] + fdread = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hread), 0) + fdwrite = c_open_osfhandle(rffi.cast(rffi.INTPTR_T, hwrite), 1) + if not (fdread == -1 or fdwrite == -1): + return (fdread, fdwrite) + rwin32.CloseHandle(hread) + rwin32.CloseHandle(hwrite) + raise WindowsError(rwin32.GetLastError_saved(), "CreatePipe failed") else: filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw') try: - handle_posix_error('pipe', c_pipe(filedes)) + if HAVE_PIPE2 and _pipe2_syscall.attempt_syscall(): + res = c_pipe2(filedes, flags) + if _pipe2_syscall.fallback(res): + res = c_pipe(filedes) + else: + res = c_pipe(filedes) # 'flags' ignored + handle_posix_error('pipe', res) return (widen(filedes[0]), widen(filedes[1])) finally: lltype.free(filedes, flavor='raw') +def pipe2(flags): + # Only available if there is really a c_pipe2 function. + # No fallback to pipe() if we get ENOSYS. + filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw') + try: + res = c_pipe2(filedes, flags) + handle_posix_error('pipe2', res) + return (widen(filedes[0]), widen(filedes[1])) + finally: + lltype.free(filedes, flavor='raw') + c_link = external('link', [rffi.CCHARP, rffi.CCHARP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO,) c_symlink = external('symlink', [rffi.CCHARP, rffi.CCHARP], rffi.INT, @@ -2079,14 +2133,47 @@ eci_inheritable = eci.merge(ExternalCompilationInfo( - separate_module_sources=[""" + separate_module_sources=[r""" +#include <errno.h> +#include <sys/ioctl.h> + RPY_EXTERN int rpy_set_inheritable(int fd, int inheritable) { - /* XXX minimal impl. XXX */ - int request = inheritable ? FIONCLEX : FIOCLEX; - return ioctl(fd, request, NULL); + static int ioctl_works = -1; + int flags; + + if (ioctl_works != 0) { + int request = inheritable ? FIONCLEX : FIOCLEX; + int err = ioctl(fd, request, NULL); + if (!err) { + ioctl_works = 1; + return 0; + } + + if (errno != ENOTTY && errno != EACCES) { + return -1; + } + else { + /* ENOTTY: The ioctl is declared but not supported by the + kernel. EACCES: SELinux policy, this can be the case on + Android. */ + ioctl_works = 0; + } + /* fallback to fcntl() if ioctl() does not work */ + } + + flags = fcntl(fd, F_GETFD); + if (flags < 0) + return -1; + + if (inheritable) + flags &= ~FD_CLOEXEC; + else + flags |= FD_CLOEXEC; + return fcntl(fd, F_SETFD, flags); } + RPY_EXTERN int rpy_get_inheritable(int fd) { @@ -2095,8 +2182,64 @@ return -1; return !(flags & FD_CLOEXEC); } - """], - post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);'])) + +RPY_EXTERN +int rpy_dup_noninheritable(int fd) +{ +#ifdef _WIN32 +#error NotImplementedError +#endif + +#ifdef F_DUPFD_CLOEXEC + return fcntl(fd, F_DUPFD_CLOEXEC, 0); +#else + fd = dup(fd); + if (fd >= 0) { + if (rpy_set_inheritable(fd, 0) != 0) { + close(fd); + return -1; + } + } + return fd; +#endif +} + +RPY_EXTERN +int rpy_dup2_noninheritable(int fd, int fd2) +{ +#ifdef _WIN32 +#error NotImplementedError +#endif + +#ifdef F_DUP2FD_CLOEXEC + return fcntl(fd, F_DUP2FD_CLOEXEC, fd2); + +#else +# if %(HAVE_DUP3)d /* HAVE_DUP3 */ + static int dup3_works = -1; + if (dup3_works != 0) { + if (dup3(fd, fd2, O_CLOEXEC) >= 0) + return 0; + if (dup3_works == -1) + dup3_works = (errno != ENOSYS); + if (dup3_works) + return -1; + } +# endif + if (dup2(fd, fd2) < 0) + return -1; + if (rpy_set_inheritable(fd2, 0) != 0) { + close(fd2); + return -1; + } + return 0; +#endif +} + """ % {'HAVE_DUP3': HAVE_DUP3}], + post_include_bits=['RPY_EXTERN int rpy_set_inheritable(int, int);\n' + 'RPY_EXTERN int rpy_get_inheritable(int);\n' + 'RPY_EXTERN int rpy_dup_noninheritable(int);\n' + 'RPY_EXTERN int rpy_dup2_noninheritable(int, int);\n'])) c_set_inheritable = external('rpy_set_inheritable', [rffi.INT, rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO, @@ -2104,12 +2247,56 @@ c_get_inheritable = external('rpy_get_inheritable', [rffi.INT], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO, compilation_info=eci_inheritable) +c_dup_noninheritable = external('rpy_dup_noninheritable', [rffi.INT], + rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eci_inheritable) +c_dup2_noninheritable = external('rpy_dup2_noninheritable', [rffi.INT,rffi.INT], + rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO, + compilation_info=eci_inheritable) def set_inheritable(fd, inheritable): - error = c_set_inheritable(fd, inheritable) - handle_posix_error('set_inheritable', error) + result = c_set_inheritable(fd, inheritable) + handle_posix_error('set_inheritable', result) def get_inheritable(fd): res = c_get_inheritable(fd) res = handle_posix_error('get_inheritable', res) return res != 0 + +class SetNonInheritableCache(object): + """Make one prebuilt instance of this for each path that creates + file descriptors, where you don't necessarily know if that function + returns inheritable or non-inheritable file descriptors. + """ + _immutable_fields_ = ['cached_inheritable?'] + cached_inheritable = -1 # -1 = don't know yet; 0 = off; 1 = on + + def set_non_inheritable(self, fd): + if self.cached_inheritable == -1: + self.cached_inheritable = get_inheritable(fd) + if self.cached_inheritable == 1: + # 'fd' is inheritable; we must manually turn it off + set_inheritable(fd, False) + + def _cleanup_(self): + self.cached_inheritable = -1 + +class ENoSysCache(object): + """Cache whether a system call returns ENOSYS or not.""" + _immutable_fields_ = ['cached_nosys?'] + cached_nosys = -1 # -1 = don't know; 0 = no; 1 = yes, getting ENOSYS + + def attempt_syscall(self): + return self.cached_nosys != 1 + + def fallback(self, res): + nosys = self.cached_nosys + if nosys == -1: + nosys = (res < 0 and get_saved_errno() == errno.ENOSYS) + self.cached_nosys = nosys + return nosys + + def _cleanup_(self): + self.cached_nosys = -1 + +_pipe2_syscall = ENoSysCache() diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py --- a/rpython/rlib/rposix_scandir.py +++ b/rpython/rlib/rposix_scandir.py @@ -50,3 +50,6 @@ if rposix.HAVE_D_TYPE: return rffi.getintfield(direntp, 'c_d_type') return DT_UNKNOWN + +def get_inode(direntp): + return rffi.getintfield(direntp, 'c_d_ino') diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -31,7 +31,7 @@ signal_names.append('CTRL_BREAK_EVENT') CTRL_C_EVENT = 0 CTRL_BREAK_EVENT = 1 -includes = ['stdlib.h', 'src/signals.h'] +includes = ['stdlib.h', 'src/signals.h', 'signal.h'] if sys.platform != 'win32': includes.append('sys/time.h') @@ -47,7 +47,9 @@ _compilation_info_ = eci if sys.platform != 'win32': - for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split(): + for name in """ + ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF + SIG_BLOCK SIG_UNBLOCK SIG_SETMASK""".split(): setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) CConfig.timeval = rffi_platform.Struct( @@ -71,7 +73,8 @@ pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void) pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void) pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void) -pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT) +pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', + [rffi.INT, rffi.INT], rffi.INT) pypysig_poll = external('pypysig_poll', [], rffi.INT, releasegil=False) # don't bother releasing the GIL around a call to pypysig_poll: it's # pointless and a performance issue @@ -98,3 +101,20 @@ [rffi.INT, itimervalP, itimervalP], rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO) c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT) + +c_pthread_kill = external('pthread_kill', [lltype.Signed, rffi.INT], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) + +if sys.platform != 'win32': + c_sigset_t = rffi.COpaquePtr('sigset_t', compilation_info=eci) + c_sigemptyset = external('sigemptyset', [c_sigset_t], rffi.INT) + c_sigaddset = external('sigaddset', [c_sigset_t, rffi.INT], rffi.INT) + c_sigismember = external('sigismember', [c_sigset_t, rffi.INT], rffi.INT) + c_sigwait = external('sigwait', [c_sigset_t, rffi.INTP], rffi.INT, + releasegil=True, + save_err=rffi.RFFI_SAVE_ERRNO) + c_sigpending = external('sigpending', [c_sigset_t], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) + c_pthread_sigmask = external('pthread_sigmask', + [rffi.INT, c_sigset_t, c_sigset_t], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -8,10 +8,11 @@ # XXX this does not support yet the least common AF_xxx address families # supported by CPython. See http://bugs.pypy.org/issue1942 +from errno import EINVAL from rpython.rlib import _rsocket_rffi as _c, jit, rgc from rpython.rlib.objectmodel import instantiate, keepalive_until_here from rpython.rlib.rarithmetic import intmask, r_uint -from rpython.rlib import rthread +from rpython.rlib import rthread, rposix from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.rffi import sizeof, offsetof from rpython.rtyper.extregistry import ExtRegistryEntry @@ -522,12 +523,28 @@ timeout = -1.0 def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, - fd=_c.INVALID_SOCKET): + fd=_c.INVALID_SOCKET, inheritable=True): """Create a new socket.""" if _c.invalid_socket(fd): - fd = _c.socket(family, type, proto) - if _c.invalid_socket(fd): - raise self.error_handler() + if not inheritable and 'SOCK_CLOEXEC' in constants: + # Non-inheritable: we try to call socket() with + # SOCK_CLOEXEC, which may fail. If we get EINVAL, + # then we fall back to the SOCK_CLOEXEC-less case. + fd = _c.socket(family, type | SOCK_CLOEXEC, proto) + if fd < 0: + if _c.geterrno() == EINVAL: + # Linux older than 2.6.27 does not support + # SOCK_CLOEXEC. An EINVAL might be caused by + # random other things, though. Don't cache. + pass + else: + raise self.error_handler() + if _c.invalid_socket(fd): + fd = _c.socket(family, type, proto) + if _c.invalid_socket(fd): + raise self.error_handler() + if not inheritable: + sock_set_inheritable(fd, False) # PLAT RISCOS self.fd = fd self.family = family @@ -630,20 +647,33 @@ return addr, addr.addr_p, addrlen_p @jit.dont_look_inside - def accept(self): + def accept(self, inheritable=True): """Wait for an incoming connection. Return (new socket fd, client address).""" if self._select(False) == 1: raise SocketTimeout address, addr_p, addrlen_p = self._addrbuf() try: - newfd = _c.socketaccept(self.fd, addr_p, addrlen_p) + remove_inheritable = not inheritable + if (not inheritable and 'SOCK_CLOEXEC' in constants + and _c.HAVE_ACCEPT4 + and _accept4_syscall.attempt_syscall()): + newfd = _c.socketaccept4(self.fd, addr_p, addrlen_p, + SOCK_CLOEXEC) + if _accept4_syscall.fallback(newfd): + newfd = _c.socketaccept(self.fd, addr_p, addrlen_p) + else: + remove_inheritable = False + else: + newfd = _c.socketaccept(self.fd, addr_p, addrlen_p) addrlen = addrlen_p[0] finally: lltype.free(addrlen_p, flavor='raw') address.unlock() if _c.invalid_socket(newfd): raise self.error_handler() + if remove_inheritable: + sock_set_inheritable(newfd, False) address.addrlen = rffi.cast(lltype.Signed, addrlen) return (newfd, address) @@ -1032,6 +1062,12 @@ return result make_socket._annspecialcase_ = 'specialize:arg(4)' +def sock_set_inheritable(fd, inheritable): + try: + rposix.set_inheritable(fd, inheritable) + except OSError as e: + raise CSocketError(e.errno) + class SocketError(Exception): applevelerrcls = 'error' def __init__(self): @@ -1090,7 +1126,7 @@ if hasattr(_c, 'socketpair'): def socketpair(family=socketpair_default_family, type=SOCK_STREAM, proto=0, - SocketClass=RSocket): + SocketClass=RSocket, inheritable=True): """socketpair([family[, type[, proto]]]) -> (socket object, socket object) Create a pair of socket objects from the sockets returned by the platform @@ -1100,18 +1136,41 @@ """ result = lltype.malloc(_c.socketpair_t, 2, flavor='raw') try: - res = _c.socketpair(family, type, proto, result) + res = -1 + remove_inheritable = not inheritable + if not inheritable and 'SOCK_CLOEXEC' in constants: + # Non-inheritable: we try to call socketpair() with + # SOCK_CLOEXEC, which may fail. If we get EINVAL, + # then we fall back to the SOCK_CLOEXEC-less case. + res = _c.socketpair(family, type | SOCK_CLOEXEC, + proto, result) + if res < 0: + if _c.geterrno() == EINVAL: + # Linux older than 2.6.27 does not support + # SOCK_CLOEXEC. An EINVAL might be caused by + # random other things, though. Don't cache. + pass + else: + raise last_error() + else: + remove_inheritable = False + # if res < 0: - raise last_error() + res = _c.socketpair(family, type, proto, result) + if res < 0: + raise last_error() fd0 = rffi.cast(lltype.Signed, result[0]) fd1 = rffi.cast(lltype.Signed, result[1]) finally: lltype.free(result, flavor='raw') + if remove_inheritable: + sock_set_inheritable(fd0, False) + sock_set_inheritable(fd1, False) return (make_socket(fd0, family, type, proto, SocketClass), make_socket(fd1, family, type, proto, SocketClass)) if _c.WIN32: - def dup(fd): + def dup(fd, inheritable=True): with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info: if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info): raise last_error() @@ -1122,15 +1181,16 @@ raise last_error() return result else: - def dup(fd): - return _c.dup(fd) - _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
