[pypy-commit] pypy buffer-interface2: merge default into branch
Author: Matti Picus Branch: buffer-interface2 Changeset: r87830:36f5b3ab1251 Date: 2016-10-16 14:55 +0300 http://bitbucket.org/pypy/pypy/changeset/36f5b3ab1251/ Log:merge default into branch diff too long, truncating to 2000 out of 44282 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -362,14 +362,25 @@ message = short explain = long self.log_error("code %d, message %s", code, message) -# using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) -content = (self.error_message_format % - {'code': code, 'message': _quote_html(message), 'explain': explain}) self.send_response(code, message) -self.send_header("Content-Type", self.error_content_type) self.send_header('Connection', 'close') + +# Message body is omitted for cases described in: +# - RFC7230: 3.3. 1xx, 204(No Content), 304(Not Modified) +# - RFC7231: 6.3.6. 205(Reset Content) +content = None +if code >= 200 and code not in (204, 205, 304): +# HTML encode to prevent Cross Site Scripting attacks +# (see bug #1100201) +content = (self.error_message_format % { +'code': code, +'message': _quote_html(message), +'explain': explain +}) +self.send_header("Content-Type", self.error_content_type) self.end_headers() -if self.command != 'HEAD' and code >= 200 and code not in (204, 304): + +if self.command != 'HEAD' and content: self.wfile.write(content) error_message_format = DEFAULT_ERROR_MESSAGE diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -84,7 +84,7 @@ path begins with one of the strings in self.cgi_directories (and the next character is a '/' or the end of the string). """ -collapsed_path = _url_collapse_path(urllib.unquote(self.path)) +collapsed_path = _url_collapse_path(self.path) dir_sep = collapsed_path.find('/', 1) head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] if head in self.cgi_directories: @@ -120,11 +120,7 @@ break # find an explicit query string, if present. -i = rest.rfind('?') -if i >= 0: -rest, query = rest[:i], rest[i+1:] -else: -query = '' +rest, _, query = rest.partition('?') # dissect the part after the directory name into a script name & # a possible additional path, to be stored in PATH_INFO. @@ -308,13 +304,15 @@ The utility of this function is limited to is_cgi method and helps preventing some security attacks. -Returns: A tuple of (head, tail) where tail is everything after the final / -and head is everything before it. Head will always start with a '/' and, -if it contains anything else, never have a trailing '/'. +Returns: The reconstituted URL, which will always start with a '/'. Raises: IndexError if too many '..' occur within the path. """ +# Query component should not be involved. +path, _, query = path.partition('?') +path = urllib.unquote(path) + # Similar to os.path.split(os.path.normpath(path)) but specific to URL # path semantics rather than local operating system semantics. path_parts = path.split('/') @@ -335,6 +333,9 @@ else: tail_part = '' +if query: +tail_part = '?'.join((tail_part, query)) + splitpath = ('/' + '/'.join(head_parts), tail_part) collapsed_path = "/".join(splitpath) diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -190,7 +190,7 @@ Backwards Compatibility --- -In order to keep compatibilty with earlier versions of Cookie.py, +In order to keep compatibility with earlier versions of Cookie.py, it is still possible to use Cookie.Cookie() to create a Cookie. In fact, this simply returns a SmartCookie. diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py --- a/lib-python/2.7/SimpleHTTPServer.py +++ b/lib-python/2.7/SimpleHTTPServer.py @@ -167,9 +167,9 @@ words = filter(None, words) path = os.getcwd() for word in words: -drive, word = os.path.splitdrive(word) -head, word = os.path.split(word) -if word in (os.curdir, os.pardir): continue +if os.path.dirname(word) or word in (os.curdir, os.pardir): +# Ignore components that are not a simple file/directory name +continue path = os.path.join(path, word) if trailin
[pypy-commit] pypy buffer-interface2: merge default into branch
Author: Matti Picus Branch: buffer-interface2 Changeset: r87582:ed7457dceb76 Date: 2016-10-04 20:33 +0300 http://bitbucket.org/pypy/pypy/changeset/ed7457dceb76/ Log:merge default into branch diff too long, truncating to 2000 out of 3661 lines diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -43,6 +43,7 @@ try: if detect_cpu.autodetect().startswith('x86'): working_modules.add('_vmprof') +working_modules.add('faulthandler') except detect_cpu.ProcessorAutodetectError: pass @@ -89,6 +90,7 @@ ('objspace.usemodules.thread', True)], 'cpyext': [('objspace.usemodules.array', True)], 'cppyy': [('objspace.usemodules.cpyext', True)], +'faulthandler': [('objspace.usemodules._vmprof', True)], } module_suggests = { # the reason you want _rawffi is for ctypes, which @@ -114,7 +116,8 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], -"_vmprof" : ["pypy.module._vmprof.interp_vmprof"], +"_vmprof" : ["pypy.module._vmprof.interp_vmprof"], +"faulthandler" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -33,7 +33,7 @@ from somewhere else. You no longer have to do the same with the ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library. -.. branch: _warning +.. branch: _warnings CPython allows warning.warn(('something', 1), Warning), on PyPy this produced a "expected a readable buffer object" error. Test and fix. @@ -42,3 +42,16 @@ CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are allowed as arguments. Test and fix for str and unicode + +.. branch: faulthandler + +Port the 'faulthandler' module to PyPy default. This module is standard +in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI. + +.. branch: test-cpyext + +Refactor cpyext testing to be more pypy3-friendly. + +.. branch: better-error-missing-self + +Improve the error message when the user forgot the "self" argument of a method. diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -33,6 +33,7 @@ --info : print translation information about this PyPy executable -X track-resources : track the creation of files and sockets and display a warning if they are not closed explicitly +-X faulthandler: attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME, PYTHONCASEOK USAGE2 = """ @@ -233,12 +234,22 @@ import pypyjit pypyjit.set_param(jitparam) +def run_faulthandler(): +if 'faulthandler' in sys.builtin_module_names: +import faulthandler +try: +faulthandler.enable(2) # manually set to stderr +except ValueError: +pass # ignore "2 is not a valid file descriptor" + def set_runtime_options(options, Xparam, *args): if Xparam == 'track-resources': sys.pypy_set_track_resources(True) +elif Xparam == 'faulthandler': +run_faulthandler() else: print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),) -print >> sys.stderr, '[options] can be: track-resources' +print >> sys.stderr, '[options] can be: track-resources, faulthandler' raise SystemExit class CommandLineError(Exception): @@ -527,6 +538,9 @@ print >> sys.stderr, ( "Warning: pypy does not implement py3k warnings") +if os.getenv('PYTHONFAULTHANDLER'): +run_faulthandler() + ##if not we_are_translated(): ##for key in sorted(options): ##print '%40s: %s' % (key, options[key]) diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -21,7 +21,8 @@ ### Construction ### def __init__(self, space, args_w, keywords=None, keywords_w=None, - w_stararg=None, w_starstararg=None, keyword_names_w=None): + w_stararg=None, w_starstararg=None, keyword_names_w=None, + methodcall=False): self.space = space assert isinstance(args_w, list) self.arguments_w = args_w @@ -41,6 +42,9 @@ # a flag that specifies whether the JIT can unroll loops that operate # on the keywords self._jit_few_keywords = self.keywords is None or jit.isconstant(len(self.keywords)) +# a flag whether this is likely a method call, which doesn't change the +# behaviour but produces better error messages +self.methodcall = methodcall
[pypy-commit] pypy buffer-interface2: merge default into branch
Author: Matti Picus Branch: buffer-interface2 Changeset: r87472:57f07a076df4 Date: 2016-09-30 13:16 +0300 http://bitbucket.org/pypy/pypy/changeset/57f07a076df4/ Log:merge default into branch diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -40,4 +40,4 @@ # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html cffi_imports: pypy-c - PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py + PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -32,3 +32,13 @@ ``lib-python`` and ``lib_pypy``. Of course, you can put a symlink to it from somewhere else. You no longer have to do the same with the ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library. + +.. branch: _warning + +CPython allows warning.warn(('something', 1), Warning), on PyPy this +produced a "expected a readable buffer object" error. Test and fix. + +.. branch: stricter-strip + +CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are +allowed as arguments. Test and fix for str and unicode diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -93,12 +93,10 @@ home1 = rffi.charp2str(ll_home) home = os.path.join(home1, 'x') # <- so that 'll_home' can be # directly the root directory -dynamic = False else: home1 = "pypy's shared library location" -home = pypydir -dynamic = True -w_path = pypy_find_stdlib(space, home, dynamic) +home = '*' +w_path = pypy_find_stdlib(space, home) if space.is_none(w_path): if verbose: debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'" @@ -305,37 +303,20 @@ # HACKHACKHACK # ugly hack to modify target goal from compile_* to build_cffi_imports # this should probably get cleaned up and merged with driver.create_exe +from rpython.tool.runsubprocess import run_subprocess from rpython.translator.driver import taskdef import types -class Options(object): -pass - - -def mkexename(name): -if sys.platform == 'win32': -name = name.new(ext='exe') -return name - compile_goal, = driver.backend_select_goals(['compile']) @taskdef([compile_goal], "Create cffi bindings for modules") def task_build_cffi_imports(self): -from pypy.tool.build_cffi_imports import create_cffi_import_libraries ''' Use cffi to compile cffi interfaces to modules''' -exename = mkexename(driver.compute_exe_name()) -basedir = exename -while not basedir.join('include').exists(): -_basedir = basedir.dirpath() -if _basedir == basedir: -raise ValueError('interpreter %s not inside pypy repo', - str(exename)) -basedir = _basedir -modules = self.config.objspace.usemodules.getpaths() -options = Options() -# XXX possibly adapt options using modules -failures = create_cffi_import_libraries(exename, options, basedir) -# if failures, they were already printed -print >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)' +filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py') +status, out, err = run_subprocess(str(driver.compute_exe_name()), + [filename]) +sys.stdout.write(out) +sys.stderr.write(err) +# otherwise, ignore errors driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver) driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal] driver.default_goal = 'build_cffi_imports' diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -387,12 +387,12 @@ class BufferInterfaceNotFound(Exception): pass +@specialize.memo() def wrappable_class_name(Class): try: return Class.typedef.name except AttributeError: return 'internal subclass of %s' % (Class.__name__,) -wrappable_class_name._annspecialcase_ = 'specialize:memo' class CannotHaveLock(Exception): """Raised by space.allocate_lock() if we're translating.""" @@ -841,12 +841,13 @@ assert type(s) is str return self.interned_strings.get(s) is not None +@specialize.arg(1) def
[pypy-commit] pypy buffer-interface2: merge default into branch
Author: Matti Picus Branch: buffer-interface2 Changeset: r87402:46f3011f6b83 Date: 2016-09-26 22:07 +0300 http://bitbucket.org/pypy/pypy/changeset/46f3011f6b83/ Log:merge default into branch diff too long, truncating to 2000 out of 2477 lines diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -13,6 +13,7 @@ import sys import os import shlex +import imp from distutils.errors import DistutilsPlatformError @@ -62,8 +63,7 @@ """Initialize the module as appropriate for POSIX systems.""" g = {} g['EXE'] = "" -g['SO'] = ".so" -g['SOABI'] = g['SO'].rsplit('.')[0] +g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] g['LIBDIR'] = os.path.join(sys.prefix, 'lib') g['CC'] = "gcc -pthread" # -pthread might not be valid on OS/X, check @@ -75,8 +75,7 @@ """Initialize the module as appropriate for NT""" g = {} g['EXE'] = ".exe" -g['SO'] = ".pyd" -g['SOABI'] = g['SO'].rsplit('.')[0] +g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] global _config_vars _config_vars = g diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -529,7 +529,7 @@ for suffix, mode, type_ in imp.get_suffixes(): if type_ == imp.C_EXTENSION: _CONFIG_VARS['SOABI'] = suffix.split('.')[1] -break +break if args: vals = [] diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py --- a/lib_pypy/_subprocess.py +++ b/lib_pypy/_subprocess.py @@ -22,7 +22,10 @@ code, message = _ffi.getwinerror() raise WindowsError(code, message) -_INVALID_HANDLE_VALUE = _ffi.cast("HANDLE", -1) +def _int2handle(val): +return _ffi.cast("HANDLE", val) + +_INVALID_HANDLE_VALUE = _int2handle(-1) class _handle(object): def __init__(self, c_handle): @@ -70,9 +73,9 @@ target = _ffi.new("HANDLE[1]") res = _kernel32.DuplicateHandle( -_ffi.cast("HANDLE", source_process), -_ffi.cast("HANDLE", source), -_ffi.cast("HANDLE", target_process), +_int2handle(source_process), +_int2handle(source), +_int2handle(target_process), target, access, inherit, options) if not res: @@ -119,12 +122,14 @@ if not res: raise _WinError() -return _handle(pi.hProcess), _handle(pi.hThread), pi.dwProcessId, pi.dwThreadId +return (_handle(pi.hProcess), +_handle(pi.hThread), +pi.dwProcessId, +pi.dwThreadId) def WaitForSingleObject(handle, milliseconds): # CPython: the first argument is expected to be an integer. -res = _kernel32.WaitForSingleObject(_ffi.cast("HANDLE", handle), -milliseconds) +res = _kernel32.WaitForSingleObject(_int2handle(handle), milliseconds) if res < 0: raise _WinError() @@ -134,7 +139,7 @@ # CPython: the first argument is expected to be an integer. code = _ffi.new("DWORD[1]") -res = _kernel32.GetExitCodeProcess(_ffi.cast("HANDLE", handle), code) +res = _kernel32.GetExitCodeProcess(_int2handle(handle), code) if not res: raise _WinError() @@ -144,7 +149,7 @@ def TerminateProcess(handle, exitcode): # CPython: the first argument is expected to be an integer. # The second argument is silently wrapped in a UINT. -res = _kernel32.TerminateProcess(_ffi.cast("HANDLE", handle), +res = _kernel32.TerminateProcess(_int2handle(handle), _ffi.cast("UINT", exitcode)) if not res: diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.8.3 +Version: 1.8.4 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.8.3" -__version_info__ = (1, 8, 3) +__version__ = "1.8.4" +__version_info__ = (1, 8, 4) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { P
[pypy-commit] pypy buffer-interface2: merge default into branch
Author: Matti Picus Branch: buffer-interface2 Changeset: r87198:a3e17bbc7b47 Date: 2016-09-17 23:24 +0300 http://bitbucket.org/pypy/pypy/changeset/a3e17bbc7b47/ Log:merge default into branch diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.8.2 +Version: 1.8.3 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.8.2" -__version_info__ = (1, 8, 2) +__version__ = "1.8.3" +__version_info__ = (1, 8, 3) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -233,7 +233,7 @@ f = PySys_GetObject((char *)"stderr"); if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.8.2" + "\ncompiled with cffi version: 1.8.3" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/pypy/doc/config/translation.profopt.txt b/pypy/doc/config/translation.profopt.txt --- a/pypy/doc/config/translation.profopt.txt +++ b/pypy/doc/config/translation.profopt.txt @@ -3,3 +3,14 @@ RPython program) to gather profile data. Example for pypy-c: "-c 'from richards import main;main(); from test import pystone; pystone.main()'" + +NOTE: be aware of what this does in JIT-enabled executables. What it +does is instrument and later optimize the C code that happens to run in +the example you specify, ignoring any execution of the JIT-generated +assembler. That means that you have to choose the example wisely. If +it is something that will just generate assembler and stay there, there +is little value. If it is something that exercises heavily library +routines that are anyway written in C, then it will optimize that. Most +interesting would be something that causes a lot of JIT-compilation, +like running a medium-sized test suite several times in a row, in order +to optimize the warm-up in general. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -449,6 +449,27 @@ support (see ``multiline_input()``). On the other hand, ``parse_and_bind()`` calls are ignored (issue `#2072`_). +* ``sys.getsizeof()`` always raises ``TypeError``. This is because a + memory profiler using this function is most likely to give results + inconsistent with reality on PyPy. It would be possible to have + ``sys.getsizeof()`` return a number (with enough work), but that may + or may not represent how much memory the object uses. It doesn't even + make really sense to ask how much *one* object uses, in isolation with + the rest of the system. For example, instances have maps, which are + often shared across many instances; in this case the maps would + probably be ignored by an implementation of ``sys.getsizeof()``, but + their overhead is important in some cases if they are many instances + with unique maps. Conversely, equal strings may share their internal + string data even if they are different objects---or empty containers + may share parts of their internals as long as they are empty. Even + stranger, some lists create objects as you read them; if you try to + estimate the size in memory of ``range(10**6)`` as the sum of all + items' size, that operation will by itself create one million integer + objects that never existed in the first place. Note that some of + these concerns also exist on CPython, just less so. For this reason + we explicitly don't implement ``sys.getsizeof()``. + + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -16,3 +16,8 @@ Improve merging of virtual states in the JIT in order to avoid jumping to the preamble. Accomplished by allocating virtual objects where non-virtuals are expected. + +.. branch: conditional_call_value_3 +JIT residual calls: if the called fu