Author: Carl Friedrich Bolz <[email protected]> Branch: better-error-missing-self Changeset: r87561:19f52dd10c67 Date: 2016-10-04 08:52 +0200 http://bitbucket.org/pypy/pypy/changeset/19f52dd10c67/
Log: merge default diff too long, truncating to 2000 out of 3429 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -40,4 +40,4 @@ # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html cffi_imports: pypy-c - PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py + PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -43,6 +43,7 @@ try: if detect_cpu.autodetect().startswith('x86'): working_modules.add('_vmprof') + working_modules.add('faulthandler') except detect_cpu.ProcessorAutodetectError: pass @@ -89,6 +90,7 @@ ('objspace.usemodules.thread', True)], 'cpyext': [('objspace.usemodules.array', True)], 'cppyy': [('objspace.usemodules.cpyext', True)], + 'faulthandler': [('objspace.usemodules._vmprof', True)], } module_suggests = { # the reason you want _rawffi is for ctypes, which @@ -114,7 +116,8 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], - "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "faulthandler" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -32,3 +32,22 @@ ``lib-python`` and ``lib_pypy``. Of course, you can put a symlink to it from somewhere else. You no longer have to do the same with the ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library. + +.. branch: _warnings + +CPython allows warning.warn(('something', 1), Warning), on PyPy this +produced a "expected a readable buffer object" error. Test and fix. + +.. branch: stricter-strip + +CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are +allowed as arguments. Test and fix for str and unicode + +.. branch: faulthandler + +Port the 'faulthandler' module to PyPy default. This module is standard +in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI. + +.. branch: test-cpyext + +Refactor cpyext testing to be more pypy3-friendly. diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -93,12 +93,10 @@ home1 = rffi.charp2str(ll_home) home = os.path.join(home1, 'x') # <- so that 'll_home' can be # directly the root directory - dynamic = False else: home1 = "pypy's shared library location" - home = pypydir - dynamic = True - w_path = pypy_find_stdlib(space, home, dynamic) + home = '*' + w_path = pypy_find_stdlib(space, home) if space.is_none(w_path): if verbose: debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'" @@ -305,37 +303,20 @@ # HACKHACKHACK # ugly hack to modify target goal from compile_* to build_cffi_imports # this should probably get cleaned up and merged with driver.create_exe + from rpython.tool.runsubprocess import run_subprocess from rpython.translator.driver import taskdef import types - class Options(object): - pass - - - def mkexename(name): - if sys.platform == 'win32': - name = name.new(ext='exe') - return name - compile_goal, = driver.backend_select_goals(['compile']) @taskdef([compile_goal], "Create cffi bindings for modules") def task_build_cffi_imports(self): - from pypy.tool.build_cffi_imports import create_cffi_import_libraries ''' Use cffi to compile cffi interfaces to modules''' - exename = mkexename(driver.compute_exe_name()) - basedir = exename - while not basedir.join('include').exists(): - _basedir = basedir.dirpath() - if _basedir == basedir: - raise ValueError('interpreter %s not inside pypy repo', - str(exename)) - basedir = _basedir - modules = self.config.objspace.usemodules.getpaths() - options = Options() - # XXX possibly adapt options using modules - failures = create_cffi_import_libraries(exename, options, basedir) - # if failures, they were already printed - print >> sys.stderr, str(exename),'successfully built (errors, if any, while building the above modules are ignored)' + filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py') + status, out, err = run_subprocess(str(driver.compute_exe_name()), + [filename]) + sys.stdout.write(out) + sys.stderr.write(err) + # otherwise, ignore errors driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver) driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal] driver.default_goal = 'build_cffi_imports' diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -33,6 +33,7 @@ --info : print translation information about this PyPy executable -X track-resources : track the creation of files and sockets and display a warning if they are not closed explicitly +-X faulthandler : attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME, PYTHONCASEOK USAGE2 = """ @@ -233,12 +234,22 @@ import pypyjit pypyjit.set_param(jitparam) +def run_faulthandler(): + if 'faulthandler' in sys.builtin_module_names: + import faulthandler + try: + faulthandler.enable(2) # manually set to stderr + except ValueError: + pass # ignore "2 is not a valid file descriptor" + def set_runtime_options(options, Xparam, *args): if Xparam == 'track-resources': sys.pypy_set_track_resources(True) + elif Xparam == 'faulthandler': + run_faulthandler() else: print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),) - print >> sys.stderr, '[options] can be: track-resources' + print >> sys.stderr, '[options] can be: track-resources, faulthandler' raise SystemExit class CommandLineError(Exception): @@ -527,6 +538,9 @@ print >> sys.stderr, ( "Warning: pypy does not implement py3k warnings") + if os.getenv('PYTHONFAULTHANDLER'): + run_faulthandler() + ## if not we_are_translated(): ## for key in sorted(options): ## print '%40s: %s' % (key, options[key]) diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -2,6 +2,8 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module.imp.importing import get_pyc_magic +from rpython.rlib import rtime + class BuildersModule(MixedModule): appleveldefs = {} @@ -14,16 +16,11 @@ class TimeModule(MixedModule): appleveldefs = {} interpleveldefs = {} - if sys.platform.startswith("linux") or 'bsd' in sys.platform: - from pypy.module.__pypy__ import interp_time + if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" - for name in [ - "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", - "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" - ]: - if getattr(interp_time, name) is not None: - interpleveldefs[name] = "space.wrap(interp_time.%s)" % name + for name in rtime.ALL_DEFINED_CLOCKS: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py --- a/pypy/module/__pypy__/interp_time.py +++ b/pypy/module/__pypy__/interp_time.py @@ -4,71 +4,28 @@ from pypy.interpreter.error import exception_from_saved_errno from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib import rtime +from rpython.rlib.rtime import HAS_CLOCK_GETTIME -if sys.platform == 'linux2': - libraries = ["rt"] -else: - libraries = [] - - -class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=["time.h"], - libraries=libraries, - ) - - HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime') - - CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME") - CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC") - CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW") - CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID") - CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID") - -cconfig = rffi_platform.configure(CConfig) - -HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"] - -CLOCK_REALTIME = cconfig["CLOCK_REALTIME"] -CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"] -CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"] -CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"] -CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"] if HAS_CLOCK_GETTIME: - #redo it for timespec - CConfig.TIMESPEC = rffi_platform.Struct("struct timespec", [ - ("tv_sec", rffi.TIME_T), - ("tv_nsec", rffi.LONG), - ]) - cconfig = rffi_platform.configure(CConfig) - TIMESPEC = cconfig['TIMESPEC'] - - c_clock_gettime = rffi.llexternal("clock_gettime", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) - c_clock_getres = rffi.llexternal("clock_getres", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) @unwrap_spec(clk_id="c_int") def clock_gettime(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_gettime(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_gettime(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) @unwrap_spec(clk_id="c_int") def clock_getres(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_getres(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_getres(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -140,7 +140,11 @@ stream = dispatch_filename(streamio.open_file_as_stream)( self.space, w_name, mode, buffering, signal_checker(self.space)) fd = stream.try_to_find_file_descriptor() - self.check_not_dir(fd) + try: + self.check_not_dir(fd) + except: + stream.close() + raise self.fdopenstream(stream, fd, mode) def direct___enter__(self): diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -667,3 +667,20 @@ f2.close() s2.close() s1.close() + + def test_close_fd_if_dir_check_fails(self): + from errno import EMFILE + for i in range(1700): + try: + open('/') + except IOError as e: + assert e.errno != EMFILE + else: + assert False + + @py.test.mark.skipif("os.name != 'posix'") + def test_dont_close_fd_if_dir_check_fails_in_fdopen(self): + import posix + fd = posix.open('/', posix.O_RDONLY) + raises(IOError, posix.fdopen, fd) + posix.close(fd) diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py --- a/pypy/module/_warnings/interp_warnings.py +++ b/pypy/module/_warnings/interp_warnings.py @@ -248,6 +248,10 @@ if space.isinstance_w(w_message, space.w_Warning): w_text = space.str(w_message) w_category = space.type(w_message) + elif (not space.isinstance_w(w_message, space.w_unicode) or + not space.isinstance_w(w_message, space.w_str)): + w_text = space.str(w_message) + w_message = space.call_function(w_category, w_message) else: w_text = w_message w_message = space.call_function(w_category, w_message) diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py --- a/pypy/module/_warnings/test/test_warnings.py +++ b/pypy/module/_warnings/test/test_warnings.py @@ -11,6 +11,7 @@ import _warnings _warnings.warn("some message", DeprecationWarning) _warnings.warn("some message", Warning) + _warnings.warn(("some message",1), Warning) def test_lineno(self): import warnings, _warnings, sys @@ -40,7 +41,10 @@ def test_show_source_line(self): import warnings import sys, StringIO - from test.warning_tests import inner + try: + from test.warning_tests import inner + except ImportError: + skip('no test, -A on cpython?') # With showarning() missing, make sure that output is okay. del warnings.showwarning diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -214,7 +214,9 @@ i = space.int_w(space.index(args_w[0])) j = space.int_w(space.index(args_w[1])) w_y = args_w[2] - return space.wrap(generic_cpy_call(space, func_target, w_self, i, j, w_y)) + res = generic_cpy_call(space, func_target, w_self, i, j, w_y) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) def wrap_lenfunc(space, w_self, w_args, func): func_len = rffi.cast(lenfunc, func) @@ -296,7 +298,10 @@ def wrap_hashfunc(space, w_self, w_args, func): func_target = rffi.cast(hashfunc, func) check_num_args(space, w_args, 0) - return space.wrap(generic_cpy_call(space, func_target, w_self)) + res = generic_cpy_call(space, func_target, w_self) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.wrap(res) class CPyBuffer(Buffer): # Similar to Py_buffer diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -2,6 +2,12 @@ import pytest def pytest_configure(config): + if config.option.runappdirect: + import sys + import py + from pypy import pypydir + sys.path.append(str(py.path.local(pypydir) / 'tool' / 'cpyext')) + return from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) # it's necessary to run "import time" at least once before any diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py deleted file mode 100644 --- a/pypy/module/cpyext/test/support.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import py -from sys import platform - -if os.name != 'nt': - so_ext = 'so' -else: - so_ext = 'dll' - -def c_compile(cfilenames, outputfilename, - compile_extra=None, link_extra=None, - include_dirs=None, libraries=None, library_dirs=None): - compile_extra = compile_extra or [] - link_extra = link_extra or [] - include_dirs = include_dirs or [] - libraries = libraries or [] - library_dirs = library_dirs or [] - if platform == 'win32': - link_extra = link_extra + ['/DEBUG'] # generate .pdb file - if platform == 'darwin': - # support Fink & Darwinports - for s in ('/sw/', '/opt/local/'): - if (s + 'include' not in include_dirs - and os.path.exists(s + 'include')): - include_dirs.append(s + 'include') - if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'): - library_dirs.append(s + 'lib') - - outputfilename = py.path.local(outputfilename).new(ext=so_ext) - saved_environ = os.environ.copy() - try: - _build( - cfilenames, outputfilename, - compile_extra, link_extra, - include_dirs, libraries, library_dirs) - finally: - # workaround for a distutils bugs where some env vars can - # become longer and longer every time it is used - for key, value in saved_environ.items(): - if os.environ.get(key) != value: - os.environ[key] = value - return outputfilename - -def _build(cfilenames, outputfilename, compile_extra, link_extra, - include_dirs, libraries, library_dirs): - from distutils.ccompiler import new_compiler - from distutils import sysconfig - compiler = new_compiler(force=1) - sysconfig.customize_compiler(compiler) # XXX - objects = [] - for cfile in cfilenames: - cfile = py.path.local(cfile) - old = cfile.dirpath().chdir() - try: - res = compiler.compile([cfile.basename], - include_dirs=include_dirs, extra_preargs=compile_extra) - assert len(res) == 1 - cobjfile = py.path.local(res[0]) - assert cobjfile.check() - objects.append(str(cobjfile)) - finally: - old.chdir() - - compiler.link_shared_object( - objects, str(outputfilename), - libraries=libraries, - extra_preargs=link_extra, - library_dirs=library_dirs) diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -49,6 +49,7 @@ assert arr.tolist() == [1, 21, 22, 23, 4] del arr[slice(1, 3)] assert arr.tolist() == [1, 23, 4] + raises(TypeError, 'arr[slice(1, 3)] = "abc"') def test_buffer(self): import sys diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -1,24 +1,21 @@ import sys import weakref -import os -import py, pytest +import pytest -from pypy import pypydir -from pypy.interpreter import gateway +from pypy.tool.cpyext.extbuild import ( + SystemCompilationInfo, HERE, get_sys_info_app) +from pypy.interpreter.gateway import unwrap_spec, interp2app from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.gensupp import uniquemodulename -from rpython.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import Py_DecRef from rpython.tool.identity_dict import identity_dict from rpython.tool import leakfinder from rpython.rlib import rawrefcount +from rpython.tool.udir import udir -from .support import c_compile - -only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -33,40 +30,19 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def convert_sources_to_files(sources, dirname): - files = [] - for i, source in enumerate(sources): - filename = dirname / ('source_%d.c' % i) - with filename.open('w') as f: - f.write(str(source)) - files.append(filename) - return files -def create_so(modname, include_dirs, source_strings=None, source_files=None, - compile_extra=None, link_extra=None, libraries=None): - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - if source_strings: - assert not source_files - files = convert_sources_to_files(source_strings, dirname) - source_files = files - soname = c_compile(source_files, outputfilename=str(dirname/modname), - compile_extra=compile_extra, link_extra=link_extra, - include_dirs=include_dirs, - libraries=libraries) - return soname +class SpaceCompiler(SystemCompilationInfo): + """Extension compiler for regular (untranslated PyPy) mode""" + def __init__(self, space, *args, **kwargs): + self.space = space + SystemCompilationInfo.__init__(self, *args, **kwargs) -class SystemCompilationInfo(object): - """Bundles all the generic information required to compile extensions. + def load_module(self, mod, name): + space = self.space + api.load_extension_module(space, mod, name) + return space.getitem( + space.sys.get('modules'), space.wrap(name)) - Note: here, 'system' means OS + target interpreter + test config + ... - """ - def __init__(self, include_extra=None, compile_extra=None, link_extra=None, - extra_libs=None, ext=None): - self.include_extra = include_extra or [] - self.compile_extra = compile_extra - self.link_extra = link_extra - self.extra_libs = extra_libs - self.ext = ext def get_cpyext_info(space): from pypy.module.imp.importing import get_so_extension @@ -88,7 +64,8 @@ link_extra = ["-g"] else: compile_extra = link_extra = None - return SystemCompilationInfo( + return SpaceCompiler(space, + builddir_base=udir, include_extra=api.include_dirs, compile_extra=compile_extra, link_extra=link_extra, @@ -96,59 +73,6 @@ ext=get_so_extension(space)) -def compile_extension_module(sys_info, modname, include_dirs=[], - source_files=None, source_strings=None): - """ - Build an extension module and return the filename of the resulting native - code file. - - modname is the name of the module, possibly including dots if it is a module - inside a package. - - Any extra keyword arguments are passed on to ExternalCompilationInfo to - build the module (so specify your source with one of those). - """ - modname = modname.split('.')[-1] - soname = create_so(modname, - include_dirs=sys_info.include_extra + include_dirs, - source_files=source_files, - source_strings=source_strings, - compile_extra=sys_info.compile_extra, - link_extra=sys_info.link_extra, - libraries=sys_info.extra_libs) - pydname = soname.new(purebasename=modname, ext=sys_info.ext) - soname.rename(pydname) - return str(pydname) - -def get_so_suffix(): - from imp import get_suffixes, C_EXTENSION - for suffix, mode, typ in get_suffixes(): - if typ == C_EXTENSION: - return suffix - else: - raise RuntimeError("This interpreter does not define a filename " - "suffix for C extensions!") - -def get_sys_info_app(): - from distutils.sysconfig import get_python_inc - if sys.platform == 'win32': - compile_extra = ["/we4013"] - link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] - elif sys.platform == 'darwin': - compile_extra = link_extra = None - pass - elif sys.platform.startswith('linux'): - compile_extra = [ - "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] - link_extra = None - ext = get_so_suffix() - return SystemCompilationInfo( - include_extra=[get_python_inc()], - compile_extra=compile_extra, - link_extra=link_extra, - ext=get_so_suffix()) - - def freeze_refcnts(self): rawrefcount._dont_free_any_more() return #ZZZ @@ -159,25 +83,9 @@ #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) -class FakeSpace(object): - """Like TinyObjSpace, but different""" - def __init__(self, config): - self.config = config - - def passthrough(self, arg): - return arg - listview = passthrough - str_w = passthrough - - def unwrap(self, args): - try: - return args.str_w(None) - except: - return args - class LeakCheckingTest(object): """Base class for all cpyext tests.""" - spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array', + spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array', 'itertools', 'time', 'binascii', 'micronumpy', 'mmap' ]) @@ -265,9 +173,12 @@ cls.w_libc = cls.space.wrap(get_libc_name()) def setup_method(self, meth): - freeze_refcnts(self) + if not self.runappdirect: + freeze_refcnts(self) def teardown_method(self, meth): + if self.runappdirect: + return self.cleanup_references(self.space) # XXX: like AppTestCpythonExtensionBase.teardown_method: # find out how to disable check_and_print_leaks() if the @@ -293,21 +204,82 @@ skip("Windows Python >= 2.6 only") assert isinstance(sys.dllhandle, int) + +def _unwrap_include_dirs(space, w_include_dirs): + if w_include_dirs is None: + return None + else: + return [space.str_w(s) for s in space.listview(w_include_dirs)] + +def debug_collect(space): + rawrefcount._collect() + class AppTestCpythonExtensionBase(LeakCheckingTest): def setup_class(cls): space = cls.space - space.getbuiltinmodule("cpyext") - # 'import os' to warm up reference counts - w_import = space.builtin.getdictvalue(space, '__import__') - space.call_function(w_import, space.wrap("os")) - #state = cls.space.fromcache(RefcountState) ZZZ - #state.non_heaptypes_w[:] = [] + cls.w_here = space.wrap(str(HERE)) + cls.w_udir = space.wrap(str(udir)) if not cls.runappdirect: + cls.sys_info = get_cpyext_info(space) cls.w_runappdirect = space.wrap(cls.runappdirect) + space.getbuiltinmodule("cpyext") + # 'import os' to warm up reference counts + w_import = space.builtin.getdictvalue(space, '__import__') + space.call_function(w_import, space.wrap("os")) + #state = cls.space.fromcache(RefcountState) ZZZ + #state.non_heaptypes_w[:] = [] + cls.w_debug_collect = space.wrap(interp2app(debug_collect)) + else: + def w_import_module(self, name, init=None, body='', filename=None, + include_dirs=None, PY_SSIZE_T_CLEAN=False): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.import_module( + name, init=init, body=body, filename=filename, + include_dirs=include_dirs, + PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) + cls.w_import_module = w_import_module + + def w_import_extension(self, modname, functions, prologue="", + include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.import_extension( + modname, functions, prologue=prologue, + include_dirs=include_dirs, more_init=more_init, + PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) + cls.w_import_extension = w_import_extension + + def w_compile_module(self, name, + source_files=None, source_strings=None): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.compile_extension_module(name, + source_files=source_files, source_strings=source_strings) + cls.w_compile_module = w_compile_module + + def w_load_module(self, mod, name): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.load_module(mod, name) + cls.w_load_module = w_load_module + + + def record_imported_module(self, name): + """ + Record a module imported in a test so that it can be cleaned up in + teardown before the check for leaks is done. + + name gives the name of the module in the space's sys.modules. + """ + self.imported_module_names.append(name) def setup_method(self, func): - @gateway.unwrap_spec(name=str) + if self.runappdirect: + return + + @unwrap_spec(name=str) def compile_module(space, name, w_source_files=None, w_source_strings=None): @@ -322,166 +294,54 @@ source_strings = space.listview_bytes(w_source_strings) else: source_strings = None - pydname = compile_extension_module( - self.sys_info, name, + pydname = self.sys_info.compile_extension_module( + name, source_files=source_files, source_strings=source_strings) + + # hackish, but tests calling compile_module() always end up + # importing the result + self.record_imported_module(name) + return space.wrap(pydname) - @gateway.unwrap_spec(name=str, init='str_or_None', body=str, - load_it=bool, filename='str_or_None', - PY_SSIZE_T_CLEAN=bool) - def import_module(space, name, init=None, body='', load_it=True, + @unwrap_spec(name=str, init='str_or_None', body=str, + filename='str_or_None', PY_SSIZE_T_CLEAN=bool) + def import_module(space, name, init=None, body='', filename=None, w_include_dirs=None, PY_SSIZE_T_CLEAN=False): - """ - init specifies the overall template of the module. + include_dirs = _unwrap_include_dirs(space, w_include_dirs) + w_result = self.sys_info.import_module( + name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN) + self.record_imported_module(name) + return w_result - if init is None, the module source will be loaded from a file in this - test direcory, give a name given by the filename parameter. - if filename is None, the module name will be used to construct the - filename. - """ - if w_include_dirs is None: - include_dirs = [] - else: - include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)] - if init is not None: - code = """ - %(PY_SSIZE_T_CLEAN)s - #include <Python.h> - /* fix for cpython 2.7 Python.h if running tests with -A - since pypy compiles with -fvisibility-hidden */ - #undef PyMODINIT_FUNC - #ifdef __GNUC__ - # define RPY_EXPORTED extern __attribute__((visibility("default"))) - #else - # define RPY_EXPORTED extern __declspec(dllexport) - #endif - #define PyMODINIT_FUNC RPY_EXPORTED void + @unwrap_spec(mod=str, name=str) + def load_module(space, mod, name): + return self.sys_info.load_module(mod, name) - %(body)s - - PyMODINIT_FUNC - init%(name)s(void) { - %(init)s - } - """ % dict(name=name, init=init, body=body, - PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' - if PY_SSIZE_T_CLEAN else '') - kwds = dict(source_strings=[code]) - else: - assert not PY_SSIZE_T_CLEAN - if filename is None: - filename = name - filename = py.path.local(pypydir) / 'module' \ - / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(source_files=[filename]) - mod = compile_extension_module(self.sys_info, name, - include_dirs=include_dirs, **kwds) - - if load_it: - if self.runappdirect: - import imp - return imp.load_dynamic(name, mod) - else: - api.load_extension_module(space, mod, name) - self.imported_module_names.append(name) - return space.getitem( - space.sys.get('modules'), - space.wrap(name)) - else: - path = os.path.dirname(mod) - if self.runappdirect: - return path - else: - return space.wrap(path) - - @gateway.unwrap_spec(mod=str, name=str) - def reimport_module(space, mod, name): - if self.runappdirect: - import imp - return imp.load_dynamic(name, mod) - else: - api.load_extension_module(space, mod, name) - return space.getitem( - space.sys.get('modules'), - space.wrap(name)) - - @gateway.unwrap_spec(modname=str, prologue=str, + @unwrap_spec(modname=str, prologue=str, more_init=str, PY_SSIZE_T_CLEAN=bool) def import_extension(space, modname, w_functions, prologue="", w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): functions = space.unwrap(w_functions) - methods_table = [] - codes = [] - for funcname, flags, code in functions: - cfuncname = "%s_%s" % (modname, funcname) - methods_table.append("{\"%s\", %s, %s}," % - (funcname, cfuncname, flags)) - func_code = """ - static PyObject* %s(PyObject* self, PyObject* args) - { - %s - } - """ % (cfuncname, code) - codes.append(func_code) - - body = prologue + "\n".join(codes) + """ - static PyMethodDef methods[] = { - %s - { NULL } - }; - """ % ('\n'.join(methods_table),) - init = """Py_InitModule("%s", methods);""" % (modname,) - if more_init: - init += more_init - return import_module(space, name=modname, init=init, body=body, - w_include_dirs=w_include_dirs, - PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) - - @gateway.unwrap_spec(name=str) - def record_imported_module(name): - """ - Record a module imported in a test so that it can be cleaned up in - teardown before the check for leaks is done. - - name gives the name of the module in the space's sys.modules. - """ - self.imported_module_names.append(name) - - def debug_collect(space): - rawrefcount._collect() + include_dirs = _unwrap_include_dirs(space, w_include_dirs) + w_result = self.sys_info.import_extension( + modname, functions, prologue, include_dirs, more_init, + PY_SSIZE_T_CLEAN) + self.record_imported_module(modname) + return w_result # A list of modules which the test caused to be imported (in # self.space). These will be cleaned up automatically in teardown. self.imported_module_names = [] - if self.runappdirect: - fake = FakeSpace(self.space.config) - def interp2app(func): - def run(*args, **kwargs): - for k in kwargs.keys(): - if k not in func.unwrap_spec and not k.startswith('w_'): - v = kwargs.pop(k) - kwargs['w_' + k] = v - return func(fake, *args, **kwargs) - return run - def wrap(func): - return func - self.sys_info = get_sys_info_app() - else: - interp2app = gateway.interp2app - wrap = self.space.wrap - self.sys_info = get_cpyext_info(self.space) + wrap = self.space.wrap self.w_compile_module = wrap(interp2app(compile_module)) + self.w_load_module = wrap(interp2app(load_module)) self.w_import_module = wrap(interp2app(import_module)) - self.w_reimport_module = wrap(interp2app(reimport_module)) self.w_import_extension = wrap(interp2app(import_extension)) - self.w_record_imported_module = wrap(interp2app(record_imported_module)) - self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/') - self.w_debug_collect = wrap(interp2app(debug_collect)) # create the file lock before we count allocations self.space.call_method(self.space.sys.get("stdout"), "flush") @@ -498,6 +358,8 @@ self.space.delitem(w_modules, w_name) def teardown_method(self, func): + if self.runappdirect: + return for name in self.imported_module_names: self.unimport_module(name) self.cleanup_references(self.space) @@ -632,19 +494,15 @@ If `cherry.date` is an extension module which imports `apple.banana`, the latter is added to `sys.modules` for the `"apple.banana"` key. """ - if self.runappdirect: - skip('record_imported_module not supported in runappdirect mode') + import sys, types, os # Build the extensions. banana = self.compile_module( - "apple.banana", source_files=[self.here + 'banana.c']) - self.record_imported_module("apple.banana") + "apple.banana", source_files=[os.path.join(self.here, 'banana.c')]) date = self.compile_module( - "cherry.date", source_files=[self.here + 'date.c']) - self.record_imported_module("cherry.date") + "cherry.date", source_files=[os.path.join(self.here, 'date.c')]) # Set up some package state so that the extensions can actually be # imported. - import sys, types, os cherry = sys.modules['cherry'] = types.ModuleType('cherry') cherry.__path__ = [os.path.dirname(date)] @@ -652,7 +510,6 @@ apple.__path__ = [os.path.dirname(banana)] import cherry.date - import apple.banana assert sys.modules['apple.banana'].__name__ == 'apple.banana' assert sys.modules['cherry.date'].__name__ == 'cherry.date' @@ -989,7 +846,7 @@ f.write('not again!\n') f.close() m1 = sys.modules['foo'] - m2 = self.reimport_module(m1.__file__, name='foo') + m2 = self.load_module(m1.__file__, name='foo') assert m1 is m2 assert m1 is sys.modules['foo'] diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py --- a/pypy/module/cpyext/test/test_import.py +++ b/pypy/module/cpyext/test/test_import.py @@ -1,6 +1,6 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem import rffi class TestImport(BaseApiTest): def test_import(self, space, api): @@ -39,9 +39,9 @@ class AppTestImportLogic(AppTestCpythonExtensionBase): def test_import_logic(self): - path = self.import_module(name='test_import_module', load_it=False) - import sys - sys.path.append(path) + import sys, os + path = self.compile_module('test_import_module', + source_files=[os.path.join(self.here, 'test_import_module.c')]) + sys.path.append(os.path.dirname(path)) import test_import_module assert test_import_module.TEST is None - diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -101,7 +101,7 @@ w_stdout = space.sys.get("stdout") assert api.PyFile_SoftSpace(w_stdout, 1) == 0 assert api.PyFile_SoftSpace(w_stdout, 0) == 1 - + api.PyFile_SoftSpace(w_stdout, 1) w_ns = space.newdict() space.exec_("print 1,", w_ns, w_ns) @@ -117,11 +117,9 @@ class AppTestPyFile(AppTestCpythonExtensionBase): def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.__func__(cls) from rpython.tool.udir import udir - if option.runappdirect: - cls.w_udir = str(udir) - else: - cls.w_udir = cls.space.wrap(str(udir)) + cls.w_udir = cls.space.wrap(str(udir)) def test_file_tell(self): module = self.import_extension('foo', [ @@ -158,4 +156,3 @@ t_py = fid.tell() assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py) - diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/__init__.py @@ -0,0 +1,38 @@ +import sys +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + appleveldefs = { + } + + interpleveldefs = { + 'enable': 'handler.enable', + 'disable': 'handler.disable', + 'is_enabled': 'handler.is_enabled', +# 'register': 'interp_faulthandler.register', +# + 'dump_traceback': 'handler.dump_traceback', +# + '_read_null': 'handler.read_null', + '_sigsegv': 'handler.sigsegv', + '_sigfpe': 'handler.sigfpe', + '_sigabrt': 'handler.sigabrt', + '_stack_overflow': 'handler.stack_overflow', + } + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + if self.space.config.translation.thread: + self.extra_interpdef('dump_traceback_later', + 'handler.dump_traceback_later') + self.extra_interpdef('cancel_dump_traceback_later', + 'handler.cancel_dump_traceback_later') + if sys.platform != 'win32': + self.extra_interpdef('register', 'handler.register') + self.extra_interpdef('unregister', 'handler.unregister') + + def shutdown(self, space): + from pypy.module.faulthandler import handler + handler.finish(space) + MixedModule.shutdown(self, space) diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/cintf.py @@ -0,0 +1,99 @@ +import py +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr +from rpython.translator import cdir +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('faulthandler.h')], + include_dirs=[str(cwd), cdir], + separate_module_files=[cwd.join('faulthandler.c')]) + +eci_later = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n'])) +eci_user = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_USER\n'])) + +def direct_llexternal(*args, **kwargs): + kwargs.setdefault('_nowrapper', True) + kwargs.setdefault('compilation_info', eci) + return rffi.llexternal(*args, **kwargs) + + +DUMP_CALLBACK = lltype.Ptr(lltype.FuncType( + [rffi.INT, rffi.SIGNEDP, lltype.Signed], lltype.Void)) + +pypy_faulthandler_setup = direct_llexternal( + 'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP) + +pypy_faulthandler_teardown = direct_llexternal( + 'pypy_faulthandler_teardown', [], lltype.Void) + +pypy_faulthandler_enable = direct_llexternal( + 'pypy_faulthandler_enable', [rffi.INT, rffi.INT], rffi.CCHARP) + +pypy_faulthandler_disable = direct_llexternal( + 'pypy_faulthandler_disable', [], lltype.Void) + +pypy_faulthandler_is_enabled = direct_llexternal( + 'pypy_faulthandler_is_enabled', [], rffi.INT) + +pypy_faulthandler_write = direct_llexternal( + 'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void) + +pypy_faulthandler_write_int = direct_llexternal( + 'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void) + +pypy_faulthandler_dump_traceback = direct_llexternal( + 'pypy_faulthandler_dump_traceback', + [rffi.INT, rffi.INT, llmemory.Address], lltype.Void) + +pypy_faulthandler_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_dump_traceback_later', + [rffi.LONGLONG, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_later) + +pypy_faulthandler_cancel_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_cancel_dump_traceback_later', [], lltype.Void) + +pypy_faulthandler_check_signum = direct_llexternal( + 'pypy_faulthandler_check_signum', + [rffi.LONG], rffi.INT, + compilation_info=eci_user) + +pypy_faulthandler_register = direct_llexternal( + 'pypy_faulthandler_register', + [rffi.INT, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_user) + +pypy_faulthandler_unregister = direct_llexternal( + 'pypy_faulthandler_unregister', + [rffi.INT], rffi.INT, + compilation_info=eci_user) + + +# for tests... + +pypy_faulthandler_read_null = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void) + +pypy_faulthandler_read_null_releasegil = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigsegv = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void) + +pypy_faulthandler_sigsegv_releasegil = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigfpe = direct_llexternal( + 'pypy_faulthandler_sigfpe', [], lltype.Void) + +pypy_faulthandler_sigabrt = direct_llexternal( + 'pypy_faulthandler_sigabrt', [], lltype.Void) + +pypy_faulthandler_stackoverflow = direct_llexternal( + 'pypy_faulthandler_stackoverflow', [lltype.Float], lltype.Float) diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/dumper.py @@ -0,0 +1,54 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc +from rpython.rlib.rvmprof import traceback + +from pypy.interpreter.pycode import PyCode +from pypy.module.faulthandler.cintf import pypy_faulthandler_write +from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int + + +MAX_STRING_LENGTH = 500 + +global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw', + immortal=True, zero=True) + +def _dump(fd, s): + assert isinstance(s, str) + l = len(s) + if l >= MAX_STRING_LENGTH: + l = MAX_STRING_LENGTH - 1 + i = 0 + while i < l: + global_buf[i] = s[i] + i += 1 + global_buf[l] = '\x00' + pypy_faulthandler_write(fd, global_buf) + +def _dump_int(fd, i): + pypy_faulthandler_write_int(fd, i) + + +def dump_code(pycode, loc, fd): + if pycode is None: + _dump(fd, " File ???") + else: + _dump(fd, ' File "') + _dump(fd, pycode.co_filename) + _dump(fd, '" in ') + _dump(fd, pycode.co_name) + _dump(fd, ", from line ") + _dump_int(fd, pycode.co_firstlineno) + if loc == traceback.LOC_JITTED: + _dump(fd, " [jitted]") + elif loc == traceback.LOC_JITTED_INLINED: + _dump(fd, " [jit inlined]") + _dump(fd, "\n") + + [email protected]_collect +def _dump_callback(fd, array_p, array_length): + """We are as careful as we can reasonably be here (i.e. not 100%, + but hopefully close enough). In particular, this is written as + RPython but shouldn't allocate anything. + """ + traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length) diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.c @@ -0,0 +1,679 @@ +#include "faulthandler.h" +#include <stdlib.h> +#include <stdio.h> +#include <signal.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/resource.h> +#include <math.h> + +#ifdef RPYTHON_LL2CTYPES +# include "../../../rpython/rlib/rvmprof/src/rvmprof.h" +#else +# include "common_header.h" +# include "structdef.h" +# include "rvmprof.h" +#endif +#include "src/threadlocal.h" + +#define MAX_FRAME_DEPTH 100 +#define FRAME_DEPTH_N RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH) + + +typedef struct sigaction _Py_sighandler_t; + +typedef struct { + const int signum; + volatile int enabled; + const char* name; + _Py_sighandler_t previous; +} fault_handler_t; + +static struct { + int initialized; + int enabled; + volatile int fd, all_threads; + volatile pypy_faulthandler_cb_t dump_traceback; +} fatal_error; + +static stack_t stack; + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const int faulthandler_nsignals = + sizeof(faulthandler_handlers) / sizeof(fault_handler_t); + +RPY_EXTERN +void pypy_faulthandler_write(int fd, const char *str) +{ + (void)write(fd, str, strlen(str)); +} + +RPY_EXTERN +void pypy_faulthandler_write_int(int fd, long value) +{ + char buf[48]; + sprintf(buf, "%ld", value); + pypy_faulthandler_write(fd, buf); +} + + +RPY_EXTERN +void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext) +{ + pypy_faulthandler_cb_t fn; + intptr_t array_p[FRAME_DEPTH_N], array_length; + + fn = fatal_error.dump_traceback; + if (!fn) + return; + +#ifndef RPYTHON_LL2CTYPES + if (all_threads && _RPython_ThreadLocals_AcquireTimeout(10000) == 0) { + /* This is known not to be perfectly safe against segfaults if we + don't hold the GIL ourselves. Too bad. I suspect that CPython + has issues there too. + */ + struct pypy_threadlocal_s *my, *p; + int blankline = 0; + char buf[40]; + + my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); + p = _RPython_ThreadLocals_Head(); + p = _RPython_ThreadLocals_Enum(p); + while (p != NULL) { + if (blankline) + pypy_faulthandler_write(fd, "\n"); + blankline = 1; + + pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread"); + sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident); + pypy_faulthandler_write(fd, buf); + pypy_faulthandler_write(fd, " (most recent call first):\n"); + + array_length = vmprof_get_traceback(p->vmprof_tl_stack, + my == p ? ucontext : NULL, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + + p = _RPython_ThreadLocals_Enum(p); + } + _RPython_ThreadLocals_Release(); + } + else { + pypy_faulthandler_write(fd, "Stack (most recent call first):\n"); + array_length = vmprof_get_traceback(NULL, ucontext, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + } +#else + pypy_faulthandler_write(fd, "(no traceback when untranslated)\n"); +#endif +} + +static void +faulthandler_dump_traceback(int fd, int all_threads, void *ucontext) +{ + static volatile int reentrant = 0; + + if (reentrant) + return; + reentrant = 1; + pypy_faulthandler_dump_traceback(fd, all_threads, ucontext); + reentrant = 0; +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_LATER +#include "src/thread.h" +static struct { + int fd; + long long microseconds; + int repeat, exit; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + struct RPyOpaque_ThreadLock cancel_event; + /* released by child thread when joined */ + struct RPyOpaque_ThreadLock running; +} thread_later; + +static void faulthandler_thread(void) +{ +#ifndef _WIN32 + /* we don't want to receive any signal */ + sigset_t set; + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + RPyLockStatus st; + char buf[64]; + unsigned long hour, minutes, seconds, fraction; + long long t; + + do { + st = RPyThreadAcquireLockTimed(&thread_later.cancel_event, + thread_later.microseconds, 0); + if (st == RPY_LOCK_ACQUIRED) { + RPyThreadReleaseLock(&thread_later.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == RPY_LOCK_FAILURE); + + /* getting to know which thread holds the GIL is not as simple + * as in CPython, so for now we don't */ + + t = thread_later.microseconds; + fraction = (unsigned long)(t % 1000000); + t /= 1000000; + seconds = (unsigned long)(t % 60); + t /= 60; + minutes = (unsigned long)(t % 60); + t /= 60; + hour = (unsigned long)t; + if (fraction == 0) + sprintf(buf, "Timeout (%lu:%02lu:%02lu)!\n", + hour, minutes, seconds); + else + sprintf(buf, "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, minutes, seconds, fraction); + + pypy_faulthandler_write(thread_later.fd, buf); + pypy_faulthandler_dump_traceback(thread_later.fd, 1, NULL); + + if (thread_later.exit) + _exit(1); + } while (thread_later.repeat); + + /* The only way out */ + RPyThreadReleaseLock(&thread_later.running); +} + +RPY_EXTERN +char *pypy_faulthandler_dump_traceback_later(long long microseconds, int repeat, + int fd, int exit) +{ + pypy_faulthandler_cancel_dump_traceback_later(); + + thread_later.fd = fd; + thread_later.microseconds = microseconds; + thread_later.repeat = repeat; + thread_later.exit = exit; + + RPyThreadAcquireLock(&thread_later.running, 1); + + if (RPyThreadStart(&faulthandler_thread) == -1) { + RPyThreadReleaseLock(&thread_later.running); + return "unable to start watchdog thread"; + } + return NULL; +} +#endif /* PYPY_FAULTHANDLER_LATER */ + +RPY_EXTERN +void pypy_faulthandler_cancel_dump_traceback_later(void) +{ +#ifdef PYPY_FAULTHANDLER_LATER + /* Notify cancellation */ + RPyThreadReleaseLock(&thread_later.cancel_event); + + /* Wait for thread to join (or does nothing if no thread is running) */ + RPyThreadAcquireLock(&thread_later.running, 1); + RPyThreadReleaseLock(&thread_later.running); + + /* The main thread should always hold the cancel_event lock */ + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif /* PYPY_FAULTHANDLER_LATER */ +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_USER +typedef struct { + int enabled; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; +} user_signal_t; + +static user_signal_t *user_signals; + +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext); + +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART | SA_SIGINFO; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + return sigaction(signum, &action, p_previous); +} + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext) +{ + int save_errno; + user_signal_t *user = &user_signals[signum]; + + if (!user->enabled) + return; + + save_errno = errno; + faulthandler_dump_traceback(user->fd, user->all_threads, ucontext); + + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + errno = save_errno; + + /* call the previous signal handler */ + raise(signum); + + save_errno = errno; + (void)faulthandler_register(signum, user->chain, NULL); + } + + errno = save_errno; +} + +RPY_EXTERN +int pypy_faulthandler_check_signum(long signum) +{ + unsigned int i; + + for (i = 0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + return -1; + } + } + if (signum < 1 || NSIG <= signum) { + return -2; + } + return 0; +} + +RPY_EXTERN +char *pypy_faulthandler_register(int signum, int fd, int all_threads, int chain) +{ + user_signal_t *user; + _Py_sighandler_t previous; + int err; + + if (user_signals == NULL) { + user_signals = malloc(NSIG * sizeof(user_signal_t)); + if (user_signals == NULL) + return "out of memory"; + memset(user_signals, 0, NSIG * sizeof(user_signal_t)); + } + + user = &user_signals[signum]; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); + if (err) + return strerror(errno); + + user->previous = previous; + user->enabled = 1; + } + return NULL; +} + +RPY_EXTERN +int pypy_faulthandler_unregister(int signum) +{ + user_signal_t *user; + + if (user_signals == NULL) + return 0; + + user = &user_signals[signum]; + if (user->enabled) { + user->enabled = 0; + (void)sigaction(signum, &user->previous, NULL); + user->fd = -1; + return 1; + } + else + return 0; +} +#endif /* PYPY_FAULTHANDLER_USER */ + + +/************************************************************/ + + +/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. + + Display the current Python traceback, restore the previous handler and call + the previous handler. + + On Windows, don't explicitly call the previous handler, because the Windows + signal handler would not be called (for an unknown reason). The execution of + the program continues at faulthandler_fatal_error() exit, but the same + instruction will raise the same fault (signal), and so the previous handler + will be called. + + This function is signal-safe and should only call signal-safe functions. */ + +static void +faulthandler_fatal_error(int signum, siginfo_t *info, void *ucontext) +{ + int fd = fatal_error.fd; + int i; + fault_handler_t *handler = NULL; + int save_errno = errno; + + for (i = 0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (handler->signum == signum) + break; + } + /* If not found, we use the SIGSEGV handler (the last one in the list) */ + + /* restore the previous handler */ + if (handler->enabled) { + (void)sigaction(signum, &handler->previous, NULL); + handler->enabled = 0; + } + + pypy_faulthandler_write(fd, "Fatal Python error: "); + pypy_faulthandler_write(fd, handler->name); + pypy_faulthandler_write(fd, "\n\n"); + + faulthandler_dump_traceback(fd, fatal_error.all_threads, ucontext); + + errno = save_errno; +#ifdef MS_WINDOWS + if (signum == SIGSEGV) { + /* don't explicitly call the previous handler for SIGSEGV in this signal + handler, because the Windows signal handler would not be called */ + return; + } +#endif + /* call the previous signal handler: it is called immediately if we use + sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */ + raise(signum); +} + + +RPY_EXTERN +char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback) +{ + if (fatal_error.initialized) + return NULL; + assert(!fatal_error.enabled); + fatal_error.dump_traceback = dump_callback; + + /* Try to allocate an alternate stack for faulthandler() signal handler to + * be able to allocate memory on the stack, even on a stack overflow. If it + * fails, ignore the error. */ + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + stack.ss_sp = malloc(stack.ss_size); + if (stack.ss_sp != NULL) { + int err = sigaltstack(&stack, NULL); + if (err) { + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } + +#ifdef PYPY_FAULTHANDLER_LATER + if (!RPyThreadLockInit(&thread_later.cancel_event) || + !RPyThreadLockInit(&thread_later.running)) + return "failed to initialize locks"; + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif + + fatal_error.fd = -1; + fatal_error.initialized = 1; + + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_teardown(void) +{ + if (fatal_error.initialized) { + +#ifdef PYPY_FAULTHANDLER_LATER + pypy_faulthandler_cancel_dump_traceback_later(); + RPyThreadReleaseLock(&thread_later.cancel_event); + RPyOpaqueDealloc_ThreadLock(&thread_later.running); + RPyOpaqueDealloc_ThreadLock(&thread_later.cancel_event); +#endif + +#ifdef PYPY_FAULTHANDLER_USER + int signum; + for (signum = 0; signum < NSIG; signum++) + pypy_faulthandler_unregister(signum); + /* don't free 'user_signals', the gain is very minor and it can + lead to rare crashes if another thread is still busy */ +#endif + + pypy_faulthandler_disable(); + fatal_error.initialized = 0; + if (stack.ss_sp) { + stack.ss_flags = SS_DISABLE; + sigaltstack(&stack, NULL); + free(stack.ss_sp); + stack.ss_sp = NULL; + } + } +} + +RPY_EXTERN +char *pypy_faulthandler_enable(int fd, int all_threads) +{ + /* Install the handler for fatal signals, faulthandler_fatal_error(). */ + int i; + fatal_error.fd = fd; + fatal_error.all_threads = all_threads; + + if (!fatal_error.enabled) { + fatal_error.enabled = 1; + + for (i = 0; i < faulthandler_nsignals; i++) { + int err; + struct sigaction action; + fault_handler_t *handler = &faulthandler_handlers[i]; + + action.sa_sigaction = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER | SA_SIGINFO; + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + err = sigaction(handler->signum, &action, &handler->previous); + if (err) { + return strerror(errno); + } + handler->enabled = 1; + } + } + return NULL; +} + +RPY_EXTERN +void pypy_faulthandler_disable(void) +{ + int i; + if (fatal_error.enabled) { + fatal_error.enabled = 0; + for (i = 0; i < faulthandler_nsignals; i++) { + fault_handler_t *handler = &faulthandler_handlers[i]; + if (!handler->enabled) + continue; + (void)sigaction(handler->signum, &handler->previous, NULL); + handler->enabled = 0; + } + } + fatal_error.fd = -1; +} + +RPY_EXTERN +int pypy_faulthandler_is_enabled(void) +{ + return fatal_error.enabled; +} + + +/************************************************************/ + + +/* for tests... */ + +static void +faulthandler_suppress_crash_report(void) +{ +#ifdef MS_WINDOWS + UINT mode; + + /* Configure Windows to not display the Windows Error Reporting dialog */ + mode = SetErrorMode(SEM_NOGPFAULTERRORBOX); + SetErrorMode(mode | SEM_NOGPFAULTERRORBOX); +#endif + +#ifndef MS_WINDOWS + struct rlimit rl; + + /* Disable creation of core dump */ + if (getrlimit(RLIMIT_CORE, &rl) != 0) { + rl.rlim_cur = 0; + setrlimit(RLIMIT_CORE, &rl); + } +#endif + +#ifdef _MSC_VER + /* Visual Studio: configure abort() to not display an error message nor + open a popup asking to report the fault. */ + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_read_null(void) +{ + int *volatile x; + + faulthandler_suppress_crash_report(); + x = NULL; + return *x; +} + +RPY_EXTERN +void pypy_faulthandler_sigsegv(void) +{ + faulthandler_suppress_crash_report(); +#if defined(MS_WINDOWS) + /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal + handler and then gives back the execution flow to the program (without + explicitly calling the previous error handler). In a normal case, the + SIGSEGV was raised by the kernel because of a fault, and so if the + program retries to execute the same instruction, the fault will be + raised again. + + Here the fault is simulated by a fake SIGSEGV signal raised by the + application. We have to raise SIGSEGV at lease twice: once for + faulthandler_fatal_error(), and one more time for the previous signal + handler. */ + while(1) + raise(SIGSEGV); +#else + raise(SIGSEGV); +#endif +} + +RPY_EXTERN +int pypy_faulthandler_sigfpe(void) +{ + /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on + PowerPC. Use volatile to disable compile-time optimizations. */ + volatile int x = 1, y = 0, z; + faulthandler_suppress_crash_report(); + z = x / y; + /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), + raise it manually. */ + raise(SIGFPE); + /* This line is never reached, but we pretend to make something with z + to silence a compiler warning. */ + return z; +} + +RPY_EXTERN +void pypy_faulthandler_sigabrt(void) +{ + faulthandler_suppress_crash_report(); + abort(); +} + +static double fh_stack_overflow(double levels) +{ + if (levels > 2.5) { + return (sqrt(fh_stack_overflow(levels - 1.0)) + + fh_stack_overflow(levels * 1e-10)); + } + return 1e100 + levels; +} + +RPY_EXTERN +double pypy_faulthandler_stackoverflow(double levels) +{ + faulthandler_suppress_crash_report(); + return fh_stack_overflow(levels); +} diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.h @@ -0,0 +1,40 @@ +#ifndef PYPY_FAULTHANDLER_H +#define PYPY_FAULTHANDLER_H + +#include "src/precommondefs.h" +#include <stdint.h> + + +typedef void (*pypy_faulthandler_cb_t)(int fd, intptr_t *array_p, + intptr_t length); + +RPY_EXTERN char *pypy_faulthandler_setup(pypy_faulthandler_cb_t dump_callback); +RPY_EXTERN void pypy_faulthandler_teardown(void); + +RPY_EXTERN char *pypy_faulthandler_enable(int fd, int all_threads); +RPY_EXTERN void pypy_faulthandler_disable(void); +RPY_EXTERN int pypy_faulthandler_is_enabled(void); + +RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str); +RPY_EXTERN void pypy_faulthandler_write_int(int fd, long value); + +RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext); + +RPY_EXTERN char *pypy_faulthandler_dump_traceback_later( + long long microseconds, int repeat, int fd, int exit); +RPY_EXTERN void pypy_faulthandler_cancel_dump_traceback_later(void); _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
