Author: Ronan Lamy <[email protected]> Branch: fix-2198 Changeset: r81376:86b5972fa57a Date: 2015-12-18 16:33 +0100 http://bitbucket.org/pypy/pypy/changeset/86b5972fa57a/
Log: hg merge default diff too long, truncating to 2000 out of 2340 lines diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.4.0 +Version: 1.4.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.4.0" -__version_info__ = (1, 4, 0) +__version__ = "1.4.1" +__version_info__ = (1, 4, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -83,30 +83,27 @@ **pypy-stm requires 64-bit Linux for now.** -Development is done in the branch `stmgc-c7`_. If you are only -interested in trying it out, you can download a Ubuntu binary here__ -(``pypy-stm-2.*.tar.bz2``, for Ubuntu 12.04-14.04). The current version -supports four "segments", which means that it will run up to four -threads in parallel. (Development recently switched to `stmgc-c8`_, -but that is not ready for trying out yet.) +Development is done in the branch `stmgc-c8`_. If you are only +interested in trying it out, please pester us until we upload a recent +prebuilt binary. The current version supports four "segments", which +means that it will run up to four threads in parallel. To build a version from sources, you first need to compile a custom -version of clang(!); we recommend downloading `llvm and clang like -described here`__, but at revision 201645 (use ``svn co -r 201645 <path>`` -for all checkouts). Then apply all the patches in `this directory`__: -they are fixes for a clang-only feature that hasn't been used so heavily -in the past (without the patches, you get crashes of clang). Then get -the branch `stmgc-c7`_ of PyPy and run:: +version of gcc(!). See the instructions here: +https://bitbucket.org/pypy/stmgc/src/default/gcc-seg-gs/ +(Note that these patches are being incorporated into gcc. It is likely +that future versions of gcc will not need to be patched any more.) + +Then get the branch `stmgc-c8`_ of PyPy and run:: cd pypy/goal ../../rpython/bin/rpython -Ojit --stm - PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py -.. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/ +At the end, this will try to compile the generated C code by calling +``gcc-seg-gs``, which must be the script you installed in the +instructions above. + .. _`stmgc-c8`: https://bitbucket.org/pypy/pypy/src/stmgc-c8/ -.. __: https://bitbucket.org/pypy/pypy/downloads/ -.. __: http://clang.llvm.org/get_started.html -.. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/ .. _caveats: @@ -114,6 +111,12 @@ Current status (stmgc-c7) ------------------------- +.. warning:: + + THIS PAGE IS OLD, THE REST IS ABOUT STMGC-C7 WHEREAS THE CURRENT + DEVELOPMENT WORK IS DONE ON STMGC-C8 + + * **NEW:** It seems to work fine, without crashing any more. Please `report any crash`_ you find (or other bugs). diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -81,3 +81,15 @@ Move wrappers for OS functions from `rpython/rtyper` to `rpython/rlib` and turn them into regular RPython functions. Most RPython-compatible `os.*` functions are now directly accessible as `rpython.rposix.*`. + +.. branch: always-enable-gil + +Simplify a bit the GIL handling in non-jitted code. Fixes issue #2205. + +.. branch: flowspace-cleanups + +Trivial cleanups in flowspace.operation : fix comment & duplicated method + +.. branch: test-AF_NETLINK +.. branch: small-cleanups-misc +.. branch: cpyext-slotdefs diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload, clibffi -VERSION = "1.4.0" +VERSION = "1.4.1" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py --- a/pypy/module/_cffi_backend/call_python.py +++ b/pypy/module/_cffi_backend/call_python.py @@ -40,10 +40,9 @@ at least 8 bytes in size. """ from pypy.module._cffi_backend.ccallback import reveal_callback + from rpython.rlib import rgil - after = rffi.aroundstate.after - if after: - after() + rgil.acquire() rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -71,9 +70,7 @@ cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) rffi.stackcounter.stacks_counter -= 1 - before = rffi.aroundstate.before - if before: - before() + rgil.release() def get_ll_cffi_call_python(): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.4.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.4.1", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -251,7 +251,7 @@ from pypy.module._socket.interp_socket import addr_as_object if not hasattr(rsocket._c, 'sockaddr_ll'): py.test.skip("posix specific test") - # HACK: To get the correct interface numer of lo, which in most cases is 1, + # HACK: To get the correct interface number of lo, which in most cases is 1, # but can be anything (i.e. 39), we need to call the libc function # if_nametoindex to get the correct index import ctypes @@ -513,7 +513,7 @@ def test_getsetsockopt(self): import _socket as socket import struct - # A socket sould start with reuse == 0 + # A socket should start with reuse == 0 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) reuse = s.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) assert reuse == 0 @@ -627,6 +627,26 @@ self.foo = _socket.socket() +class AppTestNetlink: + def setup_class(cls): + if not hasattr(os, 'getpid'): + py.test.skip("AF_NETLINK needs os.getpid()") + w_ok = space.appexec([], "(): import _socket; " + + "return hasattr(_socket, 'AF_NETLINK')") + if not space.is_true(w_ok): + py.test.skip("no AF_NETLINK on this platform") + cls.space = space + + def test_connect_to_kernel_netlink_routing_socket(self): + import _socket, os + s = _socket.socket(_socket.AF_NETLINK, _socket.SOCK_DGRAM, _socket.NETLINK_ROUTE) + assert s.getsockname() == (0L, 0L) + s.bind((0, 0)) + a, b = s.getsockname() + assert a == os.getpid() + assert b == 0 + + class AppTestPacket: def setup_class(cls): if not hasattr(os, 'getuid') or os.getuid() != 0: diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -124,7 +124,7 @@ METH_COEXIST METH_STATIC METH_CLASS METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS -Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE +Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES """.split() for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) @@ -602,6 +602,7 @@ # Make the wrapper for the cases (1) and (2) def make_wrapper(space, callable, gil=None): "NOT_RPYTHON" + from rpython.rlib import rgil names = callable.api_func.argnames argtypes_enum_ui = unrolling_iterable(enumerate(zip(callable.api_func.argtypes, [name.startswith("w_") for name in names]))) @@ -617,9 +618,7 @@ # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer if gil_acquire: - after = rffi.aroundstate.after - if after: - after() + rgil.acquire() rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value @@ -692,9 +691,7 @@ pypy_debug_catch_fatal_exception() rffi.stackcounter.stacks_counter -= 1 if gil_release: - before = rffi.aroundstate.before - if before: - before() + rgil.release() return retval callable._always_inline_ = 'try' wrapper.__name__ = "wrapper for %r" % (callable, ) diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -4,14 +4,14 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( - cpython_api, generic_cpy_call, PyObject, Py_ssize_t) + cpython_api, generic_cpy_call, PyObject, Py_ssize_t, Py_TPFLAGS_CHECKTYPES) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc, readbufferproc) -from pypy.module.cpyext.pyobject import from_ref +from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, oefmt @@ -65,22 +65,24 @@ func_binary = rffi.cast(binaryfunc, func) check_num_args(space, w_args, 1) args_w = space.fixedview(w_args) - - if not space.is_true(space.issubtype(space.type(args_w[0]), - space.type(w_self))): + ref = make_ref(space, w_self) + if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.is_true(space.issubtype(space.type(args_w[0]), + space.type(w_self)))): return space.w_NotImplemented - + Py_DecRef(space, ref) return generic_cpy_call(space, func_binary, w_self, args_w[0]) def wrap_binaryfunc_r(space, w_self, w_args, func): func_binary = rffi.cast(binaryfunc, func) check_num_args(space, w_args, 1) args_w = space.fixedview(w_args) - - if not space.is_true(space.issubtype(space.type(args_w[0]), - space.type(w_self))): + ref = make_ref(space, w_self) + if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.is_true(space.issubtype(space.type(args_w[0]), + space.type(w_self)))): return space.w_NotImplemented - + Py_DecRef(space, ref) return generic_cpy_call(space, func_binary, args_w[0], w_self) def wrap_inquirypred(space, w_self, w_args, func): diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -591,45 +591,92 @@ def test_binaryfunc(self): module = self.import_extension('foo', [ - ("new_obj", "METH_NOARGS", + ("newInt", "METH_VARARGS", """ - FooObject *fooObj; + IntLikeObject *intObj; + long intval; - Foo_Type.tp_as_number = &foo_as_number; - foo_as_number.nb_add = foo_nb_add_call; - if (PyType_Ready(&Foo_Type) < 0) return NULL; - fooObj = PyObject_New(FooObject, &Foo_Type); - if (!fooObj) { + if (!PyArg_ParseTuple(args, "i", &intval)) + return NULL; + + IntLike_Type.tp_as_number = &intlike_as_number; + IntLike_Type.tp_flags |= Py_TPFLAGS_CHECKTYPES; + intlike_as_number.nb_add = intlike_nb_add; + if (PyType_Ready(&IntLike_Type) < 0) return NULL; + intObj = PyObject_New(IntLikeObject, &IntLike_Type); + if (!intObj) { return NULL; } - return (PyObject *)fooObj; + intObj->ival = intval; + return (PyObject *)intObj; + """), + ("newIntNoOp", "METH_VARARGS", + """ + IntLikeObjectNoOp *intObjNoOp; + long intval; + + if (!PyArg_ParseTuple(args, "i", &intval)) + return NULL; + + IntLike_Type_NoOp.tp_flags |= Py_TPFLAGS_CHECKTYPES; + if (PyType_Ready(&IntLike_Type_NoOp) < 0) return NULL; + intObjNoOp = PyObject_New(IntLikeObjectNoOp, &IntLike_Type_NoOp); + if (!intObjNoOp) { + return NULL; + } + + intObjNoOp->ival = intval; + return (PyObject *)intObjNoOp; """)], """ typedef struct { PyObject_HEAD - } FooObject; + long ival; + } IntLikeObject; static PyObject * - foo_nb_add_call(PyObject *self, PyObject *other) + intlike_nb_add(PyObject *self, PyObject *other) { - return PyInt_FromLong(42); + long val1 = ((IntLikeObject *)(self))->ival; + if (PyInt_Check(other)) { + long val2 = PyInt_AsLong(other); + return PyInt_FromLong(val1+val2); + } + + long val2 = ((IntLikeObject *)(other))->ival; + return PyInt_FromLong(val1+val2); } - PyTypeObject Foo_Type = { + PyTypeObject IntLike_Type = { PyObject_HEAD_INIT(0) /*ob_size*/ 0, - /*tp_name*/ "Foo", - /*tp_basicsize*/ sizeof(FooObject), + /*tp_name*/ "IntLike", + /*tp_basicsize*/ sizeof(IntLikeObject), }; - static PyNumberMethods foo_as_number; + static PyNumberMethods intlike_as_number; + + typedef struct + { + PyObject_HEAD + long ival; + } IntLikeObjectNoOp; + + PyTypeObject IntLike_Type_NoOp = { + PyObject_HEAD_INIT(0) + /*ob_size*/ 0, + /*tp_name*/ "IntLikeNoOp", + /*tp_basicsize*/ sizeof(IntLikeObjectNoOp), + }; """) - a = module.new_obj() - b = module.new_obj() + a = module.newInt(1) + b = module.newInt(2) c = 3 - assert (a + b) == 42 - raises(TypeError, "b + c") + d = module.newIntNoOp(4) + assert (a + b) == 3 + assert (b + c) == 5 + assert (d + a) == 5 def test_tp_new_in_subclass_of_type(self): skip("BROKEN") diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -48,3 +48,6 @@ use_bytecode_counter=False) space.actionflag.__class__ = interp_signal.SignalActionFlag # xxx yes I know the previous line is a hack + + def startup(self, space): + space.check_signal_action.startup(space) diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -63,19 +63,25 @@ AsyncAction.__init__(self, space) self.pending_signal = -1 self.fire_in_another_thread = False - if self.space.config.objspace.usemodules.thread: - from pypy.module.thread import gil - gil.after_thread_switch = self._after_thread_switch + # + @rgc.no_collect + def _after_thread_switch(): + if self.fire_in_another_thread: + if self.space.threadlocals.signals_enabled(): + self.fire_in_another_thread = False + self.space.actionflag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. + self._after_thread_switch = _after_thread_switch + # ^^^ so that 'self._after_thread_switch' can be annotated as a + # constant - @rgc.no_collect - def _after_thread_switch(self): - if self.fire_in_another_thread: - if self.space.threadlocals.signals_enabled(): - self.fire_in_another_thread = False - self.space.actionflag.rearm_ticker() - # this occurs when we just switched to the main thread - # and there is a signal pending: we force the ticker to - # -1, which should ensure perform() is called quickly. + def startup(self, space): + # this is translated + if space.config.objspace.usemodules.thread: + from rpython.rlib import rgil + rgil.invoke_after_thread_switch(self._after_thread_switch) def perform(self, executioncontext, frame): self._poll_for_signals() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1353,8 +1353,8 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("enum foo;") from cffi import __version_info__ - if __version_info__ < (1, 4): - py.test.skip("re-enable me in version 1.4") + if __version_info__ < (1, 5): + py.test.skip("re-enable me in version 1.5") e = py.test.raises(CDefError, ffi.cast, "enum foo", -1) assert str(e.value) == ( "'enum foo' has no values explicitly defined: refusing to guess " diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -11,7 +11,6 @@ from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.executioncontext import PeriodicAsyncAction from pypy.module.thread.threadlocals import OSThreadLocals -from rpython.rlib.objectmodel import invoke_around_extcall class GILThreadLocals(OSThreadLocals): """A version of OSThreadLocals that enforces a GIL.""" @@ -23,34 +22,21 @@ space.actionflag.register_periodic_action(GILReleaseAction(space), use_bytecode_counter=True) - def _initialize_gil(self, space): - rgil.gil_allocate() - def setup_threads(self, space): """Enable threads in the object space, if they haven't already been.""" if not self.gil_ready: - self._initialize_gil(space) + # Note: this is a quasi-immutable read by module/pypyjit/interp_jit + # It must be changed (to True) only if it was really False before + rgil.allocate() self.gil_ready = True result = True else: result = False # already set up - - # add the GIL-releasing callback around external function calls. - # - # XXX we assume a single space, but this is not quite true during - # testing; for example, if you run the whole of test_lock you get - # a deadlock caused by the first test's space being reused by - # test_lock_again after the global state was cleared by - # test_compile_lock. As a workaround, we repatch these global - # fields systematically. - invoke_around_extcall(before_external_call, after_external_call) return result - def reinit_threads(self, space): - "Called in the child process after a fork()" - OSThreadLocals.reinit_threads(self, space) - if self.gil_ready: # re-initialize the gil if needed - self._initialize_gil(space) + ## def reinit_threads(self, space): + ## "Called in the child process after a fork()" + ## OSThreadLocals.reinit_threads(self, space) class GILReleaseAction(PeriodicAsyncAction): @@ -59,43 +45,4 @@ """ def perform(self, executioncontext, frame): - do_yield_thread() - - -after_thread_switch = lambda: None # hook for signal.py - -def before_external_call(): - # this function must not raise, in such a way that the exception - # transformer knows that it cannot raise! - rgil.gil_release() -before_external_call._gctransformer_hint_cannot_collect_ = True -before_external_call._dont_reach_me_in_del_ = True - -def after_external_call(): - rgil.gil_acquire() - rthread.gc_thread_run() - after_thread_switch() -after_external_call._gctransformer_hint_cannot_collect_ = True -after_external_call._dont_reach_me_in_del_ = True - -# The _gctransformer_hint_cannot_collect_ hack is needed for -# translations in which the *_external_call() functions are not inlined. -# They tell the gctransformer not to save and restore the local GC -# pointers in the shadow stack. This is necessary because the GIL is -# not held after the call to before_external_call() or before the call -# to after_external_call(). - -def do_yield_thread(): - # explicitly release the gil, in a way that tries to give more - # priority to other threads (as opposed to continuing to run in - # the same thread). - if rgil.gil_yield_thread(): - rthread.gc_thread_run() - after_thread_switch() -do_yield_thread._gctransformer_hint_close_stack_ = True -do_yield_thread._dont_reach_me_in_del_ = True -do_yield_thread._dont_inline_ = True - -# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_. -# The *_external_call() functions are themselves called only from the rffi -# module from a helper function that also has this hint. + rgil.yield_thread() diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py --- a/pypy/module/thread/test/support.py +++ b/pypy/module/thread/test/support.py @@ -5,7 +5,7 @@ import errno from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.module.thread import gil +from rpython.rlib import rgil NORMAL_TIMEOUT = 300.0 # 5 minutes @@ -15,9 +15,9 @@ adaptivedelay = 0.04 limit = time.time() + delay * NORMAL_TIMEOUT while time.time() <= limit: - gil.before_external_call() + rgil.release() time.sleep(adaptivedelay) - gil.after_external_call() + rgil.acquire() gc.collect() if space.is_true(space.call_function(w_condition)): return diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -1,5 +1,6 @@ import time from pypy.module.thread import gil +from rpython.rlib import rgil from rpython.rlib.test import test_rthread from rpython.rlib import rthread as thread from rpython.rlib.objectmodel import we_are_translated @@ -55,7 +56,7 @@ assert state.datalen3 == len(state.data) assert state.datalen4 == len(state.data) debug_print(main, i, state.datalen4) - gil.do_yield_thread() + rgil.yield_thread() assert i == j j += 1 def bootstrap(): @@ -82,9 +83,9 @@ if not still_waiting: raise ValueError("time out") still_waiting -= 1 - if not we_are_translated(): gil.before_external_call() + if not we_are_translated(): rgil.release() time.sleep(0.01) - if not we_are_translated(): gil.after_external_call() + if not we_are_translated(): rgil.acquire() debug_print("leaving!") i1 = i2 = 0 for tid, i in state.data: diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -521,7 +521,6 @@ def descr_getitem(self, space, w_index): if isinstance(w_index, W_SliceObject): - # XXX consider to extend rlist's functionality? length = self.length() start, stop, step, slicelength = w_index.indices4(space, length) assert slicelength >= 0 diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -441,7 +441,7 @@ def dict_contains(s_dct, s_element, position): s_dct.dictdef.generalize_key(s_element) if s_dct._is_empty(position): - s_bool =SomeBool() + s_bool = SomeBool() s_bool.const = False return s_bool return s_Bool diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -1,5 +1,5 @@ """ -This module defines all the SpaceOeprations used in rpython.flowspace. +This module defines all the SpaceOperations used in rpython.flowspace. """ import __builtin__ @@ -196,21 +196,6 @@ return cls._dispatch(type(s_arg)) @classmethod - def get_specialization(cls, s_arg, *_ignored): - try: - impl = getattr(s_arg, cls.opname) - - def specialized(annotator, arg, *other_args): - return impl(*[annotator.annotation(x) for x in other_args]) - try: - specialized.can_only_throw = impl.can_only_throw - except AttributeError: - pass - return specialized - except AttributeError: - return cls._dispatch(type(s_arg)) - - @classmethod def register_transform(cls, Some_cls): def decorator(func): cls._transform[Some_cls] = func diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -380,6 +380,8 @@ # the call that it is no longer equal to css. See description # in translator/c/src/thread_pthread.c. + # XXX some duplicated logic here, but note that rgil.acquire() + # does more than just RPyGilAcquire() if old_rpy_fastgil == 0: # this case occurs if some other thread stole the GIL but # released it again. What occurred here is that we changed @@ -390,9 +392,8 @@ elif old_rpy_fastgil == 1: # 'rpy_fastgil' was (and still is) locked by someone else. # We need to wait for the regular mutex. - after = rffi.aroundstate.after - if after: - after() + from rpython.rlib import rgil + rgil.acquire() else: # stole the GIL from a different thread that is also # currently in an external call from the jit. Attach @@ -421,9 +422,8 @@ # 'rpy_fastgil' contains only zero or non-zero, and this is only # called when the old value stored in 'rpy_fastgil' was non-zero # (i.e. still locked, must wait with the regular mutex) - after = rffi.aroundstate.after - if after: - after() + from rpython.rlib import rgil + rgil.acquire() _REACQGIL0_FUNC = lltype.Ptr(lltype.FuncType([], lltype.Void)) _REACQGIL2_FUNC = lltype.Ptr(lltype.FuncType([rffi.CCHARP, lltype.Signed], diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py --- a/rpython/jit/backend/llsupport/test/test_gc_integration.py +++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py @@ -17,7 +17,6 @@ from rpython.jit.backend.llsupport.test.test_regalloc_integration import BaseTestRegalloc from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.codewriter import longlong -from rpython.rlib.objectmodel import invoke_around_extcall CPU = getcpuclass() @@ -625,9 +624,6 @@ self.S = S self.cpu = cpu - def teardown_method(self, meth): - rffi.aroundstate._cleanup_() - def test_shadowstack_call(self): cpu = self.cpu cpu.gc_ll_descr.init_nursery(100) diff --git a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py @@ -1,6 +1,5 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.jit import dont_look_inside -from rpython.rlib.objectmodel import invoke_around_extcall from rpython.jit.metainterp.optimizeopt import ALL_OPTS_NAMES from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib import rposix @@ -16,20 +15,10 @@ compile_kwds = dict(enable_opts=ALL_OPTS_NAMES, thread=True) def define_simple(self): - class Glob: - def __init__(self): - self.event = 0 - glob = Glob() - # - c_strchr = rffi.llexternal('strchr', [rffi.CCHARP, lltype.Signed], rffi.CCHARP) - def func(): - glob.event += 1 - def before(n, x): - invoke_around_extcall(func, func) return (n, None, None, None, None, None, None, None, None, None, None, None) # @@ -73,7 +62,8 @@ def f42(n): length = len(glob.lst) raw = alloc1() - fn = llhelper(CALLBACK, rffi._make_wrapper_for(CALLBACK, callback)) + wrapper = rffi._make_wrapper_for(CALLBACK, callback, None, True) + fn = llhelper(CALLBACK, wrapper) if n & 1: # to create a loop and a bridge, and also pass # to run the qsort() call in the blackhole interp c_qsort(rffi.cast(rffi.VOIDP, raw), rffi.cast(rffi.SIZE_T, 2), diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1170,10 +1170,11 @@ 'GC_LOAD/3/rfi', # parameters GC_LOAD_INDEXED # 1: pointer to complex object - # 2: integer describing the offset + # 2: integer describing the index # 3: constant integer scale factor - # 4: constant integer offset + # 4: constant integer base offset (final offset is 'base + scale * index') # 5: constant integer. byte size of datatype to load (negative if it is signed) + # (GC_LOAD is equivalent to GC_LOAD_INDEXED with arg3==1, arg4==0) 'GC_LOAD_INDEXED/5/rfi', '_RAW_LOAD_FIRST', @@ -1204,8 +1205,9 @@ # same paramters as GC_LOAD, but one additional for the value to store # note that the itemsize is not signed! + # (gcptr, index, value, [scale, base_offset,] itemsize) 'GC_STORE/4d/n', - 'GC_STORE_INDEXED/5d/n', + 'GC_STORE_INDEXED/6d/n', 'INCREMENT_DEBUG_COUNTER/1/n', '_RAW_STORE_FIRST', diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4044,7 +4044,7 @@ self.interp_operations(f, []) def test_external_call(self): - from rpython.rlib.objectmodel import invoke_around_extcall + from rpython.rlib import rgil TIME_T = lltype.Signed # ^^^ some 32-bit platforms have a 64-bit rffi.TIME_T, but we @@ -4058,11 +4058,6 @@ pass state = State() - def before(): - if we_are_jitted(): - raise Oups - state.l.append("before") - def after(): if we_are_jitted(): raise Oups @@ -4070,14 +4065,14 @@ def f(): state.l = [] - invoke_around_extcall(before, after) + rgil.invoke_after_thread_switch(after) external(lltype.nullptr(T.TO)) return len(state.l) res = self.interp_operations(f, []) - assert res == 2 + assert res == 1 res = self.interp_operations(f, []) - assert res == 2 + assert res == 1 self.check_operations_history(call_release_gil_i=1, call_may_force_i=0) def test_unescaped_write_zero(self): diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1124,8 +1124,8 @@ resultvar=op.result) def gct_gc_thread_run(self, hop): - assert self.translator.config.translation.thread - if hasattr(self.root_walker, 'thread_run_ptr'): + if (self.translator.config.translation.thread and + hasattr(self.root_walker, 'thread_run_ptr')): livevars = self.push_roots(hop) assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -184,6 +184,9 @@ hdr.tid |= self.gc.gcflag_extra return (hdr.tid & self.gc.gcflag_extra) != 0 + def thread_run(self): + pass + # ____________________________________________________________ class LLInterpRootWalker: diff --git a/rpython/rlib/_rposix_repr.py b/rpython/rlib/_rposix_repr.py deleted file mode 100644 --- a/rpython/rlib/_rposix_repr.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -RTyping support for os.stat_result objects. -They are rtyped just like a tuple of the correct length supporting -only indexing and the st_xxx attributes. We need a custom StatResultRepr -because when rtyping for LL backends we have extra platform-dependent -items at the end of the tuple, but for OO backends we only want the -portable items. This allows the OO backends to assume a fixed shape for -the tuples returned by os.stat(). -""" -from rpython.annotator import model as annmodel -from rpython.rtyper.llannotation import lltype_to_annotation -from rpython.flowspace.model import Constant -from rpython.flowspace.operation import op -from rpython.tool.pairtype import pairtype -from rpython.rtyper.rmodel import Repr -from rpython.rtyper.rint import IntegerRepr -from rpython.rtyper.error import TyperError -from rpython.rlib import rposix_stat - - -class StatResultRepr(Repr): - - def __init__(self, rtyper): - self.rtyper = rtyper - self.stat_fields = rposix_stat.STAT_FIELDS - - self.stat_field_indexes = {} - for i, (name, TYPE) in enumerate(self.stat_fields): - self.stat_field_indexes[name] = i - - self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE) - for name, TYPE in self.stat_fields]) - self.r_tuple = rtyper.getrepr(self.s_tuple) - self.lowleveltype = self.r_tuple.lowleveltype - - def redispatch_getfield(self, hop, index): - rtyper = self.rtyper - s_index = rtyper.annotator.bookkeeper.immutablevalue(index) - hop2 = hop.copy() - spaceop = op.getitem(hop.args_v[0], Constant(index)) - spaceop.result = hop.spaceop.result - hop2.spaceop = spaceop - hop2.args_v = spaceop.args - hop2.args_s = [self.s_tuple, s_index] - hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] - return hop2.dispatch() - - def rtype_getattr(self, hop): - s_attr = hop.args_s[1] - attr = s_attr.const - try: - index = self.stat_field_indexes[attr] - except KeyError: - raise TyperError("os.stat().%s: field not available" % (attr,)) - return self.redispatch_getfield(hop, index) - - -class __extend__(pairtype(StatResultRepr, IntegerRepr)): - - def rtype_getitem((r_sta, r_int), hop): - s_int = hop.args_s[1] - index = s_int.const - return r_sta.redispatch_getfield(hop, index) - - -def specialize_make_stat_result(hop): - r_StatResult = hop.rtyper.getrepr(rposix_stat.s_StatResult) - [v_result] = hop.inputargs(r_StatResult.r_tuple) - # no-op conversion from r_StatResult.r_tuple to r_StatResult - hop.exception_cannot_occur() - return v_result - - -class StatvfsResultRepr(Repr): - - def __init__(self, rtyper): - self.rtyper = rtyper - self.statvfs_fields = rposix_stat.STATVFS_FIELDS - - self.statvfs_field_indexes = {} - for i, (name, TYPE) in enumerate(self.statvfs_fields): - self.statvfs_field_indexes[name] = i - - self.s_tuple = annmodel.SomeTuple([lltype_to_annotation(TYPE) - for name, TYPE in self.statvfs_fields]) - self.r_tuple = rtyper.getrepr(self.s_tuple) - self.lowleveltype = self.r_tuple.lowleveltype - - def redispatch_getfield(self, hop, index): - rtyper = self.rtyper - s_index = rtyper.annotator.bookkeeper.immutablevalue(index) - hop2 = hop.copy() - spaceop = op.getitem(hop.args_v[0], Constant(index)) - spaceop.result = hop.spaceop.result - hop2.spaceop = spaceop - hop2.args_v = spaceop.args - hop2.args_s = [self.s_tuple, s_index] - hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] - return hop2.dispatch() - - def rtype_getattr(self, hop): - s_attr = hop.args_s[1] - attr = s_attr.const - try: - index = self.statvfs_field_indexes[attr] - except KeyError: - raise TyperError("os.statvfs().%s: field not available" % (attr,)) - return self.redispatch_getfield(hop, index) - - -class __extend__(pairtype(StatvfsResultRepr, IntegerRepr)): - def rtype_getitem((r_sta, r_int), hop): - s_int = hop.args_s[1] - index = s_int.const - return r_sta.redispatch_getfield(hop, index) - - -def specialize_make_statvfs_result(hop): - r_StatvfsResult = hop.rtyper.getrepr(rposix_stat.s_StatvfsResult) - [v_result] = hop.inputargs(r_StatvfsResult.r_tuple) - hop.exception_cannot_occur() - return v_result diff --git a/rpython/rlib/entrypoint.py b/rpython/rlib/entrypoint.py --- a/rpython/rlib/entrypoint.py +++ b/rpython/rlib/entrypoint.py @@ -56,10 +56,11 @@ """ def deco(func): source = py.code.Source(""" + from rpython.rlib import rgil + def wrapper(%(args)s): # acquire the GIL - after = rffi.aroundstate.after - if after: after() + rgil.acquire() # rffi.stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -78,8 +79,7 @@ assert 0 # dead code rffi.stackcounter.stacks_counter -= 1 # release the GIL - before = rffi.aroundstate.before - if before: before() + rgil.release() # return res """ % {'args': ', '.join(['arg%d' % i for i in range(len(argtypes))])}) diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -599,22 +599,10 @@ def hlinvoke(repr, llcallable, *args): raise TypeError("hlinvoke is meant to be rtyped and not called direclty") -def invoke_around_extcall(before, after): - """Call before() before any external function call, and after() after. - At the moment only one pair before()/after() can be registered at a time. +def is_in_callback(): + """Returns True if we're currently in a callback *or* if there are + multiple threads around. """ - # NOTE: the hooks are cleared during translation! To be effective - # in a compiled program they must be set at run-time. - from rpython.rtyper.lltypesystem import rffi - rffi.aroundstate.before = before - rffi.aroundstate.after = after - # the 'aroundstate' contains regular function and not ll pointers to them, - # but let's call llhelper() anyway to force their annotation - from rpython.rtyper.annlowlevel import llhelper - llhelper(rffi.AroundFnPtr, before) - llhelper(rffi.AroundFnPtr, after) - -def is_in_callback(): from rpython.rtyper.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -186,7 +186,13 @@ """ if not obj: return False - return can_move(obj) + # XXX returning can_move() here might acidentally work for the use + # cases (see issue #2212), but this is not really safe. Now we + # just return True for any non-NULL pointer, and too bad for the + # few extra 'cond_call_gc_wb'. It could be improved e.g. to return + # False if 'obj' is a static prebuilt constant, or if we're not + # running incminimark... + return True #can_move(obj) def _heap_stats(): raise NotImplementedError # can't be run directly diff --git a/rpython/rlib/rgil.py b/rpython/rlib/rgil.py --- a/rpython/rlib/rgil.py +++ b/rpython/rlib/rgil.py @@ -2,6 +2,7 @@ from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.extregistry import ExtRegistryEntry # these functions manipulate directly the GIL, whose definition does not # escape the C code itself @@ -10,27 +11,135 @@ eci = ExternalCompilationInfo( includes = ['src/thread.h'], separate_module_files = [translator_c_dir / 'src' / 'thread.c'], - include_dirs = [translator_c_dir]) + include_dirs = [translator_c_dir], + post_include_bits = ['#define RPY_WITH_GIL']) llexternal = rffi.llexternal -gil_allocate = llexternal('RPyGilAllocate', [], lltype.Void, +_gil_allocate = llexternal('RPyGilAllocate', [], lltype.Void, + _nowrapper=True, sandboxsafe=True, + compilation_info=eci) + +_gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed, _nowrapper=True, sandboxsafe=True, compilation_info=eci) -gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed, +_gil_release = llexternal('RPyGilRelease', [], lltype.Void, _nowrapper=True, sandboxsafe=True, compilation_info=eci) -gil_release = llexternal('RPyGilRelease', [], lltype.Void, - _nowrapper=True, sandboxsafe=True, - compilation_info=eci) - -gil_acquire = llexternal('RPyGilAcquire', [], lltype.Void, +_gil_acquire = llexternal('RPyGilAcquire', [], lltype.Void, _nowrapper=True, sandboxsafe=True, compilation_info=eci) gil_fetch_fastgil = llexternal('RPyFetchFastGil', [], llmemory.Address, _nowrapper=True, sandboxsafe=True, compilation_info=eci) + +# ____________________________________________________________ + + +def invoke_after_thread_switch(callback): + """Invoke callback() after a thread switch. + + This is a hook used by pypy.module.signal. Several callbacks should + be easy to support (but not right now). + + This function should be called from the translated RPython program + (i.e. *not* at module level!), but registers the callback + statically. The exact point at which invoke_after_thread_switch() + is called has no importance: the callback() will be called anyway. + """ + print "NOTE: invoke_after_thread_switch() is meant to be translated " + print "and not called directly. Using some emulation." + global _emulated_after_thread_switch + _emulated_after_thread_switch = callback + +_emulated_after_thread_switch = None + +def _after_thread_switch(): + """NOT_RPYTHON""" + if _emulated_after_thread_switch is not None: + _emulated_after_thread_switch() + + +class Entry(ExtRegistryEntry): + _about_ = invoke_after_thread_switch + + def compute_result_annotation(self, s_callback): + assert s_callback.is_constant() + callback = s_callback.const + bk = self.bookkeeper + translator = bk.annotator.translator + if hasattr(translator, '_rgil_invoke_after_thread_switch'): + assert translator._rgil_invoke_after_thread_switch == callback, ( + "not implemented yet: several invoke_after_thread_switch()") + else: + translator._rgil_invoke_after_thread_switch = callback + bk.emulate_pbc_call("rgil.invoke_after_thread_switch", s_callback, []) + + def specialize_call(self, hop): + # the actual call is not done here + hop.exception_cannot_occur() + +class Entry(ExtRegistryEntry): + _about_ = _after_thread_switch + + def compute_result_annotation(self): + # the call has been emulated already in invoke_after_thread_switch() + pass + + def specialize_call(self, hop): + translator = hop.rtyper.annotator.translator + if hasattr(translator, '_rgil_invoke_after_thread_switch'): + func = translator._rgil_invoke_after_thread_switch + graph = translator._graphof(func) + llfn = hop.rtyper.getcallable(graph) + c_callback = hop.inputconst(lltype.typeOf(llfn), llfn) + hop.exception_is_here() + hop.genop("direct_call", [c_callback]) + else: + hop.exception_cannot_occur() + + +def allocate(): + _gil_allocate() + +def release(): + # this function must not raise, in such a way that the exception + # transformer knows that it cannot raise! + _gil_release() +release._gctransformer_hint_cannot_collect_ = True +release._dont_reach_me_in_del_ = True + +def acquire(): + from rpython.rlib import rthread + _gil_acquire() + rthread.gc_thread_run() + _after_thread_switch() +acquire._gctransformer_hint_cannot_collect_ = True +acquire._dont_reach_me_in_del_ = True + +# The _gctransformer_hint_cannot_collect_ hack is needed for +# translations in which the *_external_call() functions are not inlined. +# They tell the gctransformer not to save and restore the local GC +# pointers in the shadow stack. This is necessary because the GIL is +# not held after the call to gil.release() or before the call +# to gil.acquire(). + +def yield_thread(): + # explicitly release the gil, in a way that tries to give more + # priority to other threads (as opposed to continuing to run in + # the same thread). + if _gil_yield_thread(): + from rpython.rlib import rthread + rthread.gc_thread_run() + _after_thread_switch() +yield_thread._gctransformer_hint_close_stack_ = True +yield_thread._dont_reach_me_in_del_ = True +yield_thread._dont_inline_ = True + +# yield_thread() needs a different hint: _gctransformer_hint_close_stack_. +# The *_external_call() functions are themselves called only from the rffi +# module from a helper function that also has this hint. diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -234,9 +234,16 @@ includes = ['io.h', 'sys/utime.h', 'sys/types.h'] libraries = [] else: + if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): + _ptyh = 'util.h' + elif sys.platform.startswith('freebsd'): + _ptyh = 'libutil.h' + else: + _ptyh = 'pty.h' includes = ['unistd.h', 'sys/types.h', 'sys/wait.h', 'utime.h', 'sys/time.h', 'sys/times.h', - 'grp.h', 'dirent.h'] + 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h', + 'signal.h', 'sys/utsname.h', _ptyh] libraries = ['util'] eci = ExternalCompilationInfo( includes=includes, @@ -1269,7 +1276,8 @@ if not _WIN32: TMSP = lltype.Ptr(TMS) c_times = external('times', [TMSP], CLOCK_T, - save_err=rffi.RFFI_SAVE_ERRNO) + save_err=rffi.RFFI_SAVE_ERRNO | + rffi.RFFI_ZERO_ERRNO_BEFORE) # Here is a random extra platform parameter which is important. # Strictly speaking, this should probably be retrieved at runtime, not @@ -1291,7 +1299,13 @@ if not _WIN32: l_tmsbuf = lltype.malloc(TMSP.TO, flavor='raw') try: - result = handle_posix_error('times', c_times(l_tmsbuf)) + # note: times() can return a negative value (or even -1) + # even if there is no error + result = widen(c_times(l_tmsbuf)) + if result == -1: + errno = get_saved_errno() + if errno != 0: + raise OSError(errno, 'times() failed') return ( rffi.cast(lltype.Signed, l_tmsbuf.c_tms_utime) / CLOCK_TICKS_PER_SECOND, @@ -1607,7 +1621,8 @@ #___________________________________________________________________ c_chroot = external('chroot', [rffi.CCHARP], rffi.INT, - save_err=rffi.RFFI_SAVE_ERRNO) + save_err=rffi.RFFI_SAVE_ERRNO, + macro=_MACRO_ON_POSIX) @replace_os_function('chroot') def chroot(path): diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -5,11 +5,16 @@ import os, sys +from rpython.flowspace.model import Constant +from rpython.flowspace.operation import op from rpython.annotator import model as annmodel from rpython.rtyper import extregistry from rpython.tool.pairtype import pairtype from rpython.rtyper.tool import rffi_platform as platform from rpython.rtyper.llannotation import lltype_to_annotation +from rpython.rtyper.rmodel import Repr +from rpython.rtyper.rint import IntegerRepr +from rpython.rtyper.error import TyperError from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import lltype, rffi @@ -23,7 +28,7 @@ _LINUX = sys.platform.startswith('linux') if _WIN32: - from rpython.rlib import rwin32 + from rpython.rlib import rwin32 from rpython.rlib.rwin32file import make_win32_traits # Support for float times is here. @@ -84,8 +89,7 @@ knowntype = os.stat_result def rtyper_makerepr(self, rtyper): - from rpython.rlib import _rposix_repr - return _rposix_repr.StatResultRepr(rtyper) + return StatResultRepr(rtyper) def rtyper_makekey(self): return self.__class__, @@ -111,6 +115,83 @@ return s_reduced, stat_result_reduce, stat_result_recreate +class __extend__(pairtype(SomeStatResult, annmodel.SomeInteger)): + def getitem((s_sta, s_int)): + assert s_int.is_constant(), "os.stat()[index]: index must be constant" + index = s_int.const + assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range" + name, TYPE = STAT_FIELDS[index] + return lltype_to_annotation(TYPE) + + +class StatResultRepr(Repr): + + def __init__(self, rtyper): + self.rtyper = rtyper + self.stat_field_indexes = {} + for i, (name, TYPE) in enumerate(STAT_FIELDS): + self.stat_field_indexes[name] = i + + self.s_tuple = annmodel.SomeTuple( + [lltype_to_annotation(TYPE) for name, TYPE in STAT_FIELDS]) + self.r_tuple = rtyper.getrepr(self.s_tuple) + self.lowleveltype = self.r_tuple.lowleveltype + + def redispatch_getfield(self, hop, index): + rtyper = self.rtyper + s_index = rtyper.annotator.bookkeeper.immutablevalue(index) + hop2 = hop.copy() + spaceop = op.getitem(hop.args_v[0], Constant(index)) + spaceop.result = hop.spaceop.result + hop2.spaceop = spaceop + hop2.args_v = spaceop.args + hop2.args_s = [self.s_tuple, s_index] + hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] + return hop2.dispatch() + + def rtype_getattr(self, hop): + s_attr = hop.args_s[1] + attr = s_attr.const + try: + index = self.stat_field_indexes[attr] + except KeyError: + raise TyperError("os.stat().%s: field not available" % (attr,)) + return self.redispatch_getfield(hop, index) + + +class __extend__(pairtype(StatResultRepr, IntegerRepr)): + def rtype_getitem((r_sta, r_int), hop): + s_int = hop.args_s[1] + index = s_int.const + return r_sta.redispatch_getfield(hop, index) + +s_StatResult = SomeStatResult() + +def make_stat_result(tup): + """Turn a tuple into an os.stat_result object.""" + positional = tuple( + lltype.cast_primitive(TYPE, value) for value, (name, TYPE) in + zip(tup, STAT_FIELDS)[:N_INDEXABLE_FIELDS]) + kwds = {} + for value, (name, TYPE) in zip(tup, STAT_FIELDS)[N_INDEXABLE_FIELDS:]: + kwds[name] = lltype.cast_primitive(TYPE, value) + return os.stat_result(positional, kwds) + + +class MakeStatResultEntry(extregistry.ExtRegistryEntry): + _about_ = make_stat_result + + def compute_result_annotation(self, s_tup): + return s_StatResult + + def specialize_call(self, hop): + r_StatResult = hop.rtyper.getrepr(s_StatResult) + [v_result] = hop.inputargs(r_StatResult.r_tuple) + # no-op conversion from r_StatResult.r_tuple to r_StatResult + hop.exception_cannot_occur() + return v_result + + class SomeStatvfsResult(annmodel.SomeObject): if hasattr(os, 'statvfs_result'): knowntype = os.statvfs_result @@ -118,8 +199,7 @@ knowntype = None # will not be used def rtyper_makerepr(self, rtyper): - from rpython.rlib import _rposix_repr - return _rposix_repr.StatvfsResultRepr(rtyper) + return StatvfsResultRepr(rtyper) def rtyper_makekey(self): return self.__class__, @@ -130,15 +210,6 @@ return lltype_to_annotation(TYPE) -class __extend__(pairtype(SomeStatResult, annmodel.SomeInteger)): - def getitem((s_sta, s_int)): - assert s_int.is_constant(), "os.stat()[index]: index must be constant" - index = s_int.const - assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range" - name, TYPE = STAT_FIELDS[index] - return lltype_to_annotation(TYPE) - - class __extend__(pairtype(SomeStatvfsResult, annmodel.SomeInteger)): def getitem((s_stat, s_int)): assert s_int.is_constant() @@ -146,33 +217,55 @@ return lltype_to_annotation(TYPE) -s_StatResult = SomeStatResult() s_StatvfsResult = SomeStatvfsResult() -def make_stat_result(tup): - """Turn a tuple into an os.stat_result object.""" - positional = tup[:N_INDEXABLE_FIELDS] - kwds = {} - for i, name in enumerate(STAT_FIELD_NAMES[N_INDEXABLE_FIELDS:]): - kwds[name] = tup[N_INDEXABLE_FIELDS + i] - return os.stat_result(positional, kwds) +class StatvfsResultRepr(Repr): + def __init__(self, rtyper): + self.rtyper = rtyper + self.statvfs_field_indexes = {} + for i, (name, TYPE) in enumerate(STATVFS_FIELDS): + self.statvfs_field_indexes[name] = i + + self.s_tuple = annmodel.SomeTuple( + [lltype_to_annotation(TYPE) for name, TYPE in STATVFS_FIELDS]) + self.r_tuple = rtyper.getrepr(self.s_tuple) + self.lowleveltype = self.r_tuple.lowleveltype + + def redispatch_getfield(self, hop, index): + rtyper = self.rtyper + s_index = rtyper.annotator.bookkeeper.immutablevalue(index) + hop2 = hop.copy() + spaceop = op.getitem(hop.args_v[0], Constant(index)) + spaceop.result = hop.spaceop.result + hop2.spaceop = spaceop + hop2.args_v = spaceop.args + hop2.args_s = [self.s_tuple, s_index] + hop2.args_r = [self.r_tuple, rtyper.getrepr(s_index)] + return hop2.dispatch() + + def rtype_getattr(self, hop): + s_attr = hop.args_s[1] + attr = s_attr.const + try: + index = self.statvfs_field_indexes[attr] + except KeyError: + raise TyperError("os.statvfs().%s: field not available" % (attr,)) + return self.redispatch_getfield(hop, index) + + +class __extend__(pairtype(StatvfsResultRepr, IntegerRepr)): + def rtype_getitem((r_sta, r_int), hop): + s_int = hop.args_s[1] + index = s_int.const + return r_sta.redispatch_getfield(hop, index) def make_statvfs_result(tup): - return os.statvfs_result(tup) - - -class MakeStatResultEntry(extregistry.ExtRegistryEntry): - _about_ = make_stat_result - - def compute_result_annotation(self, s_tup): - return s_StatResult - - def specialize_call(self, hop): - from rpython.rlib import _rposix_repr - return _rposix_repr.specialize_make_stat_result(hop) - + args = tuple( + lltype.cast_primitive(TYPE, value) for value, (name, TYPE) in + zip(tup, STATVFS_FIELDS)) + return os.statvfs_result(args) class MakeStatvfsResultEntry(extregistry.ExtRegistryEntry): _about_ = make_statvfs_result @@ -181,8 +274,10 @@ return s_StatvfsResult def specialize_call(self, hop): - from rpython.rlib import _rposix_repr - return _rposix_repr.specialize_make_statvfs_result(hop) + r_StatvfsResult = hop.rtyper.getrepr(s_StatvfsResult) + [v_result] = hop.inputargs(r_StatvfsResult.r_tuple) + hop.exception_cannot_occur() + return v_result # ____________________________________________________________ # diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -79,7 +79,12 @@ @specialize.arg(0) def ll_start_new_thread(func): + from rpython.rlib import rgil _check_thread_enabled() + rgil.allocate() + # ^^^ convenience: any RPython program which uses explicitly + # rthread.start_new_thread() will initialize the GIL at that + # point. ident = c_thread_start(func) if ident == -1: raise error("can't start new thread") diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -165,9 +165,11 @@ globals().update(rffi_platform.configure(CConfigForClockGetTime)) TIMESPEC = TIMESPEC CLOCK_PROCESS_CPUTIME_ID = CLOCK_PROCESS_CPUTIME_ID + eci_with_lrt = eci.merge(ExternalCompilationInfo(libraries=['rt'])) c_clock_gettime = external('clock_gettime', [lltype.Signed, lltype.Ptr(TIMESPEC)], - rffi.INT, releasegil=False) + rffi.INT, releasegil=False, + compilation_info=eci_with_lrt) else: RUSAGE = RUSAGE RUSAGE_SELF = RUSAGE_SELF or 0 diff --git a/rpython/rlib/test/test_rgil.py b/rpython/rlib/test/test_rgil.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/test/test_rgil.py @@ -0,0 +1,47 @@ +from rpython.rlib import rgil +from rpython.translator.c.test.test_standalone import StandaloneTests + + +class BaseTestGIL(StandaloneTests): + + def test_simple(self): + def main(argv): + rgil.release() + # don't have the GIL here + rgil.acquire() + rgil.yield_thread() + print "OK" # there is also a release/acquire pair here + return 0 + + main([]) + + t, cbuilder = self.compile(main) + data = cbuilder.cmdexec('') + assert data == "OK\n" + + def test_after_thread_switch(self): + class Foo: + pass + foo = Foo() + foo.counter = 0 + def seeme(): + foo.counter += 1 + def main(argv): + rgil.invoke_after_thread_switch(seeme) + print "Test" # one release/acquire pair here + print foo.counter + print foo.counter + return 0 + + t, cbuilder = self.compile(main) + data = cbuilder.cmdexec('') + assert data == "Test\n1\n2\n" + + +class TestGILAsmGcc(BaseTestGIL): + gc = 'minimark' + gcrootfinder = 'asmgcc' + +class TestGILShadowStack(BaseTestGIL): + gc = 'minimark' + gcrootfinder = 'shadowstack' diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py --- a/rpython/rlib/test/test_rposix_stat.py +++ b/rpython/rlib/test/test_rposix_stat.py @@ -32,7 +32,11 @@ fname = udir.join('test_stat_large_number.txt') fname.ensure() t1 = 5000000000.0 - os.utime(str(fname), (t1, t1)) + try: + os.utime(str(fname), (t1, t1)) + except OverflowError: + py.test.skip("This platform doesn't support setting stat times " + "to large values") assert rposix_stat.stat(str(fname)).st_mtime == t1 @py.test.mark.skipif(not hasattr(os, 'statvfs'), diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -143,7 +143,7 @@ def test_simple_tcp(): - import thread + from rpython.rlib import rthread sock = RSocket() try_ports = [1023] + range(20000, 30000, 437) for port in try_ports: @@ -169,14 +169,14 @@ connected[0] = True finally: lock.release() - lock = thread.allocate_lock() - lock.acquire() - thread.start_new_thread(connecting, ()) + lock = rthread.allocate_lock() + lock.acquire(True) + rthread.start_new_thread(connecting, ()) print 'waiting for connection' fd1, addr2 = sock.accept() s1 = RSocket(fd=fd1) print 'connection accepted' - lock.acquire() + lock.acquire(True) assert connected[0] print 'connecting side knows that the connection was accepted too' assert addr.eq(s2.getpeername()) @@ -188,7 +188,9 @@ buf = s2.recv(100) assert buf == '?' print 'received ok' - thread.start_new_thread(s2.sendall, ('x'*50000,)) + def sendstuff(): + s2.sendall('x'*50000) + rthread.start_new_thread(sendstuff, ()) buf = '' while len(buf) < 50000: data = s1.recv(50100) diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -5,13 +5,6 @@ from rpython.rtyper.lltypesystem import lltype, rffi import py -def setup_module(mod): - # Hack to avoid a deadlock if the module is run after other test files :-( - # In this module, we assume that rthread.start_new_thread() is not - # providing us with a GIL equivalent, except in test_gc_locking - # which installs its own aroundstate. - rffi.aroundstate._cleanup_() - def test_lock(): l = allocate_lock() ok1 = l.acquire(True) @@ -31,6 +24,7 @@ py.test.fail("Did not raise") def test_tlref_untranslated(): + import thread class FooBar(object): pass t = ThreadLocalReference(FooBar) @@ -43,7 +37,7 @@ time.sleep(0.2) results.append(t.get() is x) for i in range(5): - start_new_thread(subthread, ()) + thread.start_new_thread(subthread, ()) time.sleep(0.5) assert results == [True] * 15 @@ -99,7 +93,6 @@ def test_gc_locking(self): import time - from rpython.rlib.objectmodel import invoke_around_extcall from rpython.rlib.debug import ll_assert class State: @@ -123,17 +116,6 @@ ll_assert(j == self.j, "2: bad j") run._dont_inline_ = True - def before_extcall(): - release_NOAUTO(state.gil) - before_extcall._gctransformer_hint_cannot_collect_ = True - # ^^^ see comments in gil.py about this hint - - def after_extcall(): - acquire_NOAUTO(state.gil, True) - gc_thread_run() - after_extcall._gctransformer_hint_cannot_collect_ = True - # ^^^ see comments in gil.py about this hint - def bootstrap(): # after_extcall() is called before we arrive here. # We can't just acquire and release the GIL manually here, @@ -154,14 +136,9 @@ start_new_thread(bootstrap, ()) def f(): - state.gil = allocate_ll_lock() - acquire_NOAUTO(state.gil, True) state.bootstrapping = allocate_lock() state.answers = [] state.finished = 0 - # the next line installs before_extcall() and after_extcall() - # to be called automatically around external function calls. - invoke_around_extcall(before_extcall, after_extcall) g(10, 1) done = False @@ -179,10 +156,7 @@ return len(state.answers) expected = 89 - try: - fn = self.getcompiled(f, []) - finally: - rffi.aroundstate._cleanup_() + fn = self.getcompiled(f, []) answers = fn() assert answers == expected diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -169,9 +169,9 @@ argnames = ', '.join(['a%d' % i for i in range(len(args))]) source = py.code.Source(""" + from rpython.rlib import rgil def call_external_function(%(argnames)s): - before = aroundstate.before - if before: before() + rgil.release() # NB. it is essential that no exception checking occurs here! if %(save_err)d: from rpython.rlib import rposix @@ -180,12 +180,10 @@ if %(save_err)d: from rpython.rlib import rposix rposix._errno_after(%(save_err)d) - after = aroundstate.after - if after: after() + rgil.acquire() return res """ % locals()) - miniglobals = {'aroundstate': aroundstate, - 'funcptr': funcptr, + miniglobals = {'funcptr': funcptr, '__name__': __name__, # for module name propagation } exec source.compile() in miniglobals @@ -205,7 +203,7 @@ # don't inline, as a hack to guarantee that no GC pointer is alive # anywhere in call_external_function else: - # if we don't have to invoke the aroundstate, we can just call + # if we don't have to invoke the GIL handling, we can just call # the low-level function pointer carelessly if macro is None and save_err == RFFI_ERR_NONE: call_external_function = funcptr @@ -270,13 +268,10 @@ freeme = arg elif _isfunctype(TARGET) and not _isllptr(arg): # XXX pass additional arguments - if invoke_around_handlers: - arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, - callbackholder, - aroundstate)) - else: - arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, - callbackholder)) + use_gil = invoke_around_handlers + arg = llhelper(TARGET, _make_wrapper_for(TARGET, arg, + callbackholder, + use_gil)) else: SOURCE = lltype.typeOf(arg) if SOURCE != TARGET: @@ -315,7 +310,7 @@ def __init__(self): self.callbacks = {} -def _make_wrapper_for(TP, callable, callbackholder=None, aroundstate=None): +def _make_wrapper_for(TP, callable, callbackholder, use_gil): """ Function creating wrappers for callbacks. Note that this is cheating as we assume constant callbacks and we just memoize wrappers """ @@ -330,11 +325,13 @@ callbackholder.callbacks[callable] = True args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" + rgil = None + if use_gil: + from rpython.rlib import rgil + def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple - if aroundstate is not None: - after = aroundstate.after - if after: - after() + if rgil is not None: + rgil.acquire() # from now on we hold the GIL stackcounter.stacks_counter += 1 llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py @@ -349,13 +346,11 @@ traceback.print_exc() result = errorcode stackcounter.stacks_counter -= 1 - if aroundstate is not None: - before = aroundstate.before - if before: - before() + if rgil is not None: + rgil.release() # here we don't hold the GIL any more. As in the wrapper() produced # by llexternal, it is essential that no exception checking occurs - # after the call to before(). + # after the call to rgil.release(). return result """ % locals()) miniglobals = locals().copy() @@ -369,13 +364,6 @@ AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -class AroundState: - def _cleanup_(self): - self.before = None # or a regular RPython function - self.after = None # or a regular RPython function -aroundstate = AroundState() -aroundstate._cleanup_() - class StackCounter: def _cleanup_(self): self.stacks_counter = 0 # number of "stack pieces": callbacks diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -688,42 +688,6 @@ assert interpret(f, []) == 4 - def test_around_extcall(self): - if sys.platform == "win32": - py.test.skip('No pipes on windows') - import os - from rpython.annotator import model as annmodel - from rpython.rlib.objectmodel import invoke_around_extcall - from rpython.rtyper.extfuncregistry import register_external - read_fd, write_fd = os.pipe() - try: - # we need an external function that is not going to get wrapped around - # before()/after() calls, in order to call it from before()/after()... - def mywrite(s): - os.write(write_fd, s) - def llimpl(s): - s = ''.join(s.chars) - os.write(write_fd, s) - register_external(mywrite, [str], annmodel.s_None, 'll_mywrite', - llfakeimpl=llimpl, sandboxsafe=True) - - def before(): - mywrite("B") - def after(): - mywrite("A") - def f(): - os.write(write_fd, "-") - invoke_around_extcall(before, after) - os.write(write_fd, "E") - - interpret(f, []) - data = os.read(read_fd, 99) - assert data == "-BEA" - - finally: - os.close(write_fd) - os.close(read_fd) - def test_external_callable(self): """ Try to call some llexternal function with llinterp """ diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -159,7 +159,7 @@ def __del__(self): delcalls[0] += 1 - os.write(1, "__del__\n") + #os.write(1, "__del__\n") def f(x=int): a = A() diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -33,6 +33,10 @@ # include <io.h> #endif +#ifdef RPY_WITH_GIL +# include <src/thread.h> +#endif + RPY_EXTERN int pypy_main_function(int argc, char *argv[]) @@ -46,6 +50,14 @@ _setmode(1, _O_BINARY); #endif +#ifdef RPY_WITH_GIL + /* Note that the GIL's mutexes are not automatically made; if the + program starts threads, it needs to call rgil.gil_allocate(). + RPyGilAcquire() still works without that, but crash if it finds + that it really needs to wait on a mutex. */ + RPyGilAcquire(); +#endif + #ifdef PYPY_USE_ASMGCC pypy_g_rpython_rtyper_lltypesystem_rffi_StackCounter.sc_inst_stacks_counter++; #endif @@ -82,6 +94,10 @@ pypy_malloc_counters_results(); +#ifdef RPY_WITH_GIL + RPyGilRelease(); +#endif + return exitcode; memory_out: diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -120,11 +120,8 @@ got += 1; fd = ((void* *) (((char *)fd) + sizeof(void*)))[0]; } - if (rpy_fastgil != 1) { - RPyAssert(rpy_fastgil != 0, - "pypy_check_stack_count doesn't have the GIL"); - got++; /* <= the extra one currently stored in rpy_fastgil */ - } + RPyAssert(rpy_fastgil == 1, + "pypy_check_stack_count doesn't have the GIL"); RPyAssert(got == stacks_counter - 1, "bad stacks_counter or non-closed stacks around"); # endif diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h --- a/rpython/translator/c/src/thread.h +++ b/rpython/translator/c/src/thread.h @@ -28,7 +28,8 @@ RPY_EXTERN void RPyGilAllocate(void); RPY_EXTERN long RPyGilYieldThread(void); -RPY_EXTERN void RPyGilAcquire(void); +RPY_EXTERN void RPyGilAcquireSlowPath(long); _______________________________________________ pypy-commit mailing list [email protected] https://mail.python.org/mailman/listinfo/pypy-commit
