Author: Armin Rigo <ar...@tunes.org> Branch: cffi-static-callback Changeset: r80680:b2f90804d8eb Date: 2015-11-15 13:17 +0100 http://bitbucket.org/pypy/pypy/changeset/b2f90804d8eb/
Log: in-progress diff --git a/pypy/module/_cffi_backend/call_python.py b/pypy/module/_cffi_backend/call_python.py new file mode 100644 --- /dev/null +++ b/pypy/module/_cffi_backend/call_python.py @@ -0,0 +1,155 @@ +import os +from rpython.rlib.objectmodel import specialize, instantiate +from rpython.rlib.rarithmetic import intmask +from rpython.rlib import jit +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.annlowlevel import llhelper + +from pypy.interpreter.error import oefmt +from pypy.interpreter.gateway import interp2app +from pypy.module._cffi_backend import parse_c_type +from pypy.module._cffi_backend import cerrno +from pypy.module._cffi_backend import cffi_opcode +from pypy.module._cffi_backend import realize_c_type +from pypy.module._cffi_backend.realize_c_type import getop, getarg + + +STDERR = 2 +CALLPY_FN = lltype.FuncType([parse_c_type.PCALLPY, rffi.CCHARP], + lltype.Void) + + +def get_printable_location(callpython): + with callpython as ptr: + callpy = rffi.cast(parse_c_type.PCALLPY, ptr) + return 'cffi_call_python ' + rffi.charp2str(callpy.g_name) + +jitdriver = jit.JitDriver(name='cffi_call_python', + greens=['callpython'], + reds=['ll_args'], + get_printable_location=get_printable_location) + +def py_invoke_callpython(callpython, ll_args): + jitdriver.jit_merge_point(callpython=callpython, ll_args=ll_args) + # the same buffer is used both for passing arguments and the result value + callpython.do_invoke(ll_args, ll_args) + + +def _cffi_call_python(ll_callpy, ll_args): + """Invoked by the helpers generated from CFFI_CALL_PYTHON in the cdef. + + 'callpy' is a static structure that describes which of the + CFFI_CALL_PYTHON is called. It has got fields 'name' and + 'type_index' describing the function, and more reserved fields + that are initially zero. These reserved fields are set up by + ffi.call_python(), which invokes init_call_python() below. + + 'args' is a pointer to an array of 8-byte entries. Each entry + contains an argument. If an argument is less than 8 bytes, only + the part at the beginning of the entry is initialized. If an + argument is 'long double' or a struct/union, then it is passed + by reference. + + 'args' is also used as the place to write the result to. In all + cases, 'args' is at least 8 bytes in size. + """ + from pypy.module._cffi_backend.ccallback import reveal_callback + + cerrno._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) + + if not ll_callpy.c_reserved1: + # Not initialized! We don't have a space at all, so just + # write the error to the file descriptor stderr. (xxx cpython's + # cffi writes it to sys.stderr) + try: + funcname = rffi.charp2str(ll_callpy.c_name) + msg = ("CFFI_CALL_PYTHON: function %s() called, but no code was " + "attached to it yet with ffi.call_python('%s'). " + "Returning 0.\n" % (funcname, funcname)) + os.write(STDERR, msg) + except: + pass + for i in range(intmask(ll_callpy.c_size_of_result)): + ll_args[i] = '\x00' + else: + callpython = reveal_callback(ll_callpy.c_reserved1) + space = callpython.space + must_leave = False + try: + must_leave = space.threadlocals.try_enter_thread(space) + py_invoke_callpython(callpython, ll_args) + # + except Exception, e: + # oups! last-level attempt to recover. + try: + os.write(STDERR, "SystemError: call_python function raised ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except: + pass + callpython.write_error_return_value(ll_res) + if must_leave: + space.threadlocals.leave_thread(space) + + cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) + + +def get_ll_cffi_call_python(): + return llhelper(lltype.Ptr(CALLPY_FN), _cffi_call_python) + + +class Cache: + def __init__(self, space): + self.cache_dict = {} + + +def callpy_deco(space, w_ffi, w_python_callable, w_name, w_error, w_onerror): + from pypy.module._cffi_backend.ffi_obj import W_FFIObject + from pypy.module._cffi_backend.ccallback import W_CallPython + + ffi = space.interp_w(W_FFIObject, w_ffi) + + if space.is_w(w_name, space.w_None): + XXX + else: + name = space.str_w(w_name) + + ctx = ffi.ctxobj.ctx + index = parse_c_type.search_in_globals(ctx, name) + if index < 0: + raise callpy_not_found(ffi, name) + + g = ctx.c_globals[index] + if getop(g.c_type_op) != cffi_opcode.OP_CALL_PYTHON: + raise callpy_not_found(ffi, name) + + w_ct = realize_c_type.realize_c_type(ffi, ctx.c_types, getarg(g.c_type_op)) + + # make a W_CallPython instance, which is nonmovable; then cast it + # to a raw pointer and assign it to the field 'reserved1' of the + # callpy object from C. We must make sure to keep it alive forever, + # or at least until ffi.call_python() is used again to change the + # binding. Note that the W_CallPython is never exposed to the user. + callpy = rffi.cast(parse_c_type.PCALLPY, g.c_address) + callpython = instantiate(W_CallPython, nonmovable=True) + callpython.__init__(space, rffi.cast(rffi.CCHARP, callpy), w_ct, + w_python_callable, w_error, w_onerror) + + key = rffi.cast(lltype.Signed, callpy) + space.fromcache(Cache).cache_dict[key] = callpython + callpy.c_reserved1 = rffi.cast(rffi.CCHARP, callpython.hide_object()) + + # return a cdata of type function-pointer, equal to the one + # obtained by reading 'lib.bar' (see lib_obj.py) + ptr = lltype.direct_fieldptr(g, 'c_size_or_direct_fn') + return w_ct.convert_to_object(rffi.cast(rffi.CCHARP, ptr)) + + +def callpy_not_found(ffi, name): + raise oefmt(ffi.w_FFIError, + "ffi.call_python('%s'): name not found as a " + "CFFI_CALL_PYTHON line from the cdef", name) + +@specialize.memo() +def get_generic_decorator(space): + return space.wrap(interp2app(callpy_deco)) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -3,7 +3,7 @@ """ import sys, os, py -from rpython.rlib import clibffi, jit, jit_libffi, rgc, objectmodel +from rpython.rlib import clibffi, jit, rgc, objectmodel from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.lltypesystem import lltype, llmemory, rffi @@ -25,46 +25,38 @@ # we can cast to a plain VOIDP. As long as the object is not freed, # we can cast the VOIDP back to a W_CDataCallback in reveal_callback(). cdata = objectmodel.instantiate(W_CDataCallback, nonmovable=True) - gcref = rgc.cast_instance_to_gcref(cdata) - raw_cdata = rgc.hide_nonmovable_gcref(gcref) - cdata.__init__(space, ctype, w_callable, w_error, w_onerror, raw_cdata) + cdata.__init__(space, ctype, w_callable, w_error, w_onerror) return cdata def reveal_callback(raw_ptr): addr = rffi.cast(llmemory.Address, raw_ptr) gcref = rgc.reveal_gcref(addr) - return rgc.try_cast_gcref_to_instance(W_CDataCallback, gcref) + return rgc.try_cast_gcref_to_instance(W_CallPython, gcref) class Closure(object): """This small class is here to have a __del__ outside any cycle.""" - ll_error = lltype.nullptr(rffi.CCHARP.TO) # set manually - def __init__(self, ptr): self.ptr = ptr def __del__(self): clibffi.closureHeap.free(rffi.cast(clibffi.FFI_CLOSUREP, self.ptr)) - if self.ll_error: - lltype.free(self.ll_error, flavor='raw') -class W_CDataCallback(W_CData): - _immutable_fields_ = ['key_pycode'] +class W_CallPython(W_CData): + """Base class for W_CDataCallback, also used from call_python.py. + """ w_onerror = None + decode_args_from_libffi = False - def __init__(self, space, ctype, w_callable, w_error, w_onerror, - raw_cdata): - raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc()) - self._closure = Closure(raw_closure) - W_CData.__init__(self, space, raw_closure, ctype) + def __init__(self, space, cdata, ctype, w_callable, w_error, w_onerror): + W_CData.__init__(self, space, cdata, ctype) # if not space.is_true(space.callable(w_callable)): raise oefmt(space.w_TypeError, "expected a callable object, not %T", w_callable) self.w_callable = w_callable - self.key_pycode = space._try_fetch_pycode(w_callable) if not space.is_none(w_onerror): if not space.is_true(space.callable(w_onerror)): raise oefmt(space.w_TypeError, @@ -74,40 +66,20 @@ # fresult = self.getfunctype().ctitem size = fresult.size - if size > 0: - if fresult.is_primitive_integer and size < SIZE_OF_FFI_ARG: - size = SIZE_OF_FFI_ARG - self._closure.ll_error = lltype.malloc(rffi.CCHARP.TO, size, - flavor='raw', zero=True) - if not space.is_none(w_error): - convert_from_object_fficallback(fresult, self._closure.ll_error, - w_error) + if fresult.is_primitive_integer and size < SIZE_OF_FFI_ARG: + size = SIZE_OF_FFI_ARG + with lltype.scoped_alloc(rffi.CCHARP.TO, size, zero=True) as ll_error: + if not space.is_none(w_error): + convert_from_object_fficallback(fresult, ll_error, w_error, + self.decode_args_from_libffi) + self.error_string = rffi.charpsize2str(ll_error, size) # # We must setup the GIL here, in case the callback is invoked in # some other non-Pythonic thread. This is the same as cffi on - # CPython. + # CPython, or ctypes. if space.config.translation.thread: from pypy.module.thread.os_thread import setup_threads setup_threads(space) - # - cif_descr = self.getfunctype().cif_descr - if not cif_descr: - raise oefmt(space.w_NotImplementedError, - "%s: callback with unsupported argument or " - "return type or with '...'", self.getfunctype().name) - with self as ptr: - closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr) - unique_id = rffi.cast(rffi.VOIDP, raw_cdata) - res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif, - invoke_callback, - unique_id) - if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: - raise OperationError(space.w_SystemError, - space.wrap("libffi failed to build this callback")) - - def _repr_extra(self): - space = self.space - return 'calling ' + space.str_w(space.repr(self.w_callable)) def getfunctype(self): ctype = self.ctype @@ -117,43 +89,118 @@ space.wrap("expected a function ctype")) return ctype + def hide_object(self): + gcref = rgc.cast_instance_to_gcref(self) + return rgc.hide_nonmovable_gcref(gcref) + + def _repr_extra(self): + space = self.space + return 'calling ' + space.str_w(space.repr(self.w_callable)) + + def write_error_return_value(self, ll_res): + error_string = self.error_string + for i in range(len(error_string)): + ll_res[i] = error_string[i] + + def do_invoke(self, ll_res, ll_args): + space = self.space + extra_line = '' + try: + w_args = self.prepare_args_tuple(ll_args) + w_res = space.call(self.w_callable, w_args) + extra_line = "Trying to convert the result back to C:\n" + self.convert_result(ll_res, w_res) + except OperationError, e: + self.handle_applevel_exception(e, ll_res, extra_line) + @jit.unroll_safe - def invoke(self, ll_args): + def prepare_args_tuple(self, ll_args): space = self.space ctype = self.getfunctype() ctype = jit.promote(ctype) args_w = [] + decode_args_from_libffi = self.decode_args_from_libffi for i, farg in enumerate(ctype.fargs): - ll_arg = rffi.cast(rffi.CCHARP, ll_args[i]) + if decode_args_from_libffi: + ll_arg = rffi.cast(rffi.CCHARP, ll_args[i]) + else: + ll_arg = rffi.ptradd(rffi.cast(rffi.CCHARP, ll_args), 8 * i) + if farg.is_indirect_arg_for_call_python: + ll_arg = rffi.cast(rffi.CCHARPP, ll_arg)[0] args_w.append(farg.convert_to_object(ll_arg)) - return space.call(self.w_callable, space.newtuple(args_w)) + return space.newtuple(args_w) def convert_result(self, ll_res, w_res): fresult = self.getfunctype().ctitem - convert_from_object_fficallback(fresult, ll_res, w_res) + convert_from_object_fficallback(fresult, ll_res, w_res, + self.decode_args_from_libffi) def print_error(self, operr, extra_line): space = self.space operr.write_unraisable(space, "cffi callback ", self.w_callable, with_traceback=True, extra_line=extra_line) - def write_error_return_value(self, ll_res): - fresult = self.getfunctype().ctitem - if fresult.size > 0: - misc._raw_memcopy(self._closure.ll_error, ll_res, fresult.size) - keepalive_until_here(self) # to keep self._closure.ll_error alive + @jit.dont_look_inside + def handle_applevel_exception(self, e, ll_res, extra_line): + space = self.space + self.write_error_return_value(ll_res) + if self.w_onerror is None: + self.print_error(e, extra_line) + else: + try: + e.normalize_exception(space) + w_t = e.w_type + w_v = e.get_w_value(space) + w_tb = space.wrap(e.get_traceback()) + w_res = space.call_function(self.w_onerror, w_t, w_v, w_tb) + if not space.is_none(w_res): + self.convert_result(ll_res, w_res) + except OperationError, e2: + # double exception! print a double-traceback... + self.print_error(e, extra_line) # original traceback + e2.write_unraisable(space, '', with_traceback=True, + extra_line="\nDuring the call to 'onerror', " + "another exception occurred:\n\n") -def convert_from_object_fficallback(fresult, ll_res, w_res): +class W_CDataCallback(W_CallPython): + _immutable_fields_ = ['key_pycode'] + decode_args_from_libffi = True + + def __init__(self, space, ctype, w_callable, w_error, w_onerror): + raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc()) + self._closure = Closure(raw_closure) + W_CallPython.__init__(self, space, raw_closure, ctype, + w_callable, w_error, w_onerror) + self.key_pycode = space._try_fetch_pycode(w_callable) + # + cif_descr = self.getfunctype().cif_descr + if not cif_descr: + raise oefmt(space.w_NotImplementedError, + "%s: callback with unsupported argument or " + "return type or with '...'", self.getfunctype().name) + with self as ptr: + closure_ptr = rffi.cast(clibffi.FFI_CLOSUREP, ptr) + unique_id = rffi.cast(rffi.VOIDP, self.hide_object()) + res = clibffi.c_ffi_prep_closure(closure_ptr, cif_descr.cif, + invoke_callback, + unique_id) + if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: + raise OperationError(space.w_SystemError, + space.wrap("libffi failed to build this callback")) + + +def convert_from_object_fficallback(fresult, ll_res, w_res, + encode_result_for_libffi): space = fresult.space - small_result = fresult.size < SIZE_OF_FFI_ARG - if small_result and isinstance(fresult, W_CTypeVoid): + if isinstance(fresult, W_CTypeVoid): if not space.is_w(w_res, space.w_None): raise OperationError(space.w_TypeError, space.wrap("callback with the return type 'void'" " must return None")) return # + small_result = encode_result_for_libffi and fresult.size < SIZE_OF_FFI_ARG if small_result and fresult.is_primitive_integer: # work work work around a libffi irregularity: for integer return # types we have to fill at least a complete 'ffi_arg'-sized result @@ -191,29 +238,6 @@ STDERR = 2 -@jit.dont_look_inside -def _handle_applevel_exception(callback, e, ll_res, extra_line): - space = callback.space - callback.write_error_return_value(ll_res) - if callback.w_onerror is None: - callback.print_error(e, extra_line) - else: - try: - e.normalize_exception(space) - w_t = e.w_type - w_v = e.get_w_value(space) - w_tb = space.wrap(e.get_traceback()) - w_res = space.call_function(callback.w_onerror, - w_t, w_v, w_tb) - if not space.is_none(w_res): - callback.convert_result(ll_res, w_res) - except OperationError, e2: - # double exception! print a double-traceback... - callback.print_error(e, extra_line) # original traceback - e2.write_unraisable(space, '', with_traceback=True, - extra_line="\nDuring the call to 'onerror', " - "another exception occurred:\n\n") - def get_printable_location(key_pycode): if key_pycode is None: return 'cffi_callback <?>' @@ -226,13 +250,7 @@ def py_invoke_callback(callback, ll_res, ll_args): jitdriver.jit_merge_point(callback=callback, ll_res=ll_res, ll_args=ll_args) - extra_line = '' - try: - w_res = callback.invoke(ll_args) - extra_line = "Trying to convert the result back to C:\n" - callback.convert_result(ll_res, w_res) - except OperationError, e: - _handle_applevel_exception(callback, e, ll_res, extra_line) + callback.do_invoke(ll_res, ll_args) def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py --- a/pypy/module/_cffi_backend/cffi1_module.py +++ b/pypy/module/_cffi_backend/cffi1_module.py @@ -17,12 +17,12 @@ def load_cffi1_module(space, name, path, initptr): # This is called from pypy.module.cpyext.api.load_extension_module() - from pypy.module._cffi_backend.call_python import get_cffi_call_python + from pypy.module._cffi_backend.call_python import get_ll_cffi_call_python initfunc = rffi.cast(initfunctype, initptr) with lltype.scoped_alloc(rffi.VOIDPP.TO, 16, zero=True) as p: p[0] = rffi.cast(rffi.VOIDP, VERSION_EXPORT) - p[1] = rffi.cast(rffi.VOIDP, get_cffi_call_python()) + p[1] = rffi.cast(rffi.VOIDP, get_ll_cffi_call_python()) initfunc(p) version = rffi.cast(lltype.Signed, p[0]) if not (VERSION_MIN <= version <= VERSION_MAX): diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -22,6 +22,7 @@ cast_anything = False is_primitive_integer = False is_nonfunc_pointer_or_array = False + is_indirect_arg_for_call_python = False kind = "?" def __init__(self, space, size, name, name_position): diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -424,6 +424,7 @@ class W_CTypePrimitiveLongDouble(W_CTypePrimitiveFloat): _attrs_ = [] + is_indirect_arg_for_call_python = True @jit.dont_look_inside def extra_repr(self, cdata): diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -18,6 +18,7 @@ class W_CTypeStructOrUnion(W_CType): _immutable_fields_ = ['alignment?', '_fields_list?[*]', '_fields_dict?', '_custom_field_pos?', '_with_var_array?'] + is_indirect_arg_for_call_python = True # three possible states: # - "opaque": for opaque C structs; self.size < 0. diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -279,6 +279,30 @@ return cbuffer.buffer(self.space, w_cdata, size) + @unwrap_spec(w_name=WrappedDefault(None), + w_error=WrappedDefault(None), + w_onerror=WrappedDefault(None)) + def descr_call_python(self, w_name, w_error, w_onerror): + """\ +A decorator. Attaches the decorated Python function to the C code +generated for the CFFI_CALL_PYTHON function of the same name. Calling +the C function will then invoke the Python function. + +Optional arguments: 'name' is the name of the C function, if +different from the Python function; and 'error' and 'onerror' +handle what occurs if the Python function raises an exception +(see the docs for details).""" + # + # returns a single-argument function + space = self.space + w_ffi = space.wrap(self) + w_decorator = call_python.get_generic_decorator(space) + return space.appexec([w_decorator, w_ffi, w_name, w_error, w_onerror], + """(decorator, ffi, name, error, onerror): + return lambda python_callable: decorator(ffi, python_callable, + name, error, onerror)""") + + @unwrap_spec(w_python_callable=WrappedDefault(None), w_error=WrappedDefault(None), w_onerror=WrappedDefault(None)) @@ -633,7 +657,7 @@ addressof = interp2app(W_FFIObject.descr_addressof), alignof = interp2app(W_FFIObject.descr_alignof), buffer = interp2app(W_FFIObject.descr_buffer), - #call_python = interp2app(W_FFIObject.descr_call_python), + call_python = interp2app(W_FFIObject.descr_call_python), callback = interp2app(W_FFIObject.descr_callback), cast = interp2app(W_FFIObject.descr_cast), dlclose = interp2app(W_FFIObject.descr_dlclose), diff --git a/pypy/module/_cffi_backend/parse_c_type.py b/pypy/module/_cffi_backend/parse_c_type.py --- a/pypy/module/_cffi_backend/parse_c_type.py +++ b/pypy/module/_cffi_backend/parse_c_type.py @@ -71,11 +71,11 @@ ('error_location', rffi.SIZE_T), ('error_message', rffi.CCHARP)) -CALLPY_S = rffi.CStruct('_cffi_callpy_s', - ('name', rffi.CCHARP), - ('size_of_result', rffi.SIZE_T), - ('reserved1', rffi.VOIDP), - ('reserved2', rffi.VOIDP)) +PCALLPY = rffi.CStructPtr('_cffi_callpy_s', + ('name', rffi.CCHARP), + ('size_of_result', rffi.SIZE_T), + ('reserved1', rffi.CCHARP), + ('reserved2', rffi.CCHARP)) GETCONST_S = rffi.CStruct('_cffi_getconst_s', ('value', rffi.ULONGLONG), _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit