Author: Armin Rigo <[email protected]>
Branch:
Changeset: r81816:6c7267731b67
Date: 2016-01-15 19:47 +0100
http://bitbucket.org/pypy/pypy/changeset/6c7267731b67/
Log: hg merge cffi-static-callback-embedding
Add embedding support, bringing CFFI to version 1.5.
diff --git a/lib-python/2.7/distutils/command/build_ext.py
b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -685,13 +685,17 @@
# the previous version of this code did. This should work for
# CPython too. The point is that on PyPy with cpyext, the
# config var 'SO' is just ".so" but we want to return
- # ".pypy-VERSION.so" instead.
- so_ext = _get_c_extension_suffix()
+ # ".pypy-VERSION.so" instead. Note a further tweak for cffi's
+ # embedding mode: if EXT_SUFFIX is also defined, use that
+ # directly.
+ so_ext = get_config_var('EXT_SUFFIX')
if so_ext is None:
- so_ext = get_config_var('SO') # fall-back
- # extensions in debug_mode are named 'module_d.pyd' under windows
- if os.name == 'nt' and self.debug:
- so_ext = '_d.pyd'
+ so_ext = _get_c_extension_suffix()
+ if so_ext is None:
+ so_ext = get_config_var('SO') # fall-back
+ # extensions in debug_mode are named 'module_d.pyd' under windows
+ if os.name == 'nt' and self.debug:
+ so_ext = '_d.pyd'
return os.path.join(*ext_path) + so_ext
def get_export_symbols (self, ext):
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.4.2
+Version: 1.5.0
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.4.2"
-__version_info__ = (1, 4, 2)
+__version__ = "1.5.0"
+__version_info__ = (1, 5, 0)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -146,8 +146,9 @@
((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
#define _cffi_convert_array_from_object \
((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+#define _CFFI_CPIDX 25
#define _cffi_call_python \
- ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[25])
+ ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
#define _CFFI_NUM_EXPORTS 26
typedef struct _ctypedescr CTypeDescrObject;
@@ -206,7 +207,8 @@
/********** end CPython-specific section **********/
#else
_CFFI_UNUSED_FN
-static void (*_cffi_call_python)(struct _cffi_externpy_s *, char *);
+static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *);
+# define _cffi_call_python _cffi_call_python_org
#endif
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -74,6 +74,7 @@
self._windows_unicode = None
self._init_once_cache = {}
self._cdef_version = None
+ self._embedding = None
if hasattr(backend, 'set_ffi'):
backend.set_ffi(self)
for name in backend.__dict__:
@@ -101,13 +102,21 @@
If 'packed' is specified as True, all structs declared inside this
cdef are packed, i.e. laid out without any field alignment at all.
"""
+ self._cdef(csource, override=override, packed=packed)
+
+ def embedding_api(self, csource, packed=False):
+ self._cdef(csource, packed=packed, dllexport=True)
+ if self._embedding is None:
+ self._embedding = ''
+
+ def _cdef(self, csource, override=False, **options):
if not isinstance(csource, str): # unicode, on Python 2
if not isinstance(csource, basestring):
raise TypeError("cdef() argument must be a string")
csource = csource.encode('ascii')
with self._lock:
self._cdef_version = object()
- self._parser.parse(csource, override=override, packed=packed)
+ self._parser.parse(csource, override=override, **options)
self._cdefsources.append(csource)
if override:
for cache in self._function_caches:
@@ -533,6 +542,31 @@
('_UNICODE', '1')]
kwds['define_macros'] = defmacros
+ def _apply_embedding_fix(self, kwds):
+ # must include an argument like "-lpython2.7" for the compiler
+ if '__pypy__' in sys.builtin_module_names:
+ if hasattr(sys, 'prefix'):
+ import os
+ libdir = os.path.join(sys.prefix, 'bin')
+ dirs = kwds.setdefault('library_dirs', [])
+ if libdir not in dirs:
+ dirs.append(libdir)
+ pythonlib = "pypy-c"
+ else:
+ if sys.platform == "win32":
+ template = "python%d%d"
+ if sys.flags.debug:
+ template = template + '_d'
+ else:
+ template = "python%d.%d"
+ pythonlib = (template %
+ (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
+ if hasattr(sys, 'abiflags'):
+ pythonlib += sys.abiflags
+ libraries = kwds.setdefault('libraries', [])
+ if pythonlib not in libraries:
+ libraries.append(pythonlib)
+
def set_source(self, module_name, source, source_extension='.c', **kwds):
if hasattr(self, '_assigned_source'):
raise ValueError("set_source() cannot be called several times "
@@ -592,14 +626,23 @@
recompile(self, module_name, source,
c_file=filename, call_c_compiler=False, **kwds)
- def compile(self, tmpdir='.', verbose=0):
+ def compile(self, tmpdir='.', verbose=0, target=None):
+ """The 'target' argument gives the final file name of the
+ compiled DLL. Use '*' to force distutils' choice, suitable for
+ regular CPython C API modules. Use a file name ending in '.*'
+ to ask for the system's default extension for dynamic libraries
+ (.so/.dll).
+
+ The default is '*' when building a non-embedded C API extension,
+ and (module_name + '.*') when building an embedded library.
+ """
from .recompiler import recompile
#
if not hasattr(self, '_assigned_source'):
raise ValueError("set_source() must be called before compile()")
module_name, source, source_extension, kwds = self._assigned_source
return recompile(self, module_name, source, tmpdir=tmpdir,
- source_extension=source_extension,
+ target=target, source_extension=source_extension,
compiler_verbose=verbose, **kwds)
def init_once(self, func, tag):
@@ -626,6 +669,32 @@
self._init_once_cache[tag] = (True, result)
return result
+ def embedding_init_code(self, pysource):
+ if self._embedding:
+ raise ValueError("embedding_init_code() can only be called once")
+ # fix 'pysource' before it gets dumped into the C file:
+ # - remove empty lines at the beginning, so it starts at "line 1"
+ # - dedent, if all non-empty lines are indented
+ # - check for SyntaxErrors
+ import re
+ match = re.match(r'\s*\n', pysource)
+ if match:
+ pysource = pysource[match.end():]
+ lines = pysource.splitlines() or ['']
+ prefix = re.match(r'\s*', lines[0]).group()
+ for i in range(1, len(lines)):
+ line = lines[i]
+ if line.rstrip():
+ while not line.startswith(prefix):
+ prefix = prefix[:-1]
+ i = len(prefix)
+ lines = [line[i:]+'\n' for line in lines]
+ pysource = ''.join(lines)
+ #
+ compile(pysource, "cffi_init", "exec")
+ #
+ self._embedding = pysource
+
def _load_backend_lib(backend, name, flags):
if name is None:
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -220,8 +220,7 @@
self._included_declarations = set()
self._anonymous_counter = 0
self._structnode2type = weakref.WeakKeyDictionary()
- self._override = False
- self._packed = False
+ self._options = None
self._int_constants = {}
self._recomplete = []
self._uses_new_feature = None
@@ -281,16 +280,15 @@
msg = 'parse error\n%s' % (msg,)
raise api.CDefError(msg)
- def parse(self, csource, override=False, packed=False):
- prev_override = self._override
- prev_packed = self._packed
+ def parse(self, csource, override=False, packed=False, dllexport=False):
+ prev_options = self._options
try:
- self._override = override
- self._packed = packed
+ self._options = {'override': override,
+ 'packed': packed,
+ 'dllexport': dllexport}
self._internal_parse(csource)
finally:
- self._override = prev_override
- self._packed = prev_packed
+ self._options = prev_options
def _internal_parse(self, csource):
ast, macros, csource = self._parse(csource)
@@ -376,10 +374,13 @@
def _declare_function(self, tp, quals, decl):
tp = self._get_type_pointer(tp, quals)
- if self._inside_extern_python:
- self._declare('extern_python ' + decl.name, tp)
+ if self._options['dllexport']:
+ tag = 'dllexport_python '
+ elif self._inside_extern_python:
+ tag = 'extern_python '
else:
- self._declare('function ' + decl.name, tp)
+ tag = 'function '
+ self._declare(tag + decl.name, tp)
def _parse_decl(self, decl):
node = decl.type
@@ -449,7 +450,7 @@
prevobj, prevquals = self._declarations[name]
if prevobj is obj and prevquals == quals:
return
- if not self._override:
+ if not self._options['override']:
raise api.FFIError(
"multiple declarations of %s (for interactive usage, "
"try cdef(xx, override=True))" % (name,))
@@ -728,7 +729,7 @@
if isinstance(tp, model.StructType) and tp.partial:
raise NotImplementedError("%s: using both bitfields and '...;'"
% (tp,))
- tp.packed = self._packed
+ tp.packed = self._options['packed']
if tp.completed: # must be re-completed: it is not opaque any more
tp.completed = 0
self._recomplete.append(tp)
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,14 @@
allsources.append(os.path.normpath(src))
return Extension(name=modname, sources=allsources, **kwds)
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, target_extension=None,
+ embedding=False):
"""Compile a C extension module using distutils."""
saved_environ = os.environ.copy()
try:
- outputfilename = _build(tmpdir, ext, compiler_verbose)
+ outputfilename = _build(tmpdir, ext, compiler_verbose,
+ target_extension, embedding)
outputfilename = os.path.abspath(outputfilename)
finally:
# workaround for a distutils bugs where some env vars can
@@ -36,7 +38,32 @@
os.environ[key] = value
return outputfilename
-def _build(tmpdir, ext, compiler_verbose=0):
+def _save_val(name):
+ import distutils.sysconfig
+ config_vars = distutils.sysconfig.get_config_vars()
+ return config_vars.get(name, Ellipsis)
+
+def _restore_val(name, value):
+ import distutils.sysconfig
+ config_vars = distutils.sysconfig.get_config_vars()
+ config_vars[name] = value
+ if value is Ellipsis:
+ del config_vars[name]
+
+def _win32_hack_for_embedding():
+ from distutils.msvc9compiler import MSVCCompiler
+ if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'):
+ MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \
+ MSVCCompiler._remove_visual_c_ref
+ MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file:
manifest_file
+
+def _win32_unhack_for_embedding():
+ from distutils.msvc9compiler import MSVCCompiler
+ MSVCCompiler._remove_visual_c_ref = \
+ MSVCCompiler._remove_visual_c_ref_CFFI_BAK
+
+def _build(tmpdir, ext, compiler_verbose=0, target_extension=None,
+ embedding=False):
# XXX compact but horrible :-(
from distutils.core import Distribution
import distutils.errors, distutils.log
@@ -49,18 +76,29 @@
options['build_temp'] = ('ffiplatform', tmpdir)
#
try:
+ if sys.platform == 'win32' and embedding:
+ _win32_hack_for_embedding()
old_level = distutils.log.set_threshold(0) or 0
+ old_SO = _save_val('SO')
+ old_EXT_SUFFIX = _save_val('EXT_SUFFIX')
try:
+ if target_extension is not None:
+ _restore_val('SO', target_extension)
+ _restore_val('EXT_SUFFIX', target_extension)
distutils.log.set_verbosity(compiler_verbose)
dist.run_command('build_ext')
+ cmd_obj = dist.get_command_obj('build_ext')
+ [soname] = cmd_obj.get_outputs()
finally:
distutils.log.set_threshold(old_level)
+ _restore_val('SO', old_SO)
+ _restore_val('EXT_SUFFIX', old_EXT_SUFFIX)
+ if sys.platform == 'win32' and embedding:
+ _win32_unhack_for_embedding()
except (distutils.errors.CompileError,
distutils.errors.LinkError) as e:
raise VerificationError('%s: %s' % (e.__class__.__name__, e))
#
- cmd_obj = dist.get_command_obj('build_ext')
- [soname] = cmd_obj.get_outputs()
return soname
try:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,6 +3,7 @@
from .cffi_opcode import *
VERSION = "0x2601"
+VERSION_EMBEDDED = "0x2701"
class GlobalExpr:
@@ -281,6 +282,29 @@
lines[i:i+1] = self._rel_readlines('parse_c_type.h')
prnt(''.join(lines))
#
+ # if we have ffi._embedding != None, we give it here as a macro
+ # and include an extra file
+ base_module_name = self.module_name.split('.')[-1]
+ if self.ffi._embedding is not None:
+ prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,))
+ prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' %
+ (self._string_literal(self.ffi._embedding),))
+ prnt('#ifdef PYPY_VERSION')
+ prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % (
+ base_module_name,))
+ prnt('#elif PY_MAJOR_VERSION >= 3')
+ prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % (
+ base_module_name,))
+ prnt('#else')
+ prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % (
+ base_module_name,))
+ prnt('#endif')
+ lines = self._rel_readlines('_embedding.h')
+ prnt(''.join(lines))
+ version = VERSION_EMBEDDED
+ else:
+ version = VERSION
+ #
# then paste the C source given by the user, verbatim.
prnt('/************************************************************/')
prnt()
@@ -365,17 +389,16 @@
prnt()
#
# the init function
- base_module_name = self.module_name.split('.')[-1]
prnt('#ifdef PYPY_VERSION')
prnt('PyMODINIT_FUNC')
prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
prnt('{')
if self._num_externpy:
prnt(' if (((intptr_t)p[0]) >= 0x0A03) {')
- prnt(' _cffi_call_python = '
+ prnt(' _cffi_call_python_org = '
'(void(*)(struct _cffi_externpy_s *, char *))p[1];')
prnt(' }')
- prnt(' p[0] = (const void *)%s;' % VERSION)
+ prnt(' p[0] = (const void *)%s;' % version)
prnt(' p[1] = &_cffi_type_context;')
prnt('}')
# on Windows, distutils insists on putting init_cffi_xyz in
@@ -394,14 +417,14 @@
prnt('PyInit_%s(void)' % (base_module_name,))
prnt('{')
prnt(' return _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, VERSION))
+ self.module_name, version))
prnt('}')
prnt('#else')
prnt('PyMODINIT_FUNC')
prnt('init%s(void)' % (base_module_name,))
prnt('{')
prnt(' _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, VERSION))
+ self.module_name, version))
prnt('}')
prnt('#endif')
@@ -1123,7 +1146,10 @@
assert isinstance(tp, model.FunctionPtrType)
self._do_collect_type(tp)
- def _generate_cpy_extern_python_decl(self, tp, name):
+ def _generate_cpy_dllexport_python_collecttype(self, tp, name):
+ self._generate_cpy_extern_python_collecttype(tp, name)
+
+ def _generate_cpy_extern_python_decl(self, tp, name, dllexport=False):
prnt = self._prnt
if isinstance(tp.result, model.VoidType):
size_of_result = '0'
@@ -1156,7 +1182,11 @@
size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % (
tp.result.get_c_name(''), size_of_a,
tp.result.get_c_name(''), size_of_a)
- prnt('static %s' % tp.result.get_c_name(name_and_arguments))
+ if dllexport:
+ tag = 'CFFI_DLLEXPORT'
+ else:
+ tag = 'static'
+ prnt('%s %s' % (tag, tp.result.get_c_name(name_and_arguments)))
prnt('{')
prnt(' char a[%s];' % size_of_a)
prnt(' char *p = a;')
@@ -1174,6 +1204,9 @@
prnt()
self._num_externpy += 1
+ def _generate_cpy_dllexport_python_decl(self, tp, name):
+ self._generate_cpy_extern_python_decl(tp, name, dllexport=True)
+
def _generate_cpy_extern_python_ctx(self, tp, name):
if self.target_is_python:
raise ffiplatform.VerificationError(
@@ -1185,6 +1218,21 @@
self._lsts["global"].append(
GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name))
+ def _generate_cpy_dllexport_python_ctx(self, tp, name):
+ self._generate_cpy_extern_python_ctx(tp, name)
+
+ def _string_literal(self, s):
+ def _char_repr(c):
+ # escape with a '\' the characters '\', '"' or (for trigraphs) '?'
+ if c in '\\"?': return '\\' + c
+ if ' ' <= c < '\x7F': return c
+ if c == '\n': return '\\n'
+ return '\\%03o' % ord(c)
+ lines = []
+ for line in s.splitlines(True):
+ lines.append('"%s"' % ''.join([_char_repr(c) for c in line]))
+ return ' \\\n'.join(lines)
+
# ----------
# emitting the opcodes for individual types
@@ -1311,12 +1359,15 @@
def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
c_file=None, source_extension='.c', extradir=None,
- compiler_verbose=1, **kwds):
+ compiler_verbose=1, target=None, **kwds):
if not isinstance(module_name, str):
module_name = module_name.encode('ascii')
if ffi._windows_unicode:
ffi._apply_windows_unicode(kwds)
if preamble is not None:
+ embedding = (ffi._embedding is not None)
+ if embedding:
+ ffi._apply_embedding_fix(kwds)
if c_file is None:
c_file, parts = _modname_to_file(tmpdir, module_name,
source_extension)
@@ -1325,13 +1376,40 @@
ext_c_file = os.path.join(*parts)
else:
ext_c_file = c_file
- ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds)
+ #
+ if target is None:
+ if embedding:
+ target = '%s.*' % module_name
+ else:
+ target = '*'
+ if target == '*':
+ target_module_name = module_name
+ target_extension = None # use default
+ else:
+ if target.endswith('.*'):
+ target = target[:-2]
+ if sys.platform == 'win32':
+ target += '.dll'
+ else:
+ target += '.so'
+ # split along the first '.' (not the last one, otherwise the
+ # preceeding dots are interpreted as splitting package names)
+ index = target.find('.')
+ if index < 0:
+ raise ValueError("target argument %r should be a file name "
+ "containing a '.'" % (target,))
+ target_module_name = target[:index]
+ target_extension = target[index:]
+ #
+ ext = ffiplatform.get_extension(ext_c_file, target_module_name, **kwds)
updated = make_c_source(ffi, module_name, preamble, c_file)
if call_c_compiler:
cwd = os.getcwd()
try:
os.chdir(tmpdir)
- outputfilename = ffiplatform.compile('.', ext,
compiler_verbose)
+ outputfilename = ffiplatform.compile('.', ext,
compiler_verbose,
+ target_extension,
+ embedding=embedding)
finally:
os.chdir(cwd)
return outputfilename
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -10,6 +10,15 @@
with a ``libpypy-c.so`` or ``pypy-c.dll`` file. This is the default in
recent versions of PyPy.
+.. note::
+
+ The interface described in this page is kept for backward compatibility.
+ From PyPy 4.1, it is recommended to use instead CFFI's `native embedding
+ support,`__ which gives a simpler approach that works on CPython as well
+ as PyPy.
+
+.. __: http://cffi.readthedocs.org/en/latest/embedding.html
+
The resulting shared library exports very few functions, however they are
enough to accomplish everything you need, provided you follow a few principles.
The API is:
diff --git a/pypy/goal/targetpypystandalone.py
b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -84,13 +84,6 @@
from rpython.rlib.entrypoint import entrypoint_highlevel
from rpython.rtyper.lltypesystem import rffi, lltype
- w_pathsetter = space.appexec([], """():
- def f(path):
- import sys
- sys.path[:] = path
- return f
- """)
-
@entrypoint_highlevel('main', [rffi.CCHARP, rffi.INT],
c_name='pypy_setup_home')
def pypy_setup_home(ll_home, verbose):
@@ -109,7 +102,10 @@
" not found in '%s' or in any parent directory" % home1)
return rffi.cast(rffi.INT, 1)
space.startup()
- space.call_function(w_pathsetter, w_path)
+ space.appexec([w_path], """(path):
+ import sys
+ sys.path[:] = path
+ """)
# import site
try:
space.setattr(space.getbuiltinmodule('sys'),
@@ -149,6 +145,9 @@
return
os_thread.setup_threads(space)
os_thread.bootstrapper.acquire(space, None, None)
+ # XXX this doesn't really work. Don't use os.fork(), and
+ # if your embedder program uses fork(), don't use any PyPy
+ # code in the fork
rthread.gc_thread_start()
os_thread.bootstrapper.nbthreads += 1
os_thread.bootstrapper.release()
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -1,8 +1,9 @@
import sys
from pypy.interpreter.mixedmodule import MixedModule
-from rpython.rlib import rdynload, clibffi
+from rpython.rlib import rdynload, clibffi, entrypoint
+from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.4.2"
+VERSION = "1.5.0"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
@@ -65,6 +66,10 @@
if has_stdcall:
interpleveldefs['FFI_STDCALL'] = 'space.wrap(%d)' % FFI_STDCALL
+ def startup(self, space):
+ from pypy.module._cffi_backend import embedding
+ embedding.glob.space = space
+
def get_dict_rtld_constants():
found = {}
@@ -78,3 +83,11 @@
for _name, _value in get_dict_rtld_constants().items():
Module.interpleveldefs[_name] = 'space.wrap(%d)' % _value
+
+
+# write this entrypoint() here, to make sure it is registered early enough
[email protected]_highlevel('main', [rffi.INT, rffi.VOIDP],
+ c_name='pypy_init_embedded_cffi_module')
+def pypy_init_embedded_cffi_module(version, init_struct):
+ from pypy.module._cffi_backend import embedding
+ return embedding.pypy_init_embedded_cffi_module(version, init_struct)
diff --git a/pypy/module/_cffi_backend/cffi1_module.py
b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -9,18 +9,18 @@
VERSION_MIN = 0x2601
-VERSION_MAX = 0x26FF
+VERSION_MAX = 0x27FF
VERSION_EXPORT = 0x0A03
-initfunctype = lltype.Ptr(lltype.FuncType([rffi.VOIDPP], lltype.Void))
+INITFUNCPTR = lltype.Ptr(lltype.FuncType([rffi.VOIDPP], lltype.Void))
def load_cffi1_module(space, name, path, initptr):
# This is called from pypy.module.cpyext.api.load_extension_module()
from pypy.module._cffi_backend.call_python import get_ll_cffi_call_python
- initfunc = rffi.cast(initfunctype, initptr)
+ initfunc = rffi.cast(INITFUNCPTR, initptr)
with lltype.scoped_alloc(rffi.VOIDPP.TO, 16, zero=True) as p:
p[0] = rffi.cast(rffi.VOIDP, VERSION_EXPORT)
p[1] = rffi.cast(rffi.VOIDP, get_ll_cffi_call_python())
@@ -41,7 +41,8 @@
w_name = space.wrap(name)
module = Module(space, w_name)
- module.setdictvalue(space, '__file__', space.wrap(path))
+ if path is not None:
+ module.setdictvalue(space, '__file__', space.wrap(path))
module.setdictvalue(space, 'ffi', space.wrap(ffi))
module.setdictvalue(space, 'lib', space.wrap(lib))
w_modules_dict = space.sys.get('modules')
diff --git a/pypy/module/_cffi_backend/embedding.py
b/pypy/module/_cffi_backend/embedding.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -0,0 +1,146 @@
+import os
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+from pypy.interpreter.error import OperationError, oefmt
+
+# ____________________________________________________________
+
+
+EMBED_VERSION_MIN = 0xB011
+EMBED_VERSION_MAX = 0xB0FF
+
+STDERR = 2
+INITSTRUCTPTR = lltype.Ptr(lltype.Struct('CFFI_INIT',
+ ('name', rffi.CCHARP),
+ ('func', rffi.VOIDP),
+ ('code', rffi.CCHARP)))
+
+def load_embedded_cffi_module(space, version, init_struct):
+ from pypy.module._cffi_backend.cffi1_module import load_cffi1_module
+ declare_c_function() # translation-time hint only:
+ # declare _cffi_carefully_make_gil()
+ #
+ version = rffi.cast(lltype.Signed, version)
+ if not (EMBED_VERSION_MIN <= version <= EMBED_VERSION_MAX):
+ raise oefmt(space.w_ImportError,
+ "cffi embedded module has got unknown version tag %s",
+ hex(version))
+ #
+ if space.config.objspace.usemodules.thread:
+ from pypy.module.thread import os_thread
+ os_thread.setup_threads(space)
+ #
+ name = rffi.charp2str(init_struct.name)
+ load_cffi1_module(space, name, None, init_struct.func)
+ code = rffi.charp2str(init_struct.code)
+ compiler = space.createcompiler()
+ pycode = compiler.compile(code, "<init code for '%s'>" % name, 'exec', 0)
+ w_globals = space.newdict(module=True)
+ space.setitem_str(w_globals, "__builtins__", space.wrap(space.builtin))
+ pycode.exec_code(space, w_globals, w_globals)
+
+
+class Global:
+ pass
+glob = Global()
+
+def pypy_init_embedded_cffi_module(version, init_struct):
+ # called from __init__.py
+ name = "?"
+ try:
+ init_struct = rffi.cast(INITSTRUCTPTR, init_struct)
+ name = rffi.charp2str(init_struct.name)
+ #
+ space = glob.space
+ must_leave = False
+ try:
+ must_leave = space.threadlocals.try_enter_thread(space)
+ load_embedded_cffi_module(space, version, init_struct)
+ res = 0
+ except OperationError, operr:
+ operr.write_unraisable(space, "initialization of '%s'" % name,
+ with_traceback=True)
+ space.appexec([], r"""():
+ import sys
+ sys.stderr.write('pypy version: %s.%s.%s\n' %
+ sys.pypy_version_info[:3])
+ sys.stderr.write('sys.path: %r\n' % (sys.path,))
+ """)
+ res = -1
+ if must_leave:
+ space.threadlocals.leave_thread(space)
+ except Exception, e:
+ # oups! last-level attempt to recover.
+ try:
+ os.write(STDERR, "From initialization of '")
+ os.write(STDERR, name)
+ os.write(STDERR, "':\n")
+ os.write(STDERR, str(e))
+ os.write(STDERR, "\n")
+ except:
+ pass
+ res = -1
+ return rffi.cast(rffi.INT, res)
+
+# ____________________________________________________________
+
+
+eci = ExternalCompilationInfo(separate_module_sources=[
+r"""
+/* XXX Windows missing */
+#include <stdio.h>
+#include <dlfcn.h>
+#include <pthread.h>
+
+RPY_EXPORTED void rpython_startup_code(void);
+RPY_EXPORTED int pypy_setup_home(char *, int);
+
+static unsigned char _cffi_ready = 0;
+static const char *volatile _cffi_module_name;
+
+static void _cffi_init_error(const char *msg, const char *extra)
+{
+ fprintf(stderr,
+ "\nPyPy initialization failure when loading module '%s':\n%s%s\n",
+ _cffi_module_name, msg, extra);
+}
+
+static void _cffi_init(void)
+{
+ Dl_info info;
+ char *home;
+
+ rpython_startup_code();
+ RPyGilAllocate();
+
+ if (dladdr(&_cffi_init, &info) == 0) {
+ _cffi_init_error("dladdr() failed: ", dlerror());
+ return;
+ }
+ home = realpath(info.dli_fname, NULL);
+ if (pypy_setup_home(home, 1) != 0) {
+ _cffi_init_error("pypy_setup_home() failed", "");
+ return;
+ }
+ _cffi_ready = 1;
+}
+
+RPY_EXPORTED
+int pypy_carefully_make_gil(const char *name)
+{
+ /* For CFFI: this initializes the GIL and loads the home path.
+ It can be called completely concurrently from unrelated threads.
+ It assumes that we don't hold the GIL before (if it exists), and we
+ don't hold it afterwards.
+ */
+ static pthread_once_t once_control = PTHREAD_ONCE_INIT;
+
+ _cffi_module_name = name; /* not really thread-safe, but better than
+ nothing */
+ pthread_once(&once_control, _cffi_init);
+ return (int)_cffi_ready - 1;
+}
+"""])
+
+declare_c_function = rffi.llexternal_use_eci(eci)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.4.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.5.0", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1353,8 +1353,8 @@
ffi = FFI(backend=self.Backend())
ffi.cdef("enum foo;")
from cffi import __version_info__
- if __version_info__ < (1, 5):
- py.test.skip("re-enable me in version 1.5")
+ if __version_info__ < (1, 6):
+ py.test.skip("re-enable me in version 1.6")
e = py.test.raises(CDefError, ffi.cast, "enum foo", -1)
assert str(e.value) == (
"'enum foo' has no values explicitly defined: refusing to guess "
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -54,3 +54,10 @@
content = open(p).read()
#v = BACKEND_VERSIONS.get(v, v)
assert (('assert __version__ == "%s"' % v) in content)
+
+def test_embedding_h():
+ parent = os.path.dirname(os.path.dirname(cffi.__file__))
+ v = cffi.__version__
+ p = os.path.join(parent, 'cffi', '_embedding.h')
+ content = open(p).read()
+ assert ('cffi version: %s"' % (v,)) in content
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1719,3 +1719,10 @@
exec("from _test_import_from_lib.lib import *", d)
assert (set(key for key in d if not key.startswith('_')) ==
set(['myfunc', 'MYFOO']))
+ #
+ # also test "import *" on the module itself, which should be
+ # equivalent to "import ffi, lib"
+ d = {}
+ exec("from _test_import_from_lib import *", d)
+ assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
+ ['ffi', 'lib'])
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -60,11 +60,16 @@
if (name.endswith('.so') or name.endswith('.pyd') or
name.endswith('.dylib')):
found_so = os.path.join(curdir, name)
- # foo.cpython-34m.so => foo
- name = name.split('.')[0]
- # foo_d.so => foo (Python 2 debug builds)
+ # foo.so => foo
+ parts = name.split('.')
+ del parts[-1]
+ if len(parts) > 1 and parts[-1] != 'bar':
+ # foo.cpython-34m.so => foo, but foo.bar.so => foo.bar
+ del parts[-1]
+ name = '.'.join(parts)
+ # foo_d => foo (Python 2 debug builds)
if name.endswith('_d') and hasattr(sys, 'gettotalrefcount'):
- name = name.rsplit('_', 1)[0]
+ name = name[:-2]
name += '.SO'
if name.startswith('pycparser') and name.endswith('.egg'):
continue # no clue why this shows up sometimes and not
others
@@ -209,6 +214,58 @@
'Release': '?'}})
@chdir_to_tmp
+ def test_api_compile_explicit_target_1(self):
+ ffi = cffi.FFI()
+ ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
+ x = ffi.compile(target="foo.bar.*")
+ if sys.platform != 'win32':
+ sofile = self.check_produced_files({
+ 'foo.bar.SO': None,
+ 'mod_name_in_package': {'mymod.c': None,
+ 'mymod.o': None}})
+ assert os.path.isabs(x) and os.path.samefile(x, sofile)
+ else:
+ self.check_produced_files({
+ 'foo.bar.SO': None,
+ 'mod_name_in_package': {'mymod.c': None},
+ 'Release': '?'})
+
+ @chdir_to_tmp
+ def test_api_compile_explicit_target_2(self):
+ ffi = cffi.FFI()
+ ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
+ x = ffi.compile(target=os.path.join("mod_name_in_package",
"foo.bar.*"))
+ if sys.platform != 'win32':
+ sofile = self.check_produced_files({
+ 'mod_name_in_package': {'foo.bar.SO': None,
+ 'mymod.c': None,
+ 'mymod.o': None}})
+ assert os.path.isabs(x) and os.path.samefile(x, sofile)
+ else:
+ self.check_produced_files({
+ 'mod_name_in_package': {'foo.bar.SO': None,
+ 'mymod.c': None},
+ 'Release': '?'})
+
+ @chdir_to_tmp
+ def test_api_compile_explicit_target_3(self):
+ ffi = cffi.FFI()
+ ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
+ x = ffi.compile(target="foo.bar.baz")
+ if sys.platform != 'win32':
+ self.check_produced_files({
+ 'foo.bar.baz': None,
+ 'mod_name_in_package': {'mymod.c': None,
+ 'mymod.o': None}})
+ sofile = os.path.join(str(self.udir), 'foo.bar.baz')
+ assert os.path.isabs(x) and os.path.samefile(x, sofile)
+ else:
+ self.check_produced_files({
+ 'foo.bar.baz': None,
+ 'mod_name_in_package': {'mymod.c': None},
+ 'Release': '?'})
+
+ @chdir_to_tmp
def test_api_distutils_extension_1(self):
ffi = cffi.FFI()
ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/__init__.py
@@ -0,0 +1,1 @@
+# Generated by pypy/tool/import_cffi.py
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
@@ -0,0 +1,13 @@
+#include <stdio.h>
+
+extern int add1(int, int);
+
+
+int main(void)
+{
+ int x, y;
+ x = add1(40, 2);
+ y = add1(100, -5);
+ printf("got: %d %d\n", x, y);
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1.py
@@ -0,0 +1,34 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ import sys, time
+ sys.stdout.write("preparing")
+ for i in range(3):
+ sys.stdout.flush()
+ time.sleep(0.02)
+ sys.stdout.write(".")
+ sys.stdout.write("\n")
+
+ from _add1_cffi import ffi
+
+ int(ord("A")) # check that built-ins are there
+
+ @ffi.def_extern()
+ def add1(x, y):
+ sys.stdout.write("adding %d and %d\n" % (x, y))
+ sys.stdout.flush()
+ return x + y
+""")
+
+ffi.set_source("_add1_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c
@@ -0,0 +1,14 @@
+#include <stdio.h>
+
+extern int add1(int, int);
+extern int add2(int, int, int);
+
+
+int main(void)
+{
+ int x, y;
+ x = add1(40, 2);
+ y = add2(100, -5, -20);
+ printf("got: %d %d\n", x, y);
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add2.py
@@ -0,0 +1,30 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add2(int, int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ import sys
+ sys.stdout.write("prepADD2\n")
+
+ assert '_add2_cffi' in sys.modules
+ m = sys.modules['_add2_cffi']
+ import _add2_cffi
+ ffi = _add2_cffi.ffi
+
+ @ffi.def_extern()
+ def add2(x, y, z):
+ sys.stdout.write("adding %d and %d and %d\n" % (x, y, z))
+ sys.stdout.flush()
+ return x + y + z
+""")
+
+ffi.set_source("_add2_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add3.py
@@ -0,0 +1,25 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add3(int, int, int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ from _add3_cffi import ffi
+ import sys
+
+ @ffi.def_extern()
+ def add3(x, y, z, t):
+ sys.stdout.write("adding %d, %d, %d, %d\n" % (x, y, z, t))
+ sys.stdout.flush()
+ return x + y + z + t
+""")
+
+ffi.set_source("_add3_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git
a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c
@@ -0,0 +1,27 @@
+#include <stdio.h>
+
+#ifdef _MSC_VER
+# define DLLIMPORT __declspec(dllimport)
+#else
+# define DLLIMPORT extern
+#endif
+
+DLLIMPORT int add_rec(int, int);
+DLLIMPORT int (*my_callback)(int);
+
+static int some_callback(int x)
+{
+ printf("some_callback(%d)\n", x);
+ fflush(stdout);
+ return add_rec(x, 9);
+}
+
+int main(void)
+{
+ int x, y;
+ my_callback = some_callback;
+ x = add_rec(40, 2);
+ y = add_rec(100, -5);
+ printf("got: %d %d\n", x, y);
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive.py
@@ -0,0 +1,34 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int (*my_callback)(int);
+ int add_rec(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ from _add_recursive_cffi import ffi, lib
+ import sys
+ print("preparing REC")
+ sys.stdout.flush()
+
+ @ffi.def_extern()
+ def add_rec(x, y):
+ print("adding %d and %d" % (x, y))
+ sys.stdout.flush()
+ return x + y
+
+ x = lib.my_callback(400)
+ print('<<< %d >>>' % (x,))
+""")
+
+ffi.set_source("_add_recursive_cffi", """
+/* use CFFI_DLLEXPORT: on windows, it expands to __declspec(dllexport),
+ which is needed to export a variable from a dll */
+CFFI_DLLEXPORT int (*my_callback)(int);
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c
@@ -0,0 +1,86 @@
+#include <stdio.h>
+#include <assert.h>
+#include <sys/time.h>
+#ifdef PTEST_USE_THREAD
+# include <pthread.h>
+# include <semaphore.h>
+static sem_t done;
+#endif
+
+
+extern int add1(int, int);
+
+
+static double time_delta(struct timeval *stop, struct timeval *start)
+{
+ return (stop->tv_sec - start->tv_sec) +
+ 1e-6 * (stop->tv_usec - start->tv_usec);
+}
+
+static double measure(void)
+{
+ long long i, iterations;
+ int result;
+ struct timeval start, stop;
+ double elapsed;
+
+ add1(0, 0); /* prepare off-line */
+
+ i = 0;
+ iterations = 1000;
+ result = gettimeofday(&start, NULL);
+ assert(result == 0);
+
+ while (1) {
+ for (; i < iterations; i++) {
+ add1(((int)i) & 0xaaaaaa, ((int)i) & 0x555555);
+ }
+ result = gettimeofday(&stop, NULL);
+ assert(result == 0);
+
+ elapsed = time_delta(&stop, &start);
+ assert(elapsed >= 0.0);
+ if (elapsed > 2.5)
+ break;
+ iterations = iterations * 3 / 2;
+ }
+
+ return elapsed / (double)iterations;
+}
+
+static void *start_routine(void *arg)
+{
+ double t = measure();
+ printf("time per call: %.3g\n", t);
+
+#ifdef PTEST_USE_THREAD
+ int status = sem_post(&done);
+ assert(status == 0);
+#endif
+
+ return arg;
+}
+
+
+int main(void)
+{
+#ifndef PTEST_USE_THREAD
+ start_routine(0);
+#else
+ pthread_t th;
+ int i, status = sem_init(&done, 0, 0);
+ assert(status == 0);
+
+ add1(0, 0); /* this is the main thread */
+
+ for (i = 0; i < PTEST_USE_THREAD; i++) {
+ status = pthread_create(&th, NULL, start_routine, NULL);
+ assert(status == 0);
+ }
+ for (i = 0; i < PTEST_USE_THREAD; i++) {
+ status = sem_wait(&done);
+ assert(status == 0);
+ }
+#endif
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/perf.py
@@ -0,0 +1,22 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ from _perf_cffi import ffi
+
+ @ffi.def_extern()
+ def add1(x, y):
+ return x + y
+""")
+
+ffi.set_source("_perf_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
@@ -0,0 +1,151 @@
+# Generated by pypy/tool/import_cffi.py
+import py
+import sys, os, re
+import shutil, subprocess, time
+from pypy.module.test_lib_pypy.cffi_tests.udir import udir
+import cffi
+
+if hasattr(sys, 'gettotalrefcount'):
+ py.test.skip("tried hard and failed to have these tests run "
+ "in a debug-mode python")
+
+
+local_dir = os.path.dirname(os.path.abspath(__file__))
+_link_error = '?'
+
+def check_lib_python_found(tmpdir):
+ global _link_error
+ if _link_error == '?':
+ ffi = cffi.FFI()
+ kwds = {}
+ ffi._apply_embedding_fix(kwds)
+ ffi.set_source("_test_lib_python_found", "", **kwds)
+ try:
+ ffi.compile(tmpdir=tmpdir, verbose=True)
+ except cffi.VerificationError as e:
+ _link_error = e
+ else:
+ _link_error = None
+ if _link_error:
+ py.test.skip(str(_link_error))
+
+
+class EmbeddingTests:
+ _compiled_modules = {}
+
+ def setup_method(self, meth):
+ check_lib_python_found(str(udir.ensure('embedding', dir=1)))
+ self._path = udir.join('embedding', meth.__name__)
+ if sys.platform == "win32":
+ self._compiled_modules.clear() # workaround
+
+ def get_path(self):
+ return str(self._path.ensure(dir=1))
+
+ def _run(self, args, env=None):
+ print(args)
+ popen = subprocess.Popen(args, env=env, cwd=self.get_path(),
+ stdout=subprocess.PIPE,
+ universal_newlines=True)
+ output = popen.stdout.read()
+ err = popen.wait()
+ if err:
+ raise OSError("popen failed with exit code %r: %r" % (
+ err, args))
+ print(output.rstrip())
+ return output
+
+ def prepare_module(self, name):
+ if name not in self._compiled_modules:
+ path = self.get_path()
+ filename = '%s.py' % name
+ # NOTE: if you have an .egg globally installed with an older
+ # version of cffi, this will not work, because sys.path ends
+ # up with the .egg before the PYTHONPATH entries. I didn't
+ # find a solution to that: we could hack sys.path inside the
+ # script run here, but we can't hack it in the same way in
+ # execute().
+ env = os.environ.copy()
+ env['PYTHONPATH'] = os.path.dirname(os.path.dirname(local_dir))
+ output = self._run([sys.executable, os.path.join(local_dir,
filename)],
+ env=env)
+ match = re.compile(r"\bFILENAME: (.+)").search(output)
+ assert match
+ dynamic_lib_name = match.group(1)
+ if sys.platform == 'win32':
+ assert dynamic_lib_name.endswith('_cffi.dll')
+ else:
+ assert dynamic_lib_name.endswith('_cffi.so')
+ self._compiled_modules[name] = dynamic_lib_name
+ return self._compiled_modules[name]
+
+ def compile(self, name, modules, opt=False, threads=False, defines={}):
+ path = self.get_path()
+ filename = '%s.c' % name
+ shutil.copy(os.path.join(local_dir, filename), path)
+ shutil.copy(os.path.join(local_dir, 'thread-test.h'), path)
+ import distutils.ccompiler
+ curdir = os.getcwd()
+ try:
+ os.chdir(self.get_path())
+ c = distutils.ccompiler.new_compiler()
+ print('compiling %s with %r' % (name, modules))
+ extra_preargs = []
+ if sys.platform == 'win32':
+ libfiles = []
+ for m in modules:
+ m = os.path.basename(m)
+ assert m.endswith('.dll')
+ libfiles.append('Release\\%s.lib' % m[:-4])
+ modules = libfiles
+ elif threads:
+ extra_preargs.append('-pthread')
+ objects = c.compile([filename], macros=sorted(defines.items()),
debug=True)
+ c.link_executable(objects + modules, name,
extra_preargs=extra_preargs)
+ finally:
+ os.chdir(curdir)
+
+ def execute(self, name):
+ path = self.get_path()
+ env = os.environ.copy()
+ env['PYTHONPATH'] = os.path.dirname(os.path.dirname(local_dir))
+ libpath = env.get('LD_LIBRARY_PATH')
+ if libpath:
+ libpath = path + ':' + libpath
+ else:
+ libpath = path
+ env['LD_LIBRARY_PATH'] = libpath
+ print('running %r in %r' % (name, path))
+ executable_name = name
+ if sys.platform == 'win32':
+ executable_name = os.path.join(path, executable_name + '.exe')
+ popen = subprocess.Popen([executable_name], cwd=path, env=env,
+ stdout=subprocess.PIPE,
+ universal_newlines=True)
+ result = popen.stdout.read()
+ err = popen.wait()
+ if err:
+ raise OSError("%r failed with exit code %r" % (name, err))
+ return result
+
+
+class TestBasic(EmbeddingTests):
+ def test_basic(self):
+ add1_cffi = self.prepare_module('add1')
+ self.compile('add1-test', [add1_cffi])
+ output = self.execute('add1-test')
+ assert output == ("preparing...\n"
+ "adding 40 and 2\n"
+ "adding 100 and -5\n"
+ "got: 42 95\n")
+
+ def test_two_modules(self):
+ add1_cffi = self.prepare_module('add1')
+ add2_cffi = self.prepare_module('add2')
+ self.compile('add2-test', [add1_cffi, add2_cffi])
+ output = self.execute('add2-test')
+ assert output == ("preparing...\n"
+ "adding 40 and 2\n"
+ "prepADD2\n"
+ "adding 100 and -5 and -20\n"
+ "got: 42 75\n")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_performance.py
@@ -0,0 +1,53 @@
+# Generated by pypy/tool/import_cffi.py
+import sys
+from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import
EmbeddingTests
+
+if sys.platform == 'win32':
+ import py
+ py.test.skip("written with POSIX functions")
+
+
+class TestPerformance(EmbeddingTests):
+ def test_perf_single_threaded(self):
+ perf_cffi = self.prepare_module('perf')
+ self.compile('perf-test', [perf_cffi], opt=True)
+ output = self.execute('perf-test')
+ print('='*79)
+ print(output.rstrip())
+ print('='*79)
+
+ def test_perf_in_1_thread(self):
+ perf_cffi = self.prepare_module('perf')
+ self.compile('perf-test', [perf_cffi], opt=True, threads=True,
+ defines={'PTEST_USE_THREAD': '1'})
+ output = self.execute('perf-test')
+ print('='*79)
+ print(output.rstrip())
+ print('='*79)
+
+ def test_perf_in_2_threads(self):
+ perf_cffi = self.prepare_module('perf')
+ self.compile('perf-test', [perf_cffi], opt=True, threads=True,
+ defines={'PTEST_USE_THREAD': '2'})
+ output = self.execute('perf-test')
+ print('='*79)
+ print(output.rstrip())
+ print('='*79)
+
+ def test_perf_in_4_threads(self):
+ perf_cffi = self.prepare_module('perf')
+ self.compile('perf-test', [perf_cffi], opt=True, threads=True,
+ defines={'PTEST_USE_THREAD': '4'})
+ output = self.execute('perf-test')
+ print('='*79)
+ print(output.rstrip())
+ print('='*79)
+
+ def test_perf_in_8_threads(self):
+ perf_cffi = self.prepare_module('perf')
+ self.compile('perf-test', [perf_cffi], opt=True, threads=True,
+ defines={'PTEST_USE_THREAD': '8'})
+ output = self.execute('perf-test')
+ print('='*79)
+ print(output.rstrip())
+ print('='*79)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_recursive.py
@@ -0,0 +1,16 @@
+# Generated by pypy/tool/import_cffi.py
+from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import
EmbeddingTests
+
+
+class TestRecursive(EmbeddingTests):
+ def test_recursive(self):
+ add_recursive_cffi = self.prepare_module('add_recursive')
+ self.compile('add_recursive-test', [add_recursive_cffi])
+ output = self.execute('add_recursive-test')
+ assert output == ("preparing REC\n"
+ "some_callback(400)\n"
+ "adding 400 and 9\n"
+ "<<< 409 >>>\n"
+ "adding 40 and 2\n"
+ "adding 100 and -5\n"
+ "got: 42 95\n")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_thread.py
@@ -0,0 +1,62 @@
+# Generated by pypy/tool/import_cffi.py
+from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import
EmbeddingTests
+
+
+class TestThread(EmbeddingTests):
+ def test_first_calls_in_parallel(self):
+ add1_cffi = self.prepare_module('add1')
+ self.compile('thread1-test', [add1_cffi], threads=True)
+ for i in range(50):
+ output = self.execute('thread1-test')
+ assert output == ("starting\n"
+ "preparing...\n" +
+ "adding 40 and 2\n" * 10 +
+ "done\n")
+
+ def _take_out(self, text, content):
+ assert content in text
+ i = text.index(content)
+ return text[:i] + text[i+len(content):]
+
+ def test_init_different_modules_in_different_threads(self):
+ add1_cffi = self.prepare_module('add1')
+ add2_cffi = self.prepare_module('add2')
+ self.compile('thread2-test', [add1_cffi, add2_cffi], threads=True)
+ output = self.execute('thread2-test')
+ output = self._take_out(output, "preparing")
+ output = self._take_out(output, ".")
+ output = self._take_out(output, ".")
+ # at least the 3rd dot should be after everything from ADD2
+ assert output == ("starting\n"
+ "prepADD2\n"
+ "adding 1000 and 200 and 30\n"
+ ".\n"
+ "adding 40 and 2\n"
+ "done\n")
+
+ def test_alt_issue(self):
+ add1_cffi = self.prepare_module('add1')
+ add2_cffi = self.prepare_module('add2')
+ self.compile('thread2-test', [add1_cffi, add2_cffi],
+ threads=True, defines={'T2TEST_AGAIN_ADD1': '1'})
+ output = self.execute('thread2-test')
+ output = self._take_out(output, "adding 40 and 2\n")
+ assert output == ("starting\n"
+ "preparing...\n"
+ "adding -1 and -1\n"
+ "prepADD2\n"
+ "adding 1000 and 200 and 30\n"
+ "done\n")
+
+ def test_load_in_parallel_more(self):
+ add2_cffi = self.prepare_module('add2')
+ add3_cffi = self.prepare_module('add3')
+ self.compile('thread3-test', [add2_cffi, add3_cffi], threads=True)
+ for i in range(150):
+ output = self.execute('thread3-test')
+ for j in range(10):
+ output = self._take_out(output, "adding 40 and 2 and 100\n")
+ output = self._take_out(output, "adding 1000, 200, 30, 4\n")
+ assert output == ("starting\n"
+ "prepADD2\n"
+ "done\n")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_tlocal.py
@@ -0,0 +1,11 @@
+# Generated by pypy/tool/import_cffi.py
+from pypy.module.test_lib_pypy.cffi_tests.embedding.test_basic import
EmbeddingTests
+
+
+class TestThreadLocal(EmbeddingTests):
+ def test_thread_local(self):
+ tlocal_cffi = self.prepare_module('tlocal')
+ self.compile('tlocal-test', [tlocal_cffi], threads=True)
+ for i in range(10):
+ output = self.execute('tlocal-test')
+ assert output == "done\n"
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h
b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h
@@ -0,0 +1,62 @@
+/************************************************************/
+#ifndef _MSC_VER
+/************************************************************/
+
+
+#include <pthread.h>
+#include <semaphore.h>
+
+
+/************************************************************/
+#else
+/************************************************************/
+
+
+/* Very quick and dirty, just what I need for these tests.
+ Don't use directly in any real code!
+*/
+
+#include <Windows.h>
+#include <assert.h>
+
+typedef HANDLE sem_t;
+typedef HANDLE pthread_t;
+
+int sem_init(sem_t *sem, int pshared, unsigned int value)
+{
+ assert(pshared == 0);
+ assert(value == 0);
+ *sem = CreateSemaphore(NULL, 0, 999, NULL);
+ return *sem ? 0 : -1;
+}
+
+int sem_post(sem_t *sem)
+{
+ return ReleaseSemaphore(*sem, 1, NULL) ? 0 : -1;
+}
+
+int sem_wait(sem_t *sem)
+{
+ WaitForSingleObject(*sem, INFINITE);
+ return 0;
+}
+
+DWORD WINAPI myThreadProc(LPVOID lpParameter)
+{
+ void *(* start_routine)(void *) = (void *(*)(void *))lpParameter;
+ start_routine(NULL);
+ return 0;
+}
+
+int pthread_create(pthread_t *thread, void *attr,
+ void *start_routine(void *), void *arg)
+{
+ assert(arg == NULL);
+ *thread = CreateThread(NULL, 0, myThreadProc, start_routine, 0, NULL);
+ return *thread ? 0 : -1;
+}
+
+
+/************************************************************/
+#endif
+/************************************************************/
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c
@@ -0,0 +1,43 @@
+#include <stdio.h>
+#include <assert.h>
+#include "thread-test.h"
+
+#define NTHREADS 10
+
+
+extern int add1(int, int);
+
+static sem_t done;
+
+
+static void *start_routine(void *arg)
+{
+ int x, status;
+ x = add1(40, 2);
+ assert(x == 42);
+
+ status = sem_post(&done);
+ assert(status == 0);
+
+ return arg;
+}
+
+int main(void)
+{
+ pthread_t th;
+ int i, status = sem_init(&done, 0, 0);
+ assert(status == 0);
+
+ printf("starting\n");
+ fflush(stdout);
+ for (i = 0; i < NTHREADS; i++) {
+ status = pthread_create(&th, NULL, start_routine, NULL);
+ assert(status == 0);
+ }
+ for (i = 0; i < NTHREADS; i++) {
+ status = sem_wait(&done);
+ assert(status == 0);
+ }
+ printf("done\n");
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c
@@ -0,0 +1,57 @@
+#include <stdio.h>
+#include <assert.h>
+#include "thread-test.h"
+
+extern int add1(int, int);
+extern int add2(int, int, int);
+
+static sem_t done;
+
+
+static void *start_routine_1(void *arg)
+{
+ int x, status;
+ x = add1(40, 2);
+ assert(x == 42);
+
+ status = sem_post(&done);
+ assert(status == 0);
+
+ return arg;
+}
+
+static void *start_routine_2(void *arg)
+{
+ int x, status;
+#ifdef T2TEST_AGAIN_ADD1
+ add1(-1, -1);
+#endif
+ x = add2(1000, 200, 30);
+ assert(x == 1230);
+
+ status = sem_post(&done);
+ assert(status == 0);
+
+ return arg;
+}
+
+int main(void)
+{
+ pthread_t th;
+ int i, status = sem_init(&done, 0, 0);
+ assert(status == 0);
+
+ printf("starting\n");
+ fflush(stdout);
+ status = pthread_create(&th, NULL, start_routine_1, NULL);
+ assert(status == 0);
+ status = pthread_create(&th, NULL, start_routine_2, NULL);
+ assert(status == 0);
+
+ for (i = 0; i < 2; i++) {
+ status = sem_wait(&done);
+ assert(status == 0);
+ }
+ printf("done\n");
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c
@@ -0,0 +1,55 @@
+#include <stdio.h>
+#include <assert.h>
+#include "thread-test.h"
+
+extern int add2(int, int, int);
+extern int add3(int, int, int, int);
+
+static sem_t done;
+
+
+static void *start_routine_2(void *arg)
+{
+ int x, status;
+ x = add2(40, 2, 100);
+ assert(x == 142);
+
+ status = sem_post(&done);
+ assert(status == 0);
+
+ return arg;
+}
+
+static void *start_routine_3(void *arg)
+{
+ int x, status;
+ x = add3(1000, 200, 30, 4);
+ assert(x == 1234);
+
+ status = sem_post(&done);
+ assert(status == 0);
+
+ return arg;
+}
+
+int main(void)
+{
+ pthread_t th;
+ int i, status = sem_init(&done, 0, 0);
+ assert(status == 0);
+
+ printf("starting\n");
+ fflush(stdout);
+ for (i = 0; i < 10; i++) {
+ status = pthread_create(&th, NULL, start_routine_2, NULL);
+ assert(status == 0);
+ status = pthread_create(&th, NULL, start_routine_3, NULL);
+ assert(status == 0);
+ }
+ for (i = 0; i < 20; i++) {
+ status = sem_wait(&done);
+ assert(status == 0);
+ }
+ printf("done\n");
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c
b/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c
@@ -0,0 +1,47 @@
+#include <stdio.h>
+#include <assert.h>
+#include "thread-test.h"
+
+#define NTHREADS 10
+
+
+extern int add1(int, int);
+
+static sem_t done;
+
+
+static void *start_routine(void *arg)
+{
+ int i, x, expected, status;
+
+ expected = add1(40, 2);
+ assert((expected % 1000) == 42);
+
+ for (i=0; i<10; i++) {
+ x = add1(50, i);
+ assert(x == expected + 8 + i);
+ }
+
+ status = sem_post(&done);
+ assert(status == 0);
+
+ return arg;
+}
+
+int main(void)
+{
+ pthread_t th;
+ int i, status = sem_init(&done, 0, 0);
+ assert(status == 0);
+
+ for (i = 0; i < NTHREADS; i++) {
+ status = pthread_create(&th, NULL, start_routine, NULL);
+ assert(status == 0);
+ }
+ for (i = 0; i < NTHREADS; i++) {
+ status = sem_wait(&done);
+ assert(status == 0);
+ }
+ printf("done\n");
+ return 0;
+}
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py
b/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal.py
@@ -0,0 +1,34 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ from _tlocal_cffi import ffi
+ import itertools
+ try:
+ import thread
+ g_seen = itertools.count().next
+ except ImportError:
+ import _thread as thread # py3
+ g_seen = itertools.count().__next__
+ tloc = thread._local()
+
+ @ffi.def_extern()
+ def add1(x, y):
+ try:
+ num = tloc.num
+ except AttributeError:
+ num = tloc.num = g_seen() * 1000
+ return x + y + num
+""")
+
+ffi.set_source("_tlocal_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
