Author: Armin Rigo <[email protected]>
Branch: static-callback
Changeset: r2426:b5b5fb2182e7
Date: 2015-11-23 16:11 +0100
http://bitbucket.org/cffi/cffi/changeset/b5b5fb2182e7/
Log: hg merge default
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -103,7 +103,11 @@
#endif
#if PY_MAJOR_VERSION < 3
-#define PyCapsule_New(pointer, name, destructor) \
+# undef PyCapsule_GetPointer
+# undef PyCapsule_New
+# define PyCapsule_GetPointer(capsule, name) \
+ (PyCObject_AsVoidPtr(capsule))
+# define PyCapsule_New(pointer, name, destructor) \
(PyCObject_FromVoidPtr(pointer, destructor))
#endif
@@ -1607,7 +1611,7 @@
PyObject_GC_UnTrack(cd);
if (cd->c_type->ct_flags & CT_IS_VOID_PTR) { /* a handle */
- PyObject *x = (PyObject *)(cd->c_data + 42);
+ PyObject *x = ((CDataObject_own_structptr *)cd)->structobj;
Py_DECREF(x);
}
else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) { /* a callback */
@@ -1627,7 +1631,7 @@
static int cdataowninggc_traverse(CDataObject *cd, visitproc visit, void *arg)
{
if (cd->c_type->ct_flags & CT_IS_VOID_PTR) { /* a handle */
- PyObject *x = (PyObject *)(cd->c_data + 42);
+ PyObject *x = ((CDataObject_own_structptr *)cd)->structobj;
Py_VISIT(x);
}
else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) { /* a callback */
@@ -1645,9 +1649,10 @@
static int cdataowninggc_clear(CDataObject *cd)
{
if (cd->c_type->ct_flags & CT_IS_VOID_PTR) { /* a handle */
- PyObject *x = (PyObject *)(cd->c_data + 42);
+ CDataObject_own_structptr *cd1 = (CDataObject_own_structptr *)cd;
+ PyObject *x = cd1->structobj;
Py_INCREF(Py_None);
- cd->c_data = ((char *)Py_None) - 42;
+ cd1->structobj = Py_None;
Py_DECREF(x);
}
else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) { /* a callback */
@@ -1835,7 +1840,7 @@
static PyObject *cdataowninggc_repr(CDataObject *cd)
{
if (cd->c_type->ct_flags & CT_IS_VOID_PTR) { /* a handle */
- PyObject *x = (PyObject *)(cd->c_data + 42);
+ PyObject *x = ((CDataObject_own_structptr *)cd)->structobj;
return _cdata_repr2(cd, "handle to", x);
}
else if (cd->c_type->ct_flags & CT_FUNCTIONPTR) { /* a callback */
@@ -5649,10 +5654,26 @@
return Py_None;
}
+static PyObject *newp_handle(CTypeDescrObject *ct_voidp, PyObject *x)
+{
+ CDataObject_own_structptr *cd;
+ cd = (CDataObject_own_structptr
*)PyObject_GC_New(CDataObject_own_structptr,
+ &CDataOwningGC_Type);
+ if (cd == NULL)
+ return NULL;
+ Py_INCREF(ct_voidp);
+ cd->head.c_type = ct_voidp;
+ cd->head.c_data = (char *)cd;
+ cd->head.c_weakreflist = NULL;
+ Py_INCREF(x);
+ cd->structobj = x;
+ PyObject_GC_Track(cd);
+ return (PyObject *)cd;
+}
+
static PyObject *b_newp_handle(PyObject *self, PyObject *args)
{
CTypeDescrObject *ct;
- CDataObject *cd;
PyObject *x;
if (!PyArg_ParseTuple(args, "O!O", &CTypeDescr_Type, &ct, &x))
return NULL;
@@ -5662,47 +5683,38 @@
return NULL;
}
- cd = (CDataObject *)PyObject_GC_New(CDataObject, &CDataOwningGC_Type);
- if (cd == NULL)
- return NULL;
- Py_INCREF(ct);
- cd->c_type = ct;
- Py_INCREF(x);
- cd->c_data = ((char *)x) - 42;
- cd->c_weakreflist = NULL;
- PyObject_GC_Track(cd);
- return (PyObject *)cd;
+ return newp_handle(ct, x);
}
static PyObject *b_from_handle(PyObject *self, PyObject *arg)
{
CTypeDescrObject *ct;
- char *raw;
+ CDataObject_own_structptr *orgcd;
PyObject *x;
if (!CData_Check(arg)) {
PyErr_SetString(PyExc_TypeError, "expected a 'cdata' object");
return NULL;
}
ct = ((CDataObject *)arg)->c_type;
- raw = ((CDataObject *)arg)->c_data;
if (!(ct->ct_flags & CT_CAST_ANYTHING)) {
PyErr_Format(PyExc_TypeError,
"expected a 'cdata' object with a 'void *' out of "
"new_handle(), got '%s'", ct->ct_name);
return NULL;
}
- if (!raw) {
+ orgcd = (CDataObject_own_structptr *)((CDataObject *)arg)->c_data;
+ if (!orgcd) {
PyErr_SetString(PyExc_RuntimeError,
"cannot use from_handle() on NULL pointer");
return NULL;
}
- x = (PyObject *)(raw + 42);
- if (Py_REFCNT(x) <= 0) {
+ if (Py_REFCNT(orgcd) <= 0 || Py_TYPE(orgcd) != &CDataOwningGC_Type) {
Py_FatalError("ffi.from_handle() detected that the address passed "
"points to garbage. If it is really the result of "
"ffi.new_handle(), then the Python object has already "
"been garbage collected");
}
+ x = orgcd->structobj;
Py_INCREF(x);
return x;
}
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -24,6 +24,7 @@
struct FFIObject_s {
PyObject_HEAD
PyObject *gc_wrefs, *gc_wrefs_freelist;
+ PyObject *init_once_cache;
struct _cffi_parse_info_s info;
char ctx_is_static, ctx_is_nonempty;
builder_c_t types_builder;
@@ -52,6 +53,7 @@
}
ffi->gc_wrefs = NULL;
ffi->gc_wrefs_freelist = NULL;
+ ffi->init_once_cache = NULL;
ffi->info.ctx = &ffi->types_builder.ctx;
ffi->info.output = internal_output;
ffi->info.output_size = FFI_COMPLEXITY_OUTPUT;
@@ -65,6 +67,7 @@
PyObject_GC_UnTrack(ffi);
Py_XDECREF(ffi->gc_wrefs);
Py_XDECREF(ffi->gc_wrefs_freelist);
+ Py_XDECREF(ffi->init_once_cache);
free_builder_c(&ffi->types_builder, ffi->ctx_is_static);
@@ -676,18 +679,8 @@
static PyObject *ffi_new_handle(FFIObject *self, PyObject *arg)
{
- CDataObject *cd;
-
- cd = (CDataObject *)PyObject_GC_New(CDataObject, &CDataOwningGC_Type);
- if (cd == NULL)
- return NULL;
- Py_INCREF(g_ct_voidp); // <ctype 'void *'>
- cd->c_type = g_ct_voidp;
- Py_INCREF(arg);
- cd->c_data = ((char *)arg) - 42;
- cd->c_weakreflist = NULL;
- PyObject_GC_Track(cd);
- return (PyObject *)cd;
+ /* g_ct_voidp is equal to <ctype 'void *'> */
+ return newp_handle(g_ct_voidp, arg);
}
PyDoc_STRVAR(ffi_from_handle_doc,
@@ -696,32 +689,8 @@
"cdata object returned by new_handle() is still alive (somewhere else\n"
"in the program). Failure to follow these rules will crash.");
-static PyObject *ffi_from_handle(PyObject *self, PyObject *arg)
-{
- CTypeDescrObject *ct;
- char *raw;
- PyObject *x;
- if (!CData_Check(arg)) {
- PyErr_SetString(PyExc_TypeError, "expected a 'cdata' object");
- return NULL;
- }
- ct = ((CDataObject *)arg)->c_type;
- raw = ((CDataObject *)arg)->c_data;
- if (!(ct->ct_flags & CT_CAST_ANYTHING)) {
- PyErr_Format(PyExc_TypeError,
- "expected a 'cdata' object with a 'void *' out of "
- "new_handle(), got '%s'", ct->ct_name);
- return NULL;
- }
- if (!raw) {
- PyErr_SetString(PyExc_RuntimeError,
- "cannot use from_handle() on NULL pointer");
- return NULL;
- }
- x = (PyObject *)(raw + 42);
- Py_INCREF(x);
- return x;
-}
+#define ffi_from_handle b_from_handle /* ffi_from_handle => b_from_handle
+ from _cffi_backend.c */
PyDoc_STRVAR(ffi_from_buffer_doc,
"Return a <cdata 'char[]'> that points to the data of the given Python\n"
@@ -916,6 +885,130 @@
#define ffi_memmove b_memmove /* ffi_memmove() => b_memmove()
from _cffi_backend.c */
+#ifdef WITH_THREAD
+# include "pythread.h"
+#else
+typedef void *PyThread_type_lock;
+# define PyThread_allocate_lock() ((void *)-1)
+# define PyThread_free_lock(lock) ((void)(lock))
+# define PyThread_acquire_lock(lock, _) ((void)(lock))
+# define PyThread_release_lock(lock) ((void)(lock))
+#endif
+
+PyDoc_STRVAR(ffi_init_once_doc,
+ "XXX document me");
+
+#if PY_MAJOR_VERSION < 3
+/* PyCapsule_New is redefined to be PyCObject_FromVoidPtr in _cffi_backend,
+ which gives 2.6 compatibility; but the destructor signature is different */
+static void _free_init_once_lock(void *lock)
+{
+ PyThread_free_lock((PyThread_type_lock)lock);
+}
+#else
+static void _free_init_once_lock(PyObject *capsule)
+{
+ PyThread_type_lock lock;
+ lock = PyCapsule_GetPointer(capsule, "cffi_init_once_lock");
+ if (lock != NULL)
+ PyThread_free_lock(lock);
+}
+#endif
+
+static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *keywords[] = {"func", "tag", NULL};
+ PyObject *cache, *func, *tag, *tup, *res, *x, *lockobj;
+ PyThread_type_lock lock;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO", keywords, &func, &tag))
+ return NULL;
+
+ /* a lot of fun with reference counting and error checking
+ in this function */
+
+ /* atomically get or create a new dict (no GIL release) */
+ cache = self->init_once_cache;
+ if (cache == NULL) {
+ cache = PyDict_New();
+ if (cache == NULL)
+ return NULL;
+ self->init_once_cache = cache;
+ }
+
+ /* get the tuple from cache[tag], or make a new one: (False, lock) */
+ tup = PyDict_GetItem(cache, tag);
+ if (tup == NULL) {
+ lock = PyThread_allocate_lock();
+ if (lock == NULL)
+ return NULL;
+ x = PyCapsule_New(lock, "cffi_init_once_lock", _free_init_once_lock);
+ if (x == NULL) {
+ PyThread_free_lock(lock);
+ return NULL;
+ }
+ tup = PyTuple_Pack(2, Py_False, x);
+ Py_DECREF(x);
+ if (tup == NULL)
+ return NULL;
+ x = tup;
+
+ /* Possible corner case if 'tag' is an object overriding __eq__
+ in pure Python: the GIL may be released when we are running it.
+ We really need to call dict.setdefault(). */
+ tup = PyObject_CallMethod(cache, "setdefault", "OO", tag, x);
+ Py_DECREF(x);
+ if (tup == NULL)
+ return NULL;
+
+ Py_DECREF(tup); /* there is still a ref inside the dict */
+ }
+
+ res = PyTuple_GET_ITEM(tup, 1);
+ Py_INCREF(res);
+
+ if (PyTuple_GET_ITEM(tup, 0) == Py_True) {
+ /* tup == (True, result): return the result. */
+ return res;
+ }
+
+ /* tup == (False, lock) */
+ lockobj = res;
+ lock = (PyThread_type_lock)PyCapsule_GetPointer(lockobj,
+ "cffi_init_once_lock");
+ if (lock == NULL) {
+ Py_DECREF(lockobj);
+ return NULL;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ PyThread_acquire_lock(lock, WAIT_LOCK);
+ Py_END_ALLOW_THREADS
+
+ x = PyDict_GetItem(cache, tag);
+ if (x != NULL && PyTuple_GET_ITEM(x, 0) == Py_True) {
+ /* the real result was put in the dict while we were waiting
+ for PyThread_acquire_lock() above */
+ res = PyTuple_GET_ITEM(x, 1);
+ Py_INCREF(res);
+ }
+ else {
+ res = PyObject_CallFunction(func, "");
+ if (res != NULL) {
+ tup = PyTuple_Pack(2, Py_True, res);
+ if (tup == NULL || PyDict_SetItem(cache, tag, tup) < 0) {
+ Py_XDECREF(tup);
+ Py_DECREF(res);
+ res = NULL;
+ }
+ }
+ }
+
+ PyThread_release_lock(lock);
+ Py_DECREF(lockobj);
+ return res;
+}
+
#define METH_VKW (METH_VARARGS | METH_KEYWORDS)
static PyMethodDef ffi_methods[] = {
@@ -934,6 +1027,7 @@
#ifdef MS_WIN32
{"getwinerror",(PyCFunction)ffi_getwinerror,METH_VKW,
ffi_getwinerror_doc},
#endif
+ {"init_once", (PyCFunction)ffi_init_once, METH_VKW, ffi_init_once_doc},
{"integer_const",(PyCFunction)ffi_int_const,METH_VKW, ffi_int_const_doc},
{"memmove", (PyCFunction)ffi_memmove, METH_VKW, ffi_memmove_doc},
{"new", (PyCFunction)ffi_new, METH_VKW, ffi_new_doc},
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -72,6 +72,7 @@
self._cdefsources = []
self._included_ffis = []
self._windows_unicode = None
+ self._init_once_cache = {}
if hasattr(backend, 'set_ffi'):
backend.set_ffi(self)
for name in backend.__dict__:
@@ -598,6 +599,30 @@
return recompile(self, module_name, source, tmpdir=tmpdir,
source_extension=source_extension, **kwds)
+ def init_once(self, func, tag):
+ # Read _init_once_cache[tag], which is either (False, lock) if
+ # we're calling the function now in some thread, or (True, result).
+ # Don't call setdefault() in most cases, to avoid allocating and
+ # immediately freeing a lock; but still use setdefaut() to avoid
+ # races.
+ try:
+ x = self._init_once_cache[tag]
+ except KeyError:
+ x = self._init_once_cache.setdefault(tag, (False, allocate_lock()))
+ # Common case: we got (True, result), so we return the result.
+ if x[0]:
+ return x[1]
+ # Else, it's a lock. Acquire it to serialize the following tests.
+ with x[1]:
+ # Read again from _init_once_cache the current status.
+ x = self._init_once_cache[tag]
+ if x[0]:
+ return x[1]
+ # Call the function and store the result back.
+ result = func()
+ self._init_once_cache[tag] = (True, result)
+ return result
+
def _load_backend_lib(backend, name, flags):
if name is None:
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -1046,16 +1046,18 @@
*Calling ffi.from_handle(p) is invalid and will likely crash if
the cdata object returned by new_handle() is not kept alive!*
-(In case you are wondering, this ``void *`` is not a ``PyObject *``
+(In case you are wondering, this ``void *`` is not the ``PyObject *``
pointer. This wouldn't make sense on PyPy anyway.)
The ``ffi.new_handle()/from_handle()`` functions *conceptually* work
like this:
-* ``new_handle()`` returns a cdata object that contains a reference to
- the Python object; we call them collectively the "handle" cdata
- objects. The ``void *`` value in this handle cdata object is random
- but unique.
+* ``new_handle()`` returns cdata objects that contains references to
+ the Python objects; we call them collectively the "handle" cdata
+ objects. The ``void *`` value in these handle cdata objects are
+ random but unique. *New in version 1.4:* two calls to
+ ``new_handle(x)`` are guaranteed to return cdata objects with
+ different ``void *`` values, even with the same ``x``.
* ``from_handle(p)`` searches all live "handle" cdata objects for the
one that has the same value ``p`` as its ``void *`` value. It then
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,18 @@
======================
+v1.4.0
+======
+
+* ``ffi.new_handle()`` is now guaranteed to return unique ``void *``
+ values, even if called twice on the same object. Previously, in
+ that case, CPython (but not PyPy) would return different ``cdata``
+ objects with the same ``void *`` value. This is useful to add and
+ remove handles from a global set without worrying about duplicates.
+
+* ``ffi.init_once()`` XXX
+
+
v1.3.1
======
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -1809,3 +1809,35 @@
assert lib.EE1 == 0
assert lib.EE2 == 0
assert lib.EE3 == 1
+
+ def test_init_once(self):
+ def do_init():
+ seen.append(1)
+ return 42
+ ffi = FFI()
+ seen = []
+ for i in range(3):
+ res = ffi.init_once(do_init, "tag1")
+ assert res == 42
+ assert seen == [1]
+ for i in range(3):
+ res = ffi.init_once(do_init, "tag2")
+ assert res == 42
+ assert seen == [1, 1]
+
+ def test_init_once_multithread(self):
+ import thread, time
+ def do_init():
+ seen.append('init!')
+ time.sleep(1)
+ seen.append('init done')
+ return 7
+ ffi = FFI()
+ seen = []
+ for i in range(6):
+ def f():
+ res = ffi.init_once(do_init, "tag")
+ seen.append(res)
+ thread.start_new_thread(f, ())
+ time.sleep(1.5)
+ assert seen == ['init!', 'init done'] + 6 * [7]
diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py
--- a/testing/cffi0/test_function.py
+++ b/testing/cffi0/test_function.py
@@ -464,7 +464,7 @@
ffi = FFI(backend=self.Backend())
ffi.cdef("double __stdcall sin(double x);") # stdcall ignored
m = ffi.dlopen(lib_m)
- if (sys.platform == 'win32' and sys.maxint < 2**32 and
+ if (sys.platform == 'win32' and sys.maxsize < 2**32 and
self.Backend is not CTypesBackend):
assert "double(__stdcall *)(double)" in str(ffi.typeof(m.sin))
else:
diff --git a/testing/cffi1/test_ffi_obj.py b/testing/cffi1/test_ffi_obj.py
--- a/testing/cffi1/test_ffi_obj.py
+++ b/testing/cffi1/test_ffi_obj.py
@@ -193,6 +193,11 @@
yp = ffi.new_handle([6, 4, 2])
assert ffi.from_handle(yp) == [6, 4, 2]
+def test_handle_unique():
+ ffi = _cffi1_backend.FFI()
+ assert ffi.new_handle(None) is not ffi.new_handle(None)
+ assert ffi.new_handle(None) != ffi.new_handle(None)
+
def test_ffi_cast():
ffi = _cffi1_backend.FFI()
assert ffi.cast("int(*)(int)", 0) == ffi.NULL
@@ -415,3 +420,37 @@
assert int(ffi.cast("_Bool", ffi.cast(type, 42))) == 1
assert int(ffi.cast("bool", ffi.cast(type, 42))) == 1
assert int(ffi.cast("_Bool", ffi.cast(type, 0))) == 0
+
+def test_init_once():
+ def do_init():
+ seen.append(1)
+ return 42
+ ffi = _cffi1_backend.FFI()
+ seen = []
+ for i in range(3):
+ res = ffi.init_once(do_init, "tag1")
+ assert res == 42
+ assert seen == [1]
+ for i in range(3):
+ res = ffi.init_once(do_init, "tag2")
+ assert res == 42
+ assert seen == [1, 1]
+
+def test_init_once_multithread():
+ import thread, time
+ def do_init():
+ print 'init!'
+ seen.append('init!')
+ time.sleep(1)
+ seen.append('init done')
+ print 'init done'
+ return 7
+ ffi = _cffi1_backend.FFI()
+ seen = []
+ for i in range(6):
+ def f():
+ res = ffi.init_once(do_init, "tag")
+ seen.append(res)
+ thread.start_new_thread(f, ())
+ time.sleep(1.5)
+ assert seen == ['init!', 'init done'] + 6 * [7]
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
