Author: Armin Rigo <ar...@tunes.org> Branch: Changeset: r3178:a62ab002583f Date: 2019-01-05 11:07 +0100 http://bitbucket.org/cffi/cffi/changeset/a62ab002583f/
Log: ffi.release() diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -1958,7 +1958,6 @@ Py_XDECREF(origobj); } -#ifdef Py_TPFLAGS_HAVE_FINALIZE /* CPython >= 3.4 */ static void cdatagcp_finalize(CDataObject_gcp *cd) { PyObject *destructor = cd->destructor; @@ -1967,7 +1966,6 @@ cd->origobj = NULL; gcp_finalize(destructor, origobj); } -#endif static void cdatagcp_dealloc(CDataObject_gcp *cd) { @@ -3134,6 +3132,74 @@ return NULL; } +static int explicit_release_case(PyObject *cd) +{ + CTypeDescrObject *ct = ((CDataObject *)cd)->c_type; + if (Py_TYPE(cd) == &CDataOwning_Type) { + if ((ct->ct_flags & (CT_POINTER | CT_ARRAY)) != 0) /* ffi.new() */ + return 0; + } + else if (Py_TYPE(cd) == &CDataOwningGC_Type) { + if (ct->ct_flags & CT_IS_UNSIZED_CHAR_A) /* ffi.from_buffer() */ + return 1; + } + else if (Py_TYPE(cd) == &CDataGCP_Type) { + return 2; /* ffi.gc() */ + } + PyErr_SetString(PyExc_ValueError, + "only 'cdata' object from ffi.new(), ffi.gc() or ffi.from_buffer() " + "can be used with the 'with' keyword or ffi.release()"); + return -1; +} + +static PyObject *cdata_enter(PyObject *cd, PyObject *noarg) +{ + if (explicit_release_case(cd) < 0) /* only to check the ctype */ + return NULL; + Py_INCREF(cd); + return cd; +} + +static PyObject *cdata_exit(PyObject *cd, PyObject *args) +{ + /* 'args' ignored */ + CTypeDescrObject *ct; + Py_buffer *view; + switch (explicit_release_case(cd)) + { + case 0: /* ffi.new() */ + /* no effect on CPython: raw memory is allocated with the + same malloc() as the object itself, so it can't be + released independently. If we use a custom allocator, + then it's implemented with ffi.gc(). */ + ct = ((CDataObject *)cd)->c_type; + if (ct->ct_flags & CT_IS_PTR_TO_OWNED) { + PyObject *x = ((CDataObject_own_structptr *)cd)->structobj; + if (Py_TYPE(x) == &CDataGCP_Type) { + /* this is a special case for + ffi.new_allocator()("struct-or-union") */ + cdatagcp_finalize((CDataObject_gcp *)x); + } + } + break; + + case 1: /* ffi.from_buffer() */ + view = ((CDataObject_owngc_frombuf *)cd)->bufferview; + PyBuffer_Release(view); + break; + + case 2: /* ffi.gc() or ffi.new_allocator()("not-struct-nor-union") */ + /* call the destructor immediately */ + cdatagcp_finalize((CDataObject_gcp *)cd); + break; + + default: + return NULL; + } + Py_INCREF(Py_None); + return Py_None; +} + static PyObject *cdata_iter(CDataObject *); static PyNumberMethods CData_as_number = { @@ -3185,6 +3251,8 @@ static PyMethodDef cdata_methods[] = { {"__dir__", cdata_dir, METH_NOARGS}, {"__complex__", cdata_complex, METH_NOARGS}, + {"__enter__", cdata_enter, METH_NOARGS}, + {"__exit__", cdata_exit, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -6891,6 +6959,15 @@ return (PyObject *)cd; } +static PyObject *b_release(PyObject *self, PyObject *arg) +{ + if (!CData_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "expected a 'cdata' object"); + return NULL; + } + return cdata_exit(arg, NULL); +} + /************************************************************/ static char _testfunc0(char a, char b) @@ -7216,6 +7293,7 @@ {"from_buffer", b_from_buffer, METH_VARARGS}, {"memmove", (PyCFunction)b_memmove, METH_VARARGS | METH_KEYWORDS}, {"gcp", (PyCFunction)b_gcp, METH_VARARGS | METH_KEYWORDS}, + {"release", b_release, METH_O}, #ifdef MS_WIN32 {"getwinerror", (PyCFunction)b_getwinerror, METH_VARARGS | METH_KEYWORDS}, #endif diff --git a/c/ffi_obj.c b/c/ffi_obj.c --- a/c/ffi_obj.c +++ b/c/ffi_obj.c @@ -1069,6 +1069,21 @@ return res; } +PyDoc_STRVAR(ffi_release_doc, +"Release now the resources held by a 'cdata' object from ffi.new(),\n" +"ffi.gc() or ffi.from_buffer(). The cdata object must not be used\n" +"afterwards.\n" +"\n" +"'ffi.release(cdata)' is equivalent to 'cdata.__exit__()'.\n" +"\n" +"Note that on CPython this method has no effect (so far) on objects\n" +"returned by ffi.new(), because the memory is allocated inline with the\n" +"cdata object and cannot be freed independently. It might be fixed in\n" +"future releases of cffi."); + +#define ffi_release b_release /* ffi_release() => b_release() + from _cffi_backend.c */ + #define METH_VKW (METH_VARARGS | METH_KEYWORDS) static PyMethodDef ffi_methods[] = { @@ -1094,6 +1109,7 @@ {"new_allocator",(PyCFunction)ffi_new_allocator,METH_VKW,ffi_new_allocator_doc}, {"new_handle", (PyCFunction)ffi_new_handle, METH_O, ffi_new_handle_doc}, {"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc}, + {"release", (PyCFunction)ffi_release, METH_O, ffi_release_doc}, {"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc}, {"string", (PyCFunction)ffi_string, METH_VKW, ffi_string_doc}, {"typeof", (PyCFunction)ffi_typeof, METH_O, ffi_typeof_doc}, diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -4085,3 +4085,114 @@ assert_eq(cast(t5, 7.0), cast(t3, 7)) assert_lt(cast(t5, 3.1), 3.101) assert_gt(cast(t5, 3.1), 3) + +def test_explicit_release_new(): + # release() on a ffi.new() object has no effect on CPython, but + # really releases memory on PyPy. We can't test that effect + # though, because a released cdata is not marked. + BIntP = new_pointer_type(new_primitive_type("int")) + p = newp(BIntP) + p[0] = 42 + py.test.raises(IndexError, "p[1]") + release(p) + # here, reading p[0] might give garbage or segfault... + release(p) # no effect + # + BStruct = new_struct_type("struct foo") + BStructP = new_pointer_type(BStruct) + complete_struct_or_union(BStruct, [('p', BIntP, -1)]) + pstruct = newp(BStructP) + assert pstruct.p == cast(BIntP, 0) + release(pstruct) + # here, reading pstruct.p might give garbage or segfault... + release(pstruct) # no effect + +def test_explicit_release_new_contextmgr(): + BIntP = new_pointer_type(new_primitive_type("int")) + with newp(BIntP) as p: + p[0] = 42 + assert p[0] == 42 + # here, reading p[0] might give garbage or segfault... + release(p) # no effect + +def test_explicit_release_badtype(): + BIntP = new_pointer_type(new_primitive_type("int")) + p = cast(BIntP, 12345) + py.test.raises(ValueError, release, p) + py.test.raises(ValueError, release, p) + BStruct = new_struct_type("struct foo") + BStructP = new_pointer_type(BStruct) + complete_struct_or_union(BStruct, [('p', BIntP, -1)]) + pstruct = newp(BStructP) + py.test.raises(ValueError, release, pstruct[0]) + +def test_explicit_release_badtype_contextmgr(): + BIntP = new_pointer_type(new_primitive_type("int")) + p = cast(BIntP, 12345) + py.test.raises(ValueError, "with p: pass") + py.test.raises(ValueError, "with p: pass") + +def test_explicit_release_gc(): + BIntP = new_pointer_type(new_primitive_type("int")) + seen = [] + intp1 = newp(BIntP, 12345) + p1 = cast(BIntP, intp1) + p = gcp(p1, seen.append) + assert seen == [] + release(p) + assert seen == [p1] + assert p1[0] == 12345 + assert p[0] == 12345 # true so far, but might change to raise RuntimeError + release(p) # no effect + +def test_explicit_release_gc_contextmgr(): + BIntP = new_pointer_type(new_primitive_type("int")) + seen = [] + intp1 = newp(BIntP, 12345) + p1 = cast(BIntP, intp1) + p = gcp(p1, seen.append) + with p: + assert p[0] == 12345 + assert seen == [] + assert seen == [p1] + assert p1[0] == 12345 + assert p[0] == 12345 # true so far, but might change to raise RuntimeError + release(p) # no effect + +def test_explicit_release_from_buffer(): + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p = from_buffer(BCharA, a) + assert p[2] == b"z" + release(p) + assert p[2] == b"z" # true so far, but might change to raise RuntimeError + release(p) # no effect + +def test_explicit_release_from_buffer_contextmgr(): + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + p = from_buffer(BCharA, a) + with p: + assert p[2] == b"z" + assert p[2] == b"z" # true so far, but might change to raise RuntimeError + release(p) # no effect + +def test_explicit_release_bytearray_on_cpython(): + if '__pypy__' in sys.builtin_module_names: + py.test.skip("pypy's bytearray are never locked") + a = bytearray(b"xyz") + BChar = new_primitive_type("char") + BCharP = new_pointer_type(BChar) + BCharA = new_array_type(BCharP, None) + a += b't' * 10 + p = from_buffer(BCharA, a) + py.test.raises(BufferError, "a += b'u' * 100") + release(p) + a += b'v' * 100 + release(p) # no effect + a += b'w' * 1000 + assert a == bytearray(b"xyz" + b't' * 10 + b'v' * 100 + b'w' * 1000) diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -530,6 +530,9 @@ def from_handle(self, x): return self._backend.from_handle(x) + def release(self, x): + self._backend.release(x) + def set_unicode(self, enabled_flag): """Windows: if 'enabled_flag' is True, enable the UNICODE and _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -60,6 +60,8 @@ `ffi.new_allocator()`_ for a way to allocate non-zero-initialized memory. +*New in version 1.12:* see ``ffi.release()``. + ffi.cast() ++++++++++ @@ -229,6 +231,8 @@ if you set ``require_writable`` to False explicitly, you still get a regular read-write cdata pointer. +*New in version 1.12:* see ``ffi.release()``. + ffi.memmove() +++++++++++++ @@ -383,6 +387,8 @@ which means the destructor is called as soon as *this* exact returned object is garbage-collected. +*New in version 1.12:* see ``ffi.release()``. + **ffi.gc(ptr, None, size=0)**: removes the ownership on a object returned by a regular call to ``ffi.gc``, and no destructor will be called when it @@ -399,7 +405,7 @@ some C libraries. In these cases, consider writing a wrapper class with custom ``__enter__()`` and ``__exit__()`` methods, allocating and freeing the C data at known points in time, and using it in a ``with`` -statement. +statement. In cffi 1.12, see also ``ffi.release()``. *New in version 1.11:* the ``size`` argument. If given, this should be an estimate of the size (in bytes) that ``ptr`` keeps alive. This @@ -571,6 +577,45 @@ lib.free(p) +ffi.release() ++++++++++++++ + +**ffi.release(cdata)**: release now the resources held by a cdata object from + ``ffi.new()``, ``ffi.gc()``, ``ffi.from_buffer()`` or + ``ffi.new_allocator()()``. The cdata object must not be used afterwards. + *New in version 1.12.* + +``ffi.release(cdata)`` is equivalent to ``cdata.__exit__()``, which means that +you can use the ``with`` statement to ensure that the cdata is released at the +end of a block (in version 1.12 and above):: + + with ffi.from_buffer(...) as p: + do something with p + +* on an object returned from ``ffi.gc(destructor)``, ``ffi.release()`` will + cause the ``destructor`` to be called immediately. + +* on an object returned from a custom allocator, the custom free function + is called immediately. + +* on CPython, ``ffi.from_buffer(buf)`` locks the buffer, so ``ffi.release()`` + unlocks it at a deterministic point. On PyPy, there is no locking (so far) + so this has no effect. + +* on CPython this method has no effect (so far) on objects returned by + ``ffi.new()``, because the memory is allocated inline with the cdata object + and cannot be freed independently. It might be fixed in future releases of + cffi. + +* on PyPy, ``ffi.release()`` frees the ``ffi.new()`` memory immediately. It is + useful because otherwise the memory is kept alive until the next GC occurs. + If you allocate large amounts of memory with ``ffi.new()`` and don't free + them with ``ffi.release()``, PyPy (>= 5.7) runs its GC more often to + compensate, so the total memory allocated should be kept within bounds + anyway; but calling ``ffi.release()`` explicitly should improve performance + by reducing the frequency of GC runs. + + ffi.init_once() +++++++++++++++ diff --git a/doc/source/using.rst b/doc/source/using.rst --- a/doc/source/using.rst +++ b/doc/source/using.rst @@ -502,7 +502,7 @@ * If you use a ``__del__()`` method to call the freeing function. -* If you use ``ffi.gc()``. +* If you use ``ffi.gc()`` without also using ``ffi.release()``. * This does not occur if you call the freeing function at a deterministic time, like in a regular ``try: finally:`` block. It diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -27,6 +27,10 @@ When set to True, it asks the object passed in to raise an exception if it is read-only. +* ``ffi.new()``, ``ffi.gc()`` or ``ffi.from_buffer()`` cdata objects + can now be released at known times, either by using the ``with`` + keyword or be calling the new ``ffi.release()``. + v1.11.5 ======= diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py --- a/testing/cffi0/test_ffi_backend.py +++ b/testing/cffi0/test_ffi_backend.py @@ -337,6 +337,13 @@ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", require_writable=True) + def test_release(self): + ffi = FFI() + p = ffi.new("int[]", 123) + ffi.release(p) + # here, reading p[0] might give garbage or segfault... + ffi.release(p) # no effect + def test_memmove(self): ffi = FFI() p = ffi.new("short[]", [-1234, -2345, -3456, -4567, -5678]) diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py --- a/testing/cffi1/test_new_ffi_1.py +++ b/testing/cffi1/test_new_ffi_1.py @@ -1456,6 +1456,35 @@ import gc; gc.collect(); gc.collect(); gc.collect() assert seen == [3] + def test_release(self): + p = ffi.new("int[]", 123) + ffi.release(p) + # here, reading p[0] might give garbage or segfault... + ffi.release(p) # no effect + + def test_release_new_allocator(self): + seen = [] + def myalloc(size): + seen.append(size) + return ffi.new("char[]", b"X" * size) + def myfree(raw): + seen.append(raw) + alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree) + p = alloc2("int[]", 15) + assert seen == [15 * 4] + ffi.release(p) + assert seen == [15 * 4, p] + ffi.release(p) # no effect + assert seen == [15 * 4, p] + # + del seen[:] + p = alloc2("struct ab *") + assert seen == [2 * 4] + ffi.release(p) + assert seen == [2 * 4, p] + ffi.release(p) # no effect + assert seen == [2 * 4, p] + def test_CData_CType(self): assert isinstance(ffi.cast("int", 0), ffi.CData) assert isinstance(ffi.new("int *"), ffi.CData) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit