Author: Armin Rigo <[email protected]>
Branch:
Changeset: r78468:5fcc7ff9de54
Date: 2015-07-06 18:34 +0200
http://bitbucket.org/pypy/pypy/changeset/5fcc7ff9de54/
Log: hg merge cffi-new-allocator
Support ffi.new_allocator()
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -37,7 +37,6 @@
'from_handle': 'handle.from_handle',
'_get_types': 'func._get_types',
'from_buffer': 'func.from_buffer',
- 'gcp': 'func.gcp',
'string': 'func.string',
'buffer': 'cbuffer.buffer',
diff --git a/pypy/module/_cffi_backend/allocator.py
b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,86 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+ _immutable_ = True
+
+ def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
+ self.ffi = ffi # may be None
+ self.w_alloc = w_alloc
+ self.w_free = w_free
+ self.should_clear_after_alloc = should_clear_after_alloc
+
+ def allocate(self, space, datasize, ctype, length=-1):
+ from pypy.module._cffi_backend import cdataobj, ctypeptr
+ if self.w_alloc is None:
+ if self.should_clear_after_alloc:
+ ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+ flavor='raw', zero=True)
+ else:
+ ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+ flavor='raw', zero=False)
+ return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+ else:
+ w_raw_cdata = space.call_function(self.w_alloc,
+ space.wrap(datasize))
+ if not isinstance(w_raw_cdata, cdataobj.W_CData):
+ raise oefmt(space.w_TypeError,
+ "alloc() must return a cdata object (got %T)",
+ w_raw_cdata)
+ if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+ raise oefmt(space.w_TypeError,
+ "alloc() must return a cdata pointer, not '%s'",
+ w_raw_cdata.ctype.name)
+ #
+ ptr = w_raw_cdata.unsafe_escaping_ptr()
+ if not ptr:
+ raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+ #
+ if self.should_clear_after_alloc:
+ rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+ rffi.cast(rffi.SIZE_T, datasize))
+ #
+ if self.w_free is None:
+ # use this class which does not have a __del__, but still
+ # keeps alive w_raw_cdata
+ res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype,
length)
+ else:
+ res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+ res.w_free = self.w_free
+ res.w_raw_cdata = w_raw_cdata
+ return res
+
+ @unwrap_spec(w_init=WrappedDefault(None))
+ def descr_call(self, space, w_arg, w_init):
+ ffi = self.ffi
+ assert ffi is not None
+ w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
+ return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+ 'FFIAllocator',
+ __call__ = interp2app(W_Allocator.descr_call),
+ )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+ space = ffi.space
+ if space.is_none(w_alloc):
+ w_alloc = None
+ if space.is_none(w_free):
+ w_free = None
+ if w_alloc is None and w_free is not None:
+ raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+ alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+ return space.wrap(alloc)
+
+
+default_allocator = W_Allocator(None, None, None,
should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None,
None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/cdataobj.py
b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -369,14 +369,13 @@
class W_CDataMem(W_CData):
- """This is the base class used for cdata objects that own and free
- their memory. Used directly by the results of cffi.cast('int', x)
- or other primitive explicitly-casted types. It is further subclassed
- by W_CDataNewOwning."""
+ """This is used only by the results of cffi.cast('int', x)
+ or other primitive explicitly-casted types."""
_attrs_ = []
- def __init__(self, space, size, ctype):
- cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)
+ def __init__(self, space, ctype):
+ cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw',
+ zero=False)
W_CData.__init__(self, space, cdata, ctype)
@rgc.must_be_light_finalizer
@@ -384,34 +383,63 @@
lltype.free(self._ptr, flavor='raw')
-class W_CDataNewOwning(W_CDataMem):
- """This is the class used for the cata objects created by newp()."""
- _attrs_ = []
+class W_CDataNewOwning(W_CData):
+ """This is the abstract base class used for cdata objects created
+ by newp(). They create and free their own memory according to an
+ allocator."""
+
+ # the 'length' is either >= 0 for arrays, or -1 for pointers.
+ _attrs_ = ['length']
+ _immutable_fields_ = ['length']
+
+ def __init__(self, space, cdata, ctype, length=-1):
+ W_CData.__init__(self, space, cdata, ctype)
+ self.length = length
def _repr_extra(self):
return self._repr_extra_owning()
-
-class W_CDataNewOwningLength(W_CDataNewOwning):
- """Subclass with an explicit length, for allocated instances of
- the C type 'foo[]'."""
- _attrs_ = ['length']
- _immutable_fields_ = ['length']
-
- def __init__(self, space, size, ctype, length):
- W_CDataNewOwning.__init__(self, space, size, ctype)
- self.length = length
-
def _sizeof(self):
- from pypy.module._cffi_backend import ctypearray
ctype = self.ctype
- assert isinstance(ctype, ctypearray.W_CTypeArray)
- return self.length * ctype.ctitem.size
+ if self.length >= 0:
+ from pypy.module._cffi_backend import ctypearray
+ assert isinstance(ctype, ctypearray.W_CTypeArray)
+ return self.length * ctype.ctitem.size
+ else:
+ return ctype.size
def get_array_length(self):
return self.length
+class W_CDataNewStd(W_CDataNewOwning):
+ """Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
+ _attrs_ = []
+
+ @rgc.must_be_light_finalizer
+ def __del__(self):
+ lltype.free(self._ptr, flavor='raw')
+
+
+class W_CDataNewNonStdNoFree(W_CDataNewOwning):
+ """Subclass using a non-standard allocator, no free()"""
+ _attrs_ = ['w_raw_cdata']
+
+class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree):
+ """Subclass using a non-standard allocator, with a free()"""
+ _attrs_ = ['w_free']
+
+ def __del__(self):
+ self.clear_all_weakrefs()
+ self.enqueue_for_destruction(self.space,
+ W_CDataNewNonStdFree.call_destructor,
+ 'destructor of ')
+
+ def call_destructor(self):
+ assert isinstance(self, W_CDataNewNonStdFree)
+ self.space.call_function(self.w_free, self.w_raw_cdata)
+
+
class W_CDataPtrToStructOrUnion(W_CData):
"""This subclass is used for the pointer returned by new('struct foo *').
It has a strong reference to a W_CDataNewOwning that really owns the
diff --git a/pypy/module/_cffi_backend/ctypearray.py
b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -28,7 +28,7 @@
def _alignof(self):
return self.ctitem.alignof()
- def newp(self, w_init):
+ def newp(self, w_init, allocator):
space = self.space
datasize = self.size
#
@@ -40,12 +40,10 @@
except OverflowError:
raise OperationError(space.w_OverflowError,
space.wrap("array size would overflow a ssize_t"))
- #
- cdata = cdataobj.W_CDataNewOwningLength(space, datasize,
- self, length)
+ else:
+ length = self.length
#
- else:
- cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+ cdata = allocator.allocate(space, datasize, self, length)
#
if not space.is_w(w_init, space.w_None):
with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypeobj.py
b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -55,7 +55,7 @@
def pack_list_of_items(self, cdata, w_ob):
return False
- def newp(self, w_init):
+ def newp(self, w_init, allocator):
space = self.space
raise oefmt(space.w_TypeError,
"expected a pointer or array ctype, got '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py
b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -63,7 +63,7 @@
value = self._cast_result(value)
else:
value = self._cast_generic(w_ob)
- w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+ w_cdata = cdataobj.W_CDataMem(space, self)
self.write_raw_integer_data(w_cdata, value)
return w_cdata
@@ -353,7 +353,7 @@
value = self.cast_unicode(w_ob)
else:
value = space.float_w(w_ob)
- w_cdata = cdataobj.W_CDataMem(space, self.size, self)
+ w_cdata = cdataobj.W_CDataMem(space, self)
if not isinstance(self, W_CTypePrimitiveLongDouble):
w_cdata.write_raw_float_data(value)
else:
@@ -446,7 +446,7 @@
return self.space.wrap(value)
def convert_to_object(self, cdata):
- w_cdata = cdataobj.W_CDataMem(self.space, self.size, self)
+ w_cdata = cdataobj.W_CDataMem(self.space, self)
with w_cdata as ptr:
self._copy_longdouble(cdata, ptr)
return w_cdata
diff --git a/pypy/module/_cffi_backend/ctypeptr.py
b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -185,7 +185,7 @@
self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid)
W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
- def newp(self, w_init):
+ def newp(self, w_init, allocator):
from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
space = self.space
ctitem = self.ctitem
@@ -205,14 +205,14 @@
datasize = ctitem.convert_struct_from_object(
lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
#
- cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
+ cdatastruct = allocator.allocate(space, datasize, ctitem)
ptr = cdatastruct.unsafe_escaping_ptr()
cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr,
self, cdatastruct)
else:
if self.is_char_or_unichar_ptr_or_array():
datasize *= 2 # forcefully add a null character
- cdata = cdataobj.W_CDataNewOwning(space, datasize, self)
+ cdata = allocator.allocate(space, datasize, self)
#
if not space.is_w(w_init, space.w_None):
with cdata as ptr:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py
b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -81,10 +81,9 @@
def copy_and_convert_to_object(self, source):
space = self.space
self.check_complete()
- ob = cdataobj.W_CDataNewOwning(space, self.size, self)
- with ob as target:
- misc._raw_memcopy(source, target, self.size)
- return ob
+ ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw',
zero=False)
+ misc._raw_memcopy(source, ptr, self.size)
+ return cdataobj.W_CDataNewStd(space, ptr, self)
def typeoffsetof_field(self, fieldname, following):
self.force_lazy_struct()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py
b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -11,7 +11,7 @@
from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray
from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle
from pypy.module._cffi_backend import cbuffer, func, wrapper
-from pypy.module._cffi_backend import cffi_opcode
+from pypy.module._cffi_backend import cffi_opcode, allocator
from pypy.module._cffi_backend.ctypeobj import W_CType
from pypy.module._cffi_backend.cdataobj import W_CData
@@ -44,6 +44,10 @@
class W_FFIObject(W_Root):
+ ACCEPT_STRING = ACCEPT_STRING
+ ACCEPT_CTYPE = ACCEPT_CTYPE
+ ACCEPT_CDATA = ACCEPT_CDATA
+
w_gc_wref_remove = None
@jit.dont_look_inside
@@ -411,7 +415,31 @@
pointer to the memory somewhere else, e.g. into another structure."""
#
w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE)
- return w_ctype.newp(w_init)
+ return w_ctype.newp(w_init, allocator.default_allocator)
+
+
+ @unwrap_spec(w_alloc=WrappedDefault(None),
+ w_free=WrappedDefault(None),
+ should_clear_after_alloc=int)
+ def descr_new_allocator(self, w_alloc, w_free,
+ should_clear_after_alloc=1):
+ """\
+Return a new allocator, i.e. a function that behaves like ffi.new()
+but uses the provided low-level 'alloc' and 'free' functions.
+
+'alloc' is called with the size as argument. If it returns NULL, a
+MemoryError is raised. 'free' is called with the result of 'alloc'
+as argument. Both can be either Python function or directly C
+functions. If 'free' is None, then no free function is called.
+If both 'alloc' and 'free' are None, the default is used.
+
+If 'should_clear_after_alloc' is set to False, then the memory
+returned by 'alloc' is assumed to be already cleared (or you are
+fine with garbage); otherwise CFFI will clear it.
+ """
+ #
+ return allocator.new_allocator(self, w_alloc, w_free,
+ should_clear_after_alloc)
def descr_new_handle(self, w_arg):
@@ -596,6 +624,7 @@
getctype = interp2app(W_FFIObject.descr_getctype),
integer_const = interp2app(W_FFIObject.descr_integer_const),
new = interp2app(W_FFIObject.descr_new),
+ new_allocator = interp2app(W_FFIObject.descr_new_allocator),
new_handle = interp2app(W_FFIObject.descr_new_handle),
offsetof = interp2app(W_FFIObject.descr_offsetof),
sizeof = interp2app(W_FFIObject.descr_sizeof),
diff --git a/pypy/module/_cffi_backend/func.py
b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -1,13 +1,13 @@
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._cffi_backend import ctypeobj, cdataobj
+from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator
# ____________________________________________________________
@unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None))
def newp(space, w_ctype, w_init):
- return w_ctype.newp(w_init)
+ return w_ctype.newp(w_init, allocator.default_allocator)
# ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py
b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -265,10 +265,105 @@
assert p1[0] == 123
seen.append(1)
ffi.gc(p, destructor=destructor) # instantly forgotten
- _cffi1_backend.gcp(p, destructor=destructor)
for i in range(5):
if seen:
break
import gc
gc.collect()
- assert seen == [1, 1]
+ assert seen == [1]
+
+ def test_ffi_new_allocator_1(self):
+ import _cffi_backend as _cffi1_backend
+ ffi = _cffi1_backend.FFI()
+ alloc1 = ffi.new_allocator()
+ alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
+ for retry in range(100):
+ p1 = alloc1("int[10]")
+ p2 = alloc2("int[10]")
+ combination = 0
+ for i in range(10):
+ assert p1[i] == 0
+ combination |= p2[i]
+ p1[i] = -42
+ p2[i] = -43
+ if combination != 0:
+ break
+ del p1, p2
+ import gc; gc.collect()
+ else:
+ raise AssertionError("cannot seem to get an int[10] not "
+ "completely cleared")
+
+ def test_ffi_new_allocator_2(self):
+ import _cffi_backend as _cffi1_backend
+ ffi = _cffi1_backend.FFI()
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", "X" * size)
+ def myfree(raw):
+ seen.append(raw)
+ alloc1 = ffi.new_allocator(myalloc, myfree)
+ alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
+ should_clear_after_alloc=False)
+ p1 = alloc1("int[10]")
+ p2 = alloc2("int[]", 10)
+ assert seen == [40, 40]
+ assert ffi.typeof(p1) == ffi.typeof("int[10]")
+ assert ffi.sizeof(p1) == 40
+ assert ffi.typeof(p2) == ffi.typeof("int[]")
+ assert ffi.sizeof(p2) == 40
+ assert p1[5] == 0
+ assert p2[6] == ord('X') * 0x01010101
+ raw1 = ffi.cast("char *", p1)
+ raw2 = ffi.cast("char *", p2)
+ del p1, p2
+ retries = 0
+ while len(seen) != 4:
+ retries += 1
+ assert retries <= 5
+ import gc; gc.collect()
+ assert seen == [40, 40, raw1, raw2]
+ assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
+ assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
+
+ def test_ffi_new_allocator_3(self):
+ import _cffi_backend as _cffi1_backend
+ ffi = _cffi1_backend.FFI()
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", "X" * size)
+ alloc1 = ffi.new_allocator(myalloc) # no 'free'
+ p1 = alloc1("int[10]")
+ assert seen == [40]
+ assert ffi.typeof(p1) == ffi.typeof("int[10]")
+ assert ffi.sizeof(p1) == 40
+ assert p1[5] == 0
+
+ def test_ffi_new_allocator_4(self):
+ import _cffi_backend as _cffi1_backend
+ ffi = _cffi1_backend.FFI()
+ raises(TypeError, ffi.new_allocator, free=lambda x: None)
+ #
+ def myalloc2(size):
+ raise LookupError
+ alloc2 = ffi.new_allocator(myalloc2)
+ raises(LookupError, alloc2, "int[5]")
+ #
+ def myalloc3(size):
+ return 42
+ alloc3 = ffi.new_allocator(myalloc3)
+ e = raises(TypeError, alloc3, "int[5]")
+ assert str(e.value) == "alloc() must return a cdata object (got int)"
+ #
+ def myalloc4(size):
+ return ffi.cast("int", 42)
+ alloc4 = ffi.new_allocator(myalloc4)
+ e = raises(TypeError, alloc4, "int[5]")
+ assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
+ #
+ def myalloc5(size):
+ return ffi.NULL
+ alloc5 = ffi.new_allocator(myalloc5)
+ raises(MemoryError, alloc5, "int[5]")
diff --git a/pypy/module/_cffi_backend/wrapper.py
b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -9,6 +9,7 @@
from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
+from pypy.module._cffi_backend import allocator
class W_FunctionWrapper(W_Root):
@@ -74,7 +75,8 @@
# then internally allocate the struct and pass a pointer to it as
# a first argument.
if locs[0] == 'R':
- w_result_cdata = ctype.fargs[0].newp(space.w_None)
+ w_result_cdata = ctype.fargs[0].newp(space.w_None,
+
allocator.nonzero_allocator)
args_w = [w_result_cdata] + args_w
prepare_args(space, rawfunctype, args_w, 1)
#
@@ -116,7 +118,7 @@
# the equivalent of ffi.new()
if space.is_w(w_arg, space.w_None):
continue
- w_arg = farg.newp(w_arg)
+ w_arg = farg.newp(w_arg, allocator.default_allocator)
args_w[i] = w_arg
diff --git a/rpython/rtyper/lltypesystem/rffi.py
b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1249,3 +1249,8 @@
lltype.Void,
releasegil=False
)
+c_memset = llexternal("memset",
+ [VOIDP, lltype.Signed, SIZE_T],
+ lltype.Void,
+ releasegil=False
+ )
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
