Author: Armin Rigo <[email protected]>
Branch:
Changeset: r97018:19e211d4c76b
Date: 2019-07-24 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/19e211d4c76b/
Log: Change the API of rffi.get_nonmovingbuffer(), now renamed to
rffi.get_nonmovingbuffer_ll(), because it is not safe: it needs to
return the real llstr object on which it operated, and this is the
object that needs to be passed to free_nonmovingbuffer_ll().
Otherwise, it's possible to come up with rpython code that would,
say, convert a char to a string once for get() and once for free(),
which defeats pinning.
diff --git a/pypy/module/_cffi_backend/ctypefunc.py
b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -10,6 +10,7 @@
from rpython.rlib.objectmodel import we_are_translated, instantiate
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.annlowlevel import llstr
from pypy.interpreter.error import OperationError, oefmt
from pypy.module import _cffi_backend
@@ -163,9 +164,9 @@
cif_descr = self.cif_descr # 'self' should have been promoted here
size = cif_descr.exchange_size
mustfree_max_plus_1 = 0
+ keepalives = [llstr(None)] * len(args_w) # llstrings
buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw')
try:
- keepalives = [None] * len(args_w) # None or strings
for i in range(len(args_w)):
data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
w_obj = args_w[i]
@@ -191,9 +192,10 @@
if flag == 1:
lltype.free(raw_cdata, flavor='raw')
elif flag >= 4:
- value = keepalives[i]
- assert value is not None
- rffi.free_nonmovingbuffer(value, raw_cdata, chr(flag))
+ llobj = keepalives[i]
+ assert llobj # not NULL
+ rffi.free_nonmovingbuffer_ll(raw_cdata,
+ llobj, chr(flag))
lltype.free(buffer, flavor='raw')
keepalive_until_here(args_w)
return w_res
diff --git a/pypy/module/_cffi_backend/ctypeptr.py
b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -303,16 +303,8 @@
else:
return lltype.nullptr(rffi.CCHARP.TO)
- def _prepare_pointer_call_argument(self, w_init, cdata, keepalives, i):
+ def _prepare_pointer_call_argument(self, w_init, cdata):
space = self.space
- if self.accept_str and space.isinstance_w(w_init, space.w_bytes):
- # special case to optimize strings passed to a "char *" argument
- value = space.bytes_w(w_init)
- if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
- self._must_be_string_of_zero_or_one(value)
- keepalives[i] = value
- return misc.write_string_as_charp(cdata, value)
- #
if (space.isinstance_w(w_init, space.w_list) or
space.isinstance_w(w_init, space.w_tuple)):
length = space.int_w(space.len(w_init))
@@ -358,14 +350,27 @@
return 1
def convert_argument_from_object(self, cdata, w_ob, keepalives, i):
+ # writes the pointer to cdata[0], writes the must-free flag in
+ # the byte just before cdata[0], and returns True if something
+ # must be done later to free.
from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
- result = (not isinstance(w_ob, cdataobj.W_CData) and
- self._prepare_pointer_call_argument(w_ob, cdata,
- keepalives, i))
+ if isinstance(w_ob, cdataobj.W_CData):
+ result = 0
+ else:
+ space = self.space
+ if self.accept_str and space.isinstance_w(w_ob, space.w_bytes):
+ # special case to optimize strings passed to a "char *"
argument
+ value = space.bytes_w(w_ob)
+ if isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveBool):
+ self._must_be_string_of_zero_or_one(value)
+ keepalives[i] = misc.write_string_as_charp(cdata, value)
+ return True
+ result = self._prepare_pointer_call_argument(w_ob, cdata)
+
if result == 0:
self.convert_from_object(cdata, w_ob)
set_mustfree_flag(cdata, result)
- return result
+ return result == 1 # 0 or 2 => False, nothing to do later
def getcfield(self, attr):
from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
diff --git a/pypy/module/_cffi_backend/misc.py
b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -111,11 +111,13 @@
def write_raw_longdouble_data(target, source):
rffi.cast(rffi.LONGDOUBLEP, target)[0] = source
[email protected]_look_inside # lets get_nonmovingbuffer_final_null be inlined
[email protected]_look_inside # lets get_nonmovingbuffer_ll_final_null be inlined
def write_string_as_charp(target, string):
- buf, buf_flag = rffi.get_nonmovingbuffer_final_null(string)
+ from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
+ buf, llobj, buf_flag = rffi.get_nonmovingbuffer_ll_final_null(string)
+ set_mustfree_flag(target, ord(buf_flag)) # 4, 5 or 6
rffi.cast(rffi.CCHARPP, target)[0] = buf
- return ord(buf_flag) # 4, 5 or 6
+ return llobj
# ____________________________________________________________
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -140,7 +140,7 @@
def __init__(self, ctx, protos):
self.protos = protos
- self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
+ self.buf, self.llobj, self.bufflag =
rffi.get_nonmovingbuffer_ll(protos)
NPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
# set both server and client callbacks, because the context
@@ -151,8 +151,8 @@
ctx, self.selectNPN_cb, self.buf)
def __del__(self):
- rffi.free_nonmovingbuffer(
- self.protos, self.buf, self.bufflag)
+ rffi.free_nonmovingbuffer_ll(
+ self.buf, self.llobj, self.bufflag)
@staticmethod
def advertiseNPN_cb(s, data_ptr, len_ptr, args):
@@ -186,7 +186,7 @@
def __init__(self, ctx, protos):
self.protos = protos
- self.buf, self.bufflag = rffi.get_nonmovingbuffer(protos)
+ self.buf, self.llobj, self.bufflag =
rffi.get_nonmovingbuffer_ll(protos)
ALPN_STORAGE.set(rffi.cast(lltype.Unsigned, self.buf), self)
with rffi.scoped_str2charp(protos) as protos_buf:
@@ -197,8 +197,8 @@
ctx, self.selectALPN_cb, self.buf)
def __del__(self):
- rffi.free_nonmovingbuffer(
- self.protos, self.buf, self.bufflag)
+ rffi.free_nonmovingbuffer_ll(
+ self.buf, self.llobj, self.bufflag)
@staticmethod
def selectALPN_cb(s, out_ptr, outlen_ptr, client, client_len, args):
diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py
--- a/rpython/rlib/rdtoa.py
+++ b/rpython/rlib/rdtoa.py
@@ -60,14 +60,14 @@
try:
# note: don't use the class scoped_view_charp here, it
# break some tests because this function is used by the GC
- ll_input, flag = rffi.get_nonmovingbuffer_final_null(input)
+ ll_input, llobj, flag = rffi.get_nonmovingbuffer_ll_final_null(input)
try:
result = dg_strtod(ll_input, end_ptr)
endpos = (rffi.cast(lltype.Signed, end_ptr[0]) -
rffi.cast(lltype.Signed, ll_input))
finally:
- rffi.free_nonmovingbuffer(input, ll_input, flag)
+ rffi.free_nonmovingbuffer_ll(ll_input, llobj, flag)
finally:
lltype.free(end_ptr, flavor='raw')
diff --git a/rpython/rtyper/lltypesystem/rffi.py
b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -7,7 +7,7 @@
from rpython.rtyper.lltypesystem.llmemory import itemoffsetof
from rpython.rtyper.llannotation import lltype_to_annotation
from rpython.tool.sourcetools import func_with_new_name
-from rpython.rlib.objectmodel import Symbolic, specialize
+from rpython.rlib.objectmodel import Symbolic, specialize, not_rpython
from rpython.rlib.objectmodel import keepalive_until_here, enforceargs
from rpython.rlib import rarithmetic, rgc
from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -269,10 +269,11 @@
arg = args[i]
if TARGET == CCHARP or TARGET is VOIDP:
if arg is None:
+ from rpython.rtyper.annlowlevel import llstr
arg = lltype.nullptr(CCHARP.TO) # None => (char*)NULL
- to_free = to_free + (arg, '\x04')
+ to_free = to_free + (arg, llstr(None), '\x04')
elif isinstance(arg, str):
- tup = get_nonmovingbuffer_final_null(arg)
+ tup = get_nonmovingbuffer_ll_final_null(arg)
to_free = to_free + tup
arg = tup[0]
elif isinstance(arg, unicode):
@@ -306,10 +307,10 @@
arg = args[i]
if TARGET == CCHARP or TARGET is VOIDP:
if arg is None:
- to_free = to_free[2:]
+ to_free = to_free[3:]
elif isinstance(arg, str):
- free_nonmovingbuffer(arg, to_free[0], to_free[1])
- to_free = to_free[2:]
+ free_nonmovingbuffer_ll(to_free[0], to_free[1], to_free[2])
+ to_free = to_free[3:]
elif TARGET == CWCHARP:
if arg is None:
to_free = to_free[1:]
@@ -835,84 +836,85 @@
return assert_str0(charpsize2str(cp, size))
charp2str._annenforceargs_ = [lltype.SomePtr(TYPEP)]
- # str -> char*, flag
+ # str -> (buf, llobj, flag)
# Can't inline this because of the raw address manipulation.
@jit.dont_look_inside
- def get_nonmovingbuffer(data):
+ def get_nonmovingbuffer_ll(data):
"""
Either returns a non-moving copy or performs neccessary pointer
arithmetic to return a pointer to the characters of a string if the
string is already nonmovable or could be pinned. Must be followed by a
- free_nonmovingbuffer call.
+ free_nonmovingbuffer_ll call.
- Also returns a char:
- * \4: no pinning, returned pointer is inside 'data' which is
nonmovable
- * \5: 'data' was pinned, returned pointer is inside
+ The return type is a 3-tuple containing the "char *" result,
+ a pointer to the low-level string object, and a flag as a char:
+
+ * \4: no pinning, returned pointer is inside nonmovable 'llobj'
+ * \5: 'llobj' was pinned, returned pointer is inside
* \6: pinning failed, returned pointer is raw malloced
For strings (not unicodes), the len()th character of the resulting
raw buffer is available, but not initialized. Use
- get_nonmovingbuffer_final_null() instead of get_nonmovingbuffer()
+ get_nonmovingbuffer_ll_final_null() instead of get_nonmovingbuffer_ll()
to get a regular null-terminated "char *".
"""
- lldata = llstrtype(data)
+ llobj = llstrtype(data)
count = len(data)
if rgc.must_split_gc_address_space():
flag = '\x06' # always make a copy in this case
- elif we_are_translated_to_c() and not rgc.can_move(data):
+ elif we_are_translated_to_c() and not rgc.can_move(llobj):
flag = '\x04' # no copy needed
else:
- if we_are_translated_to_c() and rgc.pin(data):
+ if we_are_translated_to_c() and rgc.pin(llobj):
flag = '\x05' # successfully pinned
else:
flag = '\x06' # must still make a copy
if flag == '\x06':
buf = lltype.malloc(TYPEP.TO, count + (TYPEP is CCHARP),
flavor='raw')
- copy_string_to_raw(lldata, buf, 0, count)
- return buf, '\x06'
+ copy_string_to_raw(llobj, buf, 0, count)
+ return buf, llobj, '\x06'
# ^^^ raw malloc used to get a nonmovable copy
#
# following code is executed after we're translated to C, if:
# - rgc.can_move(data) and rgc.pin(data) both returned true
# - rgc.can_move(data) returned false
- data_start = cast_ptr_to_adr(lldata) + \
+ data_start = cast_ptr_to_adr(llobj) + \
offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
- return cast(TYPEP, data_start), flag
+ return cast(TYPEP, data_start), llobj, flag
# ^^^ already nonmovable. Therefore it's not raw allocated nor
# pinned.
- get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
- get_nonmovingbuffer._annenforceargs_ = [strtype]
+ get_nonmovingbuffer_ll._always_inline_ = 'try' # get rid of the returned
tuple
+ get_nonmovingbuffer_ll._annenforceargs_ = [strtype]
@jit.dont_look_inside
- def get_nonmovingbuffer_final_null(data):
- tup = get_nonmovingbuffer(data)
- buf, flag = tup
+ def get_nonmovingbuffer_ll_final_null(data):
+ tup = get_nonmovingbuffer_ll(data)
+ buf = tup[0]
buf[len(data)] = lastchar
return tup
- get_nonmovingbuffer_final_null._always_inline_ = 'try'
- get_nonmovingbuffer_final_null._annenforceargs_ = [strtype]
+ get_nonmovingbuffer_ll_final_null._always_inline_ = 'try'
+ get_nonmovingbuffer_ll_final_null._annenforceargs_ = [strtype]
- # (str, char*, char) -> None
+ # args-from-tuple-returned-by-get_nonmoving_buffer() -> None
# Can't inline this because of the raw address manipulation.
@jit.dont_look_inside
- def free_nonmovingbuffer(data, buf, flag):
+ def free_nonmovingbuffer_ll(buf, llobj, flag):
"""
- Keep 'data' alive and unpin it if it was pinned (flag==\5).
+ Keep 'llobj' alive and unpin it if it was pinned (flag==\5).
Otherwise free the non-moving copy (flag==\6).
"""
if flag == '\x05':
- rgc.unpin(data)
+ rgc.unpin(llobj)
if flag == '\x06':
lltype.free(buf, flavor='raw')
# if flag == '\x04': data was already nonmovable,
# we have nothing to clean up
- keepalive_until_here(data)
- free_nonmovingbuffer._annenforceargs_ = [strtype, None, None]
+ keepalive_until_here(llobj)
# int -> (char*, str, int)
# Can't inline this because of the raw address manipulation.
@@ -1002,24 +1004,41 @@
charpsize2str._annenforceargs_ = [None, int]
return (str2charp, free_charp, charp2str,
- get_nonmovingbuffer, free_nonmovingbuffer,
- get_nonmovingbuffer_final_null,
+ get_nonmovingbuffer_ll, free_nonmovingbuffer_ll,
+ get_nonmovingbuffer_ll_final_null,
alloc_buffer, str_from_buffer, keep_buffer_alive_until_here,
charp2strn, charpsize2str, str2chararray, str2rawmem,
)
(str2charp, free_charp, charp2str,
- get_nonmovingbuffer, free_nonmovingbuffer, get_nonmovingbuffer_final_null,
+ get_nonmovingbuffer_ll, free_nonmovingbuffer_ll,
+ get_nonmovingbuffer_ll_final_null,
alloc_buffer, str_from_buffer, keep_buffer_alive_until_here,
charp2strn, charpsize2str, str2chararray, str2rawmem,
) = make_string_mappings(str)
(unicode2wcharp, free_wcharp, wcharp2unicode,
- get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, __not_usable,
+ get_nonmoving_unicodebuffer_ll, free_nonmoving_unicodebuffer_ll, __not_usable,
alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here,
wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, unicode2rawmem,
) = make_string_mappings(unicode)
+
+@not_rpython
+def _deprecated_get_nonmovingbuffer(*args):
+ raise Exception(
+"""The function rffi.get_nonmovingbuffer() has been removed because
+it was unsafe. Use rffi.get_nonmovingbuffer_ll() instead. It returns
+a 3-tuple instead of a 2-tuple, and all three arguments must be passed
+to rffi.free_nonmovingbuffer_ll() (instead of the original string and the
+two tuple items). Or else, use a high-level API like
+'with rffi.scoped_nonmovingbuffer()'.""")
+
+get_nonmovingbuffer = _deprecated_get_nonmovingbuffer
+get_nonmovingbuffer_final_null = _deprecated_get_nonmovingbuffer
+free_nonmovingbuffer = _deprecated_get_nonmovingbuffer
+
+
def wcharpsize2utf8(w, size):
""" Helper to convert WCHARP pointer to utf8 in one go.
Equivalent to wcharpsize2unicode().encode("utf8")
@@ -1315,14 +1334,13 @@
class scoped_nonmovingbuffer:
def __init__(self, data):
- self.data = data
+ self.buf, self.llobj, self.flag = get_nonmovingbuffer_ll(data)
__init__._annenforceargs_ = [None, annmodel.SomeString(can_be_None=False)]
def __enter__(self):
- self.buf, self.flag = get_nonmovingbuffer(self.data)
return self.buf
def __exit__(self, *args):
- free_nonmovingbuffer(self.data, self.buf, self.flag)
+ free_nonmovingbuffer_ll(self.buf, self.llobj, self.flag)
__init__._always_inline_ = 'try'
__enter__._always_inline_ = 'try'
__exit__._always_inline_ = 'try'
@@ -1334,13 +1352,13 @@
content of the 'char[]' array will not be modified.
"""
def __init__(self, data):
- self.data = data
+ self.buf, self.llobj, self.flag = get_nonmovingbuffer_ll_final_null(
+ data)
__init__._annenforceargs_ = [None, annmodel.SomeString(can_be_None=False)]
def __enter__(self):
- self.buf, self.flag = get_nonmovingbuffer_final_null(self.data)
return self.buf
def __exit__(self, *args):
- free_nonmovingbuffer(self.data, self.buf, self.flag)
+ free_nonmovingbuffer_ll(self.buf, self.llobj, self.flag)
__init__._always_inline_ = 'try'
__enter__._always_inline_ = 'try'
__exit__._always_inline_ = 'try'
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py
b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -535,7 +535,7 @@
def test_nonmovingbuffer(self):
d = 'some cool data that should not move'
def f():
- buf, flag = get_nonmovingbuffer(d)
+ buf, llobj, flag = get_nonmovingbuffer_ll(d)
try:
counter = 0
for i in range(len(d)):
@@ -543,7 +543,7 @@
counter += 1
return counter
finally:
- free_nonmovingbuffer(d, buf, flag)
+ free_nonmovingbuffer_ll(buf, llobj, flag)
assert f() == len(d)
fn = self.compile(f, [], gcpolicy='ref')
assert fn() == len(d)
@@ -553,13 +553,13 @@
def f():
counter = 0
for n in range(32):
- buf, flag = get_nonmovingbuffer(d)
+ buf, llobj, flag = get_nonmovingbuffer_ll(d)
try:
for i in range(len(d)):
if buf[i] == d[i]:
counter += 1
finally:
- free_nonmovingbuffer(d, buf, flag)
+ free_nonmovingbuffer_ll(buf, llobj, flag)
return counter
fn = self.compile(f, [], gcpolicy='semispace')
# The semispace gc uses raw_malloc for its internal data structs
@@ -574,13 +574,13 @@
def f():
counter = 0
for n in range(32):
- buf, flag = get_nonmovingbuffer(d)
+ buf, llobj, flag = get_nonmovingbuffer_ll(d)
try:
for i in range(len(d)):
if buf[i] == d[i]:
counter += 1
finally:
- free_nonmovingbuffer(d, buf, flag)
+ free_nonmovingbuffer_ll(buf, llobj, flag)
return counter
fn = self.compile(f, [], gcpolicy='incminimark')
# The incminimark gc uses raw_malloc for its internal data structs
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
