Author: Armin Rigo <ar...@tunes.org>
Branch: cffi-char16-char32
Changeset: r91507:b061a2e969ec
Date: 2017-06-04 10:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b061a2e969ec/
Log: import cffi/6c465c147687
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -62,11 +62,16 @@
typedef unsigned char _Bool;
# endif
# endif
+# if _MSC_VER < 1900 || !defined(__cplusplus) /* MSVC < 2015, or plain C */
+ typedef uint16_t char16_t;
+ typedef uint32_t char32_t;
+# endif
#else
# include <stdint.h>
# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
# include <alloca.h>
# endif
+# include <uchar.h>
#endif
#ifdef __GNUC__
@@ -159,9 +164,9 @@
#define _cffi_from_c_struct \
((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
#define _cffi_to_c_wchar_t \
- ((wchar_t(*)(PyObject *))_cffi_exports[19])
+ ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
#define _cffi_from_c_wchar_t \
- ((PyObject *(*)(wchar_t))_cffi_exports[20])
+ ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
#define _cffi_to_c_long_double \
((long double(*)(PyObject *))_cffi_exports[21])
#define _cffi_to_c__Bool \
@@ -174,7 +179,11 @@
#define _CFFI_CPIDX 25
#define _cffi_call_python \
((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t \
+ ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t \
+ ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
struct _cffi_ctypedescr;
@@ -215,6 +224,46 @@
return NULL;
}
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t; /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+ if (sizeof(_cffi_wchar_t) == 2)
+ return (uint16_t)_cffi_to_c_wchar_t(o);
+ else
+ return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+ if (sizeof(_cffi_wchar_t) == 2)
+ return _cffi_from_c_wchar_t(x);
+ else
+ return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+ if (sizeof(_cffi_wchar_t) == 4)
+ return (int)_cffi_to_c_wchar_t(o);
+ else
+ return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+ if (sizeof(_cffi_wchar_t) == 4)
+ return _cffi_from_c_wchar_t(x);
+ else
+ return _cffi_from_c_wchar3216_t(x);
+}
+
+
/********** end CPython-specific section **********/
#else
_CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
PRIM_UINTMAX = 47
PRIM_FLOATCOMPLEX = 48
PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16 = 50
+PRIM_CHAR32 = 51
-
-_NUM_PRIM = 50
+_NUM_PRIM = 52
_UNKNOWN_PRIM = -1
_UNKNOWN_FLOAT_PRIM = -2
_UNKNOWN_LONG_DOUBLE = -3
@@ -135,6 +136,8 @@
'double _Complex': PRIM_DOUBLECOMPLEX,
'_Bool': PRIM_BOOL,
'wchar_t': PRIM_WCHAR,
+ 'char16_t': PRIM_CHAR16,
+ 'char32_t': PRIM_CHAR32,
'int8_t': PRIM_INT8,
'uint8_t': PRIM_UINT8,
'int16_t': PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -122,6 +122,8 @@
'_Bool': 'i',
# the following types are not primitive in the C sense
'wchar_t': 'c',
+ 'char16_t': 'c',
+ 'char32_t': 'c',
'int8_t': 'i',
'uint8_t': 'i',
'int16_t': 'i',
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
#define _CFFI_PRIM_UINTMAX 47
#define _CFFI_PRIM_FLOATCOMPLEX 48
#define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16 50
+#define _CFFI_PRIM_CHAR32 51
-#define _CFFI__NUM_PRIM 50
+#define _CFFI__NUM_PRIM 52
#define _CFFI__UNKNOWN_PRIM (-1)
#define _CFFI__UNKNOWN_FLOAT_PRIM (-2)
#define _CFFI__UNKNOWN_LONG_DOUBLE (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
from .error import VerificationError
from .cffi_opcode import *
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
class GlobalExpr:
@@ -126,6 +127,10 @@
self.ffi = ffi
self.module_name = module_name
self.target_is_python = target_is_python
+ self._version = VERSION_BASE
+
+ def needs_version(self, ver):
+ self._version = max(self._version, ver)
def collect_type_table(self):
self._typesdict = {}
@@ -304,9 +309,7 @@
prnt('#endif')
lines = self._rel_readlines('_embedding.h')
prnt(''.join(lines))
- version = VERSION_EMBEDDED
- else:
- version = VERSION
+ self.needs_version(VERSION_EMBEDDED)
#
# then paste the C source given by the user, verbatim.
prnt('/************************************************************/')
@@ -405,7 +408,7 @@
prnt(' _cffi_call_python_org = '
'(void(*)(struct _cffi_externpy_s *, char *))p[1];')
prnt(' }')
- prnt(' p[0] = (const void *)%s;' % version)
+ prnt(' p[0] = (const void *)0x%x;' % self._version)
prnt(' p[1] = &_cffi_type_context;')
prnt('}')
# on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
prnt('PyMODINIT_FUNC')
prnt('PyInit_%s(void)' % (base_module_name,))
prnt('{')
- prnt(' return _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, version))
+ prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+ self.module_name, self._version))
prnt('}')
prnt('#else')
prnt('PyMODINIT_FUNC')
prnt('init%s(void)' % (base_module_name,))
prnt('{')
- prnt(' _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, version))
+ prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+ self.module_name, self._version))
prnt('}')
prnt('#endif')
prnt()
prnt('#ifdef __GNUC__')
prnt('# pragma GCC visibility pop')
prnt('#endif')
+ self._version = None
def _to_py(self, x):
if isinstance(x, str):
@@ -476,7 +480,8 @@
prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
prnt()
prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
- prnt(" _version = %s," % (VERSION,))
+ prnt(" _version = 0x%x," % (self._version,))
+ self._version = None
#
# the '_types' keyword argument
self.cffi_types = tuple(self.cffi_types) # don't change any more
@@ -515,8 +520,11 @@
# double' here, and _cffi_to_c_double would loose precision
converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
else:
- converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+ cname = tp.get_c_name('')
+ converter = '(%s)_cffi_to_c_%s' % (cname,
tp.name.replace(' ', '_'))
+ if cname in ('char16_t', 'char32_t'):
+ self.needs_version(VERSION_CHAR16CHAR32)
errvalue = '-1'
#
elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
elif isinstance(tp, model.UnknownFloatType):
return '_cffi_from_c_double(%s)' % (var,)
elif tp.name != 'long double' and not tp.is_complex_type():
- return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+ cname = tp.name.replace(' ', '_')
+ if cname in ('char16_t', 'char32_t'):
+ self.needs_version(VERSION_CHAR16CHAR32)
+ return '_cffi_from_c_%s(%s)' % (cname, var)
else:
return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
var, self._gettypenum(tp))
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2254,6 +2254,11 @@
py.test.raises(TypeError, newp, BChar16A, [x])
x = cast(BChar16, 'A')
py.test.raises(TypeError, newp, BChar32A, [x])
+ #
+ a = newp(BChar16A, u+'\U00012345')
+ assert len(a) == 3
+ a = newp(BChar32A, u+'\U00012345')
+ assert len(a) == 2 # even if the Python unicode string above is 2 chars
def test_keepalive_struct():
# exception to the no-keepalive rule: p=newp(BStructPtr) returns a
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -2,6 +2,7 @@
import py, sys, platform
import pytest
from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests,
test_function, test_ownlib
+from pypy.module.test_lib_pypy.cffi_tests.support import u
from cffi import FFI
import _cffi_backend
@@ -398,6 +399,8 @@
"double",
"long double",
"wchar_t",
+ "char16_t",
+ "char32_t",
"_Bool",
"int8_t",
"uint8_t",
@@ -509,3 +512,43 @@
py.test.raises(TypeError, cd)
py.test.raises(TypeError, cd, ffi.NULL)
py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+ def test_explicitly_defined_char16_t(self):
+ ffi = FFI()
+ ffi.cdef("typedef uint16_t char16_t;")
+ x = ffi.cast("char16_t", 1234)
+ assert ffi.typeof(x) is ffi.typeof("uint16_t")
+
+ def test_char16_t(self):
+ ffi = FFI()
+ x = ffi.new("char16_t[]", 5)
+ assert len(x) == 5 and ffi.sizeof(x) == 10
+ x[2] = u+'\u1324'
+ assert x[2] == u+'\u1324'
+ y = ffi.new("char16_t[]", u+'\u1234\u5678')
+ assert len(y) == 3
+ assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+ assert ffi.string(y) == u+'\u1234\u5678'
+ z = ffi.new("char16_t[]", u+'\U00012345')
+ assert len(z) == 3
+ assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+ assert ffi.string(z) == u+'\U00012345'
+
+ def test_char32_t(self):
+ ffi = FFI()
+ x = ffi.new("char32_t[]", 5)
+ assert len(x) == 5 and ffi.sizeof(x) == 20
+ x[3] = u+'\U00013245'
+ assert x[3] == u+'\U00013245'
+ y = ffi.new("char32_t[]", u+'\u1234\u5678')
+ assert len(y) == 3
+ assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+ py_uni = u+'\U00012345'
+ z = ffi.new("char32_t[]", py_uni)
+ assert len(z) == 2
+ assert list(z) == [py_uni, u+'\x00'] # maybe a 2-unichars string
+ assert ffi.string(z) == py_uni
+ if len(py_uni) == 1: # 4-bytes unicodes in Python
+ s = ffi.new("char32_t[]", u+'\ud808\udf00')
+ assert len(s) == 3
+ assert list(s) == [u+'\ud808', u+'\udf00', u+'\x00']
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -3,6 +3,7 @@
import subprocess, weakref
from cffi import FFI
from cffi.backend_ctypes import CTypesBackend
+from pypy.module.test_lib_pypy.cffi_tests.support import u
SOURCE = """\
@@ -93,6 +94,15 @@
}
EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
+
+EXPORT unsigned short foo_2bytes(unsigned short a)
+{
+ return (unsigned short)(a + 42);
+}
+EXPORT unsigned int foo_4bytes(unsigned int a)
+{
+ return (unsigned int)(a + 42);
+}
"""
class TestOwnLib(object):
@@ -301,3 +311,18 @@
pfn = ffi.addressof(lib, "test_getting_errno")
assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
assert pfn == lib.test_getting_errno
+
+ def test_char16_char32_t(self):
+ if self.module is None:
+ py.test.skip("fix the auto-generation of the tiny test lib")
+ if self.Backend is CTypesBackend:
+ py.test.skip("not implemented with the ctypes backend")
+ ffi = FFI(backend=self.Backend())
+ ffi.cdef("""
+ char16_t foo_2bytes(char16_t);
+ char32_t foo_4bytes(char32_t);
+ """)
+ lib = ffi.dlopen(self.module)
+ assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+ assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+ assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -242,7 +242,7 @@
F = tp.is_float_type()
X = tp.is_complex_type()
I = tp.is_integer_type()
- assert C == (typename in ('char', 'wchar_t'))
+ assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
assert F == (typename in ('float', 'double', 'long double'))
assert X == (typename in ('float _Complex', 'double _Complex'))
assert I + F + C + X == 1 # one and only one of them is true
@@ -385,6 +385,10 @@
lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
assert lib.foo(uniexample1) == uniexample2
+def test_char16_char32_type():
+ py.test.skip("XXX test or fully prevent char16_t and char32_t from "
+ "working in ffi.verify() mode")
+
def test_no_argument():
ffi = FFI()
ffi.cdef("int foo(void);")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1673,6 +1673,8 @@
"double",
"long double",
"wchar_t",
+ "char16_t",
+ "char32_t",
"_Bool",
"int8_t",
"uint8_t",
@@ -1743,3 +1745,30 @@
exec("from _test_import_from_lib import *", d)
assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
['ffi', 'lib'])
+
+ def test_char16_t(self):
+ x = ffi.new("char16_t[]", 5)
+ assert len(x) == 5 and ffi.sizeof(x) == 10
+ x[2] = u+'\u1324'
+ assert x[2] == u+'\u1324'
+ y = ffi.new("char16_t[]", u+'\u1234\u5678')
+ assert len(y) == 3
+ assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+ assert ffi.string(y) == u+'\u1234\u5678'
+ z = ffi.new("char16_t[]", u+'\U00012345')
+ assert len(z) == 3
+ assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+ assert ffi.string(z) == u+'\U00012345'
+
+ def test_char32_t(self):
+ x = ffi.new("char32_t[]", 5)
+ assert len(x) == 5 and ffi.sizeof(x) == 20
+ x[3] = u+'\U00013245'
+ assert x[3] == u+'\U00013245'
+ y = ffi.new("char32_t[]", u+'\u1234\u5678')
+ assert len(y) == 3
+ assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+ z = ffi.new("char32_t[]", u+'\U00012345')
+ assert len(z) == 2
+ assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars strin
+ assert ffi.string(z) == u+'\U00012345'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,13 +25,14 @@
assert ''.join(map(str, recomp.cffi_types)) == expected_output
def verify(ffi, module_name, source, *args, **kwds):
+ no_cpp = kwds.pop('no_cpp', False)
kwds.setdefault('undef_macros', ['NDEBUG'])
module_name = '_CFFI_' + module_name
ffi.set_source(module_name, source)
- if not os.environ.get('NO_CPP'): # test the .cpp mode too
+ if not os.environ.get('NO_CPP') and not no_cpp: # test the .cpp mode too
kwds.setdefault('source_extension', '.cpp')
source = 'extern "C" {\n%s\n}' % (source,)
- else:
+ elif sys.platform != 'win32':
# add '-Werror' to the existing 'extra_compile_args' flags
kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
['-Werror'])
@@ -2251,3 +2252,29 @@
int f(int a) { return a + 40; }
""", extra_compile_args=['-fvisibility=hidden'])
assert lib.f(2) == 42
+
+def test_override_default_definition():
+ ffi = FFI()
+ ffi.cdef("typedef long int16_t, char16_t;")
+ lib = verify(ffi, "test_override_default_definition", "")
+ assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is
ffi.typeof("long")
+
+def test_char16_char32_type(no_cpp=False):
+ ffi = FFI()
+ ffi.cdef("""
+ char16_t foo_2bytes(char16_t);
+ char32_t foo_4bytes(char32_t);
+ """)
+ lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+ char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
+ char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
+ """, no_cpp=no_cpp)
+ assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+ assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+ assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+ py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
+ py.test.raises(TypeError, lib.foo_2bytes, 1234)
+ py.test.raises(TypeError, lib.foo_4bytes, 1234)
+
+def test_char16_char32_plain_c():
+ test_char16_char32_type(no_cpp=True)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -222,7 +222,7 @@
F = tp.is_float_type()
X = tp.is_complex_type()
I = tp.is_integer_type()
- assert C == (typename in ('char', 'wchar_t'))
+ assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
assert F == (typename in ('float', 'double', 'long double'))
assert X == (typename in ('float _Complex', 'double _Complex'))
assert I + F + C + X == 1 # one and only one of them is true
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
