Author: Philip Jenvey <pjen...@underboss.org>
Branch: py3k
Changeset: r86855:83f439eeb461
Date: 2016-09-03 12:07 -0700
http://bitbucket.org/pypy/pypy/changeset/83f439eeb461/
Log: merge default
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -498,7 +498,10 @@
""" Collector for test methods. """
def collect(self):
if hasinit(self.obj):
- pytest.skip("class %s.%s with __init__ won't get collected" % (
+ # XXX used to be skip(), but silently skipping classes
+ # XXX just because they have been written long ago is
+ # XXX imho a very, very, very bad idea
+ pytest.fail("class %s.%s with __init__ won't get collected" % (
self.obj.__module__,
self.obj.__name__,
))
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.8.0
+Version: 1.8.1
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.8.0"
-__version_info__ = (1, 8, 0)
+__version__ = "1.8.1"
+__version_info__ = (1, 8, 1)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -1,4 +1,20 @@
#define _CFFI_
+
+/* We try to define Py_LIMITED_API before including Python.h.
+
+ Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and
+ Py_REF_DEBUG are not defined. This is a best-effort approximation:
+ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if
+ the same works for the other two macros. Py_DEBUG implies them,
+ but not the other way around.
+*/
+#ifndef _CFFI_USE_EMBEDDING
+# include <pyconfig.h>
+# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+# define Py_LIMITED_API
+# endif
+#endif
+
#include <Python.h>
#ifdef __cplusplus
extern "C" {
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
f = PySys_GetObject((char *)"stderr");
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.8.0"
+ "\ncompiled with cffi version: 1.8.1"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -652,7 +652,7 @@
recompile(self, module_name, source,
c_file=filename, call_c_compiler=False, **kwds)
- def compile(self, tmpdir='.', verbose=0, target=None):
+ def compile(self, tmpdir='.', verbose=0, target=None, debug=None):
"""The 'target' argument gives the final file name of the
compiled DLL. Use '*' to force distutils' choice, suitable for
regular CPython C API modules. Use a file name ending in '.*'
@@ -669,7 +669,7 @@
module_name, source, source_extension, kwds = self._assigned_source
return recompile(self, module_name, source, tmpdir=tmpdir,
target=target, source_extension=source_extension,
- compiler_verbose=verbose, **kwds)
+ compiler_verbose=verbose, debug=debug, **kwds)
def init_once(self, func, tag):
# Read _init_once_cache[tag], which is either (False, lock) if
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -997,29 +997,37 @@
assert onerror is None # XXX not implemented
return BType(source, error)
+ _weakref_cache_ref = None
+
def gcp(self, cdata, destructor):
- BType = self.typeof(cdata)
+ if self._weakref_cache_ref is None:
+ import weakref
+ class MyRef(weakref.ref):
+ def __eq__(self, other):
+ myref = self()
+ return self is other or (
+ myref is not None and myref is other())
+ def __ne__(self, other):
+ return not (self == other)
+ self._weakref_cache_ref = {}, MyRef
+ weak_cache, MyRef = self._weakref_cache_ref
if destructor is None:
- if not (hasattr(BType, '_gcp_type') and
- BType._gcp_type is BType):
+ try:
+ del weak_cache[MyRef(cdata)]
+ except KeyError:
raise TypeError("Can remove destructor only on a object "
"previously returned by ffi.gc()")
- cdata._destructor = None
return None
- try:
- gcp_type = BType._gcp_type
- except AttributeError:
- class CTypesDataGcp(BType):
- __slots__ = ['_orig', '_destructor']
- def __del__(self):
- if self._destructor is not None:
- self._destructor(self._orig)
- gcp_type = BType._gcp_type = CTypesDataGcp
- new_cdata = self.cast(gcp_type, cdata)
- new_cdata._orig = cdata
- new_cdata._destructor = destructor
+ def remove(k):
+ cdata, destructor = weak_cache.pop(k, (None, None))
+ if destructor is not None:
+ destructor(cdata)
+
+ new_cdata = self.cast(self.typeof(cdata), cdata)
+ assert new_cdata is not cdata
+ weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor)
return new_cdata
typeof = type
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -21,12 +21,12 @@
allsources.append(os.path.normpath(src))
return Extension(name=modname, sources=allsources, **kwds)
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, debug=None):
"""Compile a C extension module using distutils."""
saved_environ = os.environ.copy()
try:
- outputfilename = _build(tmpdir, ext, compiler_verbose)
+ outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
outputfilename = os.path.abspath(outputfilename)
finally:
# workaround for a distutils bugs where some env vars can
@@ -36,7 +36,7 @@
os.environ[key] = value
return outputfilename
-def _build(tmpdir, ext, compiler_verbose=0):
+def _build(tmpdir, ext, compiler_verbose=0, debug=None):
# XXX compact but horrible :-(
from distutils.core import Distribution
import distutils.errors, distutils.log
@@ -44,6 +44,9 @@
dist = Distribution({'ext_modules': [ext]})
dist.parse_config_files()
options = dist.get_option_dict('build_ext')
+ if debug is None:
+ debug = sys.flags.debug
+ options['debug'] = ('ffiplatform', debug)
options['force'] = ('ffiplatform', True)
options['build_lib'] = ('ffiplatform', tmpdir)
options['build_temp'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -275,8 +275,8 @@
def write_c_source_to_f(self, f, preamble):
self._f = f
prnt = self._prnt
- if self.ffi._embedding is None:
- prnt('#define Py_LIMITED_API')
+ if self.ffi._embedding is not None:
+ prnt('#define _CFFI_USE_EMBEDDING')
#
# first the '#include' (actually done by inlining the file's content)
lines = self._rel_readlines('_cffi_include.h')
@@ -1431,7 +1431,7 @@
def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
c_file=None, source_extension='.c', extradir=None,
- compiler_verbose=1, target=None, **kwds):
+ compiler_verbose=1, target=None, debug=None, **kwds):
if not isinstance(module_name, str):
module_name = module_name.encode('ascii')
if ffi._windows_unicode:
@@ -1467,7 +1467,8 @@
if target != '*':
_patch_for_target(patchlist, target)
os.chdir(tmpdir)
- outputfilename = ffiplatform.compile('.', ext,
compiler_verbose)
+ outputfilename = ffiplatform.compile('.', ext,
+ compiler_verbose, debug)
finally:
os.chdir(cwd)
_unpatch_meths(patchlist)
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -69,16 +69,36 @@
else:
_add_c_module(dist, ffi, module_name, source, source_extension, kwds)
+def _set_py_limited_api(Extension, kwds):
+ """
+ Add py_limited_api to kwds if setuptools >= 26 is in use.
+ Do not alter the setting if it already exists.
+ Setuptools takes care of ignoring the flag on Python 2 and PyPy.
+ """
+ if 'py_limited_api' not in kwds:
+ import setuptools
+ try:
+ setuptools_major_version =
int(setuptools.__version__.partition('.')[0])
+ if setuptools_major_version >= 26:
+ kwds['py_limited_api'] = True
+ except ValueError: # certain development versions of setuptools
+ # If we don't know the version number of setuptools, we
+ # try to set 'py_limited_api' anyway. At worst, we get a
+ # warning.
+ kwds['py_limited_api'] = True
+ return kwds
def _add_c_module(dist, ffi, module_name, source, source_extension, kwds):
from distutils.core import Extension
- from distutils.command.build_ext import build_ext
+ # We are a setuptools extension. Need this build_ext for py_limited_api.
+ from setuptools.command.build_ext import build_ext
from distutils.dir_util import mkpath
from distutils import log
from cffi import recompiler
allsources = ['$PLACEHOLDER']
allsources.extend(kwds.pop('sources', []))
+ kwds = _set_py_limited_api(Extension, kwds)
ext = Extension(name=module_name, sources=allsources, **kwds)
def make_mod(tmpdir, pre_run=None):
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi, entrypoint
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.8.0"
+VERSION = "1.8.1"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.8.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.8.1", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py
b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -187,6 +187,43 @@
""")
+ def test_oldstyle_methcall(self):
+ def main():
+ def g(): pass
+ class A:
+ def f(self):
+ return self.x + 1
+ class I(A):
+ pass
+ class J(I):
+ pass
+
+
+ class B(J):
+ def __init__(self, x):
+ self.x = x
+
+ i = 0
+ b = B(1)
+ while i < 1000:
+ g()
+ v = b.f() # ID: meth
+ i += v
+ return i
+
+ log = self.run(main, [], threshold=80)
+ loop, = log.loops_by_filename(self.filepath, is_entry_bridge=True)
+ assert loop.match_by_id('meth',
+ '''
+ guard_nonnull_class(p18, ..., descr=...)
+ p52 = getfield_gc_r(p18, descr=...) # read map
+ guard_value(p52, ConstPtr(ptr53), descr=...)
+ p54 = getfield_gc_r(p18, descr=...) # read class
+ guard_value(p54, ConstPtr(ptr55), descr=...)
+ p56 = force_token() # done
+ ''')
+
+
def test_oldstyle_newstyle_mix(self):
def main():
class A:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -1479,6 +1479,7 @@
assert p1[0] == 123
seen.append(1)
q = ffi.gc(p, destructor)
+ assert ffi.typeof(q) is ffi.typeof(p)
import gc; gc.collect()
assert seen == []
del q
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py
@@ -149,3 +149,25 @@
p = snip_setuptools_verify2.C.getpwuid(0)
assert snip_setuptools_verify2.ffi.string(p.pw_name) == b"root"
''')
+
+ def test_set_py_limited_api(self):
+ from cffi.setuptools_ext import _set_py_limited_api
+ try:
+ import setuptools
+ orig_version = setuptools.__version__
+ setuptools.__version__ = '26.0.0'
+ from setuptools import Extension
+
+ kwds = _set_py_limited_api(Extension, {})
+ assert kwds['py_limited_api'] == True
+
+ setuptools.__version__ = '25.0'
+ kwds = _set_py_limited_api(Extension, {})
+ assert not kwds
+
+ setuptools.__version__ = 'development'
+ kwds = _set_py_limited_api(Extension, {})
+ assert kwds['py_limited_api'] == True
+
+ finally:
+ setuptools.__version__ = orig_version
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1975,9 +1975,9 @@
def test_function_returns_partial_struct():
ffi = FFI()
- ffi.cdef("struct a { int a; ...; }; struct a f1(int);")
+ ffi.cdef("struct aaa { int a; ...; }; struct aaa f1(int);")
lib = verify(ffi, "test_function_returns_partial_struct", """
- struct a { int b, a, c; };
- static struct a f1(int x) { struct a s = {0}; s.a = x; return s; }
+ struct aaa { int b, a, c; };
+ static struct aaa f1(int x) { struct aaa s = {0}; s.a = x; return s; }
""")
assert lib.f1(52).a == 52
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_numbers.py
@@ -194,6 +194,29 @@
_fields_ = [('t', enum)]
assert isinstance(S().t, enum)
+ def test_no_missing_shape_to_ffi_type(self):
+ # whitebox test
+ import sys
+ if '__pypy__' not in sys.builtin_module_names:
+ skip("only for pypy's ctypes")
+ skip("re-enable after adding 'g' to _shape_to_ffi_type.typemap, "
+ "which I think needs fighting all the way up from "
+ "rpython.rlib.libffi")
+ from _ctypes.basics import _shape_to_ffi_type
+ from _rawffi import Array
+ for i in range(1, 256):
+ try:
+ Array(chr(i))
+ except ValueError:
+ pass
+ else:
+ assert chr(i) in _shape_to_ffi_type.typemap
+
+ @py.test.mark.xfail
+ def test_pointer_to_long_double(self):
+ import ctypes
+ ctypes.POINTER(ctypes.c_longdouble)
+
## def test_perf(self):
## check_perf()
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -164,8 +164,15 @@
# annotations that are passed in, and don't annotate the old
# graph -- it's already low-level operations!
for a, s_newarg in zip(block.inputargs, cells):
- s_oldarg = self.binding(a)
- assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg
+ s_oldarg = a.annotation
+ # XXX: Should use s_oldarg.contains(s_newarg) but that breaks
+ # PyPy translation
+ if annmodel.unionof(s_oldarg, s_newarg) != s_oldarg:
+ raise annmodel.AnnotatorError(
+ "Late-stage annotation is not allowed to modify the "
+ "existing annotation for variable %s: %s" %
+ (a, s_oldarg))
+
else:
assert not self.frozen
if block not in self.annotated:
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -17,7 +17,7 @@
from rpython.flowspace.model import Variable, Constant, const
from rpython.flowspace.operation import op
from rpython.rlib import rarithmetic
-from rpython.annotator.model import AnnotatorError
+from rpython.annotator.model import AnnotatorError, TLS
BINARY_OPERATIONS = set([oper.opname for oper in op.__dict__.values()
if oper.dispatch == 2])
@@ -436,6 +436,11 @@
class __extend__(pairtype(SomeFloat, SomeFloat)):
def union((flt1, flt2)):
+ if not TLS.allow_int_to_float:
+ # in this mode, if one of the two is actually the
+ # subclass SomeInteger, complain
+ if isinstance(flt1, SomeInteger) or isinstance(flt2, SomeInteger):
+ raise UnionError(flt1, flt2)
return SomeFloat()
add = sub = mul = union
diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py
--- a/rpython/annotator/model.py
+++ b/rpython/annotator/model.py
@@ -44,6 +44,7 @@
# A global attribute :-( Patch it with 'True' to enable checking of
# the no_nul attribute...
check_str_without_nul = False
+ allow_int_to_float = True
TLS = State()
class SomeObject(object):
@@ -749,6 +750,7 @@
s1 = pair(s1, s2).union()
else:
# this is just a performance shortcut
+ # XXX: This is a lie! Grep for no_side_effects_in_union and weep.
if s1 != s2:
s1 = pair(s1, s2).union()
return s1
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -47,6 +47,8 @@
cls.ulonglong = clibffi.cast_type_to_ffitype(rffi.ULONGLONG)
cls.signed = clibffi.cast_type_to_ffitype(rffi.SIGNED)
cls.wchar_t = clibffi.cast_type_to_ffitype(lltype.UniChar)
+ # XXX long double support: clibffi.ffi_type_longdouble, but then
+ # XXX fix the whole rest of this file to add a case for long double
del cls._import
@staticmethod
diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py
--- a/rpython/rlib/rdynload.py
+++ b/rpython/rlib/rdynload.py
@@ -98,8 +98,15 @@
try:
ctypes.CDLL(name)
except OSError as e:
+ # common case: ctypes fails too, with the real dlerror()
+ # message in str(e). Return that error message.
return str(e)
else:
+ # uncommon case: may happen if 'name' is a linker script
+ # (which the C-level dlopen() can't handle) and we are
+ # directly running on pypy (whose implementation of ctypes
+ # or cffi will resolve linker scripts). In that case,
+ # unsure what we can do.
return ("opening %r with ctypes.CDLL() works, "
"but not with c_dlopen()??" % (name,))
@@ -160,7 +167,18 @@
mode = _dlopen_default_mode()
elif (mode & (RTLD_LAZY | RTLD_NOW)) == 0:
mode |= RTLD_NOW
+ #
+ # haaaack for 'pypy py.test -A' if libm.so is a linker script
+ # (see reason in _dlerror_on_dlopen_untranslated())
+ must_free = False
+ if not we_are_translated() and platform.name == "linux":
+ if name and rffi.charp2str(name) == 'libm.so':
+ name = rffi.str2charp('libm.so.6')
+ must_free = True
+ #
res = c_dlopen(name, rffi.cast(rffi.INT, mode))
+ if must_free:
+ rffi.free_charp(name)
if not res:
if not we_are_translated():
err = _dlerror_on_dlopen_untranslated(name)
diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py
--- a/rpython/rlib/rmarshal.py
+++ b/rpython/rlib/rmarshal.py
@@ -346,11 +346,15 @@
# on s_bigger. It relies on the fact that s_bigger was created with
# an expression like 'annotation([s_item])' which returns a ListDef with
# no bookkeeper, on which side-effects are not allowed.
+ saved = annmodel.TLS.allow_int_to_float
try:
+ annmodel.TLS.allow_int_to_float = False
s_union = annmodel.unionof(s_bigger, s_smaller)
return s_bigger.contains(s_union)
except (annmodel.UnionError, TooLateForChange):
return False
+ finally:
+ annmodel.TLS.allow_int_to_float = saved
class __extend__(pairtype(MTag, annmodel.SomeObject)):
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -137,7 +137,27 @@
result=result)
return result.build(), pos
-@specialize.argtype(6)
+def _invalid_cont_byte(ordch):
+ return ordch>>6 != 0x2 # 0b10
+
+_invalid_byte_2_of_2 = _invalid_cont_byte
+_invalid_byte_3_of_3 = _invalid_cont_byte
+_invalid_byte_3_of_4 = _invalid_cont_byte
+_invalid_byte_4_of_4 = _invalid_cont_byte
+
+@specialize.arg(2)
+def _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
+ return (ordch2>>6 != 0x2 or # 0b10
+ (ordch1 == 0xe0 and ordch2 < 0xa0)
+ # surrogates shouldn't be valid UTF-8!
+ or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f))
+
+def _invalid_byte_2_of_4(ordch1, ordch2):
+ return (ordch2>>6 != 0x2 or # 0b10
+ (ordch1 == 0xf0 and ordch2 < 0x90) or
+ (ordch1 == 0xf4 and ordch2 > 0x8f))
+
+@specialize.arg(5)
def str_decode_utf_8_impl(s, size, errors, final, errorhandler,
allow_surrogates, result):
if size == 0:
@@ -157,22 +177,23 @@
if pos + n > size:
if not final:
break
+ # argh, this obscure block of code is mostly a copy of
+ # what follows :-(
charsleft = size - pos - 1 # either 0, 1, 2
- # note: when we get the 'unexpected end of data' we don't care
- # about the pos anymore and we just ignore the value
+ # note: when we get the 'unexpected end of data' we need
+ # to care about the pos returned; it can be lower than size,
+ # in case we need to continue running this loop
if not charsleft:
# there's only the start byte and nothing else
r, pos = errorhandler(errors, 'utf8',
'unexpected end of data',
s, pos, pos+1)
result.append(r)
- break
+ continue
ordch2 = ord(s[pos+1])
if n == 3:
# 3-bytes seq with only a continuation byte
- if (ordch2>>6 != 0x2 or # 0b10
- (ordch1 == 0xe0 and ordch2 < 0xa0)):
- # or (ordch1 == 0xed and ordch2 > 0x9f)
+ if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
# second byte invalid, take the first and continue
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
@@ -185,19 +206,17 @@
'unexpected end of data',
s, pos, pos+2)
result.append(r)
- break
+ continue
elif n == 4:
# 4-bytes seq with 1 or 2 continuation bytes
- if (ordch2>>6 != 0x2 or # 0b10
- (ordch1 == 0xf0 and ordch2 < 0x90) or
- (ordch1 == 0xf4 and ordch2 > 0x8f)):
+ if _invalid_byte_2_of_4(ordch1, ordch2):
# second byte invalid, take the first and continue
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+1)
result.append(r)
continue
- elif charsleft == 2 and ord(s[pos+2])>>6 != 0x2: # 0b10
+ elif charsleft == 2 and _invalid_byte_3_of_4(ord(s[pos+2])):
# third byte invalid, take the first two and continue
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
@@ -210,7 +229,8 @@
'unexpected end of data',
s, pos, pos+charsleft+1)
result.append(r)
- break
+ continue
+ raise AssertionError("unreachable")
if n == 0:
r, pos = errorhandler(errors, 'utf8',
@@ -223,7 +243,7 @@
elif n == 2:
ordch2 = ord(s[pos+1])
- if ordch2>>6 != 0x2: # 0b10
+ if _invalid_byte_2_of_2(ordch2):
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+1)
@@ -237,17 +257,13 @@
elif n == 3:
ordch2 = ord(s[pos+1])
ordch3 = ord(s[pos+2])
- if (ordch2>>6 != 0x2 or # 0b10
- (ordch1 == 0xe0 and ordch2 < 0xa0)
- # surrogates shouldn't be valid UTF-8!
- or (not allow_surrogates and ordch1 == 0xed and ordch2 > 0x9f)
- ):
+ if _invalid_byte_2_of_3(ordch1, ordch2, allow_surrogates):
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+1)
result.append(r)
continue
- elif ordch3>>6 != 0x2: # 0b10
+ elif _invalid_byte_3_of_3(ordch3):
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+2)
@@ -263,21 +279,19 @@
ordch2 = ord(s[pos+1])
ordch3 = ord(s[pos+2])
ordch4 = ord(s[pos+3])
- if (ordch2>>6 != 0x2 or # 0b10
- (ordch1 == 0xf0 and ordch2 < 0x90) or
- (ordch1 == 0xf4 and ordch2 > 0x8f)):
+ if _invalid_byte_2_of_4(ordch1, ordch2):
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+1)
result.append(r)
continue
- elif ordch3>>6 != 0x2: # 0b10
+ elif _invalid_byte_3_of_4(ordch3):
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+2)
result.append(r)
continue
- elif ordch4>>6 != 0x2: # 0b10
+ elif _invalid_byte_4_of_4(ordch4):
r, pos = errorhandler(errors, 'utf8',
'invalid continuation byte',
s, pos, pos+3)
diff --git a/rpython/rlib/test/test_rmarshal.py
b/rpython/rlib/test/test_rmarshal.py
--- a/rpython/rlib/test/test_rmarshal.py
+++ b/rpython/rlib/test/test_rmarshal.py
@@ -128,10 +128,12 @@
def test_llinterp_marshal():
from rpython.rtyper.test.test_llinterp import interpret
- marshaller = get_marshaller([(int, str, float)])
+ marshaller1 = get_marshaller([(int, str, float)])
+ marshaller2 = get_marshaller([(int, str, int)])
def f():
buf = []
- marshaller(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+ marshaller1(buf, [(5, "hello", -0.5), (7, "world", 1E100)])
+ marshaller2(buf, [(5, "hello", 1)])
return ''.join(buf)
res = interpret(f, [])
res = ''.join(res.chars)
@@ -139,14 +141,20 @@
assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
'i\x07\x00\x00\x00s\x05\x00\x00\x00world'
- 'f\x061e+100')
+ 'f\x061e+100'
+ '[\x01\x00\x00\x00(\x03\x00\x00\x00i\x05\x00\x00\x00'
+ 's\x05\x00\x00\x00helloi\x01\x00\x00\x00')
else:
assert res == ('[\x02\x00\x00\x00(\x03\x00\x00\x00'
'I\x05\x00\x00\x00\x00\x00\x00\x00'
's\x05\x00\x00\x00hellof\x04-0.5(\x03\x00\x00\x00'
'I\x07\x00\x00\x00\x00\x00\x00\x00'
's\x05\x00\x00\x00world'
- 'f\x061e+100')
+ 'f\x061e+100'
+ '[\x01\x00\x00\x00(\x03\x00\x00\x00'
+ 'I\x05\x00\x00\x00\x00\x00\x00\x00'
+ 's\x05\x00\x00\x00hello'
+ 'I\x01\x00\x00\x00\x00\x00\x00\x00')
def test_llinterp_unmarshal():
from rpython.rtyper.test.test_llinterp import interpret
diff --git a/rpython/rlib/test/test_runicode.py
b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -286,9 +286,15 @@
class TestUTF8Decoding(UnicodeTests):
- def __init__(self):
+ def setup_method(self, meth):
self.decoder = self.getdecoder('utf-8')
+ def custom_replace(self, errors, encoding, msg, s, startingpos, endingpos):
+ assert errors == 'custom'
+ # returns FOO, but consumes only one character (not up to endingpos)
+ FOO = u'\u1234'
+ return FOO, startingpos + 1
+
def to_bytestring(self, bytes):
return ''.join(chr(int(c, 16)) for c in bytes.split())
@@ -309,6 +315,7 @@
E.g. <80> is a continuation byte and can appear only after a start
byte.
"""
FFFD = u'\ufffd'
+ FOO = u'\u1234'
for byte in '\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
py.test.raises(UnicodeDecodeError, self.decoder, byte, 1, None,
final=True)
self.checkdecodeerror(byte, 'utf-8', 0, 1, addstuff=False,
@@ -320,6 +327,11 @@
assert self.decoder(byte, 1, 'ignore', final=True) == (u'', 1)
assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'ignore',
final=True) == (u'aaaabbbb', 9))
+ assert self.decoder(byte, 1, 'custom', final=True,
+ errorhandler=self.custom_replace) == (FOO, 1)
+ assert (self.decoder('aaaa' + byte + 'bbbb', 9, 'custom',
+ final=True, errorhandler=self.custom_replace) ==
+ (u'aaaa'+ FOO + u'bbbb', 9))
def test_unexpected_end_of_data(self):
"""
@@ -343,6 +355,7 @@
'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
]
FFFD = u'\ufffd'
+ FOO = u'\u1234'
for seq in sequences:
seq = self.to_bytestring(seq)
py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq),
@@ -358,6 +371,12 @@
) == (u'', len(seq))
assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'ignore',
final=True) == (u'aaaabbbb', len(seq) + 8))
+ assert (self.decoder(seq, len(seq), 'custom', final=True,
+ errorhandler=self.custom_replace) ==
+ (FOO * len(seq), len(seq)))
+ assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'custom',
+ final=True, errorhandler=self.custom_replace) ==
+ (u'aaaa'+ FOO * len(seq) + u'bbbb', len(seq) + 8))
def test_invalid_cb_for_2bytes_seq(self):
"""
diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py
b/rpython/rtyper/lltypesystem/test/test_rffi.py
--- a/rpython/rtyper/lltypesystem/test/test_rffi.py
+++ b/rpython/rtyper/lltypesystem/test/test_rffi.py
@@ -38,6 +38,24 @@
xf = self.compile(f, [])
assert xf() == 8+3
+ def test_no_float_to_int_conversion(self):
+ c_source = py.code.Source("""
+ int someexternalfunction(int x)
+ {
+ return (x + 3);
+ }
+ """)
+
+ eci = ExternalCompilationInfo(separate_module_sources=[c_source])
+ z = llexternal('someexternalfunction', [Signed], Signed,
+ compilation_info=eci)
+
+ def f():
+ return z(8.2)
+
+ py.test.raises(TypeError, f)
+ py.test.raises(TypeError, self.compile, f, [])
+
def test_hashdefine(self):
h_source = """
#define X(i) (i+3)
diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py
--- a/rpython/rtyper/rint.py
+++ b/rpython/rtyper/rint.py
@@ -540,7 +540,7 @@
def ll_ullong_py_mod_zer(x, y):
if y == 0:
raise ZeroDivisionError
- return llop.ullong_mod(UnsignedLongLong, x, y)
+ return ll_ullong_py_mod(x, y)
@jit.dont_look_inside
def ll_lllong_py_mod(x, y):
diff --git a/rpython/translator/backendopt/inline.py
b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -532,8 +532,7 @@
return sys.maxint
else:
res = Solution[blockmap[graph.startblock]]
- assert res >= 0
- return res
+ return max(res, 0.0)
def static_instruction_count(graph):
count = 0
diff --git a/rpython/translator/c/test/test_exception.py
b/rpython/translator/c/test/test_exception.py
--- a/rpython/translator/c/test/test_exception.py
+++ b/rpython/translator/c/test/test_exception.py
@@ -9,7 +9,7 @@
getcompiledopt = test_backendoptimized.TestTypedOptimizedTestCase().getcompiled
-class TestException(Exception):
+class InTestException(Exception):
pass
class MyException(Exception):
@@ -18,7 +18,7 @@
def test_simple1():
def raise_(i):
if i == 0:
- raise TestException()
+ raise InTestException()
elif i == 1:
raise MyException()
else:
@@ -29,7 +29,7 @@
b = raise_(i) + 12
c = raise_(i) + 13
return a+b+c
- except TestException:
+ except InTestException:
return 7
except MyException:
return 123
diff --git a/rpython/translator/goal/translate.py
b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -213,6 +213,7 @@
log.WARNING(warning)
def main():
+ sys.setrecursionlimit(2000) # PyPy can't translate within cpython's 1k
limit
targetspec_dic, translateconfig, config, args =
parse_options_and_load_target()
from rpython.translator import translator
from rpython.translator import driver
diff --git a/rpython/translator/sandbox/test/test_sandbox.py
b/rpython/translator/sandbox/test/test_sandbox.py
--- a/rpython/translator/sandbox/test/test_sandbox.py
+++ b/rpython/translator/sandbox/test/test_sandbox.py
@@ -29,6 +29,7 @@
assert msg == fnname
msg = read_message(f)
assert msg == args
+ assert [type(x) for x in msg] == [type(x) for x in args]
if isinstance(result, Exception):
write_exception(g, result)
else:
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
