Author: Matti Picus <matti.pi...@gmail.com> Branch: release-pypy3.5-5.x Changeset: r92452:631824fbda33 Date: 2017-09-24 12:00 +0300 http://bitbucket.org/pypy/pypy/changeset/631824fbda33/
Log: merge py3.5 into release diff too long, truncating to 2000 out of 31743 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,6 +1,6 @@ syntax: glob *.py[co] -*.sw[po] +*.sw[pon] *~ .*.swp .idea @@ -10,6 +10,8 @@ .venv .cache +.cache/ +.gdb_history syntax: regexp ^testresult$ ^site-packages$ @@ -25,16 +27,17 @@ ^pypy/module/cpyext/test/.+\.manifest$ ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$ ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ -^pypy/module/cppyy/src/.+\.o$ -^pypy/module/cppyy/bench/.+\.so$ -^pypy/module/cppyy/bench/.+\.root$ -^pypy/module/cppyy/bench/.+\.d$ -^pypy/module/cppyy/src/.+\.errors$ -^pypy/module/cppyy/test/.+_rflx\.cpp$ -^pypy/module/cppyy/test/.+\.so$ -^pypy/module/cppyy/test/.+\.rootmap$ -^pypy/module/cppyy/test/.+\.exe$ -^pypy/module/cppyy/test/.+_cint.h$ +^pypy/module/_cppyy/src/.+\.o$ +^pypy/module/_cppyy/bench/.+\.so$ +^pypy/module/_cppyy/bench/.+\.root$ +^pypy/module/_cppyy/bench/.+\.d$ +^pypy/module/_cppyy/src/.+\.errors$ +^pypy/module/_cppyy/test/.+_rflx\.cpp$ +^pypy/module/_cppyy/test/.+\.so$ +^pypy/module/_cppyy/test/.+\.rootmap$ +^pypy/module/_cppyy/test/.+\.exe$ +^pypy/module/_cppyy/test/.+_cint.h$ +^pypy/module/_cppyy/.+/*\.pcm$ ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$ ^pypy/doc/.+\.html$ ^pypy/doc/config/.+\.rst$ @@ -90,8 +93,4 @@ .hypothesis/ ^release/ ^rpython/_cache$ -^\.cache$ -pypy/module/cppyy/.+/*\.pcm - - diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -38,3 +38,5 @@ b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1 +c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0 +a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0 diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ RUNINTERP = $(PYPY_EXECUTABLE) endif -.PHONY: cffi_imports +.PHONY: pypy-c cffi_imports pypy-c: @echo @@ -32,7 +32,7 @@ @echo "====================================================================" @echo @sleep 5 - $(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + cd pypy/goal && $(RUNINTERP) ../../rpython/bin/rpython -Ojit targetpypystandalone.py # Note: the -jN option, or MAKEFLAGS=-jN, are not usable. They are # replaced with an opaque --jobserver option by the time this Makefile @@ -40,4 +40,4 @@ # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html cffi_imports: pypy-c - PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true + PYTHONPATH=. pypy/goal/pypy-c pypy/tool/build_cffi_imports.py || /bin/true diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py --- a/extra_tests/test_decimal.py +++ b/extra_tests/test_decimal.py @@ -1,3 +1,6 @@ +import pytest +from hypothesis import example, settings, given, strategies as st + import pickle import sys @@ -8,52 +11,112 @@ # import _decimal as C # import _pydecimal as P +@pytest.yield_fixture(params=[C, P], ids=['_decimal', '_pydecimal']) +def module(request): + yield request.param -class TestPythonAPI: +# Translate symbols. +CondMap = { + C.Clamped: P.Clamped, + C.ConversionSyntax: P.ConversionSyntax, + C.DivisionByZero: P.DivisionByZero, + C.DivisionImpossible: P.InvalidOperation, + C.DivisionUndefined: P.DivisionUndefined, + C.Inexact: P.Inexact, + C.InvalidContext: P.InvalidContext, + C.InvalidOperation: P.InvalidOperation, + C.Overflow: P.Overflow, + C.Rounded: P.Rounded, + C.Subnormal: P.Subnormal, + C.Underflow: P.Underflow, + C.FloatOperation: P.FloatOperation, +} - def check_equal(self, val, proto): - d = C.Decimal(val) - p = pickle.dumps(d, proto) - assert d == pickle.loads(p) +def check_same_flags(flags_C, flags_P): + for signal in flags_C: + assert flags_C[signal] == flags_P[CondMap[signal]] - def test_C(self): + +def test_C(): + sys.modules["decimal"] = C + import decimal + d = decimal.Decimal('1') + assert isinstance(d, C.Decimal) + assert isinstance(d, decimal.Decimal) + assert isinstance(d.as_tuple(), C.DecimalTuple) + + assert d == C.Decimal('1') + +def check_round_trip(val, proto): + d = C.Decimal(val) + p = pickle.dumps(d, proto) + assert d == pickle.loads(p) + +def test_pickle(): + v = '-3.123e81723' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): sys.modules["decimal"] = C - import decimal - d = decimal.Decimal('1') - assert isinstance(d, C.Decimal) - assert isinstance(d, decimal.Decimal) - assert isinstance(d.as_tuple(), C.DecimalTuple) + check_round_trip('-3.141590000', proto) + check_round_trip(v, proto) - assert d == C.Decimal('1') + cd = C.Decimal(v) + pd = P.Decimal(v) + cdt = cd.as_tuple() + pdt = pd.as_tuple() + assert cdt.__module__ == pdt.__module__ - def test_pickle(self): - v = '-3.123e81723' - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - sys.modules["decimal"] = C - self.check_equal('-3.141590000', proto) - self.check_equal(v, proto) + p = pickle.dumps(cdt, proto) + r = pickle.loads(p) + assert isinstance(r, C.DecimalTuple) + assert cdt == r - cd = C.Decimal(v) - pd = P.Decimal(v) - cdt = cd.as_tuple() - pdt = pd.as_tuple() - assert cdt.__module__ == pdt.__module__ + sys.modules["decimal"] = C + p = pickle.dumps(cd, proto) + sys.modules["decimal"] = P + r = pickle.loads(p) + assert isinstance(r, P.Decimal) + assert r == pd - p = pickle.dumps(cdt, proto) - r = pickle.loads(p) - assert isinstance(r, C.DecimalTuple) - assert cdt == r + sys.modules["decimal"] = C + p = pickle.dumps(cdt, proto) + sys.modules["decimal"] = P + r = pickle.loads(p) + assert isinstance(r, P.DecimalTuple) + assert r == pdt - sys.modules["decimal"] = C - p = pickle.dumps(cd, proto) - sys.modules["decimal"] = P - r = pickle.loads(p) - assert isinstance(r, P.Decimal) - assert r == pd +def test_compare_total(module): + assert module.Decimal('12').compare_total(module.Decimal('12.0')) == 1 + assert module.Decimal('4367').compare_total(module.Decimal('NaN')) == -1 - sys.modules["decimal"] = C - p = pickle.dumps(cdt, proto) - sys.modules["decimal"] = P - r = pickle.loads(p) - assert isinstance(r, P.DecimalTuple) - assert r == pdt +def test_compare_total_mag(module): + assert module.Decimal(1).compare_total_mag(-2) == -1 + +def convert_arg(module, arg): + if isinstance(arg, module.Decimal): + return arg + elif type(arg).__name__ == 'Decimal': + return module.Decimal(str(arg)) + else: + return arg + +from fractions import Fraction +from decimal import Decimal + +@given(st.decimals(), st.decimals() | st.fractions()) +def test_lt(d1, d2): + with C.localcontext(C.ExtendedContext) as ctx_C: + d1_C = convert_arg(C, d1) + d2_C = convert_arg(C, d2) + try: + res_C = d1_C < d2_C + except Exception as e: + res_C = str(type(e)) + with P.localcontext(P.ExtendedContext) as ctx_P: + d1_P = convert_arg(P, d1) + d2_P = convert_arg(P, d2) + try: + res_P = d1_P < d2_P + except Exception as e: + res_P = str(type(e)) + assert res_C == res_P + check_same_flags(ctx_C.flags, ctx_P.flags) diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -361,17 +361,20 @@ if handle is None: if flags & _FUNCFLAG_CDECL: - self._handle = _ffi.CDLL(name, mode) + pypy_dll = _ffi.CDLL(name, mode) else: - self._handle = _ffi.WinDLL(name, mode) - else: - self._handle = handle + pypy_dll = _ffi.WinDLL(name, mode) + self.__pypy_dll__ = pypy_dll + handle = int(pypy_dll) + if _sys.maxint > 2 ** 32: + handle = int(handle) # long -> int + self._handle = handle def __repr__(self): - return "<%s '%s', handle %r at 0x%x>" % ( - self.__class__.__name__, self._name, self._handle, - id(self) & (_sys.maxint * 2 + 1)) - + return "<%s '%s', handle %x at %x>" % \ + (self.__class__.__name__, self._name, + (self._handle & (_sys.maxint*2 + 1)), + id(self) & (_sys.maxint*2 + 1)) def __getattr__(self, name): if name.startswith('__') and name.endswith('__'): diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py --- a/lib-python/2.7/ctypes/test/test_byteswap.py +++ b/lib-python/2.7/ctypes/test/test_byteswap.py @@ -23,7 +23,6 @@ setattr(bits, "i%s" % i, 1) dump(bits) - @xfail def test_endian_short(self): if sys.byteorder == "little": self.assertIs(c_short.__ctype_le__, c_short) @@ -51,7 +50,6 @@ self.assertEqual(bin(s), "3412") self.assertEqual(s.value, 0x1234) - @xfail def test_endian_int(self): if sys.byteorder == "little": self.assertIs(c_int.__ctype_le__, c_int) @@ -80,7 +78,6 @@ self.assertEqual(bin(s), "78563412") self.assertEqual(s.value, 0x12345678) - @xfail def test_endian_longlong(self): if sys.byteorder == "little": self.assertIs(c_longlong.__ctype_le__, c_longlong) @@ -109,7 +106,6 @@ self.assertEqual(bin(s), "EFCDAB9078563412") self.assertEqual(s.value, 0x1234567890ABCDEF) - @xfail def test_endian_float(self): if sys.byteorder == "little": self.assertIs(c_float.__ctype_le__, c_float) @@ -128,7 +124,6 @@ self.assertAlmostEqual(s.value, math.pi, 6) self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s)) - @xfail def test_endian_double(self): if sys.byteorder == "little": self.assertIs(c_double.__ctype_le__, c_double) @@ -156,7 +151,6 @@ self.assertIs(c_char.__ctype_le__, c_char) self.assertIs(c_char.__ctype_be__, c_char) - @xfail def test_struct_fields_1(self): if sys.byteorder == "little": base = BigEndianStructure @@ -192,7 +186,6 @@ pass self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)]) - @xfail def test_struct_struct(self): # nested structures with different byteorders @@ -221,7 +214,6 @@ self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) - @xfail def test_struct_fields_2(self): # standard packing in struct uses no alignment. # So, we have to align using pad bytes. @@ -245,7 +237,6 @@ s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) self.assertEqual(bin(s1), bin(s2)) - @xfail def test_unaligned_nonnative_struct_fields(self): if sys.byteorder == "little": base = BigEndianStructure diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py --- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py +++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py @@ -37,10 +37,7 @@ for typ in byteswapped_structures: ## print >> sys.stderr, typ.value self.assertEqual(typ.value.offset, 1) - try: - o = typ() - except NotImplementedError as e: - self.skipTest(str(e)) # for PyPy + o = typ() o.value = 4 self.assertEqual(o.value, 4) diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -218,6 +218,10 @@ compiler.shared_lib_extension = so_ext +def get_config_h_filename(): + """Returns the path of pyconfig.h.""" + inc_dir = get_python_inc(plat_specific=1) + return os.path.join(inc_dir, 'pyconfig.h') from sysconfig_cpython import ( parse_makefile, _variable_rx, expand_makefile_vars) diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -226,7 +226,19 @@ return "-L" + dir def _is_gcc(self, compiler_name): - return "gcc" in compiler_name or "g++" in compiler_name + # XXX PyPy workaround, look at the big comment below for more + # context. On CPython, the hack below works fine because + # `compiler_name` contains the name of the actual compiler which was + # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine). + # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end + # result is that we pass the wrong option to the compiler. + # + # The workaround is to *always* pretend to be GCC if we are on Linux: + # this should cover the vast majority of real systems, including the + # ones which use clang (which understands the '-Wl,-rpath' syntax as + # well) + return (sys.platform == "linux2" or + "gcc" in compiler_name or "g++" in compiler_name) def runtime_library_dir_option(self, dir): # XXX Hackish, at the very least. See Python bug #445902: diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py --- a/lib-python/2.7/multiprocessing/heap.py +++ b/lib-python/2.7/multiprocessing/heap.py @@ -62,7 +62,7 @@ self.size = size self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next()) self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - assert win32.GetLastError() == 0, 'tagname already in use' + #assert win32.GetLastError() == 0, 'tagname already in use' self._state = (self.size, self.name) def __getstate__(self): @@ -72,7 +72,7 @@ def __setstate__(self, state): self.size, self.name = self._state = state self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS + #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS else: diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py --- a/lib-python/2.7/string.py +++ b/lib-python/2.7/string.py @@ -75,7 +75,7 @@ for i in range(256): buf[i] = i for i in range(n): - buf[ord(fromstr[i])] = tostr[i] + buf[ord(fromstr[i])] = ord(tostr[i]) return str(buf) diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py --- a/lib-python/2.7/test/test_os.py +++ b/lib-python/2.7/test/test_os.py @@ -580,6 +580,7 @@ "getentropy() does not use a file descriptor") class URandomFDTests(unittest.TestCase): @unittest.skipUnless(resource, "test requires the resource module") + @test_support.impl_detail(pypy=False) # on Linux, may use getrandom() def test_urandom_failure(self): # Check urandom() failing when it is not able to open /dev/random. # We spawn a new process to make the test more robust (if getrlimit() diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -309,9 +309,12 @@ def __init__(self, message, category, filename, lineno, file=None, line=None): - local_values = locals() - for attr in self._WARNING_DETAILS: - setattr(self, attr, local_values[attr]) + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.file = file + self.line = line self._category_name = category.__name__ if category else None def __str__(self): diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py --- a/lib-python/3/ctypes/__init__.py +++ b/lib-python/3/ctypes/__init__.py @@ -346,16 +346,18 @@ if handle is None: if flags & _FUNCFLAG_CDECL: - self._handle = _ffi.CDLL(name, mode) + pypy_dll = _ffi.CDLL(name, mode) else: - self._handle = _ffi.WinDLL(name, mode) - else: - self._handle = handle + pypy_dll = _ffi.WinDLL(name, mode) + self.__pypy_dll__ = pypy_dll + handle = int(pypy_dll) + self._handle = handle def __repr__(self): - return "<%s '%s', handle %r at 0x%x>" % ( - self.__class__.__name__, self._name, self._handle, - id(self) & (_sys.maxsize * 2 + 1)) + return "<%s '%s', handle %x at 0x%x>" % \ + (self.__class__.__name__, self._name, + (self._handle & (_sys.maxsize*2 + 1)), + id(self) & (_sys.maxsize*2 + 1)) def __getattr__(self, name): if name.startswith('__') and name.endswith('__'): diff --git a/lib-python/3/ctypes/test/test_byteswap.py b/lib-python/3/ctypes/test/test_byteswap.py --- a/lib-python/3/ctypes/test/test_byteswap.py +++ b/lib-python/3/ctypes/test/test_byteswap.py @@ -2,7 +2,6 @@ from binascii import hexlify from ctypes import * -from ctypes.test import xfail def bin(s): return hexlify(memoryview(s)).decode().upper() @@ -43,7 +42,6 @@ with self.assertRaises(AttributeError): little.z = 24 - @xfail def test_endian_short(self): if sys.byteorder == "little": self.assertIs(c_short.__ctype_le__, c_short) @@ -71,7 +69,6 @@ self.assertEqual(bin(s), "3412") self.assertEqual(s.value, 0x1234) - @xfail def test_endian_int(self): if sys.byteorder == "little": self.assertIs(c_int.__ctype_le__, c_int) @@ -100,7 +97,6 @@ self.assertEqual(bin(s), "78563412") self.assertEqual(s.value, 0x12345678) - @xfail def test_endian_longlong(self): if sys.byteorder == "little": self.assertIs(c_longlong.__ctype_le__, c_longlong) @@ -129,7 +125,6 @@ self.assertEqual(bin(s), "EFCDAB9078563412") self.assertEqual(s.value, 0x1234567890ABCDEF) - @xfail def test_endian_float(self): if sys.byteorder == "little": self.assertIs(c_float.__ctype_le__, c_float) @@ -148,7 +143,6 @@ self.assertAlmostEqual(s.value, math.pi, places=6) self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s)) - @xfail def test_endian_double(self): if sys.byteorder == "little": self.assertIs(c_double.__ctype_le__, c_double) @@ -176,7 +170,6 @@ self.assertIs(c_char.__ctype_le__, c_char) self.assertIs(c_char.__ctype_be__, c_char) - @xfail def test_struct_fields_1(self): if sys.byteorder == "little": base = BigEndianStructure @@ -212,7 +205,6 @@ pass self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)]) - @xfail def test_struct_struct(self): # nested structures with different byteorders @@ -241,7 +233,6 @@ self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) - @xfail def test_struct_fields_2(self): # standard packing in struct uses no alignment. # So, we have to align using pad bytes. @@ -265,7 +256,6 @@ s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) self.assertEqual(bin(s1), bin(s2)) - @xfail def test_unaligned_nonnative_struct_fields(self): if sys.byteorder == "little": base = BigEndianStructure diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -810,7 +810,8 @@ month = self._month if day is None: day = self._day - return date(year, month, day) + # PyPy fix: returns type(self)() instead of date() + return type(self)(year, month, day) # Comparisons of date objects with other. @@ -1285,7 +1286,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return time(hour, minute, second, microsecond, tzinfo) + # PyPy fix: returns type(self)() instead of time() + return type(self)(hour, minute, second, microsecond, tzinfo) # Pickle support. @@ -1497,7 +1499,8 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return datetime(year, month, day, hour, minute, second, microsecond, + # PyPy fix: returns type(self)() instead of datetime() + return type(self)(year, month, day, hour, minute, second, microsecond, tzinfo) def astimezone(self, tz=None): diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -73,7 +73,7 @@ g['CCSHARED'] = "-fPIC" g['LDSHARED'] = "cc -pthread -shared" g['EXT_SUFFIX'] = so_ext - g['SHLIB_SUFFIX'] = so_ext + g['SHLIB_SUFFIX'] = ".so" g['SO'] = so_ext # deprecated in Python 3, for backward compatibility g['AR'] = "ar" g['ARFLAGS'] = "rc" @@ -81,6 +81,19 @@ g['LIBDIR'] = os.path.join(sys.prefix, 'lib') g['VERSION'] = get_python_version() + if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() + g['LDSHARED'] += ' -undefined dynamic_lookup' + g['CC'] += ' -arch %s' % (arch,) + global _config_vars _config_vars = g diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py --- a/lib-python/3/stat.py +++ b/lib-python/3/stat.py @@ -139,13 +139,21 @@ def filemode(mode): """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" perm = [] + + # The first group gets a question mark if none of the bits match the mode. + empty = "?" + for table in _filemode_table: for bit, char in table: if mode & bit == bit: perm.append(char) break else: - perm.append("-") + perm.append(empty) + + # All the rest of the positions get a - if the bits don't match. + empty = "-" + return "".join(perm) diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py --- a/lib-python/3/test/test_asyncio/test_base_events.py +++ b/lib-python/3/test/test_asyncio/test_base_events.py @@ -1588,9 +1588,15 @@ sock.getsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR)) if reuseport_supported: - self.assertFalse( - sock.getsockopt( - socket.SOL_SOCKET, socket.SO_REUSEPORT)) + try: + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + except OSError: + # Python's socket module was compiled using modern headers + # thus defining SO_REUSEPORT but this process is running + # under an older kernel that does not support SO_REUSEPORT. + reuseport_supported = False self.assertFalse( sock.getsockopt( socket.SOL_SOCKET, socket.SO_BROADCAST)) diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py --- a/lib-python/3/test/test_descr.py +++ b/lib-python/3/test/test_descr.py @@ -1663,7 +1663,8 @@ self.assertEqual(b.foo, 3) self.assertEqual(b.__class__, D) - @unittest.expectedFailure + #@unittest.expectedFailure --- on CPython. On PyPy, the test passes + @support.impl_detail(cpython=False) def test_bad_new(self): self.assertRaises(TypeError, object.__new__) self.assertRaises(TypeError, object.__new__, '') diff --git a/lib-python/3/test/test_importlib/builtin/test_loader.py b/lib-python/3/test/test_importlib/builtin/test_loader.py --- a/lib-python/3/test/test_importlib/builtin/test_loader.py +++ b/lib-python/3/test/test_importlib/builtin/test_loader.py @@ -1,6 +1,8 @@ from .. import abc from .. import util +from importlib.machinery import BuiltinImporter + machinery = util.import_importlib('importlib.machinery') import sys @@ -14,7 +16,7 @@ def setUp(self): self.verification = {'__name__': 'errno', '__package__': '', - '__loader__': self.machinery.BuiltinImporter} + '__loader__': BuiltinImporter} # PyPy change def verify(self, module): """Verify that the module matches against what it should have.""" diff --git a/lib-python/3/test/test_importlib/extension/test_loader.py b/lib-python/3/test/test_importlib/extension/test_loader.py --- a/lib-python/3/test/test_importlib/extension/test_loader.py +++ b/lib-python/3/test/test_importlib/extension/test_loader.py @@ -88,6 +88,7 @@ def setUp(self): self.name = '_testmultiphase' + __import__(self.name) # PyPy hack finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec @@ -145,7 +146,8 @@ importlib.reload(module) self.assertIs(ex_class, module.Example) - def test_try_registration(self): + # XXX: PyPy doesn't support the PyState_* functions yet + def XXXtest_try_registration(self): '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work''' module = self.load_module() with self.subTest('PyState_FindModule'): diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py --- a/lib-python/3/test/test_marshal.py +++ b/lib-python/3/test/test_marshal.py @@ -271,6 +271,11 @@ if n is not None and n > 4: n += 10**6 return n + def read(self, n): # PyPy calls read(), not readinto() + result = super().read(n) + if len(result) > 4: + result += b'\x00' * (10**6) + return result for value in (1.0, 1j, b'0123456789', '0123456789'): self.assertRaises(ValueError, marshal.load, BadReader(marshal.dumps(value))) @@ -348,7 +353,8 @@ strobj = "abcde"*3 dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"} - def helper3(self, rsample, recursive=False, simple=False): + def helper3(self, rsample, recursive=False, simple=False, + check_sharing=True, check_non_sharing=True): #we have two instances sample = (rsample, rsample) @@ -358,28 +364,35 @@ n3 = CollectObjectIDs(set(), marshal.loads(s3)) #same number of instances generated - self.assertEqual(n3, n0) + # except in one corner case on top of pypy, for code objects + if check_sharing: + self.assertEqual(n3, n0) if not recursive: #can compare with version 2 s2 = marshal.dumps(sample, 2) n2 = CollectObjectIDs(set(), marshal.loads(s2)) #old format generated more instances - self.assertGreater(n2, n0) + # except on pypy where equal ints or floats always have + # the same id anyway + if check_non_sharing: + self.assertGreater(n2, n0) #if complex objects are in there, old format is larger - if not simple: + if check_non_sharing and not simple: self.assertGreater(len(s2), len(s3)) else: self.assertGreaterEqual(len(s2), len(s3)) def testInt(self): self.helper(self.intobj) - self.helper3(self.intobj, simple=True) + self.helper3(self.intobj, simple=True, + check_non_sharing=support.check_impl_detail()) def testFloat(self): self.helper(self.floatobj) - self.helper3(self.floatobj) + self.helper3(self.floatobj, + check_non_sharing=support.check_impl_detail()) def testStr(self): self.helper(self.strobj) @@ -395,7 +408,7 @@ if __file__.endswith(".py"): code = compile(code, __file__, "exec") self.helper(code) - self.helper3(code) + self.helper3(code, check_sharing=support.check_impl_detail()) def testRecursion(self): d = dict(self.dictobj) diff --git a/lib-python/3/test/test_pyexpat.py b/lib-python/3/test/test_pyexpat.py --- a/lib-python/3/test/test_pyexpat.py +++ b/lib-python/3/test/test_pyexpat.py @@ -11,7 +11,7 @@ from xml.parsers import expat from xml.parsers.expat import errors -from test.support import sortdict +from test.support import sortdict, impl_detail class SetAttributeTest(unittest.TestCase): @@ -446,6 +446,7 @@ self.assertEqual(os.path.basename(entry[0]), filename) self.assertEqual(entry[2], funcname) + @impl_detail("PyPy does not have pyexpat.c", pypy=False) def test_exception(self): parser = expat.ParserCreate() parser.StartElementHandler = self.StartElementHandler diff --git a/lib-python/3/test/test_stat.py b/lib-python/3/test/test_stat.py --- a/lib-python/3/test/test_stat.py +++ b/lib-python/3/test/test_stat.py @@ -138,6 +138,10 @@ self.assertS_IS("REG", st_mode) self.assertEqual(modestr, '-r--r--r--') self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) + + # If there are only permission bits, no type bytes, a question + # mark is rendered in the type field. + self.assertEqual(self.statmod.filemode(0o420), '?r---w----') else: os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py --- a/lib-python/3/test/test_sysconfig.py +++ b/lib-python/3/test/test_sysconfig.py @@ -397,9 +397,16 @@ self.assertTrue('linux' in suffix, suffix) if re.match('(i[3-6]86|x86_64)$', machine): if ctypes.sizeof(ctypes.c_char_p()) == 4: - self.assertTrue(suffix.endswith('i386-linux-gnu.so') \ - or suffix.endswith('x86_64-linux-gnux32.so'), - suffix) + self.assertTrue( + suffix.endswith(( + 'i386-linux-gnu.so', + 'i486-linux-gnu.so', + 'i586-linux-gnu.so', + 'i686-linux-gnu.so', + 'x86_64-linux-gnux32.so', + )), + suffix, + ) else: # 8 byte pointer size self.assertTrue(suffix.endswith('x86_64-linux-gnu.so'), suffix) diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md --- a/lib_pypy/_cffi_ssl/README.md +++ b/lib_pypy/_cffi_ssl/README.md @@ -5,8 +5,15 @@ it renames the compiled shared object to _pypy_openssl.so (which means that cryptography can ship their own cffi backend) -NOTE: currently, we have changed ``_cffi_src/openssl/callbacks.py`` to -not rely on the CPython C API. +NOTE: currently, we have the following changes: + +* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API + (this change is now backported) + +* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4) + +* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1) + # Tests? diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn deleted file mode 100644 index 180c02ff82d3363f34a334aae22c9876d4c96481..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [cut] diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py @@ -221,10 +221,16 @@ static const long X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM = 0; static const long X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED = 0; static const long X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 = 0; +#ifndef X509_V_ERR_HOSTNAME_MISMATCH static const long X509_V_ERR_HOSTNAME_MISMATCH = 0; +#endif +#ifndef X509_V_ERR_EMAIL_MISMATCH static const long X509_V_ERR_EMAIL_MISMATCH = 0; +#endif +#ifndef X509_V_ERR_IP_ADDRESS_MISMATCH static const long X509_V_ERR_IP_ADDRESS_MISMATCH = 0; #endif +#endif /* OpenSSL 1.0.2beta2+ verification parameters */ #if CRYPTOGRAPHY_OPENSSL_102BETA2_OR_GREATER && \ diff --git a/lib_pypy/_cffi_ssl/_cffi_src/utils.py b/lib_pypy/_cffi_ssl/_cffi_src/utils.py --- a/lib_pypy/_cffi_ssl/_cffi_src/utils.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/utils.py @@ -47,9 +47,19 @@ # is legal, but the following will fail to compile: # int foo(int); # int foo(short); + # + # XXX <arigo> No, it is a bad idea. OpenSSL itself tends to tweak + # the definitions, like adding a 'const' (see issue #2575). Every + # time they do so, it makes a gratuitous break in this code. It is + # better to rely on the C compiler for that, which is a little bit + # more flexible. That's the point of set_source(). We can still + # re-enable the line ``#functions +`` below to get the original + # behavior. (I would enable it during tests, but I don't find any + # custom test at all..??) + # verify_source = "\n".join( includes + - functions + + #functions + customizations ) ffi = build_ffi( diff --git a/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn b/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn deleted file mode 100644 index 40344f6cee5cb001b73dd3a9a203015568831391..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [cut] diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py --- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py +++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py @@ -173,14 +173,13 @@ return tuple(dn) -STATIC_BIO_BUF = ffi.new("char[]", 2048) - def _bio_get_str(biobuf): - length = lib.BIO_gets(biobuf, STATIC_BIO_BUF, len(STATIC_BIO_BUF)-1) + bio_buf = ffi.new("char[]", 2048) + length = lib.BIO_gets(biobuf, bio_buf, len(bio_buf)-1) if length < 0: if biobuf: lib.BIO_free(biobuf) raise ssl_error(None) - return _str_with_len(STATIC_BIO_BUF, length) + return _str_with_len(bio_buf, length) def _decode_certificate(certificate): retval = {} diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py --- a/lib_pypy/_cffi_ssl/_stdssl/error.py +++ b/lib_pypy/_cffi_ssl/_stdssl/error.py @@ -1,4 +1,5 @@ import sys +import os import traceback from _pypy_openssl import ffi from _pypy_openssl import lib @@ -100,18 +101,17 @@ errval = SSL_ERROR_WANT_CONNECT elif err == SSL_ERROR_SYSCALL: if e == 0: - if ret == 0 or obj.socket is not None: + if ret == 0 or obj.socket is None: errtype = SSLEOFError errstr = "EOF occurred in violation of protocol" errval = SSL_ERROR_EOF elif ret == -1 and obj.socket is not None: # the underlying BIO reported an I/0 error lib.ERR_clear_error() - s = obj.get_socket_or_None() - s.errorhandler() - assert 0, "must not get here" - #errno = ffi.errno - #return IOError(errno) + # s = obj.get_socket_or_None() + # XXX: Windows? + errno = ffi.errno + return OSError(errno, os.strerror(errno)) else: errtype = SSLSyscallError errstr = "Some I/O error occurred" diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -74,12 +74,16 @@ return self._type_._alignmentofinstances() def _CData_output(self, resarray, base=None, index=-1): - # this seems to be a string if we're array of char, surprise! - from ctypes import c_char, c_wchar - if self._type_ is c_char: - return _rawffi.charp2string(resarray.buffer, self._length_) - if self._type_ is c_wchar: - return _rawffi.wcharp2unicode(resarray.buffer, self._length_) + from _rawffi.alt import types + # If a char_p or unichar_p is received, skip the string interpretation + if base._ffiargtype != types.Pointer(types.char_p) and \ + base._ffiargtype != types.Pointer(types.unichar_p): + # this seems to be a string if we're array of char, surprise! + from ctypes import c_char, c_wchar + if self._type_ is c_char: + return _rawffi.charp2string(resarray.buffer, self._length_) + if self._type_ is c_wchar: + return _rawffi.wcharp2unicode(resarray.buffer, self._length_) res = self.__new__(self) ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_) res._buffer = ffiarray diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -82,7 +82,7 @@ return False def in_dll(self, dll, name): - return self.from_address(dll._handle.getaddressindll(name)) + return self.from_address(dll.__pypy_dll__.getaddressindll(name)) def from_buffer(self, obj, offset=0): size = self._sizeofinstances() diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -430,7 +430,7 @@ ffires = restype.get_ffi_argtype() return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_) - cdll = self.dll._handle + cdll = self.dll.__pypy_dll__ try: ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes] ffi_restype = restype.get_ffi_argtype() diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -141,6 +141,10 @@ ptr._buffer = tp._ffiarray(1, autofree=True) ptr._buffer[0] = obj._buffer result = ptr + elif isinstance(obj, bytes): + result = tp() + result._buffer[0] = memoryview(obj)._pypy_raw_address() + return result elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -61,6 +61,54 @@ pyobj_container = GlobalPyobjContainer() +def swap_bytes(value, sizeof, typeof, get_or_set): + def swap_2(): + return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00) + + def swap_4(): + return ((value & 0x000000FF) << 24) | \ + ((value & 0x0000FF00) << 8) | \ + ((value & 0x00FF0000) >> 8) | \ + ((value >> 24) & 0xFF) + + def swap_8(): + return ((value & 0x00000000000000FF) << 56) | \ + ((value & 0x000000000000FF00) << 40) | \ + ((value & 0x0000000000FF0000) << 24) | \ + ((value & 0x00000000FF000000) << 8) | \ + ((value & 0x000000FF00000000) >> 8) | \ + ((value & 0x0000FF0000000000) >> 24) | \ + ((value & 0x00FF000000000000) >> 40) | \ + ((value >> 56) & 0xFF) + + def swap_double_float(typ): + from struct import pack, unpack + if get_or_set == 'set': + if sys.byteorder == 'little': + st = pack(''.join(['>', typ]), value) + else: + st = pack(''.join(['<', typ]), value) + return unpack(typ, st)[0] + else: + packed = pack(typ, value) + if sys.byteorder == 'little': + st = unpack(''.join(['>', typ]), packed) + else: + st = unpack(''.join(['<', typ]), packed) + return st[0] + + if typeof in ('c_float', 'c_float_le', 'c_float_be'): + return swap_double_float('f') + elif typeof in ('c_double', 'c_double_le', 'c_double_be'): + return swap_double_float('d') + else: + if sizeof == 2: + return swap_2() + elif sizeof == 4: + return swap_4() + elif sizeof == 8: + return swap_8() + def generic_xxx_p_from_param(cls, value): if value is None: return cls(None) @@ -265,6 +313,31 @@ def _as_ffi_pointer_(self, ffitype): return as_ffi_pointer(self, ffitype) result._as_ffi_pointer_ = _as_ffi_pointer_ + if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \ + and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'): + from sys import byteorder + if byteorder == 'big': + name += '_le' + swapped = self.__new__(self, name, bases, dct) + result.__ctype_le__ = swapped + result.__ctype_be__ = result + swapped.__ctype_be__ = result + swapped.__ctype_le__ = swapped + else: + name += '_be' + swapped = self.__new__(self, name, bases, dct) + result.__ctype_be__ = swapped + result.__ctype_le__ = result + swapped.__ctype_le__ = result + swapped.__ctype_be__ = swapped + from _ctypes import sizeof + def _getval(self): + return swap_bytes(self._buffer[0], sizeof(self), name, 'get') + def _setval(self, value): + d = result() + d.value = value + self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set') + swapped.value = property(_getval, _setval) return result diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -40,6 +40,22 @@ else: rawfields.append((f[0], f[1]._ffishape_)) + # hack for duplicate field names + already_seen = set() + names1 = names + names = [] + for f in names1: + if f not in already_seen: + names.append(f) + already_seen.add(f) + already_seen = set() + for i in reversed(range(len(rawfields))): + if rawfields[i][0] in already_seen: + rawfields[i] = (('$DUP%d$%s' % (i, rawfields[i][0]),) + + rawfields[i][1:]) + already_seen.add(rawfields[i][0]) + # /hack + _set_shape(self, rawfields, self._is_union) fields = {} @@ -130,6 +146,7 @@ obj._buffer.__setattr__(self.name, arg) + def _set_shape(tp, rawfields, is_union=False): tp._ffistruct_ = _rawffi.Structure(rawfields, is_union, getattr(tp, '_pack_', 0)) @@ -224,18 +241,26 @@ res.__dict__['_index'] = -1 return res - class StructOrUnion(_CData, metaclass=StructOrUnionMeta): def __new__(cls, *args, **kwds): from _ctypes import union - self = super(_CData, cls).__new__(cls) - if ('_abstract_' in cls.__dict__ or cls is Structure + if ('_abstract_' in cls.__dict__ or cls is Structure or cls is union.Union): raise TypeError("abstract class") if hasattr(cls, '_swappedbytes_'): - raise NotImplementedError("missing in PyPy: structure/union with " - "swapped (non-native) byte ordering") + fields = [None] * len(cls._fields_) + for i in range(len(cls._fields_)): + if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None): + swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1]) + else: + swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1]) + if len(cls._fields_[i]) < 3: + fields[i] = (cls._fields_[i][0], swapped) + else: + fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2]) + names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None)) + self = super(_CData, cls).__new__(cls) if hasattr(cls, '_ffistruct_'): self.__dict__['_buffer'] = self._ffistruct_(autofree=True) return self diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -8,6 +8,9 @@ from _curses_cffi import ffi, lib +version = b"2.2" +__version__ = b"2.2" + def _copy_to_globals(name): globals()[name] = getattr(lib, name) @@ -60,10 +63,6 @@ _setup() -# Do we want this? -# version = "2.2" -# __version__ = "2.2" - # ____________________________________________________________ @@ -404,6 +403,17 @@ raise error("getch requires 0 or 2 arguments") return val + def get_wch(self, *args): + wch = ffi.new("int[1]") + if len(args) == 0: + val = lib.wget_wch(self._win, wch) + elif len(args) == 2: + val = lib.mvwget_wch(self._win, *args, wch) + else: + raise error("get_wch requires 0 or 2 arguments") + _check_ERR(val, "get_wch") + return wch[0] + def getkey(self, *args): if len(args) == 0: val = lib.wgetch(self._win) @@ -919,101 +929,29 @@ return None -# XXX: Do something about the following? -# /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES -# * and _curses.COLS */ -# #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) -# static int -# update_lines_cols(void) -# { -# PyObject *o; -# PyObject *m = PyImport_ImportModuleNoBlock("curses"); +# Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES +# and _curses.COLS +def update_lines_cols(): + globals()["LINES"] = lib.LINES + globals()["COLS"] = lib.COLS + try: + m = sys.modules["curses"] + m.LINES = lib.LINES + m.COLS = lib.COLS + except (KeyError, AttributeError): + pass -# if (!m) -# return 0; -# o = PyInt_FromLong(LINES); -# if (!o) { -# Py_DECREF(m); -# return 0; -# } -# if (PyObject_SetAttrString(m, "LINES", o)) { -# Py_DECREF(m); -# Py_DECREF(o); -# return 0; -# } -# if (PyDict_SetItemString(ModDict, "LINES", o)) { -# Py_DECREF(m); -# Py_DECREF(o); -# return 0; -# } -# Py_DECREF(o); -# o = PyInt_FromLong(COLS); -# if (!o) { -# Py_DECREF(m); -# return 0; -# } -# if (PyObject_SetAttrString(m, "COLS", o)) { -# Py_DECREF(m); -# Py_DECREF(o); -# return 0; -# } -# if (PyDict_SetItemString(ModDict, "COLS", o)) { -# Py_DECREF(m); -# Py_DECREF(o); -# return 0; -# } -# Py_DECREF(o); -# Py_DECREF(m); -# return 1; -# } -# #endif +def resizeterm(lines, columns): + _ensure_initialised() + _check_ERR(lib.resizeterm(lines, columns), "resizeterm") + update_lines_cols() -# #ifdef HAVE_CURSES_RESIZETERM -# static PyObject * -# PyCurses_ResizeTerm(PyObject *self, PyObject *args) -# { -# int lines; -# int columns; -# PyObject *result; -# PyCursesInitialised; - -# if (!PyArg_ParseTuple(args,"ii:resizeterm", &lines, &columns)) -# return NULL; - -# result = PyCursesCheckERR(resizeterm(lines, columns), "resizeterm"); -# if (!result) -# return NULL; -# if (!update_lines_cols()) -# return NULL; -# return result; -# } - -# #endif - -# #ifdef HAVE_CURSES_RESIZE_TERM -# static PyObject * -# PyCurses_Resize_Term(PyObject *self, PyObject *args) -# { -# int lines; -# int columns; - -# PyObject *result; - -# PyCursesInitialised; - -# if (!PyArg_ParseTuple(args,"ii:resize_term", &lines, &columns)) -# return NULL; - -# result = PyCursesCheckERR(resize_term(lines, columns), "resize_term"); -# if (!result) -# return NULL; -# if (!update_lines_cols()) -# return NULL; -# return result; -# } -# #endif /* HAVE_CURSES_RESIZE_TERM */ +def resize_term(lines, columns): + _ensure_initialised() + _check_ERR(lib.resize_term(lines, columns), "resize_term") + update_lines_cols() def setsyx(y, x): @@ -1078,6 +1016,11 @@ return _check_ERR(lib.ungetch(_chtype(ch)), "ungetch") +def unget_wch(ch): + _ensure_initialised() + return _check_ERR(lib.unget_wch(_chtype(ch)), "unget_wch") + + def use_env(flag): lib.use_env(flag) return None diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py --- a/lib_pypy/_curses_build.py +++ b/lib_pypy/_curses_build.py @@ -1,3 +1,4 @@ +import os from cffi import FFI, VerificationError @@ -17,6 +18,11 @@ # error message raise e_last +def find_curses_include_dirs(): + if os.path.exists('/usr/include/ncursesw'): + return ['/usr/include/ncursesw'] + return [] + ffi = FFI() @@ -59,7 +65,8 @@ void _m_getsyx(int *yx) { getsyx(yx[0], yx[1]); } -""", libraries=[find_curses_library(), 'panel']) +""", libraries=[find_curses_library(), 'panel'], + include_dirs=find_curses_include_dirs()) ffi.cdef(""" @@ -70,6 +77,8 @@ typedef unsigned long... chtype; typedef chtype attr_t; +typedef int... wint_t; + typedef struct { short id; /* ID to distinguish multiple devices */ @@ -105,6 +114,13 @@ static const chtype A_CHARTEXT; static const chtype A_COLOR; +static const chtype A_HORIZONTAL; +static const chtype A_LEFT; +static const chtype A_LOW; +static const chtype A_RIGHT; +static const chtype A_TOP; +static const chtype A_VERTICAL; + static const int BUTTON1_RELEASED; static const int BUTTON1_PRESSED; static const int BUTTON1_CLICKED; @@ -160,6 +176,8 @@ void filter(void); int flash(void); int flushinp(void); +int wget_wch(WINDOW *, wint_t *); +int mvwget_wch(WINDOW *, int, int, wint_t *); chtype getbkgd(WINDOW *); WINDOW * getwin(FILE *); int halfdelay(int); @@ -220,6 +238,8 @@ int resetty(void); int reset_prog_mode(void); int reset_shell_mode(void); +int resizeterm(int, int); +int resize_term(int, int); int savetty(void); int scroll(WINDOW *); int scrollok(WINDOW *, bool); @@ -233,6 +253,7 @@ int touchwin(WINDOW *); int typeahead(int); int ungetch(int); +int unget_wch(const wchar_t); int untouchwin(WINDOW *); void use_env(bool); int waddch(WINDOW *, const chtype); diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py --- a/lib_pypy/_decimal.py +++ b/lib_pypy/_decimal.py @@ -489,13 +489,16 @@ vv.exp = 0 multiplied = Decimal._new_empty() denom = Decimal(other.denominator) - with _CatchStatus(context) as (ctx, status_ptr): - _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd, - ctx, status_ptr) - multiplied._mpd.exp += exp # XXX probably a bug - # in _decimal.c + maxctx = _ffi.new("struct mpd_context_t*") + _mpdec.mpd_maxcontext(maxctx) + status_ptr = _ffi.new("uint32_t*") + _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd, + maxctx, status_ptr) + multiplied._mpd.exp = exp finally: _mpdec.mpd_del(vv) + if status_ptr[0] != 0: + raise ValueError("exact conversion for comparison failed") return multiplied, numerator else: @@ -719,8 +722,8 @@ compare = _make_binary_operation('compare') compare_signal = _make_binary_operation('compare_signal') - compare_total = _make_binary_operation('compare') - compare_total_mag = _make_binary_operation('compare') + compare_total = _make_binary_operation('compare_total') + compare_total_mag = _make_binary_operation('compare_total_mag') logical_and = _make_binary_operation('logical_and') logical_or = _make_binary_operation('logical_or') logical_xor = _make_binary_operation('logical_xor') diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py --- a/lib_pypy/_lzma.py +++ b/lib_pypy/_lzma.py @@ -10,6 +10,7 @@ import weakref import sys import io +import __pypy__ from _lzma_cffi import ffi, lib as m @@ -63,6 +64,10 @@ m._pylzma_stream_init(ret) return ffi.gc(ret, m.lzma_end) +def _release_lzma_stream(st): + ffi.gc(st, None) + m.lzma_end(st) + def add_constant(c): globals()[c] = getattr(m, 'LZMA_' + c) @@ -148,39 +153,39 @@ def parse_filter_spec_lzma(id, preset=m.LZMA_PRESET_DEFAULT, **kwargs): ret = ffi.new('lzma_options_lzma*') if m.lzma_lzma_preset(ret, preset): - raise LZMAError("Invalid...") + raise LZMAError("Invalid compression preset: %s" % preset) for arg, val in kwargs.items(): if arg in ('dict_size', 'lc', 'lp', 'pb', 'nice_len', 'depth'): setattr(ret, arg, val) elif arg in ('mf', 'mode'): setattr(ret, arg, int(val)) else: - raise ValueError("Invalid...") + raise ValueError("Invalid filter specifier for LZMA filter") return ret def parse_filter_spec(spec): if not isinstance(spec, collections.Mapping): - raise TypeError("Filter...") + raise TypeError("Filter specifier must be a dict or dict-like object") ret = ffi.new('lzma_filter*') try: ret.id = spec['id'] except KeyError: - raise ValueError("Filter...") + raise ValueError("Filter specifier must have an \"id\" entry") if ret.id in (m.LZMA_FILTER_LZMA1, m.LZMA_FILTER_LZMA2): try: options = parse_filter_spec_lzma(**spec) except TypeError: - raise ValueError("Invalid...") + raise ValueError("Invalid filter specifier for LZMA filter") elif ret.id == m.LZMA_FILTER_DELTA: try: options = parse_filter_spec_delta(**spec) except TypeError: - raise ValueError("Invalid...") + raise ValueError("Invalid filter specifier for delta filter") elif ret.id in BCJ_FILTERS: try: options = parse_filter_spec_bcj(**spec) except TypeError: - raise ValueError("Invalid...") + raise ValueError("Invalid filter specifier for BCJ filter") else: raise ValueError("Invalid %d" % (ret.id,)) @@ -204,7 +209,9 @@ def parse_filter_chain_spec(filterspecs): if len(filterspecs) > m.LZMA_FILTERS_MAX: - raise ValueError("Too...") + raise ValueError( + "Too many filters - liblzma supports a maximum of %s" % + m.LZMA_FILTERS_MAX) filters = ffi.new('lzma_filter[]', m.LZMA_FILTERS_MAX+1) _owns[filters] = children = [] for i in range(m.LZMA_FILTERS_MAX+1): @@ -236,7 +243,7 @@ elif filter.id in BCJ_FILTERS: add_opts('lzma_options_bcj', 'start_offset') else: - raise ValueError("Invalid...") + raise ValueError("Invalid filter ID: %s" % filter.id) return spec def _decode_filter_properties(filter_id, encoded_props): @@ -420,25 +427,26 @@ For one-shot decompression, use the decompress() function instead. """ - def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None, header=None, check=None, unpadded_size=None): + def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None, + header=None, check=None, unpadded_size=None): decoder_flags = m.LZMA_TELL_ANY_CHECK | m.LZMA_TELL_NO_CHECK - #decoder_flags = 0 if memlimit is not None: if format == FORMAT_RAW: - raise ValueError("Cannot sp...") - #memlimit = long(memlimit) + raise ValueError("Cannot specify memory limit with FORMAT_RAW") else: memlimit = m.UINT64_MAX if format == FORMAT_RAW and filters is None: - raise ValueError("Must...") + raise ValueError("Must specify filters for FORMAT_RAW") elif format != FORMAT_RAW and filters is not None: - raise ValueError("Cannot...") + raise ValueError("Cannot specify filters except with FORMAT_RAW") if format == FORMAT_BLOCK and (header is None or unpadded_size is None or check is None): - raise ValueError("Must...") + raise ValueError("Must specify header, unpadded_size and check " + "with FORMAT_BLOCK") elif format != FORMAT_BLOCK and (header is not None or unpadded_size is not None or check is not None): - raise ValueError("Cannot...") + raise ValueError("Cannot specify header, unpadded_size or check " + "except with FORMAT_BLOCK") format = _parse_format(format) self.lock = threading.Lock() @@ -476,7 +484,7 @@ self.expected_size = block.compressed_size catch_lzma_error(m.lzma_block_decoder, self.lzs, block) else: - raise ValueError("invalid...") + raise ValueError("invalid container format: %s" % format) def pre_decompress_left_data(self, buf, buf_size): # in this case there is data left that needs to be processed before the first @@ -551,7 +559,7 @@ raise TypeError("max_length parameter object cannot be interpreted as an integer") with self.lock: if self.eof: - raise EOFError("Already...") + raise EOFError("Already at end of stream") lzs = self.lzs data = to_bytes(data) buf = ffi.new('uint8_t[]', data) @@ -648,6 +656,16 @@ raise TypeError("cannot serialize '%s' object" % self.__class__.__name__) + +# Issue #2579: Setting up the stream for encoding takes around 17MB of +# RAM on my Linux 64 system. So we call add_memory_pressure(17MB) when +# we create the stream. In flush(), we actively free the stream even +# though we could just leave it to the GC (but 17MB is too much for +# doing that sanely); at this point we call add_memory_pressure(-17MB) +# to cancel the original increase. +COMPRESSION_STREAM_SIZE = 1024*1024*17 + + class LZMACompressor(object): """ LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None) @@ -679,15 +697,16 @@ """ def __init__(self, format=FORMAT_XZ, check=-1, preset=None, filters=None): if format != FORMAT_XZ and check not in (-1, m.LZMA_CHECK_NONE): - raise ValueError("Integrity...") + raise ValueError("Integrity checks are only supported by FORMAT_XZ") if preset is not None and filters is not None: - raise ValueError("Cannot...") + raise ValueError("Cannot specify both preset and filter chain") if preset is None: preset = m.LZMA_PRESET_DEFAULT format = _parse_format(format) self.lock = threading.Lock() self.flushed = 0 self.lzs = _new_lzma_stream() + __pypy__.add_memory_pressure(COMPRESSION_STREAM_SIZE) if format == FORMAT_XZ: if filters is None: if check == -1: @@ -702,19 +721,19 @@ if filters is None: options = ffi.new('lzma_options_lzma*') if m.lzma_lzma_preset(options, preset): - raise LZMAError("Invalid...") + raise LZMAError("Invalid compression preset: %s" % preset) catch_lzma_error(m.lzma_alone_encoder, self.lzs, options) else: raise NotImplementedError elif format == FORMAT_RAW: if filters is None: - raise ValueError("Must...") + raise ValueError("Must specify filters for FORMAT_RAW") filters = parse_filter_chain_spec(filters) catch_lzma_error(m.lzma_raw_encoder, self.lzs, filters) else: - raise ValueError("Invalid...") + raise ValueError("invalid container format: %s" % format) def compress(self, data): """ @@ -728,7 +747,7 @@ """ with self.lock: if self.flushed: - raise ValueError("Compressor...") + raise ValueError("Compressor has been flushed") return self._compress(data) def _compress(self, data, action=m.LZMA_RUN): @@ -769,9 +788,12 @@ def flush(self): with self.lock: if self.flushed: - raise ValueError("Repeated...") + raise ValueError("Repeated call to flush()") self.flushed = 1 - return self._compress(b'', action=m.LZMA_FINISH) + result = self._compress(b'', action=m.LZMA_FINISH) + __pypy__.add_memory_pressure(-COMPRESSION_STREAM_SIZE) + _release_lzma_stream(self.lzs) + return result def __getstate__(self): raise TypeError("cannot serialize '%s' object" % diff --git a/lib_pypy/_testmultiphase.c b/lib_pypy/_testmultiphase.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_testmultiphase.c @@ -0,0 +1,627 @@ +/* Copied from CPython's Modules/_testmultiphase.c */ +/***************************************************/ + +/* Testing module for multi-phase initialization of extension modules (PEP 489) + */ + +#include "Python.h" + +/* Example objects */ +typedef struct { + PyObject_HEAD + PyObject *x_attr; /* Attributes dictionary */ +} ExampleObject; + +/* Example methods */ + +static int +Example_traverse(ExampleObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->x_attr); + return 0; +} + +static int +Example_finalize(ExampleObject *self) +{ + Py_CLEAR(self->x_attr); + return 0; +} + +static PyObject * +Example_demo(ExampleObject *self, PyObject *args) +{ + PyObject *o = NULL; + if (!PyArg_ParseTuple(args, "|O:demo", &o)) + return NULL; + if (o != NULL && PyUnicode_Check(o)) { + Py_INCREF(o); + return o; + } + Py_INCREF(Py_None); + return Py_None; +} + + +static PyMethodDef Example_methods[] = { + {"demo", (PyCFunction)Example_demo, METH_VARARGS, + PyDoc_STR("demo() -> None")}, + {NULL, NULL} /* sentinel */ +}; + +static PyObject * +Example_getattro(ExampleObject *self, PyObject *name) +{ + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItem(self->x_attr, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + } + return PyObject_GenericGetAttr((PyObject *)self, name); +} + +static int +Example_setattr(ExampleObject *self, char *name, PyObject *v) +{ + if (self->x_attr == NULL) { + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) + return -1; + } + if (v == NULL) { + int rv = PyDict_DelItemString(self->x_attr, name); + if (rv < 0) + PyErr_SetString(PyExc_AttributeError, + "delete non-existing Example attribute"); + return rv; + } + else + return PyDict_SetItemString(self->x_attr, name, v); +} + +static PyType_Slot Example_Type_slots[] = { + {Py_tp_doc, "The Example type"}, +// {Py_tp_finalize, Example_finalize}, + {Py_tp_traverse, Example_traverse}, + {Py_tp_getattro, Example_getattro}, + {Py_tp_setattr, Example_setattr}, + {Py_tp_methods, Example_methods}, + {0, 0}, +}; + +static PyType_Spec Example_Type_spec = { + "_testimportexec.Example", + sizeof(ExampleObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, + Example_Type_slots +}; + +/* Function of two integers returning integer */ + +PyDoc_STRVAR(testexport_foo_doc, +"foo(i,j)\n\ +\n\ +Return the sum of i and j."); + +static PyObject * +testexport_foo(PyObject *self, PyObject *args) +{ + long i, j; + long res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + return NULL; + res = i + j; + return PyLong_FromLong(res); +} + +/* Test that PyState registration fails */ + +//PyDoc_STRVAR(call_state_registration_func_doc, +//"register_state(0): call PyState_FindModule()\n\ +//register_state(1): call PyState_AddModule()\n\ +//register_state(2): call PyState_RemoveModule()"); +// +//static PyObject * +//call_state_registration_func(PyObject *mod, PyObject *args) +//{ +// int i, ret; +// PyModuleDef *def = PyModule_GetDef(mod); +// if (def == NULL) { +// return NULL; +// } +// if (!PyArg_ParseTuple(args, "i:call_state_registration_func", &i)) +// return NULL; +// switch (i) { +// case 0: +// mod = PyState_FindModule(def); +// if (mod == NULL) { +// Py_RETURN_NONE; +// } +// return mod; +// case 1: +// ret = PyState_AddModule(mod, def); +// if (ret != 0) { +// return NULL; +// } +// break; +// case 2: +// ret = PyState_RemoveModule(def); +// if (ret != 0) { +// return NULL; +// } +// break; +// } +// Py_RETURN_NONE; +//} + + +static PyType_Slot Str_Type_slots[] = { + {Py_tp_base, NULL}, /* filled out in module exec function */ + {0, 0}, +}; + +static PyType_Spec Str_Type_spec = { + "_testimportexec.Str", + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + Str_Type_slots +}; + +static PyMethodDef testexport_methods[] = { + {"foo", testexport_foo, METH_VARARGS, + testexport_foo_doc}, +// {"call_state_registration_func", call_state_registration_func, +// METH_VARARGS, call_state_registration_func_doc}, + {NULL, NULL} /* sentinel */ +}; + +static int execfunc(PyObject *m) +{ + PyObject *temp = NULL; + + /* Due to cross platform compiler issues the slots must be filled + * here. It's required for portability to Windows without requiring + * C++. */ + Str_Type_slots[0].pfunc = &PyUnicode_Type; + + /* Add a custom type */ + temp = PyType_FromSpec(&Example_Type_spec); + if (temp == NULL) + goto fail; + if (PyModule_AddObject(m, "Example", temp) != 0) + goto fail; + + /* Add an exception type */ + temp = PyErr_NewException("_testimportexec.error", NULL, NULL); + if (temp == NULL) + goto fail; + if (PyModule_AddObject(m, "error", temp) != 0) + goto fail; + + /* Add Str */ + temp = PyType_FromSpec(&Str_Type_spec); + if (temp == NULL) + goto fail; + if (PyModule_AddObject(m, "Str", temp) != 0) + goto fail; + + if (PyModule_AddIntConstant(m, "int_const", 1969) != 0) + goto fail; + + if (PyModule_AddStringConstant(m, "str_const", "something different") != 0) + goto fail; + + return 0; + fail: + return -1; +} + +/* Helper for module definitions; there'll be a lot of them */ +#define TEST_MODULE_DEF(name, slots, methods) { \ + PyModuleDef_HEAD_INIT, /* m_base */ \ + name, /* m_name */ \ + PyDoc_STR("Test module " name), /* m_doc */ \ + 0, /* m_size */ \ + methods, /* m_methods */ \ + slots, /* m_slots */ \ + NULL, /* m_traverse */ \ + NULL, /* m_clear */ \ + NULL, /* m_free */ \ +} + +PyModuleDef_Slot main_slots[] = { + {Py_mod_exec, execfunc}, + {0, NULL}, +}; _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit