Author: Matti Picus <[email protected]>
Branch: stdlib-2.7.16
Changeset: r97099:5e792e528dc7
Date: 2019-06-26 17:06 +0300
http://bitbucket.org/pypy/pypy/changeset/5e792e528dc7/
Log: merge default into branch
diff too long, truncating to 2000 out of 9174 lines
diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py
b/extra_tests/cffi_tests/cffi0/test_parsing.py
--- a/extra_tests/cffi_tests/cffi0/test_parsing.py
+++ b/extra_tests/cffi_tests/cffi0/test_parsing.py
@@ -410,7 +410,17 @@
def test_enum():
ffi = FFI()
ffi.cdef("""
- enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1};
+ enum Enum {
+ POS = +1,
+ TWO = 2,
+ NIL = 0,
+ NEG = -1,
+ ADDSUB = (POS+TWO)-1,
+ DIVMULINT = (3 * 3) / 2,
+ SHIFT = (1 << 3) >> 1,
+ BINOPS = (0x7 & 0x1) | 0x8,
+ XOR = 0xf ^ 0xa
+ };
""")
needs_dlopen_none()
C = ffi.dlopen(None)
@@ -418,7 +428,11 @@
assert C.TWO == 2
assert C.NIL == 0
assert C.NEG == -1
- assert C.OP == 2
+ assert C.ADDSUB == 2
+ assert C.DIVMULINT == 4
+ assert C.SHIFT == 4
+ assert C.BINOPS == 0b1001
+ assert C.XOR == 0b0101
def test_stdcall():
ffi = FFI()
diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py
b/extra_tests/cffi_tests/cffi0/test_verify.py
--- a/extra_tests/cffi_tests/cffi0/test_verify.py
+++ b/extra_tests/cffi_tests/cffi0/test_verify.py
@@ -2535,3 +2535,29 @@
x.p = p
x.cyclic = x
del p, x
+
+def test_arithmetic_in_cdef():
+ for a in [0, 11, 15]:
+ ffi = FFI()
+ ffi.cdef("""
+ enum FOO {
+ DIVNN = ((-?) / (-3)),
+ DIVNP = ((-?) / (+3)),
+ DIVPN = ((+?) / (-3)),
+ MODNN = ((-?) % (-3)),
+ MODNP = ((-?) % (+3)),
+ MODPN = ((+?) % (-3)),
+ };
+ """.replace('?', str(a)))
+ lib = ffi.verify("""
+ enum FOO {
+ DIVNN = ((-?) / (-3)),
+ DIVNP = ((-?) / (+3)),
+ DIVPN = ((+?) / (-3)),
+ MODNN = ((-?) % (-3)),
+ MODNP = ((-?) % (+3)),
+ MODPN = ((+?) % (-3)),
+ };
+ """.replace('?', str(a)))
+ # the verify() crashes if the values in the enum are different from
+ # the values we computed ourselves from the cdef()
diff --git a/extra_tests/cffi_tests/cffi0/test_zintegration.py
b/extra_tests/cffi_tests/cffi0/test_zintegration.py
--- a/extra_tests/cffi_tests/cffi0/test_zintegration.py
+++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py
@@ -2,11 +2,13 @@
import py, os, sys, shutil
import subprocess
from extra_tests.cffi_tests.udir import udir
+import pytest
if sys.platform == 'win32':
- py.test.skip('snippets do not run on win32')
+ pytestmark = pytest.mark.skip('snippets do not run on win32')
if sys.version_info < (2, 7):
- py.test.skip('fails e.g. on a Debian/Ubuntu which patches virtualenv'
+ pytestmark = pytest.mark.skip(
+ 'fails e.g. on a Debian/Ubuntu which patches virtualenv'
' in a non-2.6-friendly way')
def create_venv(name):
diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py
b/extra_tests/cffi_tests/cffi1/test_recompiler.py
--- a/extra_tests/cffi_tests/cffi1/test_recompiler.py
+++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py
@@ -2339,3 +2339,77 @@
typedef int foo_t; struct foo_s { void (*x)(foo_t); };
""")
py.test.raises(TypeError, ffi.new, "struct foo_s *")
+
+def test_from_buffer_struct():
+ ffi = FFI()
+ ffi.cdef("""struct foo_s { int a, b; };""")
+ lib = verify(ffi, "test_from_buffer_struct_p", """
+ struct foo_s { int a, b; };
+ """)
+ p = ffi.new("struct foo_s *", [-219239, 58974983])
+ q = ffi.from_buffer("struct foo_s[]", ffi.buffer(p))
+ assert ffi.typeof(q) == ffi.typeof("struct foo_s[]")
+ assert len(q) == 1
+ assert q[0].a == p.a
+ assert q[0].b == p.b
+ assert q == p
+ q = ffi.from_buffer("struct foo_s *", ffi.buffer(p))
+ assert ffi.typeof(q) == ffi.typeof("struct foo_s *")
+ assert q.a == p.a
+ assert q.b == p.b
+ assert q[0].a == p.a
+ assert q[0].b == p.b
+ assert q == p
+
+def test_unnamed_bitfield_1():
+ ffi = FFI()
+ ffi.cdef("""struct A { char : 1; };""")
+ lib = verify(ffi, "test_unnamed_bitfield_1", """
+ struct A { char : 1; };
+ """)
+ p = ffi.new("struct A *")
+ assert ffi.sizeof(p[0]) == 1
+ # Note: on gcc, the type name is ignored for anonymous bitfields
+ # and that's why the result is 1. On MSVC, the result is
+ # sizeof("char") which is also 1.
+
+def test_unnamed_bitfield_2():
+ ffi = FFI()
+ ffi.cdef("""struct A {
+ short c : 1; short : 1; short d : 1; short : 1; };""")
+ lib = verify(ffi, "test_unnamed_bitfield_2", """
+ struct A {
+ short c : 1; short : 1; short d : 1; short : 1;
+ };
+ """)
+ p = ffi.new("struct A *")
+ assert ffi.sizeof(p[0]) == ffi.sizeof("short")
+
+def test_unnamed_bitfield_3():
+ ffi = FFI()
+ ffi.cdef("""struct A { struct { char : 1; char : 1; } b; };""")
+ lib = verify(ffi, "test_unnamed_bitfield_3", """
+ struct A { struct { char : 1; char : 1; } b; };
+ """)
+ p = ffi.new("struct A *")
+ assert ffi.sizeof(p[0]) == 1
+ # Note: on gcc, the type name is ignored for anonymous bitfields
+ # and that's why the result is 1. On MSVC, the result is
+ # sizeof("char") which is also 1.
+
+def test_unnamed_bitfield_4():
+ ffi = FFI()
+ ffi.cdef("""struct A { struct {
+ unsigned c : 1; unsigned : 1; unsigned d : 1; unsigned : 1; } a;
+ };
+ struct B { struct A a; };""")
+ lib = verify(ffi, "test_unnamed_bitfield_4", """
+ struct A { struct {
+ unsigned c : 1; unsigned : 1; unsigned d : 1; unsigned : 1; } a;
+ };
+ struct B { struct A a; };
+ """)
+ b = ffi.new("struct B *")
+ a = ffi.new("struct A *")
+ assert ffi.sizeof(a[0]) == ffi.sizeof("unsigned")
+ assert ffi.sizeof(b[0]) == ffi.sizeof(a[0])
diff --git a/extra_tests/cffi_tests/embedding/test_basic.py
b/extra_tests/cffi_tests/embedding/test_basic.py
--- a/extra_tests/cffi_tests/embedding/test_basic.py
+++ b/extra_tests/cffi_tests/embedding/test_basic.py
@@ -64,8 +64,8 @@
output = popen.stdout.read()
err = popen.wait()
if err:
- raise OSError("popen failed with exit code %r: %r" % (
- err, args))
+ raise OSError(("popen failed with exit code %r: %r\n\n%s" % (
+ err, args, output)).rstrip())
print(output.rstrip())
return output
diff --git a/extra_tests/cffi_tests/embedding/test_performance.py
b/extra_tests/cffi_tests/embedding/test_performance.py
--- a/extra_tests/cffi_tests/embedding/test_performance.py
+++ b/extra_tests/cffi_tests/embedding/test_performance.py
@@ -3,8 +3,8 @@
from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests
if sys.platform == 'win32':
- import py
- py.test.skip("written with POSIX functions")
+ import pytest
+ pytestmark = pytest.mark.skip("written with POSIX functions")
class TestPerformance(EmbeddingTests):
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py
b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -2,7 +2,6 @@
from binascii import hexlify
from ctypes import *
-from ctypes.test import xfail
def bin(s):
return hexlify(memoryview(s)).upper()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py
b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -2,7 +2,7 @@
import sys, unittest
import os
from ctypes.util import find_library
-from ctypes.test import is_resource_enabled, xfail
+from ctypes.test import is_resource_enabled
import test.test_support as support
libc_name = None
@@ -87,7 +87,6 @@
self.assertRaises(AttributeError, dll.__getitem__, 1234)
- @xfail
@unittest.skipUnless(os.name == "nt", 'Windows-specific test')
def test_1703286_A(self):
from _ctypes import LoadLibrary, FreeLibrary
@@ -99,7 +98,6 @@
handle = LoadLibrary("advapi32")
FreeLibrary(handle)
- @xfail
@unittest.skipUnless(os.name == "nt", 'Windows-specific test')
def test_1703286_B(self):
# Since on winXP 64-bit advapi32 loads like described
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py
b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -86,6 +86,7 @@
arch = platform.machine()
g['LDSHARED'] += ' -undefined dynamic_lookup'
g['CC'] += ' -arch %s' % (arch,)
+ g['MACOSX_DEPLOYMENT_TARGET'] = '10.14'
global _config_vars
_config_vars = g
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -801,7 +801,11 @@
ctx.set_ciphers("^$:,;?*'dorothyx")
@skip_if_broken_ubuntu_ssl
- def test_options(self):
+ def _test_options(self):
+ '''
+ Disable this test, it is too flaky. Different platforms define
+ different defaults
+ '''
ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
# OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value
default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3)
diff --git a/lib-python/2.7/test/test_timeit.py
b/lib-python/2.7/test/test_timeit.py
--- a/lib-python/2.7/test/test_timeit.py
+++ b/lib-python/2.7/test/test_timeit.py
@@ -317,9 +317,9 @@
def test_main_recommends_perf(self):
s = self.run_main(seconds_per_increment=2.0, switches=['-n35', '-s',
'print("CustomSetup")'])
self.assertIn(dedent("""\
- WARNING: timeit is a very unreliable tool. use perf or something
else for real measurements
+ WARNING: timeit is a very unreliable tool. use pyperf or something
else for real measurements
"""), s)
- self.assertIn("-m pip install perf", s)
+ self.assertIn("-m pip install pyperf", s)
diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py
--- a/lib-python/2.7/timeit.py
+++ b/lib-python/2.7/timeit.py
@@ -308,10 +308,10 @@
return 0
setup = "\n".join(setup) or "pass"
- print "WARNING: timeit is a very unreliable tool. use perf or something
else for real measurements"
+ print "WARNING: timeit is a very unreliable tool. use pyperf or something
else for real measurements"
executable = os.path.basename(sys.executable)
- print "%s -m pip install perf" % executable
- print "%s -m perf timeit %s" % (
+ print "%s -m pip install pyperf" % executable
+ print "%s -m pyperf timeit %s" % (
executable,
" ".join([(arg if arg.startswith("-") else repr(arg))
for arg in origargs]), )
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -177,7 +177,7 @@
RegrTest('test_copy_reg.py', core=True),
RegrTest('test_cpickle.py', core=True),
RegrTest('test_cprofile.py'),
- RegrTest('test_crypt.py', usemodules='crypt'),
+ RegrTest('test_crypt.py'),
RegrTest('test_csv.py', usemodules='_csv'),
RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"),
RegrTest('test_curses.py'),
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py
@@ -40,6 +40,11 @@
* supported
*/
static const long Cryptography_HAS_OP_NO_COMPRESSION;
+/* Internally invented symbol to tell us if SSL_OP_ENABLE_MIDDLEBOX_COMPAT is
+ * supported
+ */
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT;
+
static const long Cryptography_HAS_SSL_OP_MSIE_SSLV2_RSA_PADDING;
static const long Cryptography_HAS_SSL_SET_SSL_CTX;
static const long Cryptography_HAS_SSL_OP_NO_TICKET;
@@ -73,6 +78,7 @@
static const long SSL_OP_NETSCAPE_CHALLENGE_BUG;
static const long SSL_OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG;
static const long SSL_OP_SSLREF2_REUSE_CERT_TYPE_BUG;
+static const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT;
static const long SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER;
static const long SSL_OP_MSIE_SSLV2_RSA_PADDING;
static const long SSL_OP_SSLEAY_080_CLIENT_DH_BUG;
@@ -562,6 +568,13 @@
const long SSL_OP_NO_COMPRESSION = 0;
#endif
+#ifdef SSL_OP_ENABLE_MIDDLEBOX_COMPAT
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 1;
+#else
+static const long Cryptography_HAS_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+const long SSL_OP_ENABLE_MIDDLEBOX_COMPAT = 0;
+#endif
+
#ifdef SSL_OP_NO_TLSv1_1
static const long Cryptography_HAS_TLSv1_1 = 1;
#else
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -87,12 +87,12 @@
PROTOCOL_TLSv1 = 3
PROTOCOL_TLSv1_1 = 4
PROTOCOL_TLSv1_2 = 5
+# PROTOCOL_TLS_CLIENT = 0x10
+# PROTOCOL_TLS_SERVER = 0x11
if lib.Cryptography_HAS_TLSv1_3:
HAS_TLSv1_3 = True
else:
HAS_TLSv1_3 = False
-PROTOCOL_TLS_CLIENT = 0x10
-PROTOCOL_TLS_SERVER = 0x11
_PROTOCOL_NAMES = (name for name in dir(lib) if name.startswith('PROTOCOL_'))
@@ -219,6 +219,7 @@
def _new__ssl_socket(sslctx, sock, socket_type, server_hostname, ssl_sock):
self = _SSLSocket(sslctx)
ctx = sslctx.ctx
+ self.owner = ssl_sock # weakref
if server_hostname:
if isinstance(server_hostname, unicode):
@@ -289,7 +290,8 @@
def owner(self, value):
if value is None:
self._owner = None
- self._owner = weakref.ref(value)
+ else:
+ self._owner = weakref.ref(value)
@property
def context(self):
@@ -337,7 +339,7 @@
sockstate = SOCKET_OPERATION_OK
if sockstate == SOCKET_HAS_TIMED_OUT:
- raise socket.timeout("The handshake operation timed out")
+ raise SSLError("The handshake operation timed out")
elif sockstate == SOCKET_HAS_BEEN_CLOSED:
raise SSLError("Underlying socket has been closed.")
elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
@@ -781,10 +783,10 @@
method = lib.SSLv2_method()
elif protocol == PROTOCOL_SSLv23:
method = lib.SSLv23_method()
- elif protocol == PROTOCOL_TLS_CLIENT:
- method = lib.SSLv23_client_method()
- elif protocol == PROTOCOL_TLS_SERVER:
- method = lib.SSLv23_server_method()
+ # elif protocol == PROTOCOL_TLS_CLIENT:
+ # method = lib.SSLv23_client_method()
+ # elif protocol == PROTOCOL_TLS_SERVER:
+ # method = lib.SSLv23_server_method()
else:
raise ValueError("invalid protocol version")
@@ -795,7 +797,7 @@
# Don't check host name by default
self._check_hostname = False
- if protocol == PROTOCOL_TLS_CLIENT:
+ if 0 and protocol == PROTOCOL_TLS_CLIENT:
self._check_hostname = True
self.verify_mode = CERT_REQUIRED
else:
@@ -811,7 +813,7 @@
# Minimal security flags for server and client side context.
# Client sockets ignore server-side parameters.
options |= lib.SSL_OP_NO_COMPRESSION
- options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE
+ # options |= lib.SSL_OP_CIPHER_SERVER_PREFERENCE
options |= lib.SSL_OP_SINGLE_DH_USE
options |= lib.SSL_OP_SINGLE_ECDH_USE
lib.SSL_CTX_set_options(self.ctx, options)
diff --git a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
--- a/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/win32_extra.py
@@ -1,101 +1,101 @@
from _pypy_openssl import lib, ffi
-
-
-def enum_certificates(store_name):
- """Retrieve certificates from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide
-more cert storages, too. The function returns a list of (bytes,
-encoding_type, trust) tuples. The encoding_type flag can be interpreted
-with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
-a set of OIDs or the boolean True.
- """
- hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
- lib.CERT_STORE_READONLY_FLAG |
lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
- bytes(store_name, "ascii"))
- if hStore == ffi.NULL:
- raise WindowsError(*ffi.getwinerror())
-
- result = []
- pCertCtx = ffi.NULL
- try:
- while True:
- pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
- if pCertCtx == ffi.NULL:
- break
- cert = ffi.buffer(pCertCtx.pbCertEncoded,
pCertCtx.cbCertEncoded)[:]
- enc = certEncodingType(pCertCtx.dwCertEncodingType)
- keyusage = parseKeyUsage(pCertCtx,
lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
- if keyusage is True:
- keyusage = parseKeyUsage(pCertCtx,
lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
- result.append((cert, enc, keyusage))
- finally:
- if pCertCtx != ffi.NULL:
- lib.CertFreeCertificateContext(pCertCtx)
- if not lib.CertCloseStore(hStore, 0):
- # This error case might shadow another exception.
- raise WindowsError(*ffi.getwinerror())
- return result
-
-
-def enum_crls(store_name):
- """Retrieve CRLs from Windows' cert store.
-
-store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide
-more cert storages, too. The function returns a list of (bytes,
-encoding_type) tuples. The encoding_type flag can be interpreted with
-X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
- hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
- lib.CERT_STORE_READONLY_FLAG |
lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
- bytes(store_name, "ascii"))
- if hStore == ffi.NULL:
- raise WindowsError(*ffi.getwinerror())
-
- result = []
- pCrlCtx = ffi.NULL
- try:
- while True:
- pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
- if pCrlCtx == ffi.NULL:
- break
- crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
- enc = certEncodingType(pCrlCtx.dwCertEncodingType)
- result.append((crl, enc))
- finally:
- if pCrlCtx != ffi.NULL:
- lib.CertFreeCRLContext(pCrlCtx)
- if not lib.CertCloseStore(hStore, 0):
- # This error case might shadow another exception.
- raise WindowsError(*ffi.getwinerror())
- return result
-
-
-def certEncodingType(encodingType):
- if encodingType == lib.X509_ASN_ENCODING:
- return "x509_asn"
- if encodingType == lib.PKCS_7_ASN_ENCODING:
- return "pkcs_7_asn"
- return encodingType
-
-def parseKeyUsage(pCertCtx, flags):
- pSize = ffi.new("DWORD *")
- if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
- error_with_message = ffi.getwinerror()
- if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
- return True
- raise WindowsError(*error_with_message)
-
- pUsageMem = ffi.new("char[]", pSize[0])
- pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
- if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
- error_with_message = ffi.getwinerror()
- if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
- return True
- raise WindowsError(*error_with_message)
-
- retval = set()
- for i in range(pUsage.cUsageIdentifier):
- if pUsage.rgpszUsageIdentifier[i]:
- oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
- retval.add(oid)
- return retval
+
+
+def enum_certificates(store_name):
+ """Retrieve certificates from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide
+more cert storages, too. The function returns a list of (bytes,
+encoding_type, trust) tuples. The encoding_type flag can be interpreted
+with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either
+a set of OIDs or the boolean True.
+ """
+ hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+ lib.CERT_STORE_READONLY_FLAG |
lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+ bytes(store_name))
+ if hStore == ffi.NULL:
+ raise WindowsError(*ffi.getwinerror())
+
+ result = []
+ pCertCtx = ffi.NULL
+ try:
+ while True:
+ pCertCtx = lib.CertEnumCertificatesInStore(hStore, pCertCtx)
+ if pCertCtx == ffi.NULL:
+ break
+ cert = ffi.buffer(pCertCtx.pbCertEncoded,
pCertCtx.cbCertEncoded)[:]
+ enc = certEncodingType(pCertCtx.dwCertEncodingType)
+ keyusage = parseKeyUsage(pCertCtx,
lib.CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+ if keyusage is True:
+ keyusage = parseKeyUsage(pCertCtx,
lib.CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+ result.append((cert, enc, keyusage))
+ finally:
+ if pCertCtx != ffi.NULL:
+ lib.CertFreeCertificateContext(pCertCtx)
+ if not lib.CertCloseStore(hStore, 0):
+ # This error case might shadow another exception.
+ raise WindowsError(*ffi.getwinerror())
+ return result
+
+
+def enum_crls(store_name):
+ """Retrieve CRLs from Windows' cert store.
+
+store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide
+more cert storages, too. The function returns a list of (bytes,
+encoding_type) tuples. The encoding_type flag can be interpreted with
+X509_ASN_ENCODING or PKCS_7_ASN_ENCODING."""
+ hStore = lib.CertOpenStore(lib.CERT_STORE_PROV_SYSTEM_A, 0, ffi.NULL,
+ lib.CERT_STORE_READONLY_FLAG |
lib.CERT_SYSTEM_STORE_LOCAL_MACHINE,
+ bytes(store_name))
+ if hStore == ffi.NULL:
+ raise WindowsError(*ffi.getwinerror())
+
+ result = []
+ pCrlCtx = ffi.NULL
+ try:
+ while True:
+ pCrlCtx = lib.CertEnumCRLsInStore(hStore, pCrlCtx)
+ if pCrlCtx == ffi.NULL:
+ break
+ crl = ffi.buffer(pCrlCtx.pbCrlEncoded, pCrlCtx.cbCrlEncoded)[:]
+ enc = certEncodingType(pCrlCtx.dwCertEncodingType)
+ result.append((crl, enc))
+ finally:
+ if pCrlCtx != ffi.NULL:
+ lib.CertFreeCRLContext(pCrlCtx)
+ if not lib.CertCloseStore(hStore, 0):
+ # This error case might shadow another exception.
+ raise WindowsError(*ffi.getwinerror())
+ return result
+
+
+def certEncodingType(encodingType):
+ if encodingType == lib.X509_ASN_ENCODING:
+ return "x509_asn"
+ if encodingType == lib.PKCS_7_ASN_ENCODING:
+ return "pkcs_7_asn"
+ return encodingType
+
+def parseKeyUsage(pCertCtx, flags):
+ pSize = ffi.new("DWORD *")
+ if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, ffi.NULL, pSize):
+ error_with_message = ffi.getwinerror()
+ if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+ return True
+ raise WindowsError(*error_with_message)
+
+ pUsageMem = ffi.new("char[]", pSize[0])
+ pUsage = ffi.cast("PCERT_ENHKEY_USAGE", pUsageMem)
+ if not lib.CertGetEnhancedKeyUsage(pCertCtx, flags, pUsage, pSize):
+ error_with_message = ffi.getwinerror()
+ if error_with_message[0] == lib.CRYPT_E_NOT_FOUND:
+ return True
+ raise WindowsError(*error_with_message)
+
+ retval = set()
+ for i in range(pUsage.cUsageIdentifier):
+ if pUsage.rgpszUsageIdentifier[i]:
+ oid = ffi.string(pUsage.rgpszUsageIdentifier[i]).decode('ascii')
+ retval.add(oid)
+ return retval
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.12.3
+Version: 1.13.0
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
from .error import CDefError, FFIError, VerificationError, VerificationMissing
from .error import PkgConfigError
-__version__ = "1.12.3"
-__version_info__ = (1, 12, 3)
+__version__ = "1.13.0"
+__version_info__ = (1, 13, 0)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -145,6 +145,7 @@
int result;
PyGILState_STATE state;
PyObject *pycode=NULL, *global_dict=NULL, *x;
+ PyObject *builtins;
state = PyGILState_Ensure();
@@ -169,7 +170,7 @@
global_dict = PyDict_New();
if (global_dict == NULL)
goto error;
- PyObject *builtins = PyEval_GetBuiltins();
+ builtins = PyEval_GetBuiltins();
if (builtins == NULL)
goto error;
if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0)
@@ -223,7 +224,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.12.3"
+ "\ncompiled with cffi version: 1.13.0"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -858,19 +858,39 @@
"the actual array length in this context"
% exprnode.coord.line)
#
- if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and
- exprnode.op == '+'):
- return (self._parse_constant(exprnode.left) +
- self._parse_constant(exprnode.right))
- #
- if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and
- exprnode.op == '-'):
- return (self._parse_constant(exprnode.left) -
- self._parse_constant(exprnode.right))
+ if isinstance(exprnode, pycparser.c_ast.BinaryOp):
+ left = self._parse_constant(exprnode.left)
+ right = self._parse_constant(exprnode.right)
+ if exprnode.op == '+':
+ return left + right
+ elif exprnode.op == '-':
+ return left - right
+ elif exprnode.op == '*':
+ return left * right
+ elif exprnode.op == '/':
+ return self._c_div(left, right)
+ elif exprnode.op == '%':
+ return left - self._c_div(left, right) * right
+ elif exprnode.op == '<<':
+ return left << right
+ elif exprnode.op == '>>':
+ return left >> right
+ elif exprnode.op == '&':
+ return left & right
+ elif exprnode.op == '|':
+ return left | right
+ elif exprnode.op == '^':
+ return left ^ right
#
raise FFIError(":%d: unsupported expression: expected a "
"simple numeric constant" % exprnode.coord.line)
+ def _c_div(self, a, b):
+ result = a // b
+ if ((a < 0) ^ (b < 0)) and (a % b) != 0:
+ result += 1
+ return result
+
def _build_enum_type(self, explicit_name, decls):
if decls is not None:
partial = False
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -855,8 +855,9 @@
try:
if ftype.is_integer_type() or fbitsize >= 0:
# accept all integers, but complain on float or double
- prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is "
- "an integer */" % (fname, cname, fname))
+ if fname != '':
+ prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is
"
+ "an integer */" % (fname, cname, fname))
continue
# only accept exactly the type declared, except that '[]'
# is interpreted as a '*' and so will match any array length.
diff --git a/lib_pypy/crypt/__init__.py b/lib_pypy/crypt/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/crypt/__init__.py
@@ -0,0 +1,21 @@
+"""
+CFFI based implementation of the crypt module
+"""
+
+import sys
+import cffi
+
+ffi = cffi.FFI()
+ffi.cdef('char *crypt(char *word, char *salt);')
+
+try:
+ lib = ffi.dlopen('crypt')
+except OSError:
+ raise ImportError('crypt not available')
+
+
+def crypt(word, salt):
+ res = lib.crypt(word, salt)
+ if not res:
+ return None
+ return ffi.string(res)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -31,13 +31,13 @@
working_modules = default_modules.copy()
working_modules.update([
"_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
- "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
+ "select", "zipimport", "_lsprof", "signal", "_rawffi", "termios",
"zlib", "bz2", "struct", "_md5", "_sha", "_minimal_curses",
"cStringIO", "thread", "itertools", "pyexpat", "cpyext", "array",
"binascii", "_multiprocessing", '_warnings', "_collections",
"_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
"_csv", "_cppyy", "_pypyjson", "_jitlog",
- #" _ssl", "_hashlib"
+ #" _ssl", "_hashlib", "crypt"
])
import rpython.rlib.rvmprof.cintf
@@ -65,7 +65,8 @@
working_modules.add("_winreg")
# unix only modules
for name in ["crypt", "fcntl", "pwd", "termios", "_minimal_curses"]:
- working_modules.remove(name)
+ if name in working_modules:
+ working_modules.remove(name)
if name in translation_modules:
translation_modules.remove(name)
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,14 @@
.. branch: fix-vmprof-memory-tracking
Fix a bug that prevent memory-tracking in vmprof working on PyPy.
+
+.. branch: optimizeopt-cleanup
+
+Cleanup optimizeopt
+
+.. branch: copystrcontents-in-rewrite
+
+Remove ``copystrcontent`` and ``copyunicodecontent`` in the backends.
+Instead, replace it in ``rewrite.py`` with a direct call to ``memcpy()`` and
+new basic operation, ``load_effective_address``, which the backend can
+even decide not to implement.
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.12.3"
+VERSION = "1.13.0"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py
b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -677,11 +677,14 @@
return self.length
def _repr_extra(self):
- if self.w_keepalive is not None:
- name = self.space.type(self.w_keepalive).name
+ from pypy.module._cffi_backend import ctypearray
+ if self.w_keepalive is None:
+ return "buffer RELEASED"
+ obj_tp_name = self.space.type(self.w_keepalive).name
+ if isinstance(self.ctype, ctypearray.W_CTypeArray):
+ return "buffer len %d from '%s' object" % (self.length,
obj_tp_name)
else:
- name = "(released)"
- return "buffer len %d from '%s' object" % (self.length, name)
+ return "buffer from '%s' object" % (obj_tp_name,)
def enter_exit(self, exit_now):
# for now, limited effect on PyPy
diff --git a/pypy/module/_cffi_backend/func.py
b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -112,9 +112,10 @@
@unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int)
def from_buffer(space, w_ctype, w_x, require_writable=0):
- from pypy.module._cffi_backend import ctypearray
- if not isinstance(w_ctype, ctypearray.W_CTypeArray):
- raise oefmt(space.w_TypeError, "expected an array ctype, got '%s'",
+ from pypy.module._cffi_backend import ctypeptr, ctypearray
+ if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray):
+ raise oefmt(space.w_TypeError,
+ "expected a poiunter or array ctype, got '%s'",
w_ctype.name)
if space.isinstance_w(w_x, space.w_unicode):
raise oefmt(space.w_TypeError,
@@ -135,33 +136,36 @@
"raw address on PyPy", w_x)
#
buffersize = buf.getlength()
- arraylength = w_ctype.length
- if arraylength >= 0:
- # it's an array with a fixed length; make sure that the
- # buffer contains enough bytes.
- if buffersize < w_ctype.size:
- raise oefmt(space.w_ValueError,
- "buffer is too small (%d bytes) for '%s' (%d bytes)",
- buffersize, w_ctype.name, w_ctype.size)
+ if not isinstance(w_ctype, ctypearray.W_CTypeArray):
+ arraylength = buffersize # number of bytes, not used so far
else:
- # it's an open 'array[]'
- itemsize = w_ctype.ctitem.size
- if itemsize == 1:
- # fast path, performance only
- arraylength = buffersize
- elif itemsize > 0:
- # give it as many items as fit the buffer. Ignore a
- # partial last element.
- arraylength = buffersize / itemsize
+ arraylength = w_ctype.length
+ if arraylength >= 0:
+ # it's an array with a fixed length; make sure that the
+ # buffer contains enough bytes.
+ if buffersize < w_ctype.size:
+ raise oefmt(space.w_ValueError,
+ "buffer is too small (%d bytes) for '%s' (%d bytes)",
+ buffersize, w_ctype.name, w_ctype.size)
else:
- # it's an array 'empty[]'. Unsupported obscure case:
- # the problem is that setting the length of the result
- # to anything large (like SSIZE_T_MAX) is dangerous,
- # because if someone tries to loop over it, it will
- # turn effectively into an infinite loop.
- raise oefmt(space.w_ZeroDivisionError,
- "from_buffer('%s', ..): the actual length of the array "
- "cannot be computed", w_ctype.name)
+ # it's an open 'array[]'
+ itemsize = w_ctype.ctitem.size
+ if itemsize == 1:
+ # fast path, performance only
+ arraylength = buffersize
+ elif itemsize > 0:
+ # give it as many items as fit the buffer. Ignore a
+ # partial last element.
+ arraylength = buffersize / itemsize
+ else:
+ # it's an array 'empty[]'. Unsupported obscure case:
+ # the problem is that setting the length of the result
+ # to anything large (like SSIZE_T_MAX) is dangerous,
+ # because if someone tries to loop over it, it will
+ # turn effectively into an infinite loop.
+ raise oefmt(space.w_ZeroDivisionError,
+ "from_buffer('%s', ..): the actual length of the array "
+ "cannot be computed", w_ctype.name)
#
return cdataobj.W_CDataFromBuffer(space, _cdata, arraylength,
w_ctype, buf, w_x)
diff --git a/pypy/module/_cffi_backend/newtype.py
b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -549,10 +549,11 @@
if sflags & SF_GCC_BIG_ENDIAN:
bitshift = 8 * ftype.size - fbitsize- bitshift
- fld = ctypestruct.W_CField(ftype, field_offset_bytes,
- bitshift, fbitsize, fflags)
- fields_list.append(fld)
- fields_dict[fname] = fld
+ if fname != '':
+ fld = ctypestruct.W_CField(ftype, field_offset_bytes,
+ bitshift, fbitsize, fflags)
+ fields_list.append(fld)
+ fields_dict[fname] = fld
if boffset > boffsetmax:
boffsetmax = boffset
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.12.3", ("This test_c.py file is for testing a version"
+assert __version__ == "1.13.0", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
@@ -3830,7 +3830,9 @@
BIntP = new_pointer_type(BInt)
BIntA = new_array_type(BIntP, None)
lst = [-12345678, 87654321, 489148]
- bytestring = buffer(newp(BIntA, lst))[:] + b'XYZ'
+ bytestring = bytearray(buffer(newp(BIntA, lst))[:] + b'XYZ')
+ lst2 = lst + [42, -999999999]
+ bytestring2 = bytearray(buffer(newp(BIntA, lst2))[:] + b'XYZ')
#
p1 = from_buffer(BIntA, bytestring) # int[]
assert typeof(p1) is BIntA
@@ -3844,7 +3846,19 @@
p1[-1]
#
py.test.raises(TypeError, from_buffer, BInt, bytestring)
- py.test.raises(TypeError, from_buffer, BIntP, bytestring)
+ #
+ p2 = from_buffer(BIntP, bytestring) # int *
+ assert p2 == p1 or 'PY_DOT_PY' in globals()
+ # note: on py.py ^^^, bytearray buffers are not emulated well enough
+ assert typeof(p2) is BIntP
+ assert p2[0] == lst[0]
+ assert p2[1] == lst[1]
+ assert p2[2] == lst[2]
+ # hopefully does not crash, but doesn't raise an exception:
+ p2[3]
+ p2[-1]
+ # not enough data even for one, but this is not enforced:
+ from_buffer(BIntP, b"")
#
BIntA2 = new_array_type(BIntP, 2)
p2 = from_buffer(BIntA2, bytestring) # int[2]
@@ -3856,7 +3870,7 @@
p2[2]
with pytest.raises(IndexError):
p2[-1]
- assert p2 == p1
+ assert p2 == p1 or 'PY_DOT_PY' in globals()
#
BIntA4 = new_array_type(BIntP, 4) # int[4]: too big
py.test.raises(ValueError, from_buffer, BIntA4, bytestring)
@@ -3866,13 +3880,37 @@
('a2', BInt, -1)])
BStructP = new_pointer_type(BStruct)
BStructA = new_array_type(BStructP, None)
- p1 = from_buffer(BStructA, bytestring) # struct[]
- assert len(p1) == 1
+ p1 = from_buffer(BStructA, bytestring2) # struct[]
+ assert len(p1) == 2
assert typeof(p1) is BStructA
- assert p1[0].a1 == lst[0]
- assert p1[0].a2 == lst[1]
+ assert p1[0].a1 == lst2[0]
+ assert p1[0].a2 == lst2[1]
+ assert p1[1].a1 == lst2[2]
+ assert p1[1].a2 == lst2[3]
with pytest.raises(IndexError):
- p1[1]
+ p1[2]
+ with pytest.raises(IndexError):
+ p1[-1]
+ assert repr(p1) == "<cdata 'foo[]' buffer len 2 from 'bytearray' object>"
+ #
+ p2 = from_buffer(BStructP, bytestring2) # 'struct *'
+ assert p2 == p1 or 'PY_DOT_PY' in globals()
+ assert typeof(p2) is BStructP
+ assert p2.a1 == lst2[0]
+ assert p2.a2 == lst2[1]
+ assert p2[0].a1 == lst2[0]
+ assert p2[0].a2 == lst2[1]
+ assert p2[1].a1 == lst2[2]
+ assert p2[1].a2 == lst2[3]
+ # does not crash:
+ p2[2]
+ p2[-1]
+ # not enough data even for one, but this is not enforced:
+ from_buffer(BStructP, b"")
+ from_buffer(BStructP, b"1234567")
+ #
+ release(p1)
+ assert repr(p1) == "<cdata 'foo[]' buffer RELEASED>"
#
BEmptyStruct = new_struct_type("empty")
complete_struct_or_union(BEmptyStruct, [], Ellipsis, 0)
@@ -3886,7 +3924,37 @@
p1 = from_buffer(BEmptyStructA5, bytestring) # struct empty[5]
assert typeof(p1) is BEmptyStructA5
assert len(p1) == 5
- assert cast(BIntP, p1) == from_buffer(BIntA, bytestring)
+ assert (cast(BIntP, p1) == from_buffer(BIntA, bytestring)
+ or 'PY_DOT_PY' in globals())
+ #
+ BVarStruct = new_struct_type("varfoo")
+ BVarStructP = new_pointer_type(BVarStruct)
+ complete_struct_or_union(BVarStruct, [('a1', BInt, -1),
+ ('va', BIntA, -1)])
+ with pytest.raises(TypeError):
+ from_buffer(BVarStruct, bytestring)
+ pv = from_buffer(BVarStructP, bytestring) # varfoo *
+ assert pv.a1 == lst[0]
+ assert pv.va[0] == lst[1]
+ assert pv.va[1] == lst[2]
+ assert sizeof(pv[0]) == 1 * size_of_int()
+ with pytest.raises(TypeError):
+ len(pv.va)
+ # hopefully does not crash, but doesn't raise an exception:
+ pv.va[2]
+ pv.va[-1]
+ # not enough data even for one, but this is not enforced:
+ from_buffer(BVarStructP, b"")
+ assert repr(pv) == "<cdata 'varfoo *' buffer from 'bytearray' object>"
+ assert repr(pv[0]).startswith("<cdata 'varfoo &' ")
+ #
+ release(pv)
+ assert repr(pv) == "<cdata 'varfoo *' buffer RELEASED>"
+ assert repr(pv[0]).startswith("<cdata 'varfoo &' ")
+ #
+ pv = from_buffer(BVarStructP, bytestring) # make a fresh one
+ with pytest.raises(ValueError):
+ release(pv[0])
def test_memmove():
Short = new_primitive_type("short")
@@ -4312,8 +4380,10 @@
BCharA = new_array_type(BCharP, None)
p = from_buffer(BCharA, a)
assert p[2] == b"z"
+ assert repr(p) == "<cdata 'char[]' buffer len 3 from 'bytearray' object>"
release(p)
assert p[2] == b"z" # true so far, but might change to raise RuntimeError
+ assert repr(p) == "<cdata 'char[]' buffer RELEASED>"
release(p) # no effect
def test_explicit_release_from_buffer_contextmgr():
@@ -4325,6 +4395,7 @@
with p:
assert p[2] == b"z"
assert p[2] == b"z" # true so far, but might change to raise RuntimeError
+ assert repr(p) == "<cdata 'char[]' buffer RELEASED>"
release(p) # no effect
def test_explicit_release_bytearray_on_cpython():
diff --git a/pypy/module/cpyext/bufferobject.py
b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -5,6 +5,7 @@
cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, slot_function,
PyObjectFields, PyObject)
from pypy.module.cpyext.pyobject import make_typedescr, decref, make_ref
+from pypy.module.cpyext.buffer import CBuffer
from pypy.module.array.interp_array import ArrayBuffer
from pypy.objspace.std.bufferobject import W_Buffer
@@ -33,7 +34,7 @@
def buffer_attach(space, py_obj, w_obj, w_userdata=None):
"""
- Fills a newly allocated PyBufferObject with the given (str) buffer object.
+ Fills a newly allocated PyBufferObject with the given buffer object.
"""
py_buf = rffi.cast(PyBufferObject, py_obj)
py_buf.c_b_offset = 0
@@ -60,7 +61,17 @@
py_buf.c_b_base = make_ref(space, w_base)
py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.w_array._charbuf_start())
py_buf.c_b_size = buf.getlength()
+ elif isinstance(buf, CBuffer):
+ py_buf.c_b_base = make_ref(space, buf.view.w_obj)
+ py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.view.ptr)
+ py_buf.c_b_size = buf.getlength()
else:
+ # Raising in attach will segfault.
+ # It would be nice if we could handle the error more gracefully
+ # with something like this
+ # py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+ # py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, 0)
+ # py_buf.c_b_size = buf.getlength()
raise oefmt(space.w_NotImplementedError, "buffer flavor not supported")
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2573,6 +2573,11 @@
Py_RETURN_NONE;
}
+static PyObject *
+passthrough(PyObject *self, PyObject* args) {
+ Py_INCREF(args);
+ return args;
+}
/*********************** Install Module **************************/
static PyMethodDef a_methods[] = {
@@ -2584,6 +2589,7 @@
{"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL},
{"getitem", (PyCFunction)getitem, METH_VARARGS, NULL},
{"subclass_with_attribute", (PyCFunction)subclass_with_attribute,
METH_VARARGS, NULL},
+ {"passthrough", (PyCFunction)passthrough, METH_O, NULL},
{NULL, NULL, 0, NULL} /* Sentinel */
};
diff --git a/pypy/module/cpyext/test/test_arraymodule.py
b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -79,6 +79,9 @@
assert str(buffer('a') + arr) == "a" + expected
# python2 special cases empty-buffer + obj
assert str(buffer('') + arr) == "array('i', [1, 2, 3, 4])"
+ # make sure buffer_attach is called
+ buf2 = module.passthrough(buf)
+ assert str(buf2) == str(buf)
def test_releasebuffer(self):
module = self.import_module(name='array')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -425,9 +425,11 @@
setarrayitem_raw(i153, 0, i106, descr=...)
p156 = getfield_gc_r(p48, descr=...)
i158 = getfield_raw_i(..., descr=...)
+ i160 = int_sub(i158, 16)
+ setfield_raw(#, i160, descr=...)
setfield_gc(p48, p49, descr=...)
setfield_gc(p134, ConstPtr(null), descr=...)
- i160 = int_lt(i158, 0)
+ i160 = int_lt(i160, 0)
guard_false(i160, descr=...)
jump(..., descr=...)
""")
diff --git a/pypy/module/thread/test/test_local.py
b/pypy/module/thread/test/test_local.py
--- a/pypy/module/thread/test/test_local.py
+++ b/pypy/module/thread/test/test_local.py
@@ -5,6 +5,7 @@
def test_local_1(self):
import thread
+ import gc
from thread import _local as tlsobject
freed = []
class X:
@@ -34,8 +35,9 @@
thread.start_new_thread(f, (i,))
self.waitfor(lambda: len(ok) == 20, delay=3)
assert ok == 20*[True] # see stdout/stderr for failures in the threads
+ gc.collect(); gc.collect(); gc.collect()
- self.waitfor(lambda: len(freed) >= 40)
+ self.waitfor(lambda: len(freed) >= 40, delay=20)
assert len(freed) == 40
# in theory, all X objects should have been freed by now. Note that
# Python's own thread._local objects suffer from the very same "bug"
that
diff --git a/pypy/objspace/std/dictmultiobject.py
b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -78,8 +78,9 @@
W_ModuleDictObject.__init__(w_obj, space, strategy, storage)
return w_obj
elif instance:
- from pypy.objspace.std.mapdict import MapDictStrategy
- strategy = space.fromcache(MapDictStrategy)
+ from pypy.objspace.std.mapdict import make_instance_dict
+ assert w_type is None
+ return make_instance_dict(space)
elif strdict or module:
assert w_type is None
strategy = space.fromcache(BytesDictStrategy)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -753,6 +753,7 @@
self.space = space
def get_empty_storage(self):
+ # mainly used for tests
w_result = Object()
terminator = self.space.fromcache(get_terminator_for_dicts)
w_result._mapdict_init_empty(terminator)
@@ -865,6 +866,11 @@
def iteritems(self, w_dict):
return MapDictIteratorItems(self.space, self, w_dict)
+def make_instance_dict(space):
+ w_fake_object = Object()
+ terminator = space.fromcache(get_terminator_for_dicts)
+ w_fake_object._mapdict_init_empty(terminator)
+ return w_fake_object.getdict(space)
def materialize_r_dict(space, obj, dict_w):
map = obj._get_mapdict_map()
diff --git a/pypy/objspace/std/test/test_mapdict.py
b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -897,6 +897,17 @@
d = x.__dict__
assert list(__pypy__.reversed_dict(d)) == d.keys()[::-1]
+ def test_bug_materialize_huge_dict(self):
+ import __pypy__
+ d = __pypy__.newdict("instance")
+ for i in range(100):
+ d[str(i)] = i
+ assert len(d) == 100
+
+ for key in d:
+ assert d[key] == int(key)
+
+
class AppTestWithMapDictAndCounters(object):
spaceconfig = {"objspace.std.withmethodcachecounter": True}
diff --git a/pypy/objspace/std/unicodeobject.py
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1,5 +1,7 @@
"""The builtin unicode implementation"""
+import sys
+
from rpython.rlib.objectmodel import (
compute_hash, compute_unique_id, import_from_mixin, always_inline,
enforceargs, newlist_hint, specialize, we_are_translated)
@@ -41,13 +43,12 @@
self._utf8 = utf8str
self._length = length
self._index_storage = rutf8.null_storage()
- if not we_are_translated():
+ if not we_are_translated() and not sys.platform == 'win32':
# utf8str must always be a valid utf8 string, except maybe with
# explicit surrogate characters---which .decode('utf-8') doesn't
# special-case in Python 2, which is exactly what we want here
assert length == len(utf8str.decode('utf-8'))
-
@staticmethod
def from_utf8builder(builder):
return W_UnicodeObject(
@@ -1095,11 +1096,11 @@
if rutf8.has_surrogates(utf8):
utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
return space.newbytes(utf8)
- return encode(space, w_obj, encoding, errors)
+ return encode(space, w_obj, encoding, errors)
def decode_object(space, w_obj, encoding, errors):
- from pypy.module._codecs.interp_codecs import lookup_codec, decode
+ from pypy.module._codecs.interp_codecs import lookup_codec, decode
if errors is None or errors == 'strict':
# fast paths
if encoding is None:
@@ -1109,7 +1110,7 @@
unicodehelper.check_ascii_or_raise(space, s)
return space.newutf8(s, len(s))
if encoding == 'utf-8' or encoding == 'utf8':
- if (space.isinstance_w(w_obj, space.w_unicode) or
+ if (space.isinstance_w(w_obj, space.w_unicode) or
space.isinstance_w(w_obj, space.w_bytes)):
s = space.utf8_w(w_obj)
else:
@@ -1718,34 +1719,28 @@
def unicode_to_decimal_w(space, w_unistr):
if not isinstance(w_unistr, W_UnicodeObject):
raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr)
- unistr = w_unistr._utf8
- result = ['\0'] * w_unistr._length
- digits = ['0', '1', '2', '3', '4',
- '5', '6', '7', '8', '9']
- res_pos = 0
- iter = rutf8.Utf8StringIterator(unistr)
- for uchr in iter:
+ utf8 = w_unistr._utf8
+ result = StringBuilder(w_unistr._len())
+ it = rutf8.Utf8StringIterator(utf8)
+ for uchr in it:
if W_UnicodeObject._isspace(uchr):
- result[res_pos] = ' '
- res_pos += 1
+ result.append(' ')
continue
- try:
- result[res_pos] = digits[unicodedb.decimal(uchr)]
- except KeyError:
- if 0 < uchr < 256:
- result[res_pos] = chr(uchr)
- else:
+ if not (0 < uchr < 256):
+ try:
+ uchr = ord('0') + unicodedb.decimal(uchr)
+ except KeyError:
w_encoding = space.newtext('decimal')
- pos = iter.get_pos()
+ pos = it.get_pos()
w_start = space.newint(pos)
- w_end = space.newint(pos+1)
+ w_end = space.newint(pos + 1)
w_reason = space.newtext('invalid decimal Unicode string')
raise OperationError(space.w_UnicodeEncodeError,
- space.newtuple([w_encoding, w_unistr,
- w_start, w_end,
- w_reason]))
- res_pos += 1
- return ''.join(result)
+ space.newtuple([w_encoding, w_unistr,
+ w_start, w_end,
+ w_reason]))
+ result.append(chr(uchr))
+ return result.build()
_repr_function = rutf8.make_utf8_escape_function(
diff --git a/requirements.txt b/requirements.txt
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+--only-binary vmprof
+
cffi>=1.4.0
# parse log files in rvmprof tests
diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst
--- a/rpython/doc/jit/optimizer.rst
+++ b/rpython/doc/jit/optimizer.rst
@@ -13,7 +13,7 @@
Before some optimizations are explained in more detail, it is essential to
understand how traces look like.
-The optimizer comes with a test suit. It contains many trace
+The optimizer comes with a test suite. It contains many trace
examples and you might want to take a look at it
(in `rpython/jit/metainterp/optimizeopt/test/*.py`).
The allowed operations can be found in
`rpython/jit/metainterp/resoperation.py`.
@@ -21,7 +21,7 @@
[p0,i0,i1]
label(p0, i0, i1)
- i2 = getarray_item_raw(p0, i0, descr=<Array Signed>)
+ i2 = getarrayitem_raw(p0, i0, descr=<Array Signed>)
i3 = int_add(i1,i2)
i4 = int_add(i0,1)
i5 = int_le(i4, 100) # lower-or-equal
@@ -32,7 +32,7 @@
to compare the Python code that constructed the trace::
from array import array
- a = array('i',range(101))
+ a = array('i', range(101))
sum = 0; i = 0
while i <= 100: # can be seen as label
sum += a[i]
@@ -131,20 +131,16 @@
Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is
emitted. Instead the variable y is made equal to 0
-(= ``make_equal_to(op.result, 0)``). The variables found in a trace are
-instances of Box classes that can be found in
-`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again
-and maps the boxes to the optimization values in the optimizer. When a
-value is made equal, the two variable's boxes are made to point to the same
-`OptValue` instance.
+(= ``make_constant_int(op, 0)``). The variables found in a trace are instances
+of classes that can be found in `rpython/jit/metainterp/history.py`. When a
+value is made equal to another, its box is made to point to the other one.
-**NOTE: this OptValue organization is currently being refactored in a branch.**
Pure optimization
-----------------
-Is interwoven into the basic optimizer. It saves operations, results,
-arguments to be known to have pure semantics.
+The 'pure' optimizations interwoven into the basic optimizer. It saves
+operations, results, arguments to be known to have pure semantics.
"Pure" here means the same as the ``jit.elidable`` decorator:
free of "observable" side effects and referentially transparent
diff --git a/rpython/jit/backend/arm/opassembler.py
b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -834,73 +834,12 @@
else:
assert 0
- #from ../x86/regalloc.py:928 ff.
- def emit_op_copystrcontent(self, op, arglocs, regalloc, fcond):
- assert len(arglocs) == 0
- self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False)
+ def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond):
+ static_ofs = op.getarg(2).getint()
+ scale = op.getarg(3).getint()
+ self._gen_address(arglocs[2], arglocs[0], arglocs[1], scale,
static_ofs)
return fcond
- def emit_op_copyunicodecontent(self, op, arglocs, regalloc, fcond):
- assert len(arglocs) == 0
- self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True)
- return fcond
-
- def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode):
- # compute the source address
- args = op.getarglist()
- base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args)
- ofs_loc = regalloc.rm.make_sure_var_in_reg(args[2], args)
- assert args[0] is not args[1] # forbidden case of aliasing
- srcaddr_box = TempVar()
- forbidden_vars = [args[1], args[3], args[4], srcaddr_box]
- srcaddr_loc = regalloc.rm.force_allocate_reg(srcaddr_box,
forbidden_vars)
- self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc,
- is_unicode=is_unicode)
- # compute the destination address
- base_loc = regalloc.rm.make_sure_var_in_reg(args[1], forbidden_vars)
- ofs_loc = regalloc.rm.make_sure_var_in_reg(args[3], forbidden_vars)
- forbidden_vars = [args[4], srcaddr_box]
- dstaddr_box = TempVar()
- dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box,
forbidden_vars)
- self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc,
- is_unicode=is_unicode)
- # compute the length in bytes
- length_box = args[4]
- length_loc = regalloc.loc(length_box)
- if is_unicode:
- forbidden_vars = [srcaddr_box, dstaddr_box]
- bytes_box = TempVar()
- bytes_loc = regalloc.rm.force_allocate_reg(bytes_box,
forbidden_vars)
- scale = self._get_unicode_item_scale()
- if not length_loc.is_core_reg():
- self.regalloc_mov(length_loc, bytes_loc)
- length_loc = bytes_loc
- assert length_loc.is_core_reg()
- self.mc.MOV_ri(r.ip.value, 1 << scale)
- self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value)
- length_box = bytes_box
- length_loc = bytes_loc
- # call memcpy()
- regalloc.before_call()
- self.simple_call_no_collect(imm(self.memcpy_addr),
- [dstaddr_loc, srcaddr_loc, length_loc])
- regalloc.rm.possibly_free_var(length_box)
- regalloc.rm.possibly_free_var(dstaddr_box)
- regalloc.rm.possibly_free_var(srcaddr_box)
-
- def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
- if is_unicode:
- ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
- self.cpu.translate_support_code)
- scale = self._get_unicode_item_scale()
- else:
- ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
- self.cpu.translate_support_code)
- assert itemsize == 1
- ofs_items -= 1 # for the extra null character
- scale = 0
- self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items)
-
# result = base_loc + (scaled_loc << scale) + static_offset
def _gen_address(self, result, base_loc, scaled_loc, scale=0,
static_offset=0):
assert scaled_loc.is_core_reg()
@@ -915,16 +854,6 @@
self.mc.ADD_rr(result.value, base_loc.value, scaled_loc.value)
self.mc.ADD_ri(result.value, result.value, static_offset)
- def _get_unicode_item_scale(self):
- _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE,
- self.cpu.translate_support_code)
- if itemsize == 4:
- return 2
- elif itemsize == 2:
- return 1
- else:
- raise AssertionError("bad unicode item size")
-
def store_force_descr(self, op, fail_locs, frame_depth):
pos = self.mc.currpos()
guard_token = self.build_guard_token(op, frame_depth, fail_locs, pos,
c.AL)
diff --git a/rpython/jit/backend/arm/regalloc.py
b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -873,8 +873,6 @@
prepare_op_gc_load_indexed_r = _prepare_op_gc_load_indexed
prepare_op_gc_load_indexed_f = _prepare_op_gc_load_indexed
- prepare_op_copystrcontent = void
- prepare_op_copyunicodecontent = void
prepare_op_zero_array = void
def _prepare_op_same_as(self, op, fcond):
@@ -899,6 +897,13 @@
resloc = self.force_allocate_reg(op)
return [resloc]
+ def prepare_op_load_effective_address(self, op, fcond):
+ args = op.getarglist()
+ arg0 = self.make_sure_var_in_reg(args[0], args)
+ arg1 = self.make_sure_var_in_reg(args[1], args)
+ res = self.force_allocate_reg(op)
+ return [arg0, arg1, res]
+
def prepare_op_call_malloc_nursery(self, op, fcond):
size_box = op.getarg(0)
assert isinstance(size_box, ConstInt)
diff --git a/rpython/jit/backend/arm/runner.py
b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -23,6 +23,7 @@
supports_floats = True
supports_longlong = True
supports_singlefloats = True
+ supports_load_effective_address = True
from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE
all_reg_indexes = range(len(all_regs))
diff --git a/rpython/jit/backend/llsupport/gc.py
b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -14,6 +14,7 @@
from rpython.jit.metainterp.support import ptr2int
from rpython.jit.backend.llsupport import symbolic, jitframe
from rpython.jit.backend.llsupport.symbolic import WORD
+from rpython.jit.backend.llsupport.memcpy import memcpy_fn
from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr,
FieldDescr
from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr
from rpython.jit.backend.llsupport.descr import get_array_descr
@@ -36,6 +37,11 @@
self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT,
'typeptr')
self._generated_functions = []
+ self.memcpy_fn = memcpy_fn
+ self.memcpy_descr = get_call_descr(self,
+ [lltype.Signed, lltype.Signed, lltype.Signed], lltype.Void,
+ EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE,
+ can_collect=False))
def _setup_str(self):
self.str_descr = get_array_descr(self, rstr.STR)
diff --git a/rpython/jit/backend/llsupport/rewrite.py
b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -34,6 +34,10 @@
- Add COND_CALLs to the write barrier before SETFIELD_GC and
SETARRAYITEM_GC operations.
+ - Rewrites copystrcontent to a call to memcopy
+
+ - XXX does more than that, please write it down
+
'_write_barrier_applied' contains a dictionary of variable -> None.
If a variable is in the dictionary, next setfields can be called without
a write barrier. The idea is that an object that was freshly allocated
@@ -335,6 +339,10 @@
self.emitting_an_operation_that_can_collect()
elif op.getopnum() == rop.LABEL:
self.emit_label()
+ # ---- change COPY{STR|UNICODE}CONTENT into a call ------
+ if op.opnum == rop.COPYSTRCONTENT or op.opnum ==
rop.COPYUNICODECONTENT:
+ self.rewrite_copy_str_content(op)
+ continue
# ---------- write barriers ----------
if self.gc_ll_descr.write_barrier_descr is not None:
if op.getopnum() == rop.SETFIELD_GC:
@@ -953,6 +961,61 @@
self.gcrefs_output_list.append(gcref)
return index
+ def rewrite_copy_str_content(self, op):
+ funcaddr = llmemory.cast_ptr_to_adr(self.gc_ll_descr.memcpy_fn)
+ memcpy_fn = self.cpu.cast_adr_to_int(funcaddr)
+ memcpy_descr = self.gc_ll_descr.memcpy_descr
+ if op.getopnum() == rop.COPYSTRCONTENT:
+ basesize = self.gc_ll_descr.str_descr.basesize
+ # because we have one extra item after alloc, the actual address
+ # of string start is 1 lower, from extra_item_after_malloc
+ basesize -= 1
+ assert self.gc_ll_descr.str_descr.itemsize == 1
+ itemscale = 0
+ else:
+ basesize = self.gc_ll_descr.unicode_descr.basesize
+ itemsize = self.gc_ll_descr.unicode_descr.itemsize
+ if itemsize == 2:
+ itemscale = 1
+ elif itemsize == 4:
+ itemscale = 2
+ else:
+ assert False, "unknown size of unicode"
+ i1 = self.emit_load_effective_address(op.getarg(0), op.getarg(2),
+ basesize, itemscale)
+ i2 = self.emit_load_effective_address(op.getarg(1), op.getarg(3),
+ basesize, itemscale)
+ if op.getopnum() == rop.COPYSTRCONTENT:
+ arg = op.getarg(4)
+ else:
+ # do some basic constant folding
+ if isinstance(op.getarg(4), ConstInt):
+ arg = ConstInt(op.getarg(4).getint() << itemscale)
+ else:
+ arg = ResOperation(rop.INT_LSHIFT,
+ [op.getarg(4), ConstInt(itemscale)])
+ self.emit_op(arg)
+ self.emit_op(ResOperation(rop.CALL_N,
+ [ConstInt(memcpy_fn), i2, i1, arg], descr=memcpy_descr))
+
+ def emit_load_effective_address(self, v_gcptr, v_index, base, itemscale):
+ if self.cpu.supports_load_effective_address:
+ i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS,
+ [v_gcptr, v_index, ConstInt(base),
+ ConstInt(itemscale)])
+ self.emit_op(i1)
+ return i1
+ else:
+ if itemscale > 0:
+ v_index = ResOperation(rop.INT_LSHIFT,
+ [v_index, ConstInt(itemscale)])
+ self.emit_op(v_index)
+ i1b = ResOperation(rop.INT_ADD, [v_gcptr, v_index])
+ self.emit_op(i1b)
+ i1 = ResOperation(rop.INT_ADD, [i1b, ConstInt(base)])
+ self.emit_op(i1)
+ return i1
+
def remove_constptr(self, c):
"""Remove all ConstPtrs, and replace them with load_from_gc_table.
"""
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py
b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -142,11 +142,16 @@
raw_sfdescr = get_array_descr(self.gc_ll_descr, RAW_SF)
#
strdescr = self.gc_ll_descr.str_descr
+ str_basesize = self.gc_ll_descr.str_descr.basesize - 1
unicodedescr = self.gc_ll_descr.unicode_descr
strlendescr = strdescr.lendescr
unicodelendescr = unicodedescr.lendescr
strhashdescr = self.gc_ll_descr.str_hash_descr
unicodehashdescr = self.gc_ll_descr.unicode_hash_descr
+ uni_basesize = unicodedescr.basesize
+ uni_itemscale = {2: 1, 4: 2}[unicodedescr.itemsize]
+ memcpy_fn = self.gc_ll_descr.memcpy_fn
+ memcpy_descr = self.gc_ll_descr.memcpy_descr
casmdescr = JitCellToken()
clt = FakeLoopToken()
@@ -169,6 +174,7 @@
signedframedescr = self.cpu.signedframedescr
floatframedescr = self.cpu.floatframedescr
casmdescr.compiled_loop_token = clt
+
#
guarddescr = AbstractFailDescr()
#
@@ -200,6 +206,7 @@
load_constant_offset = True
load_supported_factors = (1,2,4,8)
+ supports_load_effective_address = True
translate_support_code = None
@@ -237,6 +244,9 @@
self._cache[key] = r
return r
+ def cast_adr_to_int(self, adr):
+ return llmemory.AddressAsInt(adr)
+
class TestBoehm(RewriteTests):
def setup_method(self, meth):
class FakeCPU(BaseFakeCPU):
@@ -1436,3 +1446,57 @@
jump()
""")
assert len(self.gcrefs) == 2
+
+ def test_rewrite_copystrcontents(self):
+ self.check_rewrite("""
+ [p0, p1, i0, i1, i_len]
+ copystrcontent(p0, p1, i0, i1, i_len)
+ """, """
+ [p0, p1, i0, i1, i_len]
+ i2 = load_effective_address(p0, i0, %(str_basesize)s, 0)
+ i3 = load_effective_address(p1, i1, %(str_basesize)s, 0)
+ call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr)
+ """)
+
+ def test_rewrite_copystrcontents_without_load_effective_address(self):
+ self.cpu.supports_load_effective_address = False
+ self.check_rewrite("""
+ [p0, p1, i0, i1, i_len]
+ copystrcontent(p0, p1, i0, i1, i_len)
+ """, """
+ [p0, p1, i0, i1, i_len]
+ i2b = int_add(p0, i0)
+ i2 = int_add(i2b, %(str_basesize)s)
+ i3b = int_add(p1, i1)
+ i3 = int_add(i3b, %(str_basesize)s)
+ call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr)
+ """)
+
+ def test_rewrite_copyunicodecontents(self):
+ self.check_rewrite("""
+ [p0, p1, i0, i1, i_len]
+ copyunicodecontent(p0, p1, i0, i1, i_len)
+ """, """
+ [p0, p1, i0, i1, i_len]
+ i2 = load_effective_address(p0, i0, %(uni_basesize)s,
%(uni_itemscale)d)
+ i3 = load_effective_address(p1, i1, %(uni_basesize)s,
%(uni_itemscale)d)
+ i4 = int_lshift(i_len, %(uni_itemscale)d)
+ call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr)
+ """)
+
+ def test_rewrite_copyunicodecontents_without_load_effective_address(self):
+ self.cpu.supports_load_effective_address = False
+ self.check_rewrite("""
+ [p0, p1, i0, i1, i_len]
+ copyunicodecontent(p0, p1, i0, i1, i_len)
+ """, """
+ [p0, p1, i0, i1, i_len]
+ i0s = int_lshift(i0, %(uni_itemscale)d)
+ i2b = int_add(p0, i0s)
+ i2 = int_add(i2b, %(uni_basesize)s)
+ i1s = int_lshift(i1, %(uni_itemscale)d)
+ i3b = int_add(p1, i1s)
+ i3 = int_add(i3b, %(uni_basesize)s)
+ i4 = int_lshift(i_len, %(uni_itemscale)d)
+ call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr)
+ """)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -19,6 +19,7 @@
# Boxes and Consts are BoxFloats and ConstFloats.
supports_singlefloats = False
supports_guard_gc_type = False
+ supports_load_effective_address = False
propagate_exception_descr = None
diff --git a/rpython/jit/backend/ppc/opassembler.py
b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -966,72 +966,6 @@
pmc.overwrite()
-class StrOpAssembler(object):
-
- _mixin_ = True
-
- def emit_copystrcontent(self, op, arglocs, regalloc):
- self._emit_copycontent(arglocs, is_unicode=False)
-
- def emit_copyunicodecontent(self, op, arglocs, regalloc):
- self._emit_copycontent(arglocs, is_unicode=True)
-
- def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale):
- if src_ofs.is_imm():
- value = src_ofs.value << scale
- if value < 32768:
- self.mc.addi(dst.value, src_ptr.value, value)
- else:
- self.mc.load_imm(dst, value)
- self.mc.add(dst.value, src_ptr.value, dst.value)
- elif scale == 0:
- self.mc.add(dst.value, src_ptr.value, src_ofs.value)
- else:
- self.mc.sldi(dst.value, src_ofs.value, scale)
- self.mc.add(dst.value, src_ptr.value, dst.value)
-
- def _emit_copycontent(self, arglocs, is_unicode):
- [src_ptr_loc, dst_ptr_loc,
- src_ofs_loc, dst_ofs_loc, length_loc] = arglocs
-
- if is_unicode:
- basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE,
- self.cpu.translate_support_code)
- if itemsize == 2: scale = 1
- elif itemsize == 4: scale = 2
- else: raise AssertionError
- else:
- basesize, itemsize, _ = symbolic.get_array_token(rstr.STR,
- self.cpu.translate_support_code)
- assert itemsize == 1
- basesize -= 1 # for the extra null character
- scale = 0
-
- self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale)
- self._emit_load_for_copycontent(r.r2, dst_ptr_loc, dst_ofs_loc, scale)
-
- if length_loc.is_imm():
- length = length_loc.getint()
- self.mc.load_imm(r.r5, length << scale)
- else:
- if scale > 0:
- self.mc.sldi(r.r5.value, length_loc.value, scale)
- elif length_loc is not r.r5:
- self.mc.mr(r.r5.value, length_loc.value)
-
- self.mc.mr(r.r4.value, r.r0.value)
- self.mc.addi(r.r4.value, r.r4.value, basesize)
- self.mc.addi(r.r3.value, r.r2.value, basesize)
-
- self.mc.load_imm(self.mc.RAW_CALL_REG, self.memcpy_addr)
- self.mc.raw_call()
-
-
-class UnicodeOpAssembler(object):
- _mixin_ = True
- # empty!
-
-
class AllocOpAssembler(object):
_mixin_ = True
@@ -1336,8 +1270,7 @@
class OpAssembler(IntOpAssembler, GuardOpAssembler,
MiscOpAssembler, FieldOpAssembler,
- StrOpAssembler, CallOpAssembler,
- UnicodeOpAssembler, ForceOpAssembler,
+ CallOpAssembler, ForceOpAssembler,
AllocOpAssembler, FloatOpAssembler,
VectorAssembler):
_mixin_ = True
diff --git a/rpython/jit/backend/ppc/regalloc.py
b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -802,18 +802,6 @@
temp_loc = r.SCRATCH2
return [base_loc, temp_loc]
- def prepare_copystrcontent(self, op):
- src_ptr_loc = self.ensure_reg(op.getarg(0))
- dst_ptr_loc = self.ensure_reg(op.getarg(1))
- src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2))
- dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3))
- length_loc = self.ensure_reg_or_any_imm(op.getarg(4))
- self._spill_before_call(gc_level=0)
- return [src_ptr_loc, dst_ptr_loc,
- src_ofs_loc, dst_ofs_loc, length_loc]
-
- prepare_copyunicodecontent = prepare_copystrcontent
-
prepare_same_as_i = helper.prepare_unary_op
prepare_same_as_r = helper.prepare_unary_op
prepare_same_as_f = helper.prepare_unary_op
diff --git a/rpython/jit/backend/x86/regalloc.py
b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1222,78 +1222,16 @@
resloc = self.force_allocate_reg(op, [op.getarg(0)])
self.perform(op, [argloc], resloc)
- def consider_copystrcontent(self, op):
- self._consider_copystrcontent(op, is_unicode=False)
-
- def consider_copyunicodecontent(self, op):
- self._consider_copystrcontent(op, is_unicode=True)
-
- def _consider_copystrcontent(self, op, is_unicode):
- # compute the source address
- args = op.getarglist()
- base_loc = self.rm.make_sure_var_in_reg(args[0], args)
- ofs_loc = self.rm.make_sure_var_in_reg(args[2], args)
- assert args[0] is not args[1] # forbidden case of aliasing
- srcaddr_box = TempVar()
- forbidden_vars = [args[1], args[3], args[4], srcaddr_box]
- srcaddr_loc = self.rm.force_allocate_reg(srcaddr_box, forbidden_vars)
- self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc,
- is_unicode=is_unicode)
- # compute the destination address
- base_loc = self.rm.make_sure_var_in_reg(args[1], forbidden_vars)
- ofs_loc = self.rm.make_sure_var_in_reg(args[3], forbidden_vars)
- forbidden_vars = [args[4], srcaddr_box]
- dstaddr_box = TempVar()
- dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, forbidden_vars)
- self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc,
- is_unicode=is_unicode)
- # compute the length in bytes
- length_box = args[4]
- length_loc = self.loc(length_box)
- if is_unicode:
- forbidden_vars = [srcaddr_box, dstaddr_box]
- bytes_box = TempVar()
- bytes_loc = self.rm.force_allocate_reg(bytes_box, forbidden_vars)
- scale = self._get_unicode_item_scale()
- if not (isinstance(length_loc, ImmedLoc) or
- isinstance(length_loc, RegLoc)):
- self.assembler.mov(length_loc, bytes_loc)
- length_loc = bytes_loc
- self.assembler.load_effective_addr(length_loc, 0, scale, bytes_loc)
- length_box = bytes_box
- length_loc = bytes_loc
- # call memcpy()
- self.rm.before_call()
- self.xrm.before_call()
- self.assembler.simple_call_no_collect(imm(self.assembler.memcpy_addr),
- [dstaddr_loc, srcaddr_loc, length_loc])
- self.rm.possibly_free_var(length_box)
- self.rm.possibly_free_var(dstaddr_box)
- self.rm.possibly_free_var(srcaddr_box)
-
- def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
- if is_unicode:
- ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
- self.translate_support_code)
- scale = self._get_unicode_item_scale()
- else:
- ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR,
- self.translate_support_code)
- assert itemsize == 1
- ofs_items -= 1 # for the extra null character
- scale = 0
- self.assembler.load_effective_addr(ofsloc, ofs_items, scale,
- resloc, baseloc)
-
- def _get_unicode_item_scale(self):
- _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE,
- self.translate_support_code)
- if itemsize == 4:
- return 2
- elif itemsize == 2:
- return 1
- else:
- raise AssertionError("bad unicode item size")
+ def consider_load_effective_address(self, op):
+ p0 = op.getarg(0)
+ i0 = op.getarg(1)
+ ploc = self.make_sure_var_in_reg(p0, [i0])
+ iloc = self.make_sure_var_in_reg(i0, [p0])
+ res = self.rm.force_allocate_reg(op, [p0, i0])
+ assert isinstance(op.getarg(2), ConstInt)
+ assert isinstance(op.getarg(3), ConstInt)
+ self.assembler.load_effective_addr(iloc, op.getarg(2).getint(),
+ op.getarg(3).getint(), res, ploc)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
