Author: Manuel Jacob <m...@manueljacob.de>
Branch: py3.6
Changeset: r94007:a756aa082d6b
Date: 2018-03-19 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/a756aa082d6b/
Log: hg merge py3.5
diff too long, truncating to 2000 out of 2599 lines
diff --git a/lib-python/3/distutils/msvc9compiler.py
b/lib-python/3/distutils/msvc9compiler.py
--- a/lib-python/3/distutils/msvc9compiler.py
+++ b/lib-python/3/distutils/msvc9compiler.py
@@ -223,6 +223,7 @@
that fails it falls back to the VS90COMNTOOLS env var.
"""
vsbase = VS_BASE % version
+ batfile = 'vcvarsall.bat'
try:
productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
"productdir")
@@ -235,9 +236,14 @@
toolsdir = os.environ.get(toolskey, None)
if toolsdir and os.path.isdir(toolsdir):
- productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
- productdir = os.path.abspath(productdir)
+ if os.path.exists(os.path.join(toolsdir, 'VsDevCmd.bat')):
+ productdir = toolsdir
+ batfile = 'VsDevCmd.bat'
+ else:
+ productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
+ productdir = os.path.abspath(productdir)
if not os.path.isdir(productdir):
+
log.debug("%s is not a valid directory" % productdir)
return None
else:
@@ -245,7 +251,7 @@
if not productdir:
log.debug("No productdir found")
return None
- vcvarsall = os.path.join(productdir, "vcvarsall.bat")
+ vcvarsall = os.path.join(productdir, batfile)
if os.path.isfile(vcvarsall):
return vcvarsall
log.debug("Unable to find vcvarsall.bat")
@@ -289,6 +295,7 @@
if len(result) != len(interesting):
raise ValueError(str(list(result.keys())))
+ log.debug('Got', result)
return result
# More globals
diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py
--- a/lib_pypy/_dbm.py
+++ b/lib_pypy/_dbm.py
@@ -168,7 +168,14 @@
def open(filename, flag='r', mode=0o666):
"open a DBM database"
if not isinstance(filename, str):
- raise TypeError("expected string")
+ if sys.version_info < (3,) and isinstance(filename, unicode):
+ # unlike CPython we'll encode 'filename' with filesystemencoding
+ # instead of defaultencoding, because that seems like a far
+ # better idea. But I'm also open for saying that we should
+ # rather go for bug-to-bug compatibility instead.
+ filename = filename.encode(sys.getfilesystemencoding())
+ else:
+ raise TypeError("expected string")
filename = filename.encode(sys.getdefaultencoding())
openflag = 0
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -3,7 +3,7 @@
import importlib.machinery
-def get_hashed_dir(cfile):
+def _get_hashed_filename(cfile):
with open(cfile,'r') as fid:
content = fid.read()
# from cffi's Verifier()
@@ -23,10 +23,28 @@
username = os.environ['USERNAME'] #windows
except KeyError:
username = os.getuid()
- output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % (
+ return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % (
username, k1, k2)
- if not os.path.exists(output_dir):
+
+def get_hashed_dir(cfile):
+ hashed_fn = _get_hashed_filename(cfile)
+ try:
+ with open(hashed_fn) as f:
+ dirname = f.read(1024)
+ except IOError:
+ dirname = ''
+ tmpdir = tempfile.gettempdir()
+ if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname
+ or not os.path.isdir(os.path.join(tmpdir, dirname))):
+ dirname = binascii.hexlify(os.urandom(8))
+ if not isinstance(dirname, str): # Python 3
+ dirname = dirname.decode('ascii')
+ dirname = 'testcapi_' + dirname
+ output_dir = os.path.join(tmpdir, dirname)
+ try:
os.mkdir(output_dir)
+ except OSError:
+ pass
return output_dir
@@ -35,13 +53,12 @@
return suffixes[0] if suffixes else None
-def compile_shared(csource, modulename, output_dir=None):
+def compile_shared(csource, modulename, output_dir):
"""Compile '_testcapi.c' or '_ctypes_test.c' into an extension module,
and import it.
"""
thisdir = os.path.dirname(__file__)
- if output_dir is None:
- output_dir = tempfile.mkdtemp()
+ assert output_dir is not None
from distutils.ccompiler import new_compiler
@@ -85,4 +102,16 @@
# Now import the newly created library, it will replace the original
# module in sys.modules
fp, filename, description = imp.find_module(modulename, path=[output_dir])
- imp.load_module(modulename, fp, filename, description)
+ with fp:
+ imp.load_module(modulename, fp, filename, description)
+
+ # If everything went fine up to now, write the name of this new
+ # directory to 'hashed_fn', for future processes (and to avoid a
+ # growing number of temporary directories that are not completely
+ # obvious to clean up on Windows)
+ hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource))
+ try:
+ with open(hashed_fn, 'w') as f:
+ f.write(os.path.basename(output_dir))
+ except IOError:
+ pass
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.11.4
+Version: 1.11.5
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI
from .error import CDefError, FFIError, VerificationError, VerificationMissing
-__version__ = "1.11.4"
-__version_info__ = (1, 11, 4)
+__version__ = "1.11.5"
+__version_info__ = (1, 11, 5)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -146,32 +146,6 @@
PyGILState_STATE state;
PyObject *pycode=NULL, *global_dict=NULL, *x;
-#if PY_MAJOR_VERSION >= 3
- /* see comments in _cffi_carefully_make_gil() about the
- Python2/Python3 difference
- */
-#else
- /* Acquire the GIL. We have no threadstate here. If Python is
- already initialized, it is possible that there is already one
- existing for this thread, but it is not made current now.
- */
- PyEval_AcquireLock();
-
- _cffi_py_initialize();
-
- /* The Py_InitializeEx() sometimes made a threadstate for us, but
- not always. Indeed Py_InitializeEx() could be called and do
- nothing. So do we have a threadstate, or not? We don't know,
- but we can replace it with NULL in all cases.
- */
- (void)PyThreadState_Swap(NULL);
-
- /* Now we can release the GIL and re-acquire immediately using the
- logic of PyGILState(), which handles making or installing the
- correct threadstate.
- */
- PyEval_ReleaseLock();
-#endif
state = PyGILState_Ensure();
/* Call the initxxx() function from the present module. It will
@@ -247,7 +221,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.11.4"
+ "\ncompiled with cffi version: 1.11.5"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -278,16 +252,14 @@
that we don't hold the GIL before (if it exists), and we don't
hold it afterwards.
- What it really does is completely different in Python 2 and
- Python 3.
+ (What it really does used to be completely different in Python 2
+ and Python 3, with the Python 2 solution avoiding the spin-lock
+ around the Py_InitializeEx() call. However, after recent changes
+ to CPython 2.7 (issue #358) it no longer works. So we use the
+ Python 3 solution everywhere.)
- Python 2
- ========
-
- Initialize the GIL, without initializing the rest of Python,
- by calling PyEval_InitThreads().
-
- PyEval_InitThreads() must not be called concurrently at all.
+ This initializes Python by calling Py_InitializeEx().
+ Important: this must not be called concurrently at all.
So we use a global variable as a simple spin lock. This global
variable must be from 'libpythonX.Y.so', not from this
cffi-based extension module, because it must be shared from
@@ -297,18 +269,6 @@
string "ENDMARKER". We change it temporarily to point to the
next character in that string. (Yes, I know it's REALLY
obscure.)
-
- Python 3
- ========
-
- In Python 3, PyEval_InitThreads() cannot be called before
- Py_InitializeEx() any more. So this function calls
- Py_InitializeEx() first. It uses the same obscure logic to
- make sure we never call it concurrently.
-
- Arguably, this is less good on the spinlock, because
- Py_InitializeEx() takes much longer to run than
- PyEval_InitThreads(). But I didn't find a way around it.
*/
#ifdef WITH_THREAD
@@ -332,8 +292,7 @@
}
#endif
-#if PY_MAJOR_VERSION >= 3
- /* Python 3: call Py_InitializeEx() */
+ /* call Py_InitializeEx() */
{
PyGILState_STATE state = PyGILState_UNLOCKED;
if (!Py_IsInitialized())
@@ -344,17 +303,6 @@
PyEval_InitThreads();
PyGILState_Release(state);
}
-#else
- /* Python 2: call PyEval_InitThreads() */
-# ifdef WITH_THREAD
- if (!PyEval_ThreadsInitialized()) {
- PyEval_InitThreads(); /* makes the GIL */
- PyEval_ReleaseLock(); /* then release it */
- }
- /* else: there is already a GIL, but we still needed to do the
- spinlock dance to make sure that we see it as fully ready */
-# endif
-#endif
#ifdef WITH_THREAD
/* release the lock */
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -143,6 +143,13 @@
self._libraries.append(lib)
return lib
+ def dlclose(self, lib):
+ """Close a library obtained with ffi.dlopen(). After this call,
+ access to functions or variables from the library will fail
+ (possibly with a segmentation fault).
+ """
+ type(lib).__cffi_close__(lib)
+
def _typeof_locked(self, cdecl):
# call me with the lock!
key = cdecl
@@ -898,6 +905,9 @@
return addressof_var(name)
raise AttributeError("cffi library has no function or "
"global variable named '%s'" % (name,))
+ def __cffi_close__(self):
+ backendlib.close_lib()
+ self.__dict__.clear()
#
if libname is not None:
try:
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -352,21 +352,20 @@
self.fldquals = fldquals
self.build_c_name_with_marker()
- def has_anonymous_struct_fields(self):
- if self.fldtypes is None:
- return False
- for name, type in zip(self.fldnames, self.fldtypes):
- if name == '' and isinstance(type, StructOrUnion):
- return True
- return False
+ def anonymous_struct_fields(self):
+ if self.fldtypes is not None:
+ for name, type in zip(self.fldnames, self.fldtypes):
+ if name == '' and isinstance(type, StructOrUnion):
+ yield type
- def enumfields(self):
+ def enumfields(self, expand_anonymous_struct_union=True):
fldquals = self.fldquals
if fldquals is None:
fldquals = (0,) * len(self.fldnames)
for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes,
self.fldbitsize, fldquals):
- if name == '' and isinstance(type, StructOrUnion):
+ if (name == '' and isinstance(type, StructOrUnion)
+ and expand_anonymous_struct_union):
# nested anonymous struct/union
for result in type.enumfields():
yield result
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -836,6 +836,10 @@
def _struct_collecttype(self, tp):
self._do_collect_type(tp)
+ if self.target_is_python:
+ # also requires nested anon struct/unions in ABI mode, recursively
+ for fldtype in tp.anonymous_struct_fields():
+ self._struct_collecttype(fldtype)
def _struct_decl(self, tp, cname, approxname):
if tp.fldtypes is None:
@@ -884,7 +888,7 @@
named_ptr not in self.ffi._parser._included_declarations)):
if tp.fldtypes is None:
pass # opaque
- elif tp.partial or tp.has_anonymous_struct_fields():
+ elif tp.partial or any(tp.anonymous_struct_fields()):
pass # field layout obtained silently from the C compiler
else:
flags.append("_CFFI_F_CHECK_FIELDS")
@@ -896,7 +900,8 @@
flags = '|'.join(flags) or '0'
c_fields = []
if reason_for_not_expanding is None:
- enumfields = list(tp.enumfields())
+ expand_anonymous_struct_union = not self.target_is_python
+ enumfields = list(tp.enumfields(expand_anonymous_struct_union))
for fldname, fldtype, fbitsize, fqual in enumfields:
fldtype = self._field_type(tp, fldname, fldtype)
self._check_not_opaque(fldtype,
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,8 +81,13 @@
it doesn't so far, creating troubles. That's why we check
for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
+
+ On Windows, it's better not to use py_limited_api until issue #355
+ can be resolved (by having virtualenv copy PYTHON3.DLL). See also
+ the start of _cffi_include.h.
"""
- if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
+ if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
+ and sys.platform != 'win32'):
import setuptools
try:
setuptools_major_version =
int(setuptools.__version__.partition('.')[0])
@@ -143,8 +148,8 @@
def _add_py_module(dist, ffi, module_name):
from distutils.dir_util import mkpath
- from distutils.command.build_py import build_py
- from distutils.command.build_ext import build_ext
+ from setuptools.command.build_py import build_py
+ from setuptools.command.build_ext import build_ext
from distutils import log
from cffi import recompiler
@@ -164,6 +169,17 @@
generate_mod(os.path.join(self.build_lib, *module_path))
dist.cmdclass['build_py'] = build_py_make_mod
+ # distutils and setuptools have no notion I could find of a
+ # generated python module. If we don't add module_name to
+ # dist.py_modules, then things mostly work but there are some
+ # combination of options (--root and --record) that will miss
+ # the module. So we add it here, which gives a few apparently
+ # harmless warnings about not finding the file outside the
+ # build directory.
+ if dist.py_modules is None:
+ dist.py_modules = []
+ dist.py_modules.append(module_name)
+
# the following is only for "build_ext -i"
base_class_2 = dist.cmdclass.get('build_ext', build_ext)
class build_ext_make_mod(base_class_2):
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -567,6 +567,11 @@
versions of PyPy may have to rename the arguments if CPython starts
accepting them too.
+* PyPy3: ``distutils`` has been enhanced to allow finding ``VsDevCmd.bat`` in
the
+ directory pointed to by the ``VS%0.f0COMNTOOLS`` (typically
``VS140COMNTOOLS``)
+ environment variable. CPython searches for ``vcvarsall.bat`` somewhere
**above**
+ that value.
+
.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
.. _`little point`:
http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -48,3 +48,17 @@
.. branch: winapi
Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3
+
+.. branch: refactor-slots
+
+Refactor cpyext slots.
+
+
+.. branch: call-loopinvariant-into-bridges
+
+Speed up branchy code that does a lot of function inlining by saving one call
+to read the TLS in most bridges.
+
+.. branch: rpython-sprint
+
+Refactor in rpython signatures
diff --git a/pypy/interpreter/test/test_zpy.py
b/pypy/interpreter/test/test_zpy.py
--- a/pypy/interpreter/test/test_zpy.py
+++ b/pypy/interpreter/test/test_zpy.py
@@ -113,6 +113,7 @@
def test_pytrace():
output = run(sys.executable, pypypath, '-S',
stdin="__pytrace__ = 1\nx = 5\nx")
+ output = output.replace('\r\n', '\n')
assert ('\t<module>: LOAD_CONST 0 (5)\n'
'\t<module>: STORE_NAME 0 (x)\n'
'\t<module>: LOAD_CONST 1 (None)\n'
diff --git a/pypy/module/_cffi_backend/__init__.py
b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.11.4"
+VERSION = "1.11.5"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/cdlopen.py
b/pypy/module/_cffi_backend/cdlopen.py
--- a/pypy/module/_cffi_backend/cdlopen.py
+++ b/pypy/module/_cffi_backend/cdlopen.py
@@ -53,8 +53,7 @@
self.libhandle = rffi.cast(DLLHANDLE, 0)
if not libhandle:
- raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed",
- self.libname)
+ return
self.may_unregister_rpython_finalizer(self.ffi.space)
# Clear the dict to force further accesses to do cdlopen_fetch()
diff --git a/pypy/module/_cffi_backend/libraryobj.py
b/pypy/module/_cffi_backend/libraryobj.py
--- a/pypy/module/_cffi_backend/libraryobj.py
+++ b/pypy/module/_cffi_backend/libraryobj.py
@@ -38,9 +38,16 @@
space = self.space
return space.newtext("<clibrary '%s'>" % self.name)
+ def check_closed(self):
+ if self.handle == rffi.cast(DLLHANDLE, 0):
+ raise oefmt(self.space.w_ValueError,
+ "library '%s' has already been closed",
+ self.name)
+
@unwrap_spec(w_ctype=W_CType, name='text')
def load_function(self, w_ctype, name):
from pypy.module._cffi_backend import ctypeptr, ctypearray
+ self.check_closed()
space = self.space
#
if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray):
@@ -60,6 +67,7 @@
@unwrap_spec(w_ctype=W_CType, name='text')
def read_variable(self, w_ctype, name):
+ self.check_closed()
space = self.space
try:
cdata = dlsym(self.handle, name)
@@ -71,6 +79,7 @@
@unwrap_spec(w_ctype=W_CType, name='text')
def write_variable(self, w_ctype, name, w_value):
+ self.check_closed()
space = self.space
try:
cdata = dlsym(self.handle, name)
@@ -80,6 +89,9 @@
name, self.name)
w_ctype.convert_from_object(rffi.cast(rffi.CCHARP, cdata), w_value)
+ def close_lib(self):
+ self._finalize_()
+
W_Library.typedef = TypeDef(
'_cffi_backend.Library',
@@ -87,6 +99,7 @@
load_function = interp2app(W_Library.load_function),
read_variable = interp2app(W_Library.read_variable),
write_variable = interp2app(W_Library.write_variable),
+ close_lib = interp2app(W_Library.close_lib),
)
W_Library.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.11.4", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.5", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
@@ -395,6 +395,10 @@
# the next one is from 'libm', not 'libc', but we assume
# that it is already loaded too, so it should work
assert x.load_function(BVoidP, 'sqrt')
+ #
+ x.close_lib()
+ py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt')
+ x.close_lib()
def test_no_len_on_nonarray():
p = new_primitive_type("int")
@@ -1210,6 +1214,9 @@
ll = find_and_load_library('c')
stderr = ll.read_variable(BVoidP, "stderr")
assert stderr == cast(BVoidP, _testfunc(8))
+ #
+ ll.close_lib()
+ py.test.raises(ValueError, ll.read_variable, BVoidP, "stderr")
def test_read_variable_as_unknown_length_array():
## FIXME: this test assumes glibc specific behavior, it's not compliant
with C standard
@@ -1236,6 +1243,9 @@
assert not ll.read_variable(BVoidP, "stderr")
ll.write_variable(BVoidP, "stderr", stderr)
assert ll.read_variable(BVoidP, "stderr") == stderr
+ #
+ ll.close_lib()
+ py.test.raises(ValueError, ll.write_variable, BVoidP, "stderr", stderr)
def test_callback():
BInt = new_primitive_type("int")
diff --git a/pypy/module/_cffi_backend/test/test_re_python.py
b/pypy/module/_cffi_backend/test/test_re_python.py
--- a/pypy/module/_cffi_backend/test/test_re_python.py
+++ b/pypy/module/_cffi_backend/test/test_re_python.py
@@ -114,12 +114,10 @@
from re_python_pysrc import ffi
lib = ffi.dlopen(self.extmod)
ffi.dlclose(lib)
- e = raises(ffi.error, ffi.dlclose, lib)
- assert str(e.value) == (
- "library '%s' is already closed" % (self.extmod,))
e = raises(ffi.error, getattr, lib, 'add42')
assert str(e.value) == (
"library '%s' has been closed" % (self.extmod,))
+ ffi.dlclose(lib) # does not raise
def test_constant_via_lib(self):
self.fix_path()
diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py
b/pypy/module/_posixsubprocess/test/test_subprocess.py
--- a/pypy/module/_posixsubprocess/test/test_subprocess.py
+++ b/pypy/module/_posixsubprocess/test/test_subprocess.py
@@ -1,4 +1,9 @@
from os.path import dirname
+import py, sys
+
+if sys.platform == 'win32':
+ py.test.skip("not used on win32")
+
class AppTestSubprocess:
spaceconfig = dict(usemodules=('_posixsubprocess', 'signal',
diff --git a/pypy/module/_posixsubprocess/test/test_ztranslation.py
b/pypy/module/_posixsubprocess/test/test_ztranslation.py
--- a/pypy/module/_posixsubprocess/test/test_ztranslation.py
+++ b/pypy/module/_posixsubprocess/test/test_ztranslation.py
@@ -1,4 +1,8 @@
from pypy.objspace.fake.checkmodule import checkmodule
+import py, sys
+
+if sys.platform == 'win32':
+ py.test.skip("not used on win32")
def test_posixsubprocess_translates():
checkmodule('_posixsubprocess')
diff --git a/pypy/module/_socket/test/test_sock_app.py
b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -813,10 +813,10 @@
def test_recv_send_timeout(self):
from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF
cli = socket()
+ cli.settimeout(1.0)
cli.connect(self.serv.getsockname())
fileno, addr = self.serv._accept()
t = socket(fileno=fileno)
- cli.settimeout(1.0)
# test recv() timeout
t.send(b'*')
buf = cli.recv(100)
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -407,8 +407,11 @@
if not (last_pos == ctx.match_start
== ctx.match_end and n > 0):
# the above ignores empty matches on latest position
+ last_pos = ctx.match_end
if filter_is_callable:
w_match = self.getmatch(ctx, True)
+ # make a copy of 'ctx'; see test_sub_matches_stay_valid
+ ctx = ctx.fresh_copy(start) # match_start/match_end dropped
w_piece = space.call_function(w_filter, w_match)
if not space.is_w(w_piece, space.w_None):
assert strbuilder is None and unicodebuilder is None
@@ -425,7 +428,6 @@
unicodebuilder.append(filter_as_unicode)
else:
sublist_w.append(w_filter)
- last_pos = ctx.match_end
n += 1
elif last_pos >= ctx.end:
break # empty match at the end: finished
diff --git a/pypy/module/_sre/test/test_app_sre.py
b/pypy/module/_sre/test/test_app_sre.py
--- a/pypy/module/_sre/test/test_app_sre.py
+++ b/pypy/module/_sre/test/test_app_sre.py
@@ -395,6 +395,18 @@
KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
raises(TypeError, KEYCRE.sub, "hello", {"%(": 1})
+ def test_sub_matches_stay_valid(self):
+ import re
+ matches = []
+ def callback(match):
+ matches.append(match)
+ return "x"
+ result = re.compile(r"[ab]").sub(callback, "acb")
+ assert result == "xcx"
+ assert len(matches) == 2
+ assert matches[0].group() == "a"
+ assert matches[1].group() == "b"
+
class AppTestSreScanner:
diff --git a/pypy/module/_winreg/interp_winreg.py
b/pypy/module/_winreg/interp_winreg.py
--- a/pypy/module/_winreg/interp_winreg.py
+++ b/pypy/module/_winreg/interp_winreg.py
@@ -175,7 +175,7 @@
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
with rffi.scoped_unicode2wcharp(filename) as wide_filename:
c_filename = rffi.cast(rffi.CCHARP, wide_filename)
- ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename)
+ ret = rwinreg.RegLoadKeyW(hkey, c_subkey, c_filename)
if ret != 0:
raiseWindowsError(space, ret, 'RegLoadKey')
@@ -196,7 +196,7 @@
hkey = hkey_w(w_hkey, space)
with rffi.scoped_unicode2wcharp(filename) as wide_filename:
c_filename = rffi.cast(rffi.CCHARP, wide_filename)
- ret = rwinreg.RegSaveKey(hkey, c_filename, None)
+ ret = rwinreg.RegSaveKeyW(hkey, c_filename, None)
if ret != 0:
raiseWindowsError(space, ret, 'RegSaveKey')
@@ -226,7 +226,7 @@
c_subkey = rffi.cast(rffi.CCHARP, subkey)
with rffi.scoped_unicode2wcharp(value) as dataptr:
c_dataptr = rffi.cast(rffi.CCHARP, dataptr)
- ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ,
+ ret = rwinreg.RegSetValueW(hkey, c_subkey, rwinreg.REG_SZ,
c_dataptr, len(value))
if ret != 0:
raiseWindowsError(space, ret, 'RegSetValue')
@@ -250,7 +250,7 @@
with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p:
- ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p)
+ ret = rwinreg.RegQueryValueW(hkey, c_subkey, None, bufsize_p)
bufSize = intmask(bufsize_p[0])
if ret == rwinreg.ERROR_MORE_DATA:
bufSize = 256
@@ -259,7 +259,7 @@
while True:
with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf:
- ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p)
+ ret = rwinreg.RegQueryValueW(hkey, c_subkey, buf,
bufsize_p)
if ret == rwinreg.ERROR_MORE_DATA:
print 'bufSize was %d, too small' % bufSize
# Resize and retry
@@ -440,7 +440,7 @@
try:
with rffi.scoped_unicode2wcharp(value_name) as wide_vn:
c_vn = rffi.cast(rffi.CCHARP, wide_vn)
- ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen)
+ ret = rwinreg.RegSetValueExW(hkey, c_vn, 0, typ, buf, buflen)
finally:
lltype.free(buf, flavor='raw')
if ret != 0:
@@ -460,7 +460,7 @@
with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize:
- ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword,
null_dword,
+ ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword,
null_dword,
None, retDataSize)
bufSize = intmask(retDataSize[0])
if ret == rwinreg.ERROR_MORE_DATA:
@@ -472,7 +472,7 @@
with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf:
with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType:
- ret = rwinreg.RegQueryValueEx(hkey, c_subkey,
null_dword,
+ ret = rwinreg.RegQueryValueExW(hkey, c_subkey,
null_dword,
retType, databuf,
retDataSize)
if ret == rwinreg.ERROR_MORE_DATA:
# Resize and retry
@@ -505,7 +505,7 @@
with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
- ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey)
+ ret = rwinreg.RegCreateKeyW(hkey, c_subkey, rethkey)
if ret != 0:
raiseWindowsError(space, ret, 'CreateKey')
return W_HKEY(space, rethkey[0])
@@ -527,7 +527,7 @@
with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key:
c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key)
with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
- ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0,
+ ret = rwinreg.RegCreateKeyExW(hkey, c_subkey, reserved, None, 0,
access, None, rethkey,
lltype.nullptr(rwin32.LPDWORD.TO))
if ret != 0:
@@ -549,7 +549,7 @@
hkey = hkey_w(w_hkey, space)
with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
- ret = rwinreg.RegDeleteKey(hkey, c_subkey)
+ ret = rwinreg.RegDeleteKeyW(hkey, c_subkey)
if ret != 0:
raiseWindowsError(space, ret, 'RegDeleteKey')
@@ -562,7 +562,7 @@
hkey = hkey_w(w_hkey, space)
with rffi.scoped_unicode2wcharp(subkey) as wide_subkey:
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
- ret = rwinreg.RegDeleteValue(hkey, c_subkey)
+ ret = rwinreg.RegDeleteValueW(hkey, c_subkey)
if ret != 0:
raiseWindowsError(space, ret, 'RegDeleteValue')
@@ -582,7 +582,7 @@
with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey:
c_subkey = rffi.cast(rffi.CCHARP, wide_subkey)
with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
- ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access,
rethkey)
+ ret = rwinreg.RegOpenKeyExW(hkey, c_subkey, reserved, access,
rethkey)
if ret != 0:
raiseWindowsError(space, ret, 'RegOpenKeyEx')
return W_HKEY(space, rethkey[0])
@@ -607,7 +607,7 @@
with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize:
with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize:
- ret = rwinreg.RegQueryInfoKey(
+ ret = rwinreg.RegQueryInfoKeyW(
hkey, None, null_dword, null_dword,
null_dword, null_dword, null_dword,
null_dword, retValueSize, retDataSize,
@@ -628,7 +628,7 @@
with lltype.scoped_alloc(rwin32.LPDWORD.TO,
1) as retType:
c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf)
- ret = rwinreg.RegEnumValue(
+ ret = rwinreg.RegEnumValueW(
hkey, index, c_valuebuf, retValueSize,
null_dword, retType, databuf, retDataSize)
if ret == rwinreg.ERROR_MORE_DATA:
@@ -673,7 +673,7 @@
with lltype.scoped_alloc(rffi.CCHARP.TO, 257) as buf:
with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize:
retValueSize[0] = r_uint(257) # includes NULL terminator
- ret = rwinreg.RegEnumKeyEx(hkey, index, buf, retValueSize,
+ ret = rwinreg.RegEnumKeyExW(hkey, index, buf, retValueSize,
null_dword, None, null_dword,
lltype.nullptr(rwin32.PFILETIME.TO))
if ret != 0:
@@ -695,7 +695,7 @@
with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as nValues:
with lltype.scoped_alloc(rwin32.PFILETIME.TO, 1) as ft:
null_dword = lltype.nullptr(rwin32.LPDWORD.TO)
- ret = rwinreg.RegQueryInfoKey(
+ ret = rwinreg.RegQueryInfoKeyW(
hkey, None, null_dword, null_dword,
nSubKeys, null_dword, null_dword,
nValues, null_dword, null_dword,
@@ -722,7 +722,7 @@
machine = space.text_or_none_w(w_machine)
hkey = hkey_w(w_hkey, space)
with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
- ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)
+ ret = rwinreg.RegConnectRegistryW(machine, hkey, rethkey)
if ret != 0:
raiseWindowsError(space, ret, 'RegConnectRegistry')
return W_HKEY(space, rethkey[0])
diff --git a/pypy/module/_winreg/test/test_winreg.py
b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -207,18 +207,20 @@
except:
pass
- key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS)
- SaveKey(key, self.tmpfilename)
+ with OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) as
key:
+ SaveKey(key, self.tmpfilename)
def test_expand_environment_string(self):
from winreg import ExpandEnvironmentStrings
import nt
r = ExpandEnvironmentStrings("%windir%\\test")
assert isinstance(r, str)
- if 'WINDIR' in list(nt.environ.keys()):
+ if 'WINDIR' in nt.environ:
assert r == nt.environ["WINDIR"] + "\\test"
+ elif 'windir' in nt.environ:
+ assert r == nt.environ["windir"] + "\\test"
else:
- assert r == nt.environ["windir"] + "\\test"
+ skip('nt.environ not filled in for untranslated tests')
def test_long_key(self):
from winreg import (
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -189,33 +189,6 @@
decref(space, view.c_obj)
return 0
-def fill_buffer(space, view, pybuf, py_obj):
- view.c_buf = cts.cast('void *', pybuf.get_raw_address())
- view.c_obj = py_obj
- if py_obj:
- incref(space, py_obj)
- view.c_len = pybuf.getlength()
- view.c_itemsize = pybuf.getitemsize()
- rffi.setintfield(view, 'c_readonly', int(pybuf.readonly))
- rffi.setintfield(view, 'c_ndim', pybuf.getndim())
- view.c_format = rffi.str2charp(pybuf.getformat())
- shape = pybuf.getshape()
- if not shape:
- view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
- else:
- view.c_shape = cts.cast('Py_ssize_t*', view.c__shape)
- for i, n in enumerate(shape):
- view.c_shape[i] = n
- strides = pybuf.getstrides()
- if not strides:
- view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
- else:
- view.c_strides = cts.cast('Py_ssize_t*', view.c__strides)
- for i, n in enumerate(strides):
- view.c_strides[i] = n
- view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
- view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
DEFAULT_FMT = rffi.str2charp("B")
@cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -688,12 +688,46 @@
return space.call_args(space.get(new_fn, w_self), args)
return slot_tp_new
+@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+ rffi.INT_real, error=-1)
+def bytes_getbuffer(space, w_str, view, flags):
+ from pypy.module.cpyext.bytesobject import PyBytes_AsString
+ from pypy.module.cpyext.buffer import PyBuffer_FillInfo
+ c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
+ return PyBuffer_FillInfo(space, view, w_str, c_buf,
+ space.len_w(w_str), 1, flags)
+
+def slot_from_buffer_w(space, typedef):
+ name = 'bf_getbuffer'
+ @slot_function([PyObject, Py_bufferP, rffi.INT_real],
+ rffi.INT_real, error=-1)
+ @func_renamer("cpyext_%s_%s" % (name, typedef.name))
+ def buff_w(space, w_self, c_view, flags):
+ w_obj = w_self
+ if c_view:
+ #like PyObject_GetBuffer
+ flags = widen(flags)
+ buf = space.buffer_w(w_obj, flags)
+ try:
+ c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ c_view.c_obj = make_ref(space, w_obj)
+ except ValueError:
+ s = buf.as_str()
+ w_s = space.newbytes(s)
+ c_view.c_obj = make_ref(space, w_s)
+ c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+ s, track_allocation=False))
+ rffi.setintfield(c_view, 'c_readonly', 1)
+ ret = fill_Py_buffer(space, buf, c_view)
+ return ret
+ return 0
+ return buff_w
+
@slot_factory('tp_as_buffer.c_bf_getbuffer')
def make_bf_getbuffer(space, typedef, name, attr):
w_type = space.gettypeobject(typedef)
- buff_fn = w_type.lookup('__buffer__')
- if buff_fn is not None:
- return slot_from___buffer__(space, typedef, buff_fn)
+ if space.is_w(w_type, space.w_bytes):
+ return bytes_getbuffer
elif typedef.buffer:
return slot_from_buffer_w(space, typedef)
else:
@@ -739,59 +773,6 @@
return slot_tp_descr_set
-def slot_from___buffer__(space, typedef, buff_fn):
- name = 'bf_getbuffer'
- @slot_function([PyObject, Py_bufferP, rffi.INT_real],
- rffi.INT_real, error=-1)
- @func_renamer("cpyext_%s_%s" % (name, typedef.name))
- def buff_w(space, w_self, c_view, flags):
- args = Arguments(space, [space.newint(flags)])
- w_obj = space.call_args(space.get(buff_fn, w_self), args)
- if c_view:
- #like PyObject_GetBuffer
- flags = widen(flags)
- buf = space.buffer_w(w_obj, flags)
- try:
- c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
- c_view.c_obj = make_ref(space, w_obj)
- except ValueError:
- s = buf.as_str()
- w_s = space.newbytes(s)
- c_view.c_obj = make_ref(space, w_s)
- c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
- s, track_allocation=False))
- rffi.setintfield(c_view, 'c_readonly', 1)
- ret = fill_Py_buffer(space, buf, c_view)
- return ret
- return 0
- return buff_w
-
-def slot_from_buffer_w(space, typedef):
- name = 'bf_getbuffer'
- @slot_function([PyObject, Py_bufferP, rffi.INT_real],
- rffi.INT_real, error=-1)
- @func_renamer("cpyext_%s_%s" % (name, typedef.name))
- def buff_w(space, w_self, c_view, flags):
- w_obj = w_self
- if c_view:
- #like PyObject_GetBuffer
- flags = widen(flags)
- buf = space.buffer_w(w_obj, flags)
- try:
- c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
- c_view.c_obj = make_ref(space, w_obj)
- except ValueError:
- s = buf.as_str()
- w_s = space.newbytes(s)
- c_view.c_obj = make_ref(space, w_s)
- c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
- s, track_allocation=False))
- rffi.setintfield(c_view, 'c_readonly', 1)
- ret = fill_Py_buffer(space, buf, c_view)
- return ret
- return 0
- return buff_w
-
missing_wrappers = ['wrap_indexargfunc', 'wrap_del']
for name in missing_wrappers:
assert name not in globals()
diff --git a/pypy/module/cpyext/test/test_typeobject.py
b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -640,6 +640,33 @@
self.attr1 = 123
assert module.test_tp_getattro(C(), 123)
+ def test_issue_2760_getattr(self):
+ module = self.import_extension('foo', [
+ ("get_foo", "METH_O",
+ '''
+ char* name = "foo";
+ PyTypeObject *tp = Py_TYPE(args);
+ PyObject *res;
+ if (tp->tp_getattr != NULL) {
+ res = (*tp->tp_getattr)(args, name);
+ }
+ else if (tp->tp_getattro != NULL) {
+ PyObject *w = PyUnicode_FromString(name);
+ res = (*tp->tp_getattro)(args, w);
+ Py_DECREF(w);
+ }
+ else {
+ res = Py_None;
+ }
+ return res;
+ ''')])
+ class Passthrough(object):
+ def __getattr__(self, name):
+ return name
+
+ obj = Passthrough()
+ assert module.get_foo(obj) == 'foo'
+
def test_nb_int(self):
module = self.import_extension('foo', [
("nb_int", "METH_VARARGS",
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -236,11 +236,6 @@
def update_all_slots(space, w_type, pto):
# fill slots in pto
- # Not very sure about it, but according to
- # test_call_tp_dealloc, we should not
- # overwrite slots that are already set: these ones are probably
- # coming from a parent C type.
-
for method_name, slot_name, slot_names, slot_apifunc in
slotdefs_for_tp_slots:
slot_func_helper = None
w_descr = w_type.dict_w.get(method_name, None)
@@ -276,8 +271,7 @@
def fill_slot(space, pto, w_type, slot_names, slot_func_helper):
# XXX special case wrapper-functions and use a "specific" slot func
if len(slot_names) == 1:
- if not getattr(pto, slot_names[0]):
- setattr(pto, slot_names[0], slot_func_helper)
+ setattr(pto, slot_names[0], slot_func_helper)
elif ((w_type is space.w_list or w_type is space.w_tuple) and
slot_names[0] == 'c_tp_as_number'):
# XXX hack - how can we generalize this? The problem is method
@@ -312,8 +306,7 @@
struct = lltype.malloc(STRUCT_TYPE, flavor='raw', zero=True)
setattr(pto, slot_names[0], struct)
- if not getattr(struct, slot_names[1]):
- setattr(struct, slot_names[1], slot_func_helper)
+ setattr(struct, slot_names[1], slot_func_helper)
def add_operators(space, dict_w, pto, name):
from pypy.module.cpyext.object import PyObject_HashNotImplemented
@@ -526,33 +519,6 @@
realize=type_realize,
dealloc=type_dealloc)
-@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
rffi.INT_real, error=-1)
-def bytes_getbuffer(space, w_str, view, flags):
- from pypy.module.cpyext.bytesobject import PyBytes_AsString
- from pypy.module.cpyext.buffer import PyBuffer_FillInfo
- c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
- return PyBuffer_FillInfo(space, view, w_str, c_buf,
- space.len_w(w_str), 1, flags)
-
-@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
rffi.INT_real, error=-1)
-def bf_getbuffer(space, w_obj, view, flags):
- from pypy.module.cpyext.buffer import fill_buffer
- buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
- fill_buffer(space, view, buf, as_pyobj(space, w_obj))
- return 0
-
-def setup_buffer_procs(space, w_type, pto):
- bufspec = w_type.layout.typedef.buffer
- if not bufspec:
- return
- c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True)
- lltype.render_immortal(c_buf)
- if space.is_w(w_type, space.w_bytes):
- c_buf.c_bf_getbuffer = llslot(space, bytes_getbuffer)
- else:
- c_buf.c_bf_getbuffer = llslot(space, bf_getbuffer)
- pto.c_tp_as_buffer = c_buf
-
@slot_function([PyObject], lltype.Void)
def type_dealloc(space, obj):
from pypy.module.cpyext.object import _dealloc
@@ -611,8 +577,6 @@
pto.c_tp_itemsize = 1
elif space.is_w(w_type, space.w_tuple):
pto.c_tp_itemsize = rffi.sizeof(PyObject)
- # buffer protocol
- setup_buffer_procs(space, w_type, pto)
state = space.fromcache(State)
pto.c_tp_free = state.C.PyObject_Free
@@ -730,7 +694,6 @@
pto.c_tp_as_buffer = base.c_tp_as_buffer
if base.c_tp_as_buffer:
# inherit base.c_tp_as_buffer functions not inherited from w_type
- # note: builtin types are handled in setup_buffer_procs
pto_as = pto.c_tp_as_buffer
base_as = base.c_tp_as_buffer
if not pto_as.c_bf_getbuffer:
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -106,7 +106,7 @@
return space.getitem(w_obj1, w_obj2)
@slot_function([PyObject, PyObject], PyObject)
-def slot_tp_getattr(space, w_obj1, w_obj2):
+def slot_tp_getattr_hook(space, w_obj1, w_obj2):
return space.getattr(w_obj1, w_obj2)
@slot_function([PyObject, PyObject, PyObject], PyObject)
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -82,7 +82,7 @@
import imp
for suffix, mode, type in imp.get_suffixes():
if type == imp.PY_SOURCE:
- assert suffix == '.py'
+ assert suffix in ('.py', '.pyw')
assert mode == 'r'
elif type == imp.PY_COMPILED:
assert suffix == '.pyc'
diff --git a/pypy/module/posix/test/test_posix2.py
b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -1092,9 +1092,11 @@
if sys.platform == 'win32':
os.chmod(my_path, 0o400)
assert (os.stat(my_path).st_mode & 0o600) == 0o400
+ os.chmod(self.path, 0o700)
else:
os.chmod(my_path, 0o200)
assert (os.stat(my_path).st_mode & 0o777) == 0o200
+ os.chmod(self.path, 0o700)
if hasattr(os, 'fchmod'):
def test_fchmod(self):
@@ -1406,6 +1408,22 @@
if len(e.value.args) > 2:
assert e.value.args[2] == "\\foo\\bar\\baz"
+ @py.test.mark.skipif("sys.platform != 'win32'")
+ def test_rename(self):
+ os = self.posix
+ fname = self.path2 + 'rename.txt'
+ with open(fname, "w") as f:
+ f.write("this is a rename test")
+ unicode_name = str(self.udir) + u'/test\u03be.txt'
+ os.rename(fname, unicode_name)
+ with open(unicode_name) as f:
+ assert f.read() == 'this is a rename test'
+ os.rename(unicode_name, fname)
+ with open(fname) as f:
+ assert f.read() == 'this is a rename test'
+ os.unlink(fname)
+
+
def test_device_encoding(self):
import sys
encoding = self.posix.device_encoding(sys.stdout.fileno())
@@ -1509,6 +1527,8 @@
def test_environ(self):
import sys, os
environ = os.environ
+ if not environ:
+ skip('environ not filled in for untranslated tests')
for k, v in environ.items():
assert type(k) is str
assert type(v) is str
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
@@ -500,3 +500,21 @@
""")
m = ffi.dlopen(lib_m)
assert dir(m) == ['MYE1', 'MYE2', 'MYFOO', 'myconst', 'myfunc',
'myvar']
+
+ def test_dlclose(self):
+ if self.Backend is CTypesBackend:
+ py.test.skip("not with the ctypes backend")
+ ffi = FFI(backend=self.Backend())
+ ffi.cdef("int foobar(void); int foobaz;")
+ lib = ffi.dlopen(lib_m)
+ ffi.dlclose(lib)
+ e = py.test.raises(ValueError, getattr, lib, 'foobar')
+ assert str(e.value).startswith("library '")
+ assert str(e.value).endswith("' has already been closed")
+ e = py.test.raises(ValueError, getattr, lib, 'foobaz')
+ assert str(e.value).startswith("library '")
+ assert str(e.value).endswith("' has already been closed")
+ e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42)
+ assert str(e.value).startswith("library '")
+ assert str(e.value).endswith("' has already been closed")
+ ffi.dlclose(lib) # does not raise
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -115,8 +115,12 @@
if sys.platform == 'win32':
import os
# did we already build it?
- if os.path.exists(str(udir.join('testownlib.dll'))):
- cls.module = str(udir.join('testownlib.dll'))
+ if cls.Backend is CTypesBackend:
+ dll_path = str(udir) + '\\testownlib1.dll' # only ascii for
the ctypes backend
+ else:
+ dll_path = str(udir) + '\\' + (u+'testownlib\u03be.dll') #
non-ascii char
+ if os.path.exists(dll_path):
+ cls.module = dll_path
return
# try (not too hard) to find the version used to compile this
python
# no mingw
@@ -136,8 +140,9 @@
if os.path.isfile(vcvarsall):
cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c
' \
' /LD /Fetestownlib.dll'
- subprocess.check_call(cmd, cwd = str(udir), shell=True)
- cls.module = str(udir.join('testownlib.dll'))
+ subprocess.check_call(cmd, cwd = str(udir), shell=True)
+ os.rename(str(udir) + '\\testownlib.dll', dll_path)
+ cls.module = dll_path
else:
subprocess.check_call(
'cc testownlib.c -shared -fPIC -o testownlib.so',
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
@@ -1,9 +1,10 @@
# Generated by pypy/tool/import_cffi.py
-import sys
+import sys, os
import py
from cffi import FFI
from cffi import recompiler, ffiplatform, VerificationMissing
from pypy.module.test_lib_pypy.cffi_tests.udir import udir
+from pypy.module.test_lib_pypy.cffi_tests.support import u
def setup_module(mod):
@@ -36,6 +37,13 @@
'globalconst42', 'globalconsthello']
)
outputfilename = ffiplatform.compile(str(tmpdir), ext)
+ if sys.platform == "win32":
+ # test with a non-ascii char
+ outputfn1 = outputfilename
+ ofn, oext = os.path.splitext(outputfn1)
+ outputfilename = ofn + (u+'\u03be') + oext
+ #print(repr(outputfn1) + ' ==> ' + repr(outputfilename))
+ os.rename(outputfn1, outputfilename)
mod.extmod = outputfilename
mod.tmpdir = tmpdir
#
@@ -56,6 +64,9 @@
typedef struct bar_s { int x; signed char a[]; } bar_t;
enum foo_e { AA, BB, CC };
int strlen(const char *);
+ struct with_union { union { int a; char b; }; };
+ union with_struct { struct { int a; char b; }; };
+ struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; };
""")
ffi.set_source('re_python_pysrc', None)
ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py')))
@@ -105,12 +116,14 @@
from re_python_pysrc import ffi
lib = ffi.dlopen(extmod)
ffi.dlclose(lib)
- e = py.test.raises(ffi.error, ffi.dlclose, lib)
- assert str(e.value).startswith(
- "library '%s' is already closed" % (extmod,))
+ if type(extmod) is not str: # unicode, on python 2
+ str_extmod = extmod.encode('utf-8')
+ else:
+ str_extmod = extmod
e = py.test.raises(ffi.error, getattr, lib, 'add42')
assert str(e.value) == (
- "library '%s' has been closed" % (extmod,))
+ "library '%s' has been closed" % (str_extmod,))
+ ffi.dlclose(lib) # does not raise
def test_constant_via_lib():
from re_python_pysrc import ffi
@@ -213,3 +226,23 @@
ffi.set_source('test_partial_enum', None)
py.test.raises(VerificationMissing, ffi.emit_python_code,
str(tmpdir.join('test_partial_enum.py')))
+
+def test_anonymous_union_inside_struct():
+ # based on issue #357
+ from re_python_pysrc import ffi
+ INT = ffi.sizeof("int")
+ assert ffi.offsetof("struct with_union", "a") == 0
+ assert ffi.offsetof("struct with_union", "b") == 0
+ assert ffi.sizeof("struct with_union") == INT
+ #
+ assert ffi.offsetof("union with_struct", "a") == 0
+ assert ffi.offsetof("union with_struct", "b") == INT
+ assert ffi.sizeof("union with_struct") >= INT + 1
+ #
+ FLOAT = ffi.sizeof("float")
+ assert ffi.sizeof("struct NVGcolor") == FLOAT * 4
+ assert ffi.offsetof("struct NVGcolor", "rgba") == 0
+ assert ffi.offsetof("struct NVGcolor", "r") == 0
+ assert ffi.offsetof("struct NVGcolor", "g") == FLOAT
+ assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2
+ assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3
diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py
b/pypy/module/test_lib_pypy/test_dbm_extra.py
--- a/pypy/module/test_lib_pypy/test_dbm_extra.py
+++ b/pypy/module/test_lib_pypy/test_dbm_extra.py
@@ -1,4 +1,4 @@
-import py
+import py, os
from rpython.tool.udir import udir
try:
from lib_pypy import dbm
@@ -73,3 +73,8 @@
assert 'key_with_empty_value' in d
assert d['key_with_empty_value'] == ''
d.close()
+
+def test_unicode_filename():
+ path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename'
+ d = dbm.open(path, 'c')
+ d.close()
diff --git a/pypy/module/unicodedata/test/test_hyp.py
b/pypy/module/unicodedata/test/test_hyp.py
--- a/pypy/module/unicodedata/test/test_hyp.py
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -1,6 +1,7 @@
+import sys
import pytest
try:
- from hypothesis import given, strategies as st, example, settings
+ from hypothesis import given, strategies as st, example, settings, assume
except ImportError:
pytest.skip("hypothesis required")
@@ -40,9 +41,14 @@
@pytest.mark.parametrize('NF1, NF2, NF3', compositions)
@example(s=u'---\uafb8\u11a7---') # issue 2289
-@example(s=u'\ufacf')
@settings(max_examples=1000)
@given(s=st.text())
def test_composition(s, space, NF1, NF2, NF3):
+ # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big')
+ assume(not (s == u'\ufacf' and sys.maxunicode == 65535))
norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1,
NF2, NF3]]
assert norm2(norm1(s)) == norm3(s)
+
+if sys.maxunicode != 65535:
+ # conditionally generate the example via an unwrapped decorator
+ test_composition = example(s=u'\ufacf')(test_composition)
diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py
--- a/rpython/annotator/signature.py
+++ b/rpython/annotator/signature.py
@@ -14,16 +14,16 @@
def _annotation_key(t):
from rpython.rtyper import extregistry
- if type(t) is list:
+ if isinstance(t, list):
assert len(t) == 1
return ('list', _annotation_key(t[0]))
- elif type(t) is dict:
+ elif isinstance(t, dict):
assert len(t.keys()) == 1
return ('dict', _annotation_key(t.items()[0]))
elif isinstance(t, tuple):
return tuple([_annotation_key(i) for i in t])
elif extregistry.is_registered(t):
- # XXX should it really be always different?
+ # XXX do we want to do something in this case?
return t
return t
@@ -38,24 +38,36 @@
return t
return _compute_annotation(t, bookkeeper)
+
+def _validate_annotation_size(t):
+ try:
+ _ = iter(t)
+ except TypeError: # if it's not an iterable, just return
+ return t # (size does not matter)
+ if isinstance(t, tuple): # we accept tuples with any length, because
+ return t # their in-memory representation is predictable
+ if len(t) > 1:
+ raise TypeError("Cannot specify multiple types in a %s (try using
tuple)", type(t))
+
+
def _compute_annotation(t, bookkeeper=None):
from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.llannotation import lltype_to_annotation
+ _validate_annotation_size(t)
if isinstance(t, SomeObject):
return t
elif isinstance(t, lltype.LowLevelType):
return lltype_to_annotation(t)
elif isinstance(t, list):
- assert len(t) == 1, "We do not support type joining in list"
- listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True,
resized=True)
- return SomeList(listdef)
+ return SomeList(
+ ListDef(bookkeeper, annotation(t[0]),
+ mutated=True, resized=True))
elif isinstance(t, tuple):
return SomeTuple(tuple([annotation(i) for i in t]))
elif isinstance(t, dict):
- assert len(t) == 1, "We do not support type joining in dict"
- result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]),
- annotation(t.values()[0])))
- return result
+ return SomeDict(
+ DictDef(bookkeeper,
+ annotation(t.keys()[0]), annotation(t.values()[0])))
elif type(t) is types.NoneType:
return s_None
elif extregistry.is_registered(t):
@@ -84,13 +96,12 @@
elif t is types.NoneType:
return s_None
elif bookkeeper and extregistry.is_registered_type(t):
- entry = extregistry.lookup_type(t)
- return entry.compute_annotation_bk(bookkeeper)
+ return (extregistry.lookup_type(t)
+ .compute_annotation_bk(bookkeeper))
elif t is type:
return SomeType()
elif bookkeeper and not hasattr(t, '_freeze_'):
- classdef = bookkeeper.getuniqueclassdef(t)
- return SomeInstance(classdef)
+ return SomeInstance(bookkeeper.getuniqueclassdef(t))
else:
raise AssertionError("annotationoftype(%r)" % (t,))
diff --git a/rpython/jit/backend/llsupport/rewrite.py
b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -354,7 +354,7 @@
else:
# this is dead code, but in case we have a gc that does
# not have a write barrier and does not zero memory, we would
- # need to clal it
+ # need to call it
if op.getopnum() == rop.SETFIELD_GC:
self.consider_setfield_gc(op)
elif op.getopnum() == rop.SETARRAYITEM_GC:
diff --git a/rpython/jit/metainterp/compile.py
b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -30,7 +30,7 @@
class CompileData(object):
memo = None
log_noopt = True
-
+
def forget_optimization_info(self):
for arg in self.trace.inputargs:
arg.set_forwarded(None)
@@ -67,19 +67,26 @@
""" This represents label() ops jump with no extra info associated with
the label
"""
- def __init__(self, trace, call_pure_results=None,
+ def __init__(self, trace, resumestorage=None, call_pure_results=None,
enable_opts=None):
self.trace = trace
+ self.resumestorage = resumestorage
self.call_pure_results = call_pure_results
self.enable_opts = enable_opts
def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
+ from rpython.jit.metainterp.optimizeopt.bridgeopt import
deserialize_optimizer_knowledge
#assert not unroll
opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
- return opt.propagate_all_forward(self.trace.get_iter(),
- self.call_pure_results)
+ traceiter = self.trace.get_iter()
+ if self.resumestorage:
+ frontend_inputargs = self.trace.inputargs
+ deserialize_optimizer_knowledge(opt, self.resumestorage,
+ frontend_inputargs,
+ traceiter.inputargs)
+ return opt.propagate_all_forward(traceiter, self.call_pure_results)
class BridgeCompileData(CompileData):
""" This represents ops() with a jump at the end that goes to some
@@ -518,7 +525,7 @@
for item in lst:
item.set_forwarded(None)
# XXX we should really do it, but we need to remember the values
- # somehoe for ContinueRunningNormally
+ # somehow for ContinueRunningNormally
if reset_values:
item.reset_value()
@@ -671,38 +678,16 @@
raise jitexc.ExitFrameWithExceptionRef(cpu, value)
-class TerminatingLoopToken(JitCellToken): # FIXME: kill?
- terminating = True
-
- def __init__(self, nargs, finishdescr):
- self.finishdescr = finishdescr
-
-def make_done_loop_tokens():
- done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid()
- done_with_this_frame_descr_int = DoneWithThisFrameDescrInt()
- done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef()
- done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat()
- exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef()
-
- # pseudo loop tokens to make the life of optimize.py easier
- d = {'loop_tokens_done_with_this_frame_int': [
- TerminatingLoopToken(1, done_with_this_frame_descr_int)
- ],
- 'loop_tokens_done_with_this_frame_ref': [
- TerminatingLoopToken(1, done_with_this_frame_descr_ref)
- ],
- 'loop_tokens_done_with_this_frame_float': [
- TerminatingLoopToken(1, done_with_this_frame_descr_float)
- ],
- 'loop_tokens_done_with_this_frame_void': [
- TerminatingLoopToken(0, done_with_this_frame_descr_void)
- ],
- 'loop_tokens_exit_frame_with_exception_ref': [
- TerminatingLoopToken(1, exit_frame_with_exception_descr_ref)
- ],
- }
- d.update(locals())
- return d
+def make_and_attach_done_descrs(targets):
+ for name, cls in [
+ ("done_with_this_frame_descr_void", DoneWithThisFrameDescrVoid),
+ ("done_with_this_frame_descr_int", DoneWithThisFrameDescrInt),
+ ("done_with_this_frame_descr_ref", DoneWithThisFrameDescrRef),
+ ("done_with_this_frame_descr_float", DoneWithThisFrameDescrFloat),
+ ("exit_frame_with_exception_descr_ref",
ExitFrameWithExceptionDescrRef)]:
+ descr = cls()
+ for target in targets:
+ setattr(target, name, descr)
class ResumeDescr(AbstractFailDescr):
_attrs_ = ()
@@ -726,6 +711,9 @@
TY_REF = 0x04
TY_FLOAT = 0x06
+ def get_resumestorage(self):
+ raise NotImplementedError("abstract base class")
+
def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd)
and not rstack.stack_almost_full()):
@@ -854,15 +842,23 @@
class ResumeGuardCopiedDescr(AbstractResumeGuardDescr):
_attrs_ = ('status', 'prev')
+ def __init__(self, prev):
+ AbstractResumeGuardDescr.__init__(self)
+ assert isinstance(prev, ResumeGuardDescr)
+ self.prev = prev
+
def copy_all_attributes_from(self, other):
assert isinstance(other, ResumeGuardCopiedDescr)
self.prev = other.prev
def clone(self):
- cloned = ResumeGuardCopiedDescr()
- cloned.copy_all_attributes_from(self)
+ cloned = ResumeGuardCopiedDescr(self.prev)
return cloned
+ def get_resumestorage(self):
+ prev = self.prev
+ assert isinstance(prev, ResumeGuardDescr)
+ return prev
class ResumeGuardDescr(AbstractResumeGuardDescr):
_attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals',
@@ -873,8 +869,7 @@
rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO)
def copy_all_attributes_from(self, other):
- if isinstance(other, ResumeGuardCopiedDescr):
- other = other.prev
+ other = other.get_resumestorage()
assert isinstance(other, ResumeGuardDescr)
self.rd_consts = other.rd_consts
self.rd_pendingfields = other.rd_pendingfields
@@ -895,6 +890,9 @@
cloned.copy_all_attributes_from(self)
return cloned
+ def get_resumestorage(self):
+ return self
+
class ResumeGuardExcDescr(ResumeGuardDescr):
pass
@@ -936,22 +934,22 @@
ptr = cpu.ts.cast_to_baseclass(gcref)
return cast_base_ptr_to_instance(AllVirtuals, ptr)
-def invent_fail_descr_for_op(opnum, optimizer, copied_guard=False):
+def invent_fail_descr_for_op(opnum, optimizer, copied_from_descr=None):
if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2:
- assert not copied_guard
+ assert copied_from_descr is None
resumedescr = ResumeGuardForcedDescr()
resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd)
elif opnum in (rop.GUARD_IS_OBJECT, rop.GUARD_SUBCLASS, rop.GUARD_GC_TYPE):
# note - this only happens in tests
resumedescr = ResumeAtPositionDescr()
elif opnum in (rop.GUARD_EXCEPTION, rop.GUARD_NO_EXCEPTION):
- if copied_guard:
- resumedescr = ResumeGuardCopiedExcDescr()
+ if copied_from_descr is not None:
+ resumedescr = ResumeGuardCopiedExcDescr(copied_from_descr)
else:
resumedescr = ResumeGuardExcDescr()
else:
- if copied_guard:
- resumedescr = ResumeGuardCopiedDescr()
+ if copied_from_descr is not None:
+ resumedescr = ResumeGuardCopiedDescr(copied_from_descr)
else:
resumedescr = ResumeGuardDescr()
return resumedescr
@@ -1036,6 +1034,9 @@
self.original_greenkey, jitcell_token)
metainterp_sd.stats.add_jitcell_token(jitcell_token)
+ def get_resumestorage(self):
+ return None
+
def compile_trace(metainterp, resumekey, runtime_boxes):
"""Try to compile a new bridge leading from the beginning of the history
@@ -1067,22 +1068,15 @@
enable_opts = jitdriver_sd.warmstate.enable_opts
call_pure_results = metainterp.call_pure_results
+ resumestorage = resumekey.get_resumestorage()
if metainterp.history.ends_with_jump:
- if isinstance(resumekey, ResumeGuardCopiedDescr):
- key = resumekey.prev
- assert isinstance(key, ResumeGuardDescr)
- elif isinstance(resumekey, ResumeFromInterpDescr):
- key = None
- else:
- key = resumekey
- assert isinstance(key, ResumeGuardDescr)
- data = BridgeCompileData(trace, runtime_boxes, key,
+ data = BridgeCompileData(trace, runtime_boxes, resumestorage,
call_pure_results=call_pure_results,
enable_opts=enable_opts,
inline_short_preamble=inline_short_preamble)
else:
- data = SimpleCompileData(trace,
+ data = SimpleCompileData(trace, resumestorage,
call_pure_results=call_pure_results,
enable_opts=enable_opts)
try:
diff --git a/rpython/jit/metainterp/history.py
b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -404,7 +404,6 @@
target_tokens = None
failed_states = None
retraced_count = 0
- terminating = False # see TerminatingLoopToken in compile.py
invalidated = False
outermost_jitdriver_sd = None
# and more data specified by the backend when the loop is compiled
@@ -935,7 +934,7 @@
return insns
def check_simple_loop(self, expected=None, **check):
- """ Usefull in the simplest case when we have only one trace ending
with
+ """ Useful in the simplest case when we have only one trace ending with
a jump back to itself and possibly a few bridges.
Only the operations within the loop formed by that single jump will
be counted.
diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py
b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
--- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py
@@ -17,11 +17,17 @@
# <length>
# (<box1> <descr> <box2>) length times, if getfield(box1, descr) == box2
# both boxes should be in the liveboxes
+# (or constants)
#
# <length>
# (<box1> <index> <descr> <box2>) length times, if getarrayitem_gc(box1,
index, descr) == box2
# both boxes should be in the liveboxes
+# (or constants)
#
+# ---- call_loopinvariant knowledge
+# <length>
+# (<const> <box2>) length times, if call_loopinvariant(const) == box2
+# box2 should be in liveboxes
# ----
@@ -55,11 +61,11 @@
return box
def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes,
liveboxes_from_env, memo):
+ from rpython.jit.metainterp.history import ConstInt
available_boxes = {}
for box in liveboxes:
if box is not None and box in liveboxes_from_env:
available_boxes[box] = None
- metainterp_sd = optimizer.metainterp_sd
# class knowledge is stored as bits, true meaning the class is known, false
# means unknown. on deserializing we look at the bits, and read the runtime
@@ -106,7 +112,19 @@
numb_state.append_int(0)
numb_state.append_int(0)
+ if optimizer.optrewrite:
+ tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite(
+ available_boxes)
+ numb_state.append_int(len(tuples_loopinvariant))
+ for constarg0, box in tuples_loopinvariant:
+ numb_state.append_short(
+ tag_box(ConstInt(constarg0), liveboxes_from_env, memo))
+ numb_state.append_short(tag_box(box, liveboxes_from_env, memo))
+ else:
+ numb_state.append_int(0)
+
def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes,
liveboxes):
+ from rpython.jit.metainterp.history import ConstInt
reader = resumecode.Reader(resumestorage.rd_numb)
assert len(frontend_boxes) == len(liveboxes)
metainterp_sd = optimizer.metainterp_sd
@@ -131,8 +149,6 @@
optimizer.make_constant_class(box, cls)
# heap knowledge
- if not optimizer.optheap:
- return
length = reader.next_item()
result_struct = []
for i in range(length):
@@ -154,4 +170,19 @@
tagged = reader.next_item()
box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu)
result_array.append((box1, index, descr, box2))
- optimizer.optheap.deserialize_optheap(result_struct, result_array)
+ if optimizer.optheap:
+ optimizer.optheap.deserialize_optheap(result_struct, result_array)
+
+ # call_loopinvariant knowledge
+ length = reader.next_item()
+ result_loopinvariant = []
+ for i in range(length):
+ tagged1 = reader.next_item()
+ const = decode_box(resumestorage, tagged1, liveboxes,
metainterp_sd.cpu)
+ assert isinstance(const, ConstInt)
+ i = const.getint()
+ tagged2 = reader.next_item()
+ box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu)
+ result_loopinvariant.append((i, box))
+ if optimizer.optrewrite:
+ optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py
b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -688,12 +688,10 @@
def _copy_resume_data_from(self, guard_op, last_guard_op):
- descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self,
True)
last_descr = last_guard_op.getdescr()
+ descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self,
last_descr)
assert isinstance(last_descr, compile.ResumeGuardDescr)
- if isinstance(descr, compile.ResumeGuardCopiedDescr):
- descr.prev = last_descr
- else:
+ if not isinstance(descr, compile.ResumeGuardCopiedDescr):
descr.copy_all_attributes_from(last_descr)
guard_op.setdescr(descr)
guard_op.setfailargs(last_guard_op.getfailargs())
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py
b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -877,6 +877,18 @@
optimize_SAME_AS_R = optimize_SAME_AS_I
optimize_SAME_AS_F = optimize_SAME_AS_I
+ def serialize_optrewrite(self, available_boxes):
+ res = []
+ for i, box in self.loop_invariant_results.iteritems():
+ box = self.get_box_replacement(box)
+ if box in available_boxes:
+ res.append((i, box))
+ return res
+
+ def deserialize_optrewrite(self, tups):
+ for i, box in tups:
+ self.loop_invariant_results[i] = box
+
dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
default=OptRewrite.emit)
optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -31,8 +31,8 @@
expected = convert_old_style_to_targets(exp, jump=True)
call_pure_results = self._convert_call_pure_results(call_pure_results)
trace = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu))
- compile_data = compile.SimpleCompileData(trace,
- call_pure_results)
+ compile_data = compile.SimpleCompileData(
+ trace, call_pure_results=call_pure_results)
info, ops = self._do_optimize_loop(compile_data)
label_op = ResOperation(rop.LABEL, info.inputargs)
loop.inputargs = info.inputargs
diff --git a/rpython/jit/metainterp/pyjitpl.py
b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1854,12 +1854,7 @@
self._addr2name_keys = []
self._addr2name_values = []
- self.__dict__.update(compile.make_done_loop_tokens())
- for val in ['int', 'float', 'ref', 'void']:
- fullname = 'done_with_this_frame_descr_' + val
- setattr(self.cpu, fullname, getattr(self, fullname))
- d = self.exit_frame_with_exception_descr_ref
- self.cpu.exit_frame_with_exception_descr_ref = d
+ compile.make_and_attach_done_descrs([self, cpu])
def _freeze_(self):
return True
@@ -1909,8 +1904,8 @@
history.REF: 'ref',
history.FLOAT: 'float',
history.VOID: 'void'}[jd.result_type]
- tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' %
name)
- jd.portal_finishtoken = tokens[0].finishdescr
+ token = getattr(self, 'done_with_this_frame_descr_%s' % name)
+ jd.portal_finishtoken = token
jd.propagate_exc_descr = exc_descr
#
self.cpu.propagate_exception_descr = exc_descr
@@ -2463,10 +2458,7 @@
def handle_guard_failure(self, resumedescr, deadframe):
debug_start('jit-tracing')
self.staticdata.profiler.start_tracing()
- if isinstance(resumedescr, compile.ResumeGuardCopiedDescr):
- key = resumedescr.prev
- else:
- key = resumedescr
+ key = resumedescr.get_resumestorage()
assert isinstance(key, compile.ResumeGuardDescr)
# store the resumekey.wref_original_loop_token() on 'self' to make
# sure that it stays alive as long as this MetaInterp
@@ -2770,21 +2762,19 @@
if result_type == history.VOID:
assert exitbox is None
exits = []
- loop_tokens = sd.loop_tokens_done_with_this_frame_void
+ token = sd.done_with_this_frame_descr_void
elif result_type == history.INT:
exits = [exitbox]
- loop_tokens = sd.loop_tokens_done_with_this_frame_int
+ token = sd.done_with_this_frame_descr_int
elif result_type == history.REF:
exits = [exitbox]
- loop_tokens = sd.loop_tokens_done_with_this_frame_ref
+ token = sd.done_with_this_frame_descr_ref
elif result_type == history.FLOAT:
exits = [exitbox]
- loop_tokens = sd.loop_tokens_done_with_this_frame_float
+ token = sd.done_with_this_frame_descr_float
else:
assert False
- # FIXME: kill TerminatingLoopToken?
# FIXME: can we call compile_trace?
- token = loop_tokens[0].finishdescr
self.history.record(rop.FINISH, exits, None, descr=token)
self.history.trace.done()
target_token = compile.compile_trace(self, self.resumekey, exits)
@@ -2810,7 +2800,7 @@
def compile_exit_frame_with_exception(self, valuebox):
self.store_token_in_vable()
sd = self.staticdata
- token = sd.loop_tokens_exit_frame_with_exception_ref[0].finishdescr
+ token = sd.exit_frame_with_exception_descr_ref
self.history.record(rop.FINISH, [valuebox], None, descr=token)
self.history.trace.done()
target_token = compile.compile_trace(self, self.resumekey, [valuebox])
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py
b/rpython/jit/metainterp/test/test_bridgeopt.py
--- a/rpython/jit/metainterp/test/test_bridgeopt.py
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -1,6 +1,9 @@
# tests that check that information is fed from the optimizer into the bridges
+import pytest
+
import math
+
from rpython.rlib import jit
from rpython.jit.metainterp.test.support import LLJitMixin
from rpython.jit.metainterp.optimizeopt.bridgeopt import
serialize_optimizer_knowledge
@@ -27,6 +30,7 @@
class FakeOptimizer(object):
metainterp_sd = None
optheap = None
+ optrewrite = None
def __init__(self, dct={}, cpu=None):
self.dct = dct
@@ -61,7 +65,8 @@
serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
- assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0,
0]
+ assert unpack_numbering(numb_state.create_numbering()) == [
+ 1, 0b010000, 0, 0, 0]
rbox1 = InputArgRef()
rbox2 = InputArgRef()
@@ -100,7 +105,7 @@
serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None)
- assert len(numb_state.create_numbering().code) == 3 +
math.ceil(len(refboxes) / 6.0)
+ assert len(numb_state.create_numbering().code) == 4 +
math.ceil(len(refboxes) / 6.0)
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
