Author: Ronan Lamy <[email protected]>
Branch: test-cpyext
Changeset: r87011:df6683b8ed30
Date: 2016-09-11 21:03 +0100
http://bitbucket.org/pypy/pypy/changeset/df6683b8ed30/
Log: hg merge default
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -12,4 +12,7 @@
Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
in numpypy
-
+.. branch: force-virtual-state
+Improve merging of virtual states in the JIT in order to avoid jumping to the
+preamble. Accomplished by allocating virtual objects where non-virtuals are
+expected.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -119,7 +119,7 @@
constant_names = """
Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER
-METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE
+METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT
METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS
Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER
Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
@@ -645,7 +645,7 @@
('format', rffi.CCHARP),
('shape', Py_ssize_tP),
('strides', Py_ssize_tP),
- ('_format', rffi.UCHAR),
+ ('_format', rffi.CFixedArray(rffi.UCHAR, Py_MAX_FMT)),
('_shape', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
('_strides', rffi.CFixedArray(Py_ssize_t, Py_MAX_NDIMS)),
('suboffsets', Py_ssize_tP),
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -1,9 +1,7 @@
-from pypy.interpreter.error import oefmt
-from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.rarithmetic import widen
+from rpython.rtyper.lltypesystem import rffi
from pypy.module.cpyext.api import (
- cpython_api, CANNOT_FAIL, Py_buffer, Py_TPFLAGS_HAVE_NEWBUFFER,
Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+ cpython_api, CANNOT_FAIL, Py_TPFLAGS_HAVE_NEWBUFFER)
+from pypy.module.cpyext.pyobject import PyObject
@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
def PyObject_CheckBuffer(space, pyobj):
@@ -14,102 +12,4 @@
return 1
return 0
-@cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
- rffi.INT_real, error=-1)
-def PyObject_GetBuffer(space, w_obj, view, flags):
- """Export obj into a Py_buffer, view. These arguments must
- never be NULL. The flags argument is a bit field indicating what
- kind of buffer the caller is prepared to deal with and therefore what
- kind of buffer the exporter is allowed to return. The buffer interface
- allows for complicated memory sharing possibilities, but some caller may
- not be able to handle all the complexity but may want to see if the
- exporter will let them take a simpler view to its memory.
-
- Some exporters may not be able to share memory in every possible way and
- may need to raise errors to signal to some consumers that something is
- just not possible. These errors should be a BufferError unless
- there is another error that is actually causing the problem. The
- exporter can use flags information to simplify how much of the
- Py_buffer structure is filled in with non-default values and/or
- raise an error if the object can't support a simpler view of its memory.
-
- 0 is returned on success and -1 on error."""
- flags = widen(flags)
- buf = space.buffer_w(w_obj, flags)
- try:
- view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
- except ValueError:
- raise BufferError("could not create buffer from object")
- view.c_len = buf.getlength()
- view.c_obj = make_ref(space, w_obj)
- ndim = buf.getndim()
- view.c_itemsize = buf.getitemsize()
- rffi.setintfield(view, 'c_readonly', int(buf.readonly))
- rffi.setintfield(view, 'c_ndim', ndim)
- view.c_format = rffi.str2charp(buf.getformat())
- view.c_shape = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
- view.c_strides = lltype.malloc(Py_ssize_tP.TO, ndim, flavor='raw')
- shape = buf.getshape()
- strides = buf.getstrides()
- for i in range(ndim):
- view.c_shape[i] = shape[i]
- view.c_strides[i] = strides[i]
- view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
- view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
- return 0
-
-def _IsFortranContiguous(view):
- ndim = widen(view.c_ndim)
- if ndim == 0:
- return 1
- if not view.c_strides:
- return ndim == 1
- sd = view.c_itemsize
- if ndim == 1:
- return view.c_shape[0] == 1 or sd == view.c_strides[0]
- for i in range(view.c_ndim):
- dim = view.c_shape[i]
- if dim == 0:
- return 1
- if view.c_strides[i] != sd:
- return 0
- sd *= dim
- return 1
-
-def _IsCContiguous(view):
- ndim = widen(view.c_ndim)
- if ndim == 0:
- return 1
- if not view.c_strides:
- return ndim == 1
- sd = view.c_itemsize
- if ndim == 1:
- return view.c_shape[0] == 1 or sd == view.c_strides[0]
- for i in range(ndim - 1, -1, -1):
- dim = view.c_shape[i]
- if dim == 0:
- return 1
- if view.c_strides[i] != sd:
- return 0
- sd *= dim
- return 1
-
-
-@cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real,
error=CANNOT_FAIL)
-def PyBuffer_IsContiguous(space, view, fort):
- """Return 1 if the memory defined by the view is C-style (fortran is
- 'C') or Fortran-style (fortran is 'F') contiguous or either one
- (fortran is 'A'). Return 0 otherwise."""
- # traverse the strides, checking for consistent stride increases from
- # right-to-left (c) or left-to-right (fortran). Copied from cpython
- if not view.c_suboffsets:
- return 0
- if (fort == 'C'):
- return _IsCContiguous(view)
- elif (fort == 'F'):
- return _IsFortranContiguous(view)
- elif (fort == 'A'):
- return (_IsCContiguous(view) or _IsFortranContiguous(view))
- return 0
-
diff --git a/pypy/module/cpyext/include/object.h
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -144,6 +144,7 @@
/* Py3k buffer interface, adapted for PyPy */
#define Py_MAX_NDIMS 32
+#define Py_MAX_FMT 5
typedef struct bufferinfo {
void *buf;
PyObject *obj; /* owned reference */
@@ -158,7 +159,7 @@
Py_ssize_t *shape;
Py_ssize_t *strides;
Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/
- unsigned char _format;
+ unsigned char _format[Py_MAX_FMT];
Py_ssize_t _strides[Py_MAX_NDIMS];
Py_ssize_t _shape[Py_MAX_NDIMS];
/* static store for shape and strides of
diff --git a/pypy/module/cpyext/memoryobject.py
b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,11 +1,125 @@
from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
- Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP)
+ Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
Py_ssize_tP)
from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import widen
from pypy.objspace.std.memoryobject import W_MemoryView
PyMemoryView_Check, PyMemoryView_CheckExact =
build_type_checkers("MemoryView", "w_memoryview")
+@cpython_api([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real],
+ rffi.INT_real, error=-1)
+def PyObject_GetBuffer(space, w_obj, view, flags):
+ """Export obj into a Py_buffer, view. These arguments must
+ never be NULL. The flags argument is a bit field indicating what
+ kind of buffer the caller is prepared to deal with and therefore what
+ kind of buffer the exporter is allowed to return. The buffer interface
+ allows for complicated memory sharing possibilities, but some caller may
+ not be able to handle all the complexity but may want to see if the
+ exporter will let them take a simpler view to its memory.
+
+ Some exporters may not be able to share memory in every possible way and
+ may need to raise errors to signal to some consumers that something is
+ just not possible. These errors should be a BufferError unless
+ there is another error that is actually causing the problem. The
+ exporter can use flags information to simplify how much of the
+ Py_buffer structure is filled in with non-default values and/or
+ raise an error if the object can't support a simpler view of its memory.
+
+ 0 is returned on success and -1 on error."""
+ flags = widen(flags)
+ buf = space.buffer_w(w_obj, flags)
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ except ValueError:
+ raise BufferError("could not create buffer from object")
+ view.c_obj = make_ref(space, w_obj)
+ return fill_Py_buffer(space, buf, view)
+
+def fill_Py_buffer(space, buf, view):
+ # c_buf, c_obj have been filled in
+ ndim = buf.getndim()
+ view.c_len = buf.getlength()
+ view.c_itemsize = buf.getitemsize()
+ rffi.setintfield(view, 'c_ndim', ndim)
+ view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
+ view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+ view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+ fmt = buf.getformat()
+ n = Py_MAX_FMT - 1 # NULL terminated buffer
+ if len(fmt) > n:
+ ### WARN?
+ pass
+ else:
+ n = len(fmt)
+ for i in range(n):
+ if ord(fmt[i]) > 255:
+ view.c_format[i] = '*'
+ else:
+ view.c_format[i] = fmt[i]
+ view.c_format[n] = '\x00'
+ shape = buf.getshape()
+ strides = buf.getstrides()
+ for i in range(ndim):
+ view.c_shape[i] = shape[i]
+ view.c_strides[i] = strides[i]
+ view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+ view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+ return 0
+
+def _IsFortranContiguous(view):
+ ndim = widen(view.c_ndim)
+ if ndim == 0:
+ return 1
+ if not view.c_strides:
+ return ndim == 1
+ sd = view.c_itemsize
+ if ndim == 1:
+ return view.c_shape[0] == 1 or sd == view.c_strides[0]
+ for i in range(view.c_ndim):
+ dim = view.c_shape[i]
+ if dim == 0:
+ return 1
+ if view.c_strides[i] != sd:
+ return 0
+ sd *= dim
+ return 1
+
+def _IsCContiguous(view):
+ ndim = widen(view.c_ndim)
+ if ndim == 0:
+ return 1
+ if not view.c_strides:
+ return ndim == 1
+ sd = view.c_itemsize
+ if ndim == 1:
+ return view.c_shape[0] == 1 or sd == view.c_strides[0]
+ for i in range(ndim - 1, -1, -1):
+ dim = view.c_shape[i]
+ if dim == 0:
+ return 1
+ if view.c_strides[i] != sd:
+ return 0
+ sd *= dim
+ return 1
+
+@cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real,
error=CANNOT_FAIL)
+def PyBuffer_IsContiguous(space, view, fort):
+ """Return 1 if the memory defined by the view is C-style (fortran is
+ 'C') or Fortran-style (fortran is 'F') contiguous or either one
+ (fortran is 'A'). Return 0 otherwise."""
+ # traverse the strides, checking for consistent stride increases from
+ # right-to-left (c) or left-to-right (fortran). Copied from cpython
+ if not view.c_suboffsets:
+ return 0
+ if (fort == 'C'):
+ return _IsCContiguous(view)
+ elif (fort == 'F'):
+ return _IsFortranContiguous(view)
+ elif (fort == 'A'):
+ return (_IsCContiguous(view) or _IsFortranContiguous(view))
+ return 0
+
@cpython_api([PyObject], PyObject)
def PyMemoryView_FromObject(space, w_obj):
return space.call_method(space.builtin, "memoryview", w_obj)
@@ -38,19 +152,6 @@
view.c_obj = make_ref(space, w_s)
rffi.setintfield(view, 'c_readonly', 1)
isstr = True
- view.c_len = w_obj.getlength()
- view.c_itemsize = w_obj.buf.getitemsize()
- rffi.setintfield(view, 'c_ndim', ndim)
- view.c__format = rffi.cast(rffi.UCHAR, w_obj.buf.getformat())
- view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
- view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
- view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
- shape = w_obj.buf.getshape()
- strides = w_obj.buf.getstrides()
- for i in range(ndim):
- view.c_shape[i] = shape[i]
- view.c_strides[i] = strides[i]
- view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
- view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+ fill_Py_buffer(space, w_obj.buf, view)
return view
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -43,16 +43,20 @@
def PySequence_Fast(space, w_obj, m):
"""Returns the sequence o as a tuple, unless it is already a tuple or
list, in
which case o is returned. Use PySequence_Fast_GET_ITEM() to access the
- members of the result. Returns NULL on failure. If the object is not a
- sequence, raises TypeError with m as the message text."""
+ members of the result. Returns NULL on failure. If the object cannot be
+ converted to a sequence, and raises a TypeError, raise a new TypeError with
+ m as the message text. If the conversion otherwise, fails, reraise the
+ original exception"""
if isinstance(w_obj, W_ListObject):
# make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM
w_obj.convert_to_cpy_strategy(space)
return w_obj
try:
return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
- except OperationError:
- raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
+ except OperationError as e:
+ if e.match(space, space.w_TypeError):
+ raise OperationError(space.w_TypeError,
space.wrap(rffi.charp2str(m)))
+ raise e
@cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
def PySequence_Fast_GET_ITEM(space, w_obj, index):
diff --git a/pypy/module/cpyext/test/test_api.py
b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -1,5 +1,5 @@
import py, pytest
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
from pypy.interpreter.baseobjspace import W_Root
from pypy.module.cpyext.state import State
from pypy.module.cpyext import api
diff --git a/pypy/module/cpyext/test/test_bufferobject.py
b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import lltype
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from pypy.module.cpyext.api import PyObject
diff --git a/pypy/module/cpyext/test/test_cpyext.py
b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -18,6 +18,8 @@
from .support import c_compile
+only_pypy ="config.option.runappdirect and '__pypy__' not in
sys.builtin_module_names"
+
@api.cpython_api([], api.PyObject)
def PyPy_Crash1(space):
1/0
@@ -288,11 +290,11 @@
"the test actually passed in the first place; if it failed "
"it is likely to reach this place.")
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_only_import(self):
import cpyext
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_load_error(self):
import cpyext
raises(ImportError, cpyext.load_module, "missing.file", "foo")
@@ -873,7 +875,7 @@
])
raises(SystemError, mod.newexc, "name", Exception, {})
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy specific test')
+ @pytest.mark.skipif(only_pypy, reason='pypy specific test')
def test_hash_pointer(self):
mod = self.import_extension('foo', [
('get_hash', 'METH_NOARGS',
@@ -924,7 +926,7 @@
print p
assert 'py' in p
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_get_version(self):
mod = self.import_extension('foo', [
('get_version', 'METH_NOARGS',
diff --git a/pypy/module/cpyext/test/test_memoryobject.py
b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,3 +1,4 @@
+from rpython.rtyper.lltypesystem import rffi
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from rpython.rlib.buffer import StringBuffer
@@ -16,8 +17,12 @@
w_buf = space.newbuffer(StringBuffer("hello"))
w_memoryview = api.PyMemoryView_FromObject(w_buf)
w_view = api.PyMemoryView_GET_BUFFER(w_memoryview)
- ndim = w_view.c_ndim
- assert ndim == 1
+ assert w_view.c_ndim == 1
+ f = rffi.charp2str(w_view.c_format)
+ assert f == 'B'
+ assert w_view.c_shape[0] == 5
+ assert w_view.c_strides[0] == 1
+ assert w_view.c_len == 5
class AppTestBufferProtocol(AppTestCpythonExtensionBase):
def test_buffer_protocol(self):
@@ -38,14 +43,10 @@
from _numpypy import multiarray as np
module = self.import_module(name='buffer_test')
get_buffer_info = module.get_buffer_info
- # test_export_flags from numpy test_multiarray
raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
- # test_relaxed_strides from numpy test_multiarray
- arr = np.zeros((1, 10))
- if arr.flags.f_contiguous:
- shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
- assert strides[0] == 8
- arr = np.ones((10, 1), order='F')
- shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
- assert strides[-1] == 8
-
+ arr = np.zeros((1, 10), order='F')
+ shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+ assert strides[0] == 8
+ arr = np.zeros((10, 1), order='C')
+ shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+ assert strides[-1] == 8
diff --git a/pypy/module/cpyext/test/test_sequence.py
b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -267,3 +267,31 @@
assert module.test_fast_sequence(s[0:-1])
assert module.test_fast_sequence(s[::-1])
+ def test_fast_keyerror(self):
+ module = self.import_extension('foo', [
+ ("test_fast_sequence", "METH_VARARGS",
+ """
+ PyObject *foo;
+ PyObject * seq = PyTuple_GetItem(args, 0);
+ if (seq == NULL)
+ Py_RETURN_NONE;
+ foo = PySequence_Fast(seq, "Could not convert object to
sequence");
+ if (foo != NULL)
+ {
+ return foo;
+ }
+ if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+ PyErr_Clear();
+ return PyBool_FromLong(1);
+ }
+ return NULL;
+ """)])
+ class Map(object):
+ def __len__(self):
+ return 1
+
+ def __getitem__(self, index):
+ raise KeyError()
+
+ assert module.test_fast_sequence(Map()) is True
+
diff --git a/pypy/module/cpyext/test/test_thread.py
b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -1,12 +1,13 @@
import sys
-import py, pytest
+import pytest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+only_pypy ="config.option.runappdirect and '__pypy__' not in
sys.builtin_module_names"
class AppTestThread(AppTestCpythonExtensionBase):
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_get_thread_ident(self):
module = self.import_extension('foo', [
("get_thread_ident", "METH_NOARGS",
@@ -33,7 +34,7 @@
assert results[0][0] != results[1][0]
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_acquire_lock(self):
module = self.import_extension('foo', [
("test_acquire_lock", "METH_NOARGS",
@@ -57,7 +58,7 @@
])
module.test_acquire_lock()
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_release_lock(self):
module = self.import_extension('foo', [
("test_release_lock", "METH_NOARGS",
@@ -79,7 +80,7 @@
])
module.test_release_lock()
- @pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_tls(self):
module = self.import_extension('foo', [
("create_key", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_version.py
b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -3,6 +3,7 @@
import py, pytest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+only_pypy ="config.option.runappdirect and '__pypy__' not in
sys.builtin_module_names"
def test_pragma_version():
from pypy.module.sys.version import CPYTHON_VERSION
@@ -32,11 +33,9 @@
assert module.py_minor_version == sys.version_info.minor
assert module.py_micro_version == sys.version_info.micro
- #@pytest.mark.skipif('__pypy__' not in sys.builtin_module_names,
reason='pypy only test')
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_pypy_versions(self):
import sys
- if '__pypy__' not in sys.builtin_module_names:
- py.test.skip("pypy only test")
init = """
if (Py_IsInitialized()) {
PyObject *m = Py_InitModule("foo", NULL);
diff --git a/rpython/jit/metainterp/optimizeopt/info.py
b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -365,6 +365,13 @@
def visitor_dispatch_virtual_type(self, visitor):
raise NotImplementedError("abstract")
+ def make_guards(self, op, short, optimizer):
+ from rpython.jit.metainterp.optimizeopt.optimizer import CONST_0
+ op = ResOperation(rop.INT_EQ, [op, CONST_0])
+ short.append(op)
+ op = ResOperation(rop.GUARD_FALSE, [op])
+ short.append(op)
+
class RawBufferPtrInfo(AbstractRawPtrInfo):
buffer = None
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7592,7 +7592,7 @@
ops = """
[i0]
p1 = new_with_vtable(descr=nodesize)
- cond_call(1, 123, p1, descr=clear_vable)
+ cond_call(i0, 123, p1, descr=clear_vable)
jump(i0)
"""
expected = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py
b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -16,7 +16,7 @@
from rpython.rlib.debug import debug_print, debug_start, debug_stop,\
have_debug_prints
-class UnrollableOptimizer(Optimizer):
+class UnrollableOptimizer(Optimizer):
def force_op_from_preamble(self, preamble_op):
if isinstance(preamble_op, PreambleOp):
if self.optunroll.short_preamble_producer is None:
@@ -120,7 +120,8 @@
assert op.get_forwarded() is None
if check_newops:
assert not self.optimizer._newoperations
-
+
+
def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
info, newops = self.optimizer.propagate_all_forward(
trace.get_iter(), call_pure_results, flush=False)
@@ -156,7 +157,7 @@
current_vs = self.get_virtual_state(end_jump.getarglist())
# pick the vs we want to jump to
assert isinstance(celltoken, JitCellToken)
-
+
target_virtual_state = self.pick_virtual_state(current_vs,
state.virtual_state,
celltoken.target_tokens)
@@ -180,17 +181,27 @@
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
- self.optimizer._newoperations)
+ self.optimizer._newoperations)
try:
- new_virtual_state = self.jump_to_existing_trace(end_jump, label_op,
-
state.runtime_boxes)
+ new_virtual_state = self.jump_to_existing_trace(
+ end_jump, label_op, state.runtime_boxes, force_boxes=False)
except InvalidLoop:
# inlining short preamble failed, jump to preamble
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
+
+ if new_virtual_state is not None:
+ # Attempt to force virtual boxes in order to avoid jumping
+ # to the preamble.
+ try:
+ new_virtual_state = self.jump_to_existing_trace(
+ end_jump, label_op, state.runtime_boxes,
force_boxes=True)
+ except InvalidLoop:
+ pass
+
if new_virtual_state is not None:
self.jump_to_preamble(celltoken, end_jump, info)
return (UnrollInfo(target_token, label_op, extra_same_as,
@@ -199,7 +210,7 @@
self.disable_retracing_if_max_retrace_guards(
self.optimizer._newoperations, target_token)
-
+
return (UnrollInfo(target_token, label_op, extra_same_as,
self.optimizer.quasi_immutable_deps),
self.optimizer._newoperations)
@@ -241,7 +252,8 @@
for a in jump_op.getarglist():
self.optimizer.force_box_for_end_of_preamble(a)
try:
- vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes)
+ vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+ force_boxes=False)
except InvalidLoop:
return self.jump_to_preamble(cell_token, jump_op, info)
if vs is None:
@@ -252,6 +264,14 @@
cell_token.retraced_count += 1
debug_print('Retracing (%d/%d)' % (cell_token.retraced_count,
limit))
else:
+ # Try forcing boxes to avoid jumping to the preamble
+ try:
+ vs = self.jump_to_existing_trace(jump_op, None, runtime_boxes,
+ force_boxes=True)
+ except InvalidLoop:
+ pass
+ if vs is None:
+ return info, self.optimizer._newoperations[:]
debug_print("Retrace count reached, jumping to preamble")
return self.jump_to_preamble(cell_token, jump_op, info)
exported_state = self.export_state(info.jump_op.getarglist(),
@@ -288,7 +308,7 @@
return info, self.optimizer._newoperations[:]
- def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes):
+ def jump_to_existing_trace(self, jump_op, label_op, runtime_boxes,
force_boxes=False):
jitcelltoken = jump_op.getdescr()
assert isinstance(jitcelltoken, JitCellToken)
virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -299,7 +319,8 @@
continue
try:
extra_guards = target_virtual_state.generate_guards(
- virtual_state, args, runtime_boxes, self.optimizer)
+ virtual_state, args, runtime_boxes, self.optimizer,
+ force_boxes=force_boxes)
patchguardop = self.optimizer.patchguardop
for guard in extra_guards.extra_guards:
if isinstance(guard, GuardResOp):
@@ -308,8 +329,18 @@
self.send_extra_operation(guard)
except VirtualStatesCantMatch:
continue
- args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
- args, self.optimizer)
+
+ # When force_boxes == True, creating the virtual args can fail when
+ # components of the virtual state alias. If this occurs, we must
+ # recompute the virtual state as boxes will have been forced.
+ try:
+ args, virtuals =
target_virtual_state.make_inputargs_and_virtuals(
+ args, self.optimizer, force_boxes=force_boxes)
+ except VirtualStatesCantMatch:
+ assert force_boxes
+ virtual_state = self.get_virtual_state(args)
+ continue
+
short_preamble = target_token.short_preamble
try:
extra = self.inline_short_preamble(args + virtuals, args,
@@ -452,7 +483,7 @@
# by short preamble
label_args = exported_state.virtual_state.make_inputargs(
targetargs, self.optimizer)
-
+
self.short_preamble_producer = ShortPreambleBuilder(
label_args, exported_state.short_boxes,
exported_state.short_inputargs, exported_state.exported_infos,
@@ -497,7 +528,7 @@
* runtime_boxes - runtime values for boxes, necessary when generating
guards to jump to
"""
-
+
def __init__(self, end_args, next_iteration_args, virtual_state,
exported_infos, short_boxes, renamed_inputargs,
short_inputargs, runtime_boxes, memo):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py
b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
ArrayStructInfo, AbstractStructPtrInfo
from rpython.jit.metainterp.optimizeopt.intutils import \
MININT, MAXINT, IntBound, IntLowerBound
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
+from rpython.jit.metainterp.resoperation import rop, ResOperation, \
InputArgInt, InputArgRef, InputArgFloat
from rpython.rlib.debug import debug_print
@@ -20,7 +20,7 @@
class GenerateGuardState(object):
- def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+ def __init__(self, optimizer=None, guards=None, renum=None, bad=None,
force_boxes=False):
self.optimizer = optimizer
self.cpu = optimizer.cpu
if guards is None:
@@ -32,6 +32,7 @@
if bad is None:
bad = {}
self.bad = bad
+ self.force_boxes = force_boxes
def get_runtime_item(self, box, descr, i):
array = box.getref_base()
@@ -303,7 +304,7 @@
opinfo = state.optimizer.getptrinfo(box)
assert isinstance(opinfo, ArrayPtrInfo)
else:
- opinfo = None
+ opinfo = None
for i in range(self.length):
for descr in self.fielddescrs:
index = i * len(self.fielddescrs) + descr.get_index()
@@ -514,6 +515,8 @@
NotVirtualStateInfo.__init__(self, cpu, type, info)
def _generate_guards(self, other, box, runtime_box, state):
+ if state.force_boxes and isinstance(other, VirtualStateInfo):
+ return self._generate_virtual_guards(other, box, runtime_box,
state)
if not isinstance(other, NotVirtualStateInfoPtr):
raise VirtualStatesCantMatch(
'The VirtualStates does not match as a ' +
@@ -545,6 +548,23 @@
# to an existing compiled loop or retracing the loop. Both alternatives
# will always generate correct behaviour, but performance will differ.
+ def _generate_virtual_guards(self, other, box, runtime_box, state):
+ """
+ Generate the guards and add state information for unifying a virtual
+ object with a non-virtual. This involves forcing the object in the
+ event that unification can succeed. Since virtual objects cannot be
null,
+ this method need only check that the virtual object has the expected
type.
+ """
+ assert state.force_boxes and isinstance(other, VirtualStateInfo)
+
+ if self.level == LEVEL_CONSTANT:
+ raise VirtualStatesCantMatch(
+ "cannot unify a constant value with a virtual object")
+
+ if self.level == LEVEL_KNOWNCLASS:
+ if not self.known_class.same_constant(other.known_class):
+ raise VirtualStatesCantMatch("classes don't match")
+
def _generate_guards_nonnull(self, other, box, runtime_box, extra_guards,
state):
if not isinstance(other, NotVirtualStateInfoPtr):
@@ -617,10 +637,10 @@
return False
return True
- def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+ def generate_guards(self, other, boxes, runtime_boxes, optimizer,
force_boxes=False):
assert (len(self.state) == len(other.state) == len(boxes) ==
len(runtime_boxes))
- state = GenerateGuardState(optimizer)
+ state = GenerateGuardState(optimizer, force_boxes=force_boxes)
for i in range(len(self.state)):
self.state[i].generate_guards(other.state[i], boxes[i],
runtime_boxes[i], state)
@@ -644,8 +664,8 @@
return boxes
- def make_inputargs_and_virtuals(self, inputargs, optimizer):
- inpargs = self.make_inputargs(inputargs, optimizer)
+ def make_inputargs_and_virtuals(self, inputargs, optimizer,
force_boxes=False):
+ inpargs = self.make_inputargs(inputargs, optimizer, force_boxes)
# we append the virtuals here in case some stuff is proven
# to be not a virtual and there are getfields in the short preamble
# that will read items out of there
@@ -653,7 +673,7 @@
for i in range(len(inputargs)):
if not isinstance(self.state[i], NotVirtualStateInfo):
virtuals.append(inputargs[i])
-
+
return inpargs, virtuals
def debug_print(self, hdr='', bad=None, metainterp_sd=None):
diff --git a/rpython/jit/metainterp/test/test_ajit.py
b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4507,3 +4507,54 @@
i += 1
return i
self.meta_interp(f, [])
+
+ def test_round_trip_raw_pointer(self):
+ # The goal of this test to to get a raw pointer op into the short
preamble
+ # so we can check that the proper guards are generated
+ # In this case, the resulting short preamble contains
+ #
+ # i1 = getfield_gc_i(p0, descr=inst__ptr)
+ # i2 = int_eq(i1, 0)
+ # guard_false(i2)
+ #
+ # as opposed to what the JIT used to produce
+ #
+ # i1 = getfield_gc_i(p0, descr=inst__ptr)
+ # guard_nonnull(i1)
+ #
+ # Which will probably generate correct assembly, but the optimization
+ # pipline expects guard_nonnull arguments to be pointer ops and may
crash
+ # and may crash on other input types.
+ driver = JitDriver(greens=[], reds=['i', 'val'])
+
+ class Box(object):
+ _ptr = lltype.nullptr(rffi.CCHARP.TO)
+
+ def new_int_buffer(value):
+ data = lltype.malloc(rffi.CCHARP.TO, rffi.sizeof(rffi.INT),
flavor='raw')
+ rffi.cast(rffi.INTP, data)[0] = rffi.cast(rffi.INT, value)
+ return data
+
+ def read_int_buffer(buf):
+ return rffi.cast(rffi.INTP, buf)[0]
+
+ def f():
+ i = 0
+ val = Box()
+ val._ptr = new_int_buffer(1)
+
+ set_param(None, 'retrace_limit', -1)
+ while i < 100:
+ driver.jit_merge_point(i=i, val=val)
+ driver.can_enter_jit(i=i, val=val)
+ # Just to produce a side exit
+ if i & 0b100:
+ i += 1
+ i += int(read_int_buffer(val._ptr))
+ lltype.free(val._ptr, flavor='raw')
+ val._ptr = new_int_buffer(1)
+ lltype.free(val._ptr, flavor='raw')
+
+ self.meta_interp(f, [])
+ self.check_resops(guard_nonnull=0)
+
diff --git a/rpython/jit/metainterp/test/test_virtual.py
b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -1,8 +1,8 @@
import py
-from rpython.rlib.jit import JitDriver, promote, dont_look_inside
+from rpython.rlib.jit import JitDriver, promote, dont_look_inside, set_param
from rpython.rlib.objectmodel import compute_unique_id
from rpython.jit.codewriter.policy import StopAtXPolicy
-from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.jit.metainterp.test.support import LLJitMixin, get_stats
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.rtyper import rclass
from rpython.rtyper.lltypesystem.lloperation import llop
@@ -965,6 +965,82 @@
self.check_aborted_count(0)
self.check_target_token_count(4)
+ def test_avoid_preamble(self):
+ driver = JitDriver(greens=[], reds=['i', 'val'])
+ class X(object):
+ def __init__(self, v):
+ self.v = v
+
+ class Box(object):
+ def __init__(self, v):
+ self.unbox = v
+
+ mask = -2
+ const = Box(X(5))
+ def f():
+ # Prevent all retracing of side exits. Ensures that the unroll
+ # optimizer will attempt to jump to either the preamble or loop.
+ set_param(driver, 'retrace_limit', -1)
+ set_param(driver, 'threshold', 1)
+ val = X(0)
+ i = 0
+ const.unbox = X(5)
+ while i < 17:
+ driver.can_enter_jit(i=i, val=val)
+ driver.jit_merge_point(i=i, val=val)
+ # Logical & rather than comparison to confuse range analysis.
+ # Test only succeeds on the first 2 iterations
+ if i & -2 == 0:
+ val = const.unbox
+ else:
+ val = X(i)
+ i += 1
+ return 0
+
+ self.meta_interp(f, [])
+
+ # With retracing disable, there will be one optimized loop expecting a
+ # non-virtual X object. The side exit creates a virtual object which
must
+ # be allocated to jump to the optimized trace.
+ self.check_resops(jump=3, label=2, new_with_vtable=2)
+ self.check_target_token_count(2)
+ self.check_trace_count(3)
+
+ def test_conflated_virtual_states(self):
+ # All cases are covered when forcing one component of the virtual state
+ # also forces an as yet unseen component.
+ # i.e. expect [NotVirtual, Virtual] and given a pair of aliasing
virtual
+ # objects
+ driver = JitDriver(greens=[], reds=['i', 'v1', 'v2'])
+ class Box(object):
+ def __init__(self, v):
+ self.v = v
+
+ class X(object):
+ def __init__(self, v):
+ self.v = v
+
+ const = Box(X(0))
+ def f():
+ set_param(None, 'retrace_limit', -1)
+ set_param(None, 'threshold', 1)
+ i = 0
+ v1 = X(0)
+ v2 = X(0)
+ const.v = X(0)
+ while i < 17:
+ driver.jit_merge_point(i=i, v1=v1, v2=v2)
+ driver.can_enter_jit(i=i, v1=v1, v2=v2)
+ if i & 1 == 0:
+ v1 = const.v
+ v2 = X(i)
+ else:
+ v1 = v2 = X(i)
+ i += 1
+ return None
+ self.meta_interp(f, [])
+ # assert did not crash
+
class VirtualMiscTests:
def test_multiple_equal_virtuals(self):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -97,7 +97,8 @@
OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"]
HAVE_TLSv1_2 = OPENSSL_VERSION_NUMBER >= 0x10001000
-if OPENSSL_VERSION_NUMBER >= 0x10100000:
+if (OPENSSL_VERSION_NUMBER >= 0x10100000 and
+ OPENSSL_VERSION_NUMBER < 0x20000000): # <= libressl :-(
eci.pre_include_bits = ()
eci.post_include_bits = ()
raise Exception("""OpenSSL version >= 1.1 not supported yet.
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
