Author: Spenser Andrew Bauman <saba...@gmail.com>
Branch: clean-exported-state
Changeset: r87948:52de89fb0bdb
Date: 2016-10-24 21:00 -0400
http://bitbucket.org/pypy/pypy/changeset/52de89fb0bdb/
Log: Merge with default
diff too long, truncating to 2000 out of 2085 lines
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -397,3 +397,13 @@
in auto-generated C code, and at least some knowledge about the
various components involved, from PyPy's own RPython source code to
the GC and possibly the JIT.
+
+
+Why doesn't PyPy move to GitHub, Gitlab, ...?
+----------------------------------------------
+
+We've been quite happy with bitbucket.org. Moving version control systems and
+hosting is a lot of hard work: On the one hand, PyPy's mercurial history is
+long and gnarly. On the other hand, all our infrastructure (buildbots,
+benchmarking, etc) would have to be adapted. So unless somebody steps up and
+volunteers to do all that work, it will likely not happen.
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -1,8 +1,17 @@
Downloading and Installing PyPy
===============================
+Using a packaged PyPy
+~~~~~~~~~~~~~~~~~~~~~
+
+Some Linux distributions provide a pypy package. Note that in order to
+install additional modules that require compilation, you may need to install
+additional packages such as pypy-dev. This will manifest as an error about
+"missing Python.h". Distributions do not as of yet supply many pypy-ready
+packages, if you require additional modules we recommend creating a virtualenv
+and using pip.
+
.. _prebuilt-pypy:
-
Download a pre-built PyPy
~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -38,6 +47,9 @@
and not move the binary there, else PyPy would not be able to find its
library.
+Installing more modules
+~~~~~~~~~~~~~~~~~~~~~~~
+
If you want to install 3rd party libraries, the most convenient way is
to install pip_ using ensurepip_ (unless you want to install virtualenv as
explained below; then you can directly use pip inside virtualenvs):
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -67,11 +67,16 @@
Make optimiseopt iterative instead of recursive so it can be reasoned about
more easily and debugging is faster.
-.. branch: stdlib-2.7.11
+.. branch: Tiberiumk/fix-2412-1476011166874
+.. branch: redirect-assembler-jitlog
-Update stdlib to version 2.7.11
-.. branch: vendor/stdlib
+
.. branch: stdlib-2.7.12
Update stdlib to version 2.7.12
+
+.. branch: buffer-interface2
+
+Improve support for new buffer interface in cpyext, bf_getbuffer on built-in
+types still missing
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -207,6 +207,9 @@
def buffer_w(self, space, flags):
w_impl = space.lookup(self, '__buffer__')
+ if w_impl is None:
+ # cpyext types that may have only old buffer interface
+ w_impl = space.lookup(self, '__wbuffer__')
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(flags))
@@ -215,7 +218,10 @@
raise BufferInterfaceNotFound
def readbuf_w(self, space):
- w_impl = space.lookup(self, '__buffer__')
+ # cpyext types that may have old buffer protocol
+ w_impl = space.lookup(self, '__rbuffer__')
+ if w_impl is None:
+ w_impl = space.lookup(self, '__buffer__')
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(space.BUF_FULL_RO))
@@ -224,7 +230,10 @@
raise BufferInterfaceNotFound
def writebuf_w(self, space):
- w_impl = space.lookup(self, '__buffer__')
+ # cpyext types that may have old buffer protocol
+ w_impl = space.lookup(self, '__wbuffer__')
+ if w_impl is None:
+ w_impl = space.lookup(self, '__buffer__')
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(space.BUF_FULL))
diff --git a/pypy/module/array/test/test_array.py
b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -2,7 +2,18 @@
import pytest
-class BaseArrayTests:
+class AppTestArray(object):
+ spaceconfig = {'usemodules': ['array', 'struct', '_rawffi', 'binascii']}
+
+ def setup_class(cls):
+ cls.w_array = cls.space.appexec([], """():
+ import array
+ return array.array
+ """)
+ cls.w_tempfile = cls.space.wrap(
+ str(pytest.ensuretemp('array').join('tmpfile')))
+ cls.w_maxint = cls.space.wrap(sys.maxint)
+
def test_ctor(self):
assert len(self.array('c')) == 0
assert len(self.array('i')) == 0
@@ -22,8 +33,8 @@
a = self.array('u')
raises(TypeError, a.append, 7)
raises(TypeError, a.append, u'hi')
- a.append(unicode('h'))
- assert a[0] == unicode('h')
+ a.append(u'h')
+ assert a[0] == u'h'
assert type(a[0]) is unicode
assert len(a) == 1
@@ -67,7 +78,7 @@
('H', ( 0, 56783, 65535), int),
('i', (-32768, 30535, 32767), int),
('I', ( 0, 56783, 65535), long),
- ('l', (-2 ** 32 / 2, 34, 2 ** 32 / 2 - 1), int),
+ ('l', (-2 ** 32 // 2, 34, 2 ** 32 // 2 - 1), int),
('L', (0, 3523532, 2 ** 32 - 1), long),
):
a = self.array(tc, ok)
@@ -105,7 +116,7 @@
assert a.tolist() == vals
a = self.array(tc.lower())
- vals = [-1 * (2 ** itembits) / 2, (2 ** itembits) / 2 - 1]
+ vals = [-1 * (2 ** itembits) // 2, (2 ** itembits) // 2 - 1]
a.fromlist(vals)
assert a.tolist() == vals
@@ -137,11 +148,11 @@
for t in inttypes:
a = self.array(t, [1, 2, 3])
b = a.itemsize
- for v in (-2 ** (8 * b) / 2, 2 ** (8 * b) / 2 - 1):
+ for v in (-2 ** (8 * b) // 2, 2 ** (8 * b) // 2 - 1):
a[1] = v
assert a[0] == 1 and a[1] == v and a[2] == 3
- raises(OverflowError, a.append, -2 ** (8 * b) / 2 - 1)
- raises(OverflowError, a.append, 2 ** (8 * b) / 2)
+ raises(OverflowError, a.append, -2 ** (8 * b) // 2 - 1)
+ raises(OverflowError, a.append, 2 ** (8 * b) // 2)
a = self.array(t.upper(), [1, 2, 3])
b = a.itemsize
@@ -175,42 +186,35 @@
raises(ValueError, a.fromstring, '\x00' * (a.itemsize + 1))
raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize - 1))
raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize + 1))
- b = self.array(t, '\x00' * a.itemsize * 2)
+ b = self.array(t, b'\x00' * a.itemsize * 2)
assert len(b) == 2 and b[0] == 0 and b[1] == 0
if sys.version_info >= (2, 7, 11):
raises(ValueError, a.fromstring, a)
def test_fromfile(self):
-
- ## class myfile(object):
- ## def __init__(self, c, s):
- ## self.c = c
- ## self.s = s
- ## def read(self,n):
- ## return self.c*min(n,self.s)
def myfile(c, s):
- f = open(self.tempfile, 'w')
+ f = open(self.tempfile, 'wb')
f.write(c * s)
f.close()
- return open(self.tempfile, 'r')
+ return open(self.tempfile, 'rb')
- f = myfile('\x00', 100)
+ f = myfile(b'\x00', 100)
for t in 'bBhHiIlLfd':
a = self.array(t)
a.fromfile(f, 2)
assert len(a) == 2 and a[0] == 0 and a[1] == 0
a = self.array('b')
- a.fromfile(myfile('\x01', 20), 2)
+ a.fromfile(myfile(b'\x01', 20), 2)
assert len(a) == 2 and a[0] == 1 and a[1] == 1
a = self.array('h')
- a.fromfile(myfile('\x01', 20), 2)
+ a.fromfile(myfile(b'\x01', 20), 2)
assert len(a) == 2 and a[0] == 257 and a[1] == 257
for i in (0, 1):
a = self.array('h')
- raises(EOFError, a.fromfile, myfile('\x01', 2 + i), 2)
+ raises(EOFError, a.fromfile, myfile(b'\x01', 2 + i), 2)
assert len(a) == 1 and a[0] == 257
def test_fromlist(self):
@@ -250,12 +254,12 @@
assert repr(a) == "array('b', [1, 2, 1, 2])"
def test_fromunicode(self):
- raises(ValueError, self.array('i').fromunicode, unicode('hi'))
+ raises(ValueError, self.array('i').fromunicode, u'hi')
a = self.array('u')
- a.fromunicode(unicode('hi'))
+ a.fromunicode(u'hi')
assert len(a) == 2 and a[0] == 'h' and a[1] == 'i'
- b = self.array('u', unicode('hi'))
+ b = self.array('u', u'hi')
assert len(b) == 2 and b[0] == 'h' and b[1] == 'i'
def test_sequence(self):
@@ -357,23 +361,6 @@
except ValueError:
assert not ok
- def test_reversingslice_pre26(self):
- import sys
- if sys.version_info >= (2, 6):
- skip('arrays can handle more slice ops than lists in 2.6')
-
- for a in range(-4, 5):
- for b in range(-4, 5):
- for c in [-4, -3, -2, -1, 1, 2, 3, 4]:
- lst = [1, 2, 3]
- arr = self.array('i', lst)
- for vals in ([4, 5], [6], []):
- try:
- lst[a:b:c] = vals
- except ValueError:
- raises(ValueError,
- "arr[a:b:c]=self.array('i', vals)")
-
def test_toxxx(self):
a = self.array('i', [1, 2, 3])
l = a.tolist()
@@ -405,7 +392,7 @@
('BHILfd', (127, 0, 1, 7, 255, 169)),
('hilHILfd', (32760, 30123, 3422, 23244))):
for tc in tcodes:
- values += ((2 ** self.array(tc).itemsize) / 2 - 1, )
+ values += ((2 ** self.array(tc).itemsize) // 2 - 1, )
s = self.array(tc, values).tostring()
a = unpack(tc * len(values), s)
assert a == values
@@ -420,8 +407,7 @@
assert repr(a) == "array('c', 'hi')"
raises(ValueError, self.array('i').tounicode)
- assert self.array('u', unicode('hello')).tounicode() == \
- unicode('hello')
+ assert self.array('u', u'hello').tounicode() == u'hello'
def test_empty_tostring(self):
a = self.array('l')
@@ -493,14 +479,14 @@
def test_compare(self):
class comparable(object):
- def __cmp__(self, other):
- return 0
+ def __eq__(self, other):
+ return True
class incomparable(object):
pass
for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'),
('abc', 'acb', 'c'),
- (unicode('abc'), unicode('acb'), 'u')):
+ (u'abc', u'acb', 'u')):
for t in tt:
a = self.array(t, v1)
b = self.array(t, v1)
@@ -767,16 +753,16 @@
self.height = height
return self
- def _index(self, (x,y)):
+ def _index(self, x, y):
x = min(max(x, 0), self.width-1)
y = min(max(y, 0), self.height-1)
return y * self.width + x
def __getitem__(self, i):
- return array.__getitem__(self, self._index(i))
+ return array.__getitem__(self, self._index(*i))
def __setitem__(self, i, val):
- return array.__setitem__(self, self._index(i), val)
+ return array.__setitem__(self, self._index(*i), val)
img = Image(5, 10, 'B')
for y in range(10):
@@ -844,8 +830,8 @@
assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
def test_unicode_outofrange(self):
- a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape'))
- b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape'))
+ a = self.array('u', u'\x01\u263a\x00\ufeff')
+ b = self.array('u', u'\x01\u263a\x00\ufeff')
b.byteswap()
assert a != b
@@ -853,7 +839,7 @@
import sys
if sys.maxunicode == 0xffff:
skip("test for 32-bit unicodes")
- a = self.array('u', '\xff\xff\xff\xff')
+ a = self.array('u', b'\xff\xff\xff\xff')
assert len(a) == 1
assert repr(a[0]) == "u'\Uffffffff'"
if sys.maxint == 2147483647:
@@ -954,28 +940,6 @@
assert a[0] == u'b'
-class TestCPythonsOwnArray(BaseArrayTests):
- def setup_class(cls):
- import array
- cls.array = array.array
- import struct
- cls.struct = struct
- cls.tempfile = str(pytest.ensuretemp('array').join('tmpfile'))
- cls.maxint = sys.maxint
-
-
-class AppTestArray(BaseArrayTests):
- spaceconfig = {'usemodules': ['array', 'struct', '_rawffi', 'binascii']}
-
- def setup_class(cls):
- cls.w_array = cls.space.appexec([], """():
- import array
- return array.array
- """)
- cls.w_tempfile = cls.space.wrap(
- str(pytest.ensuretemp('array').join('tmpfile')))
- cls.w_maxint = cls.space.wrap(sys.maxint)
-
def test_buffer_info(self):
a = self.array('c', 'Hi!')
bi = a.buffer_info()
diff --git a/pypy/module/array/test/test_array_old.py
b/pypy/module/array/test/test_array_old.py
deleted file mode 100644
--- a/pypy/module/array/test/test_array_old.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# minimal tests. See also lib-python/modified-2.4.1/test/test_array.
-
-import py
-from py.test import raises
-import struct
-
-
-class BaseArrayTests:
- # XXX very incomplete
-
- native_sizes = {'l': struct.calcsize('l')}
-
- def test_attributes(self):
- a = self.array.array('c')
- assert a.typecode == 'c'
- assert a.itemsize == 1
- a = self.array.array('l')
- assert a.typecode == 'l'
- assert a.itemsize == self.native_sizes['l']
-
- def test_imul(self):
- a = self.array.array('i', [12, 34])
- a *= 3
- assert a.tolist() == [12, 34] * 3
-
- def test_unicode(self):
- a = self.array.array('u')
- a.fromunicode(unichr(9999))
- assert len(a) == 1
- assert a.tolist() == [unichr(9999)]
-
- def test_pickle(self):
- import sys
- if sys.version_info < (2, 5):
- py.test.skip("array.array not picklable before python 2.5")
- import pickle
-
- for content in [[56, -12, 34], []]:
- a = self.array.array('i', content)
- a2 = pickle.loads(pickle.dumps(a))
- assert type(a2) is self.array.array
- assert list(a2) == content
-
- def test_init_vs_new(self):
- import sys
- if sys.version_info < (2, 5):
- py.test.skip("array.array constructor changed in 2.5")
- class A(self.array.array):
- def __init__(self, *args, **kwds):
- self.args = args
- self.kwds = kwds
-
- a = A('c', foo='bar')
- assert a.args == ('c',)
- assert a.kwds == {'foo': 'bar'}
- a = A('i', range(10), some=42)
- assert a.args == ('i', range(10))
- assert a.kwds == {'some': 42}
- raises(TypeError, A)
- raises(TypeError, A, 42)
- raises(TypeError, A, 'i', [], [])
- raises(TypeError, self.array.array, 'i', [], foo='bar')
-
-
-class TestCPythonsOwnArray(BaseArrayTests):
-
- def setup_class(cls):
- import array
- cls.array = array
-
-
-## class TestArrayOnTopOfCPython(BaseArrayTests):
-
-## def setup_class(cls):
-## from pypy.tool.lib_pypy import LIB_PYPY
-## if not hasattr(struct, 'pack_into'):
-## py.test.skip("requires CPython >= 2.5")
-## import new
-## path = LIB_PYPY.join('array.py')
-## myarraymodule = new.module('array')
-## execfile(str(path), myarraymodule.__dict__)
-## cls.array = myarraymodule
-
-## def test_unicode(self):
-## py.test.skip("no 'u' type code in CPython's struct module")
-
-## def test_pickle(self):
-## py.test.skip("pickle getting confused by the hack in setup_class()")
-
-
-class AppTestArray(BaseArrayTests):
- spaceconfig = {'usemodules': ['struct', 'array', 'binascii']}
-
- def setup_class(cls):
- """Import the array module and make it available as self.array."""
- cls.w_array = cls.space.getbuiltinmodule('array')
- cls.w_native_sizes = cls.space.wrap(cls.native_sizes)
-
-
-## class AppTestArrayWithRawFFI(AppTestArray):
-## """
-## The same as the base class, but with a space that also includes the
-## _rawffi module. The array module internally uses it in this case.
-## """
-## spaceconfig = dict(usemodules=['struct', '_rawffi'])
-
-## def test_buffer_info(self):
-## a = self.array.array('l', [123, 456])
-## assert a.itemsize == self.native_sizes['l']
-## address, length = a.buffer_info()
-## assert length == 2 # and not 2 * self.native_sizes['l']
-## assert address != 0
-## # should check the address via some unsafe peeking, but it's
-## # not easy on top of py.py
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -79,11 +79,16 @@
CONST_STRING = lltype.Ptr(lltype.Array(lltype.Char,
hints={'nolength': True}),
use_cache=False)
+CONST_STRINGP = lltype.Ptr(lltype.Array(rffi.CCHARP,
+ hints={'nolength': True}),
+ use_cache=False)
CONST_WSTRING = lltype.Ptr(lltype.Array(lltype.UniChar,
hints={'nolength': True}),
use_cache=False)
assert CONST_STRING is not rffi.CCHARP
assert CONST_STRING == rffi.CCHARP
+assert CONST_STRINGP is not rffi.CCHARPP
+assert CONST_STRINGP == rffi.CCHARPP
assert CONST_WSTRING is not rffi.CWCHARP
assert CONST_WSTRING == rffi.CWCHARP
@@ -1004,6 +1009,8 @@
for i, argtype in enumerate(func.argtypes):
if argtype is CONST_STRING:
arg = 'const char *@'
+ elif argtype is CONST_STRINGP:
+ arg = 'const char **@'
elif argtype is CONST_WSTRING:
arg = 'const wchar_t *@'
else:
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
--- a/pypy/module/cpyext/buffer.py
+++ b/pypy/module/cpyext/buffer.py
@@ -10,6 +10,10 @@
flags = pyobj.c_ob_type.c_tp_flags
if (flags & Py_TPFLAGS_HAVE_NEWBUFFER and as_buffer.c_bf_getbuffer):
return 1
+ name = rffi.charp2str(pyobj.c_ob_type.c_tp_name)
+ if name in ('str', 'bytes'):
+ # XXX remove once wrapper of __buffer__ -> bf_getbuffer works
+ return 1
return 0
diff --git a/pypy/module/cpyext/include/object.h
b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -144,7 +144,7 @@
/* Py3k buffer interface, adapted for PyPy */
#define Py_MAX_NDIMS 32
-#define Py_MAX_FMT 5
+#define Py_MAX_FMT 128
typedef struct bufferinfo {
void *buf;
PyObject *obj; /* owned reference */
diff --git a/pypy/module/cpyext/memoryobject.py
b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -1,9 +1,10 @@
from pypy.module.cpyext.api import (cpython_api, Py_buffer, CANNOT_FAIL,
Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers,
Py_ssize_tP)
-from pypy.module.cpyext.pyobject import PyObject, make_ref, incref
+from pypy.module.cpyext.pyobject import PyObject, make_ref, incref, from_ref
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.rlib.rarithmetic import widen
from pypy.objspace.std.memoryobject import W_MemoryView
+from pypy.module.cpyext.import_ import PyImport_Import
PyMemoryView_Check, PyMemoryView_CheckExact =
build_type_checkers("MemoryView", "w_memoryview")
@@ -33,33 +34,43 @@
view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
except ValueError:
raise BufferError("could not create buffer from object")
+ ret = fill_Py_buffer(space, buf, view)
view.c_obj = make_ref(space, w_obj)
- return fill_Py_buffer(space, buf, view)
+ return ret
-def fill_Py_buffer(space, buf, view):
+def fill_Py_buffer(space, buf, view):
# c_buf, c_obj have been filled in
ndim = buf.getndim()
view.c_len = buf.getlength()
view.c_itemsize = buf.getitemsize()
rffi.setintfield(view, 'c_ndim', ndim)
view.c_format = rffi.cast(rffi.CCHARP, view.c__format)
- view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
- view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
fmt = buf.getformat()
n = Py_MAX_FMT - 1 # NULL terminated buffer
if len(fmt) > n:
- ### WARN?
- pass
+ w_message = space.newbytes("PyPy specific Py_MAX_FMT is %d which is
too "
+ "small for buffer format, %d needed" % (
+ Py_MAX_FMT, len(fmt)))
+ w_stacklevel = space.newint(1)
+ w_module = PyImport_Import(space, space.newbytes("warnings"))
+ w_warn = space.getattr(w_module, space.newbytes("warn"))
+ space.call_function(w_warn, w_message, space.w_None, w_stacklevel)
else:
n = len(fmt)
for i in range(n):
view.c_format[i] = fmt[i]
- view.c_format[n] = '\x00'
- shape = buf.getshape()
- strides = buf.getstrides()
- for i in range(ndim):
- view.c_shape[i] = shape[i]
- view.c_strides[i] = strides[i]
+ view.c_format[n] = '\x00'
+ if ndim > 0:
+ view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+ view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+ shape = buf.getshape()
+ strides = buf.getstrides()
+ for i in range(ndim):
+ view.c_shape[i] = shape[i]
+ view.c_strides[i] = strides[i]
+ else:
+ view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+ view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
return 0
@@ -102,12 +113,12 @@
@cpython_api([lltype.Ptr(Py_buffer), lltype.Char], rffi.INT_real,
error=CANNOT_FAIL)
def PyBuffer_IsContiguous(space, view, fort):
- """Return 1 if the memory defined by the view is C-style (fortran is
- 'C') or Fortran-style (fortran is 'F') contiguous or either one
- (fortran is 'A'). Return 0 otherwise."""
+ """Return 1 if the memory defined by the view is C-style (fort is
+ 'C') or Fortran-style (fort is 'F') contiguous or either one
+ (fort is 'A'). Return 0 otherwise."""
# traverse the strides, checking for consistent stride increases from
# right-to-left (c) or left-to-right (fortran). Copied from cpython
- if not view.c_suboffsets:
+ if view.c_suboffsets:
return 0
if (fort == 'C'):
return _IsCContiguous(view)
@@ -121,11 +132,22 @@
def PyMemoryView_FromObject(space, w_obj):
return space.call_method(space.builtin, "memoryview", w_obj)
+@cpython_api([lltype.Ptr(Py_buffer)], PyObject)
+def PyMemoryView_FromBuffer(space, view):
+ """Create a memoryview object wrapping the given buffer-info structure
view.
+ The memoryview object then owns the buffer, which means you shouldn't
+ try to release it yourself: it will be released on deallocation of the
+ memoryview object."""
+ w_obj = from_ref(space, view.c_obj)
+ if isinstance(w_obj, W_MemoryView):
+ return w_obj
+ return space.call_method(space.builtin, "memoryview", w_obj)
+
@cpython_api([PyObject], PyObject)
def PyMemoryView_GET_BASE(space, w_obj):
# return the obj field of the Py_buffer created by PyMemoryView_GET_BUFFER
# XXX needed for numpy on py3k
- raise NotImplementedError('PyMemoryView_GET_BUFFER')
+ raise NotImplementedError('PyMemoryView_GET_BASE')
@cpython_api([PyObject], lltype.Ptr(Py_buffer), error=CANNOT_FAIL)
def PyMemoryView_GET_BUFFER(space, w_obj):
@@ -139,6 +161,7 @@
if ndim >= Py_MAX_NDIMS:
# XXX warn?
return view
+ fill_Py_buffer(space, w_obj.buf, view)
try:
view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
view.c_obj = make_ref(space, w_obj)
@@ -147,8 +170,8 @@
except ValueError:
w_s = w_obj.descr_tobytes(space)
view.c_obj = make_ref(space, w_s)
+ view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(space.str_w(w_s),
track_allocation=False))
rffi.setintfield(view, 'c_readonly', 1)
isstr = True
- fill_Py_buffer(space, w_obj.buf, view)
return view
diff --git a/pypy/module/cpyext/methodobject.py
b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -311,7 +311,7 @@
def PyClassMethod_New(space, w_func):
return space.wrap(ClassMethod(w_func))
-@cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
+@cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
def PyDescr_NewMethod(space, w_type, method):
return space.wrap(W_PyCMethodObject(space, method, w_type))
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -3,7 +3,7 @@
cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
PyVarObject, Py_buffer, size_t,
Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
- Py_GE, CONST_STRING, FILEP, fwrite)
+ Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
from pypy.module.cpyext.pyobject import (
PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
get_typedescr, _Py_NewReference)
@@ -429,7 +429,7 @@
is active then NULL is returned but PyErr_Occurred() will return false."""
return space.call_function(space.builtin.get('dir'), w_o)
-@cpython_api([PyObject, rffi.CCHARPP, Py_ssize_tP], rffi.INT_real, error=-1)
+@cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
"""Returns a pointer to a read-only memory location usable as
character-based input. The obj argument must support the single-segment
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -150,12 +150,12 @@
Return value: always NULL."""
# XXX Doesn't actually do anything with PyErr_CheckSignals.
if llfilename:
- w_filename = rffi.charp2str(llfilename)
- filename = space.wrap(w_filename)
+ filename = rffi.charp2str(llfilename)
+ w_filename = space.wrap(filename)
else:
- filename = space.w_None
+ w_filename = space.w_None
- PyErr_SetFromErrnoWithFilenameObject(space, w_type, filename)
+ PyErr_SetFromErrnoWithFilenameObject(space, w_type, w_filename)
@cpython_api([PyObject, PyObject], PyObject)
@jit.dont_look_inside # direct use of _get_errno()
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -15,6 +15,7 @@
readbufferproc, getbufferproc, ssizessizeobjargproc)
from pypy.module.cpyext.pyobject import from_ref, make_ref, Py_DecRef
from pypy.module.cpyext.pyerrors import PyErr_Occurred
+from pypy.module.cpyext.memoryobject import fill_Py_buffer
from pypy.module.cpyext.state import State
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.argument import Arguments
@@ -349,6 +350,10 @@
def getndim(self):
return self.ndim
+ def setitem(self, index, char):
+ # absolutely no safety checks, what could go wrong?
+ self.ptr[index] = char
+
def wrap_getreadbuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
@@ -358,6 +363,15 @@
space.fromcache(State).check_and_raise_exception(always=True)
return space.newbuffer(CPyBuffer(ptr[0], size, w_self))
+def wrap_getwritebuffer(space, w_self, w_args, func):
+ func_target = rffi.cast(readbufferproc, func)
+ with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
+ index = rffi.cast(Py_ssize_t, 0)
+ size = generic_cpy_call(space, func_target, w_self, index, ptr)
+ if size < 0:
+ space.fromcache(State).check_and_raise_exception(always=True)
+ return space.newbuffer(CPyBuffer(ptr[0], size, w_self, readonly=False))
+
def wrap_getbuffer(space, w_self, w_args, func):
func_target = rffi.cast(getbufferproc, func)
with lltype.scoped_alloc(Py_buffer) as pybuf:
@@ -608,13 +622,27 @@
@cpython_api([PyObject, Py_bufferP, rffi.INT_real],
rffi.INT_real, header=None, error=-1)
@func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
- def buff_w(space, w_self, pybuf, flags):
- # XXX this is wrong, needs a test
- raise oefmt(space.w_NotImplemented,
- "calling bf_getbuffer on a builtin type not supported yet")
- #args = Arguments(space, [w_self],
- # w_stararg=w_args, w_starstararg=w_kwds)
- #return space.call_args(space.get(buff_fn, w_self), args)
+ def buff_w(space, w_self, view, flags):
+ args = Arguments(space, [space.newint(flags)])
+ w_obj = space.call_args(space.get(buff_fn, w_self), args)
+ if view:
+ #like PyObject_GetBuffer
+ flags = widen(flags)
+ buf = space.buffer_w(w_obj, flags)
+ try:
+ view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ view.c_obj = make_ref(space, w_obj)
+ except ValueError:
+ w_s = space.newbytes(buf.as_str())
+ view.c_obj = make_ref(space, w_s)
+ view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+ space.str_w(w_s), track_allocation=False))
+ rffi.setintfield(view, 'c_readonly', 1)
+ ret = fill_Py_buffer(space, buf, view)
+ return ret
+ return 0
+ # XXX remove this when it no longer crashes a translated PyPy
+ return
api_func = buff_w.api_func
else:
# missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
@@ -924,13 +952,13 @@
slotdefs = eval(slotdefs_str)
# PyPy addition
slotdefs += (
- # XXX that might not be what we want!
TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None,
"wrap_getbuffer", ""),
)
if not PY3:
slotdefs += (
- TPSLOT("__buffer__", "tp_as_buffer.c_bf_getreadbuffer", None,
"wrap_getreadbuffer", ""),
+ TPSLOT("__rbuffer__", "tp_as_buffer.c_bf_getreadbuffer", None,
"wrap_getreadbuffer", ""),
+ TPSLOT("__wbuffer__", "tp_as_buffer.c_bf_getwritebuffer", None,
"wrap_getwritebuffer", ""),
)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -41,14 +41,6 @@
given shape with the given number of bytes per element."""
raise NotImplementedError
-@cpython_api([Py_buffer], PyObject)
-def PyMemoryView_FromBuffer(space, view):
- """Create a memoryview object wrapping the given buffer-info structure
view.
- The memoryview object then owns the buffer, which means you shouldn't
- try to release it yourself: it will be released on deallocation of the
- memoryview object."""
- raise NotImplementedError
-
@cpython_api([PyObject, rffi.INT_real, lltype.Char], PyObject)
def PyMemoryView_GetContiguous(space, obj, buffertype, order):
"""Create a memoryview object to a contiguous chunk of memory (in either
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2394,14 +2394,29 @@
(iternextfunc)arrayiter_next, /* tp_iternext */
0, /* tp_methods */
};
+static PyObject *
+readbuffer_as_string(PyObject *self, PyObject *args)
+{
+ PyObject *obj;
+ const void *ptr;
+ Py_ssize_t size;
+
+ if (!PyArg_ParseTuple(args, "O", &obj)) {
+ return NULL;
+ }
+ if (PyObject_AsReadBuffer(obj, &ptr, &size) < 0)
+ return NULL;
+ return PyString_FromStringAndSize((char*)ptr, size);
+}
+
/*********************** Install Module **************************/
-/* No functions in array module. */
static PyMethodDef a_methods[] = {
{"_reconstruct", (PyCFunction)_reconstruct, METH_VARARGS, NULL},
{"switch_multiply", (PyCFunction)switch_multiply, METH_NOARGS, NULL},
+ {"readbuffer_as_string", (PyCFunction)readbuffer_as_string,
METH_VARARGS, NULL},
{NULL, NULL, 0, NULL} /* Sentinel */
};
diff --git a/pypy/module/cpyext/test/test_abstract.py
b/pypy/module/cpyext/test/test_abstract.py
--- a/pypy/module/cpyext/test/test_abstract.py
+++ b/pypy/module/cpyext/test/test_abstract.py
@@ -10,7 +10,7 @@
"""
char *ptr;
Py_ssize_t size;
- if (PyObject_AsCharBuffer(args, &ptr, &size) < 0)
+ if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) <
0)
return NULL;
return PyString_FromStringAndSize(ptr, size);
"""),
@@ -104,3 +104,17 @@
assert raises(TypeError, buffer_support.readbuffer_as_string, 42)
assert raises(TypeError, buffer_support.writebuffer_as_string, 42)
assert raises(TypeError, buffer_support.charbuffer_as_string, 42)
+
+ def test_user_class(self):
+ class MyBuf(str):
+ pass
+ s = 'a\0x'
+ buf = MyBuf(s)
+ buffer_support = self.get_buffer_support()
+
+ assert buffer_support.check_readbuffer(buf)
+ assert s == buffer_support.readbuffer_as_string(buf)
+ assert raises(TypeError, buffer_support.writebuffer_as_string, buf)
+ assert s == buffer_support.charbuffer_as_string(buf)
+
+
diff --git a/pypy/module/cpyext/test/test_arraymodule.py
b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -1,8 +1,17 @@
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from pypy.conftest import option
class AppTestArrayModule(AppTestCpythonExtensionBase):
- enable_leak_checking = False
+ enable_leak_checking = True
+
+ def setup_class(cls):
+ from rpython.tool.udir import udir
+ AppTestCpythonExtensionBase.setup_class.im_func(cls)
+ if option.runappdirect:
+ cls.w_udir = str(udir)
+ else:
+ cls.w_udir = cls.space.wrap(str(udir))
+
def test_basic(self):
module = self.import_module(name='array')
@@ -90,6 +99,7 @@
assert res == [2, 4, 6]
def test_subclass(self):
+ import struct
module = self.import_module(name='array')
class Sub(module.array):
pass
@@ -98,3 +108,37 @@
res = [1, 2, 3] * arr
assert res == [1, 2, 3, 1, 2, 3]
+ val = module.readbuffer_as_string(arr)
+ assert val == struct.pack('i', 2)
+
+ def test_unicode_readbuffer(self):
+ # Not really part of array, refactor
+ import struct
+ module = self.import_module(name='array')
+ val = module.readbuffer_as_string('abcd')
+ assert val == 'abcd'
+ val = module.readbuffer_as_string(u'\u03a3')
+ assert val is not None
+
+ def test_readinto(self):
+ module = self.import_module(name='array')
+ a = module.array('c')
+ a.fromstring('0123456789')
+ filename = self.udir + "/_test_file"
+ f = open(filename, 'w+b')
+ f.write('foobar')
+ f.seek(0)
+ n = f.readinto(a)
+ f.close()
+ assert n == 6
+ assert len(a) == 10
+ assert a.tostring() == 'foobar6789'
+
+ def test_iowrite(self):
+ module = self.import_module(name='array')
+ from io import BytesIO
+ a = module.array('c')
+ a.fromstring('0123456789')
+ fd = BytesIO()
+ # only test that it works
+ fd.write(a)
diff --git a/pypy/module/cpyext/test/test_bufferobject.py
b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -25,7 +25,7 @@
"""
char *ptr;
Py_ssize_t size;
- if (PyObject_AsCharBuffer(args, &ptr, &size) < 0)
+ if (PyObject_AsCharBuffer(args, (const char **)&ptr, &size) <
0)
return NULL;
return PyString_FromStringAndSize(ptr, size);
""")
diff --git a/pypy/module/cpyext/test/test_getargs.py
b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -139,6 +139,12 @@
return result;
''')
assert 'foo\0bar\0baz' == pybuffer(buffer('foo\0bar\0baz'))
+ import sys
+ if '__pypy__' not in sys.builtin_module_names:
+ class A(object):
+ def __buffer__(self, flags):
+ return buffer('123')
+ assert pybuffer(A()) == '123'
def test_pyarg_parse_string_fails(self):
diff --git a/pypy/module/cpyext/test/test_memoryobject.py
b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -1,8 +1,12 @@
+import pytest
+
from rpython.rtyper.lltypesystem import rffi
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from rpython.rlib.buffer import StringBuffer
+only_pypy ="config.option.runappdirect and '__pypy__' not in
sys.builtin_module_names"
+
class TestMemoryViewObject(BaseApiTest):
def test_fromobject(self, space, api):
w_hello = space.newbytes("hello")
@@ -23,6 +27,14 @@
assert w_view.c_shape[0] == 5
assert w_view.c_strides[0] == 1
assert w_view.c_len == 5
+ o = rffi.charp2str(w_view.c_buf)
+ assert o == 'hello'
+ w_mv = api.PyMemoryView_FromBuffer(w_view)
+ for f in ('format', 'itemsize', 'ndim', 'readonly',
+ 'shape', 'strides', 'suboffsets'):
+ w_f = space.wrap(f)
+ assert space.eq_w(space.getattr(w_mv, w_f),
+ space.getattr(w_memoryview, w_f))
class AppTestBufferProtocol(AppTestCpythonExtensionBase):
def test_buffer_protocol(self):
@@ -39,8 +51,12 @@
viewlen = module.test_buffer(arr)
assert viewlen == y.itemsize * len(y)
+ @pytest.mark.skipif(only_pypy, reason='pypy only test')
def test_buffer_info(self):
- from _numpypy import multiarray as np
+ try:
+ from _numpypy import multiarray as np
+ except ImportError:
+ skip('pypy built without _numpypy')
module = self.import_module(name='buffer_test')
get_buffer_info = module.get_buffer_info
raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
@@ -50,3 +66,29 @@
arr = np.zeros((10, 1), order='C')
shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
assert strides[-1] == 8
+ dt1 = np.dtype(
+ [('a', 'b'), ('b', 'i'),
+ ('sub0', np.dtype('b,i')),
+ ('sub1', np.dtype('b,i')),
+ ('sub2', np.dtype('b,i')),
+ ('sub3', np.dtype('b,i')),
+ ('sub4', np.dtype('b,i')),
+ ('sub5', np.dtype('b,i')),
+ ('sub6', np.dtype('b,i')),
+ ('sub7', np.dtype('b,i')),
+ ('c', 'i')],
+ )
+ x = np.arange(dt1.itemsize, dtype='int8').view(dt1)
+ # pytest can catch warnings from v2.8 and up, we ship 2.5
+ import warnings
+ warnings.filterwarnings("error")
+ try:
+ try:
+ y = get_buffer_info(x, ['SIMPLE'])
+ except UserWarning as e:
+ pass
+ else:
+ assert False ,"PyPy-specific UserWarning not raised" \
+ " on too long format string"
+ finally:
+ warnings.resetwarnings()
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -514,10 +514,13 @@
@cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
header=None, error=-1)
def bf_getreadbuffer(space, w_buf, segment, ref):
+ from rpython.rlib.buffer import StringBuffer
if segment != 0:
raise oefmt(space.w_SystemError,
"accessing non-existent segment")
buf = space.readbuf_w(w_buf)
+ if isinstance(buf, StringBuffer):
+ return str_getreadbuffer(space, w_buf, segment, ref)
address = buf.get_raw_address()
ref[0] = address
return len(buf)
@@ -533,7 +536,6 @@
if segment != 0:
raise oefmt(space.w_SystemError,
"accessing non-existent segment")
-
buf = space.writebuf_w(w_buf)
ref[0] = buf.get_raw_address()
return len(buf)
@@ -551,6 +553,20 @@
Py_DecRef(space, pyref)
return space.len_w(w_str)
+@cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed,
+ header=None, error=-1)
+def unicode_getreadbuffer(space, w_str, segment, ref):
+ from pypy.module.cpyext.unicodeobject import (
+ PyUnicode_AS_UNICODE, PyUnicode_GET_DATA_SIZE)
+ if segment != 0:
+ raise oefmt(space.w_SystemError,
+ "accessing non-existent unicode segment")
+ pyref = make_ref(space, w_str)
+ ref[0] = PyUnicode_AS_UNICODE(space, pyref)
+ # Stolen reference: the object has better exist somewhere else
+ Py_DecRef(space, pyref)
+ return PyUnicode_GET_DATA_SIZE(space, w_str)
+
@cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed,
header=None, error=-1)
def str_getcharbuffer(space, w_buf, segment, ref):
@@ -574,8 +590,8 @@
def setup_buffer_procs(space, w_type, pto):
bufspec = w_type.layout.typedef.buffer
- if bufspec is None:
- # not a buffer
+ if bufspec is None and not space.is_w(w_type, space.w_unicode):
+ # not a buffer, but let w_unicode be a read buffer
return
c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True)
lltype.render_immortal(c_buf)
@@ -591,6 +607,13 @@
c_buf.c_bf_getcharbuffer = llhelper(
str_getcharbuffer.api_func.functype,
str_getcharbuffer.api_func.get_wrapper(space))
+ elif space.is_w(w_type, space.w_unicode):
+ # Special case: unicode doesn't support get_raw_address(), so we have a
+ # custom get*buffer that instead gives the address of the char* in the
+ # PyUnicodeObject*!
+ c_buf.c_bf_getreadbuffer = llhelper(
+ unicode_getreadbuffer.api_func.functype,
+ unicode_getreadbuffer.api_func.get_wrapper(space))
elif space.is_w(w_type, space.w_buffer):
# Special case: we store a permanent address on the cpyext wrapper,
# so we'll reuse that.
@@ -706,7 +729,7 @@
# uninitialized fields:
# c_tp_print
# XXX implement
- # c_tp_compare and the following fields (see
http://docs.python.org/c-api/typeobj.html )
+ # c_tp_compare and more?
w_base = best_base(space, w_type.bases_w)
pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base))
@@ -764,7 +787,6 @@
return find_best_base(bases_w)
def inherit_slots(space, pto, w_base):
- # XXX missing: nearly everything
base_pyo = make_ref(space, w_base)
try:
base = rffi.cast(PyTypeObjectPtr, base_pyo)
@@ -783,6 +805,23 @@
pto.c_tp_getattro = base.c_tp_getattro
if not pto.c_tp_as_buffer:
pto.c_tp_as_buffer = base.c_tp_as_buffer
+ if base.c_tp_as_buffer:
+ # inherit base.c_tp_as_buffer functions not inherited from w_type
+ # note: builtin types are handled in setup_buffer_procs
+ pto_as = pto.c_tp_as_buffer
+ base_as = base.c_tp_as_buffer
+ if not pto_as.c_bf_getbuffer:
+ pto_as.c_bf_getbuffer = base_as.c_bf_getbuffer
+ if not pto_as.c_bf_getcharbuffer:
+ pto_as.c_bf_getcharbuffer = base_as.c_bf_getcharbuffer
+ if not pto_as.c_bf_getwritebuffer:
+ pto_as.c_bf_getwritebuffer = base_as.c_bf_getwritebuffer
+ if not pto_as.c_bf_getreadbuffer:
+ pto_as.c_bf_getreadbuffer = base_as.c_bf_getreadbuffer
+ if not pto_as.c_bf_getsegcount:
+ pto_as.c_bf_getsegcount = base_as.c_bf_getsegcount
+ if not pto_as.c_bf_releasebuffer:
+ pto_as.c_bf_releasebuffer = base_as.c_bf_releasebuffer
finally:
Py_DecRef(space, base_pyo)
@@ -812,13 +851,14 @@
w_obj = space.allocate_instance(W_PyCTypeObject, w_metatype)
track_reference(space, py_obj, w_obj)
- w_obj.__init__(space, py_type)
+ # __init__ wraps all slotdefs functions from py_type via add_operators
+ w_obj.__init__(space, py_type)
w_obj.ready()
finish_type_2(space, py_type, w_obj)
- # inheriting tp_as_* slots
base = py_type.c_tp_base
if base:
+ # XXX refactor - parts of this are done in finish_type_2 ->
inherit_slots
if not py_type.c_tp_as_number:
py_type.c_tp_as_number = base.c_tp_as_number
py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_CHECKTYPES
@@ -827,7 +867,7 @@
py_type.c_tp_as_sequence = base.c_tp_as_sequence
py_type.c_tp_flags |= base.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
if not py_type.c_tp_as_mapping: py_type.c_tp_as_mapping =
base.c_tp_as_mapping
- if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer =
base.c_tp_as_buffer
+ #if not py_type.c_tp_as_buffer: py_type.c_tp_as_buffer =
base.c_tp_as_buffer
return w_obj
diff --git a/pypy/module/micronumpy/test/dummy_module.py
b/pypy/module/micronumpy/test/dummy_module.py
--- a/pypy/module/micronumpy/test/dummy_module.py
+++ b/pypy/module/micronumpy/test/dummy_module.py
@@ -27,7 +27,8 @@
globals()['uint'] = dtype('uint').type
types = ['Generic', 'Number', 'Integer', 'SignedInteger', 'UnsignedInteger',
- 'Inexact', 'Floating', 'ComplexFloating', 'Flexible', 'Character']
+ 'Inexact', 'Floating', 'ComplexFloating', 'Flexible', 'Character',
+ ]
for t in types:
globals()[t.lower()] = typeinfo[t]
@@ -40,4 +41,4 @@
return a
def isscalar(a):
- return type(a) in [typeinfo[t] for t in types]
+ return any([isinstance(a, typeinfo[t]) for t in types])
diff --git a/pypy/module/micronumpy/test/test_ndarray.py
b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1486,7 +1486,7 @@
assert d[1] == 12
def test_sum(self):
- from numpy import array, zeros, float16, complex64, str_
+ from numpy import array, zeros, float16, complex64, str_, isscalar, add
a = array(range(5))
assert a.sum() == 10
assert a[:4].sum() == 6
@@ -1515,6 +1515,13 @@
assert list(zeros((0, 2)).sum(axis=1)) == []
+ a = array([1, 2, 3, 4]).sum()
+ s = isscalar(a)
+ assert s is True
+ a = add.reduce([1.0, 2, 3, 4])
+ s = isscalar(a)
+ assert s is True,'%r is not a scalar' % type(a)
+
def test_reduce_nd(self):
from numpy import arange, array
a = arange(15).reshape(5, 3)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -288,10 +288,8 @@
_, dtype, _ = self.find_specialization(space, dtype, dtype, out,
casting='unsafe')
- call__array_wrap__ = True
if shapelen == len(axes):
if out:
- call__array_wrap__ = False
if out.ndims() > 0:
raise oefmt(space.w_ValueError,
"output parameter for reduction operation %s
has "
@@ -302,15 +300,20 @@
if out:
out.set_scalar_value(res)
return out
+ w_NDimArray = space.gettypefor(W_NDimArray)
+ call__array_wrap__ = False
if keepdims:
shape = [1] * len(obj_shape)
out = W_NDimArray.from_shape(space, shape, dtype,
w_instance=obj)
out.implementation.setitem(0, res)
+ call__array_wrap__ = True
res = out
- elif not space.is_w(space.type(w_obj),
space.gettypefor(W_NDimArray)):
+ elif (space.issubtype_w(space.type(w_obj), w_NDimArray) and
+ not space.is_w(space.type(w_obj), w_NDimArray)):
# subtypes return a ndarray subtype, not a scalar
out = W_NDimArray.from_shape(space, [1], dtype, w_instance=obj)
out.implementation.setitem(0, res)
+ call__array_wrap__ = True
res = out
if call__array_wrap__:
res = space.call_method(obj, '__array_wrap__', res,
space.w_None)
@@ -359,8 +362,7 @@
return out
loop.reduce(
space, self.func, obj, axis_flags, dtype, out, self.identity)
- if call__array_wrap__:
- out = space.call_method(obj, '__array_wrap__', out,
space.w_None)
+ out = space.call_method(obj, '__array_wrap__', out, space.w_None)
return out
def descr_outer(self, space, args_w):
diff --git a/pypy/module/operator/test/test_operator.py
b/pypy/module/operator/test/test_operator.py
--- a/pypy/module/operator/test/test_operator.py
+++ b/pypy/module/operator/test/test_operator.py
@@ -251,6 +251,13 @@
exc = raises(TypeError, operator.index, "abc")
assert str(exc.value) == "'str' object cannot be interpreted as an
index"
+ def test_index_int_subclass(self):
+ import operator
+ class myint(int):
+ def __index__(self):
+ return 13289
+ assert operator.index(myint(7)) == 7
+
def test_compare_digest(self):
import operator
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -522,6 +522,23 @@
def isinstance(space, w_inst, w_type):
return space.wrap(space.isinstance_w(w_inst, w_type))
+ def index(space, w_obj):
+ if (space.isinstance_w(w_obj, space.w_int) or
+ space.isinstance_w(w_obj, space.w_long)):
+ return w_obj
+ w_impl = space.lookup(w_obj, '__index__')
+ if w_impl is None:
+ raise oefmt(space.w_TypeError,
+ "'%T' object cannot be interpreted as an index",
+ w_obj)
+ w_result = space.get_and_call_function(w_impl, w_obj)
+
+ if (space.isinstance_w(w_result, space.w_int) or
+ space.isinstance_w(w_result, space.w_long)):
+ return w_result
+ raise oefmt(space.w_TypeError,
+ "__index__ returned non-(int,long) (type '%T')", w_result)
+
# helpers
@@ -797,17 +814,13 @@
# more of the above manually-coded operations as well)
for targetname, specialname, checkerspec in [
- ('index', '__index__', ("space.w_int", "space.w_long")),
('long', '__long__', ("space.w_int", "space.w_long")),
('float', '__float__', ("space.w_float",))]:
l = ["space.isinstance_w(w_result, %s)" % x
for x in checkerspec]
checker = " or ".join(l)
- if targetname == 'index':
- msg = "'%%T' object cannot be interpreted as an index"
- else:
- msg = "unsupported operand type for %(targetname)s(): '%%T'"
+ msg = "unsupported operand type for %(targetname)s(): '%%T'"
msg = msg % locals()
source = """if 1:
def %(targetname)s(space, w_obj):
diff --git a/pypy/objspace/std/bufferobject.py
b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -32,6 +32,10 @@
def charbuf_w(self, space):
return self.buf.as_str()
+ def descr_getbuffer(self, space, w_flags):
+ space.check_buf_flags(space.int_w(w_flags), self.buf.readonly)
+ return self
+
@staticmethod
@unwrap_spec(offset=int, size=int)
def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1):
@@ -160,6 +164,7 @@
__mul__ = interp2app(W_Buffer.descr_mul),
__rmul__ = interp2app(W_Buffer.descr_mul),
__repr__ = interp2app(W_Buffer.descr_repr),
+ __buffer__ = interp2app(W_Buffer.descr_getbuffer),
_pypy_raw_address = interp2app(W_Buffer.descr_pypy_raw_address),
)
W_Buffer.typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -605,6 +605,9 @@
def descr_mod(self, space, w_values):
return mod_format(space, self, w_values, do_unicode=False)
+ def descr_rmod(self, space, w_values):
+ return mod_format(space, w_values, self, do_unicode=False)
+
def descr_eq(self, space, w_other):
if space.config.objspace.std.withstrbuf:
from pypy.objspace.std.strbufobject import W_StringBufferObject
@@ -937,6 +940,7 @@
format = interpindirect2app(W_BytesObject.descr_format),
__format__ = interpindirect2app(W_BytesObject.descr__format__),
__mod__ = interpindirect2app(W_BytesObject.descr_mod),
+ __rmod__ = interpindirect2app(W_BytesObject.descr_rmod),
__getnewargs__ = interpindirect2app(
W_AbstractBytesObject.descr_getnewargs),
_formatter_parser = interp2app(W_BytesObject.descr_formatter_parser),
diff --git a/pypy/objspace/std/memoryobject.py
b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -131,9 +131,13 @@
return space.newbool(bool(self.buf.readonly))
def w_get_shape(self, space):
+ if self.buf.getndim() == 0:
+ return space.w_None
return space.newtuple([space.wrap(x) for x in self.buf.getshape()])
def w_get_strides(self, space):
+ if self.buf.getndim() == 0:
+ return space.w_None
return space.newtuple([space.wrap(x) for x in self.buf.getstrides()])
def w_get_suboffsets(self, space):
diff --git a/pypy/objspace/std/stringmethods.py
b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -649,10 +649,10 @@
return self._starts_ends_overflow(prefix)
return endswith(value, prefix, start, end)
- def _strip(self, space, w_chars, left, right):
+ def _strip(self, space, w_chars, left, right, name='strip'):
"internal function called by str_xstrip methods"
value = self._val(space)
- chars = self._op_val(space, w_chars, strict='strip')
+ chars = self._op_val(space, w_chars, strict=name)
lpos = 0
rpos = len(value)
@@ -689,17 +689,17 @@
def descr_strip(self, space, w_chars=None):
if space.is_none(w_chars):
return self._strip_none(space, left=1, right=1)
- return self._strip(space, w_chars, left=1, right=1)
+ return self._strip(space, w_chars, left=1, right=1, name='strip')
def descr_lstrip(self, space, w_chars=None):
if space.is_none(w_chars):
return self._strip_none(space, left=1, right=0)
- return self._strip(space, w_chars, left=1, right=0)
+ return self._strip(space, w_chars, left=1, right=0, name='lstrip')
def descr_rstrip(self, space, w_chars=None):
if space.is_none(w_chars):
return self._strip_none(space, left=0, right=1)
- return self._strip(space, w_chars, left=0, right=1)
+ return self._strip(space, w_chars, left=0, right=1, name='rstrip')
def descr_swapcase(self, space):
selfvalue = self._val(space)
diff --git a/pypy/objspace/std/test/test_bytesobject.py
b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -92,6 +92,7 @@
raises(ValueError, 'a%Zb'.__mod__, ((23,),))
def test_format(self):
+ import sys
raises(TypeError, "foo".__mod__, "bar")
raises(TypeError, u"foo".__mod__, "bar")
raises(TypeError, "foo".__mod__, u"bar")
@@ -105,6 +106,22 @@
assert result == "a foo b"
assert isinstance(result, cls)
+ for format, arg, cls in [("a %s b", "foo", str),
+ (u"a %s b", u"foo", unicode)]:
+ raises(TypeError, arg.__rmod__, format[:2])
+ result = arg.__rmod__(format)
+ assert result == "a foo b"
+ assert isinstance(result, cls)
+ for format, arg, cls in [(u"a %s b", "foo", str),
+ ("a %s b", u"foo", unicode)]:
+ result = arg.__rmod__(format)
+ if '__pypy__' in sys.builtin_module_names:
+ raises(TypeError, arg.__rmod__, format[:2])
+ assert result == "a foo b"
+ assert isinstance(result, cls)
+ else:
+ assert result is NotImplemented
+
def test_format_c_overflow(self):
raises(OverflowError, b'{0:c}'.format, -1)
raises(OverflowError, b'{0:c}'.format, 256)
@@ -114,6 +131,7 @@
exc_info = raises(TypeError, int_format.__mod__, '123')
expected = int_format + ' format: a number is required, not str'
assert str(exc_info.value) == expected
+ raises(TypeError, "None % 'abc'") # __rmod__
def test_split(self):
assert b"".split() == []
@@ -258,9 +276,9 @@
exc = raises(TypeError, s.strip, buffer(' '))
assert str(exc.value) == 'strip arg must be None, str or unicode'
exc = raises(TypeError, s.rstrip, buffer(' '))
- assert str(exc.value) == 'strip arg must be None, str or unicode'
+ assert str(exc.value) == 'rstrip arg must be None, str or unicode'
exc = raises(TypeError, s.lstrip, buffer(' '))
- assert str(exc.value) == 'strip arg must be None, str or unicode'
+ assert str(exc.value) == 'lstrip arg must be None, str or unicode'
def test_zfill(self):
assert b'123'.zfill(2) == b'123'
@@ -809,6 +827,10 @@
raises(TypeError, len, iter(iterable))
def test___radd__(self):
+ raises(TypeError, "None + ''")
+ raises(AttributeError, "'abc'.__radd__('def')")
+
+
class Foo(object):
def __radd__(self, other):
return 42
diff --git a/pypy/objspace/std/test/test_unicodeobject.py
b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -335,9 +335,9 @@
exc = raises(TypeError, s.strip, buffer(' '))
assert str(exc.value) == 'strip arg must be None, unicode or str'
exc = raises(TypeError, s.rstrip, buffer(' '))
- assert str(exc.value) == 'strip arg must be None, unicode or str'
+ assert str(exc.value) == 'rstrip arg must be None, unicode or str'
exc = raises(TypeError, s.lstrip, buffer(' '))
- assert str(exc.value) == 'strip arg must be None, unicode or str'
+ assert str(exc.value) == 'lstrip arg must be None, unicode or str'
def test_strip_str_unicode(self):
x = "--abc--".strip(u"-")
@@ -748,7 +748,9 @@
assert 'abc'.__add__(u'def') == u'abcdef'
assert u'abc'.__add__(u'def') == u'abcdef'
assert u'abc'.__add__('def') == u'abcdef'
- # xxx CPython has no str.__radd__ and no unicode.__radd__
+ assert u'abc'.__rmod__(u'%s') == u'abc'
+ ret = u'abc'.__rmod__('%s')
+ raises(AttributeError, "u'abc'.__radd__(u'def')")
def test_str_unicode_concat_overrides(self):
"Test from Jython about being bug-compatible with CPython."
diff --git a/pypy/objspace/std/unicodeobject.py
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -319,6 +319,9 @@
def descr_mod(self, space, w_values):
return mod_format(space, self, w_values, do_unicode=True)
+ def descr_rmod(self, space, w_values):
+ return mod_format(space, w_values, self, do_unicode=True)
+
def descr_translate(self, space, w_table):
selfvalue = self._value
w_sys = space.getbuiltinmodule('sys')
@@ -633,6 +636,9 @@
def __mod__():
"""x.__mod__(y) <==> x%y"""
+ def __rmod__():
+ """x.__rmod__(y) <==> y%x"""
+
def __mul__():
"""x.__mul__(n) <==> x*n"""
@@ -1096,6 +1102,8 @@
doc=UnicodeDocstrings.__format__.__doc__),
__mod__ = interp2app(W_UnicodeObject.descr_mod,
doc=UnicodeDocstrings.__mod__.__doc__),
+ __rmod__ = interp2app(W_UnicodeObject.descr_rmod,
+ doc=UnicodeDocstrings.__rmod__.__doc__),
__getnewargs__ = interp2app(W_UnicodeObject.descr_getnewargs,
doc=UnicodeDocstrings.__getnewargs__.__doc__),
_formatter_parser = interp2app(W_UnicodeObject.descr_formatter_parser),
diff --git a/rpython/jit/backend/detect_cpu.py
b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -61,6 +61,7 @@
'i86pc': MODEL_X86, # Solaris/Intel
'x86': MODEL_X86, # Apple
'Power Macintosh': MODEL_PPC_64,
+ 'powerpc': MODEL_PPC_64, # freebsd
'ppc64': MODEL_PPC_64,
'ppc64le': MODEL_PPC_64,
'x86_64': MODEL_X86,
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -510,6 +510,8 @@
# ----------
+HASH_ALGORITHM = "rpython" # XXX Is there a better name?
+
def _hash_string(s):
"""The algorithm behind compute_hash() for a string or a unicode."""
from rpython.rlib.rarithmetic import intmask
diff --git a/rpython/rlib/rlocale.py b/rpython/rlib/rlocale.py
--- a/rpython/rlib/rlocale.py
+++ b/rpython/rlib/rlocale.py
@@ -195,6 +195,7 @@
isalpha = external('isalpha', [rffi.INT], rffi.INT)
isupper = external('isupper', [rffi.INT], rffi.INT)
+toupper = external('toupper', [rffi.INT], rffi.INT)
islower = external('islower', [rffi.INT], rffi.INT)
tolower = external('tolower', [rffi.INT], rffi.INT)
isalnum = external('isalnum', [rffi.INT], rffi.INT)
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -236,6 +236,8 @@
'utime.h', 'sys/time.h', 'sys/times.h',
'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
'signal.h', 'sys/utsname.h', _ptyh]
+ if sys.platform.startswith('freebsd'):
+ includes.append('sys/ttycom.h')
libraries = ['util']
eci = ExternalCompilationInfo(
includes=includes,
@@ -2306,6 +2308,7 @@
_pipe2_syscall = ENoSysCache()
+post_include_bits=['int _cpu_count(void);']
# cpu count for linux, windows and mac (+ bsds)
# note that the code is copied from cpython and split up here
if sys.platform.startswith('linux'):
@@ -2314,17 +2317,16 @@
RPY_EXTERN int _cpu_count(void) {
return sysconf(_SC_NPROCESSORS_ONLN);
}
- """])
+ """], post_include_bits=post_include_bits)
elif sys.platform == "win32":
cpucount_eci = ExternalCompilationInfo(includes=["Windows.h"],
separate_module_sources=["""
RPY_EXTERN int _cpu_count(void) {
- int ncpu = 0;
SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
return sysinfo.dwNumberOfProcessors;
}
- """])
+ """], post_include_bits=post_include_bits)
else:
cpucount_eci = ExternalCompilationInfo(includes=["sys/types.h",
"sys/sysctl.h"],
separate_module_sources=["""
@@ -2344,8 +2346,10 @@
#endif
return ncpu;
}
- """])
+ """], post_include_bits=post_include_bits)
-cpu_count = rffi.llexternal('_cpu_count', [], rffi.INT_real,
+_cpu_count = rffi.llexternal('_cpu_count', [], rffi.INT_real,
compilation_info=cpucount_eci)
+def cpu_count():
+ return rffi.cast(lltype.Signed, _cpu_count())
diff --git a/rpython/rlib/rsre/rpy/sre_constants.py
b/rpython/rlib/rsre/rpy/sre_constants.py
--- a/rpython/rlib/rsre/rpy/sre_constants.py
+++ b/rpython/rlib/rsre/rpy/sre_constants.py
@@ -58,6 +58,7 @@
REPEAT_ONE = "repeat_one"
SUBPATTERN = "subpattern"
MIN_REPEAT_ONE = "min_repeat_one"
+RANGE_IGNORE = "range_ignore"
# positions
AT_BEGINNING = "at_beginning"
@@ -119,8 +120,8 @@
REPEAT,
REPEAT_ONE,
SUBPATTERN,
- MIN_REPEAT_ONE
-
+ MIN_REPEAT_ONE,
+ RANGE_IGNORE,
]
ATCODES = [
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -2,7 +2,7 @@
Character categories and charsets.
"""
import sys
-from rpython.rlib.rlocale import tolower, isalnum
+from rpython.rlib.rlocale import tolower, toupper, isalnum
from rpython.rlib.unroll import unrolling_iterable
from rpython.rlib import jit
from rpython.rlib.rarithmetic import int_between
@@ -67,6 +67,19 @@
char_ord += ord('a') - ord('A')
return char_ord
+def getupper(char_ord, flags):
+ if flags & SRE_FLAG_LOCALE:
+ if char_ord < 256: # cheating! Well, CPython does too.
+ char_ord = toupper(char_ord)
+ return char_ord
+ elif flags & SRE_FLAG_UNICODE:
+ assert unicodedb is not None
+ char_ord = unicodedb.toupper(char_ord)
+ else:
+ if int_between(ord('a'), char_ord, ord('z') + 1): # ASCII upper
+ char_ord += ord('A') - ord('a')
+ return char_ord
+
#### Category helpers
is_a_word = [(chr(i).isalnum() or chr(i) == '_') for i in range(256)]
@@ -139,16 +152,17 @@
##### Charset evaluation
@jit.unroll_safe
-def check_charset(pattern, ppos, char_code):
+def check_charset(ctx, ppos, char_code):
"""Checks whether a character matches set of arbitrary length.
The set starts at pattern[ppos]."""
negated = False
result = False
+ pattern = ctx.pattern
while True:
opcode = pattern[ppos]
for i, function in set_dispatch_unroll:
if opcode == i:
- newresult, ppos = function(pattern, ppos, char_code)
+ newresult, ppos = function(ctx, ppos, char_code)
result |= newresult
break
else:
@@ -163,18 +177,21 @@
return not result
return result
-def set_literal(pat, index, char_code):
+def set_literal(ctx, index, char_code):
# <LITERAL> <code>
+ pat = ctx.pattern
match = pat[index+1] == char_code
return match, index + 2
-def set_category(pat, index, char_code):
+def set_category(ctx, index, char_code):
# <CATEGORY> <code>
+ pat = ctx.pattern
match = category_dispatch(pat[index+1], char_code)
return match, index + 2
-def set_charset(pat, index, char_code):
+def set_charset(ctx, index, char_code):
# <CHARSET> <bitmap> (16 bits per code word)
+ pat = ctx.pattern
if CODESIZE == 2:
match = char_code < 256 and \
(pat[index+1+(char_code >> 4)] & (1 << (char_code & 15)))
@@ -184,13 +201,25 @@
(pat[index+1+(char_code >> 5)] & (1 << (char_code & 31)))
return match, index + 9 # skip bitmap
-def set_range(pat, index, char_code):
+def set_range(ctx, index, char_code):
# <RANGE> <lower> <upper>
+ pat = ctx.pattern
match = int_between(pat[index+1], char_code, pat[index+2] + 1)
return match, index + 3
-def set_bigcharset(pat, index, char_code):
+def set_range_ignore(ctx, index, char_code):
+ # <RANGE_IGNORE> <lower> <upper>
+ # the char_code is already lower cased
+ pat = ctx.pattern
+ lower = pat[index + 1]
+ upper = pat[index + 2]
+ match1 = int_between(lower, char_code, upper + 1)
+ match2 = int_between(lower, getupper(char_code, ctx.flags), upper + 1)
+ return match1 | match2, index + 3
+
+def set_bigcharset(ctx, index, char_code):
# <BIGCHARSET> <blockcount> <256 blockindices> <blocks>
+ pat = ctx.pattern
count = pat[index+1]
index += 2
@@ -224,7 +253,7 @@
index += count * (32 / CODESIZE) # skip blocks
return match, index
-def set_unicode_general_category(pat, index, char_code):
+def set_unicode_general_category(ctx, index, char_code):
# Unicode "General category property code" (not used by Python).
# A general category is two letters. 'pat[index+1]' contains both
# the first character, and the second character shifted by 8.
@@ -233,6 +262,7 @@
# Negative matches are triggered by bit number 7.
assert unicodedb is not None
cat = unicodedb.category(char_code)
+ pat = ctx.pattern
category_code = pat[index + 1]
first_character = category_code & 0x7F
second_character = (category_code >> 8) & 0x7F
@@ -260,6 +290,7 @@
11: set_bigcharset,
19: set_literal,
27: set_range,
+ 32: set_range_ignore,
70: set_unicode_general_category,
}
set_dispatch_unroll = unrolling_iterable(sorted(set_dispatch_table.items()))
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -40,6 +40,7 @@
OPCODE_REPEAT_ONE = 29
#OPCODE_SUBPATTERN = 30
OPCODE_MIN_REPEAT_ONE = 31
+OPCODE_RANGE_IGNORE = 32
# not used by Python itself
OPCODE_UNICODE_GENERAL_CATEGORY = 70
@@ -640,8 +641,7 @@
elif op == OPCODE_IN:
# match set member (or non_member)
# <IN> <skip> <set>
- if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern,
- ppos+1,
+ if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1,
ctx.str(ptr)):
return
ppos += ctx.pat(ppos)
@@ -650,8 +650,7 @@
elif op == OPCODE_IN_IGNORE:
# match set member (or non_member), ignoring case
# <IN> <skip> <set>
- if ptr >= ctx.end or not rsre_char.check_charset(ctx.pattern,
- ppos+1,
+ if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1,
ctx.lowstr(ptr)):
return
ppos += ctx.pat(ppos)
@@ -871,10 +870,10 @@
return True # match anything (including a newline)
@specializectx
def match_IN(ctx, ptr, ppos):
- return rsre_char.check_charset(ctx.pattern, ppos+2, ctx.str(ptr))
+ return rsre_char.check_charset(ctx, ppos+2, ctx.str(ptr))
@specializectx
def match_IN_IGNORE(ctx, ptr, ppos):
- return rsre_char.check_charset(ctx.pattern, ppos+2, ctx.lowstr(ptr))
+ return rsre_char.check_charset(ctx, ppos+2, ctx.lowstr(ptr))
@specializectx
def match_LITERAL(ctx, ptr, ppos):
return ctx.str(ptr) == ctx.pat(ppos+1)
@@ -1134,7 +1133,7 @@
while start < ctx.end:
ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start,
base=base)
- if rsre_char.check_charset(ctx.pattern, 5, ctx.str(start)):
+ if rsre_char.check_charset(ctx, 5, ctx.str(start)):
if sre_match(ctx, base, start, None) is not None:
ctx.match_start = start
return True
diff --git a/rpython/rlib/rsre/test/test_char.py
b/rpython/rlib/rsre/test/test_char.py
--- a/rpython/rlib/rsre/test/test_char.py
+++ b/rpython/rlib/rsre/test/test_char.py
@@ -34,6 +34,22 @@
assert rsre_char.getlower(UPPER_PI, SRE_FLAG_LOCALE | SRE_FLAG_UNICODE) \
== UPPER_PI
+def test_getupper():
+ assert rsre_char.getupper(ord('A'), 0) == ord('A')
+ assert rsre_char.getupper(ord('b'), 0) == ord('B')
+ assert rsre_char.getupper(10, 0) == 10
+ assert rsre_char.getupper(LOWER_PI, 0) == LOWER_PI
+ #
+ assert rsre_char.getupper(ord('a'), SRE_FLAG_UNICODE) == ord('A')
+ assert rsre_char.getupper(ord('2'), SRE_FLAG_UNICODE) == ord('2')
+ assert rsre_char.getupper(10, SRE_FLAG_UNICODE) == 10
+ assert rsre_char.getupper(LOWER_PI, SRE_FLAG_UNICODE) == UPPER_PI
+ #
+ assert rsre_char.getupper(LOWER_PI, SRE_FLAG_LOCALE) == LOWER_PI
+ assert rsre_char.getupper(LOWER_PI, SRE_FLAG_LOCALE | SRE_FLAG_UNICODE) \
+ == LOWER_PI
+
+
def test_is_word():
assert rsre_char.is_word(ord('A'))
assert rsre_char.is_word(ord('_'))
@@ -128,6 +144,10 @@
assert cat(CHCODES["category_uni_not_digit"], DINGBAT_CIRCLED)
+class Ctx:
+ def __init__(self, pattern):
+ self.pattern = pattern
+
def test_general_category():
from rpython.rlib.unicodedata import unicodedb
@@ -137,12 +157,12 @@
pat_neg = [70, ord(cat) | 0x80, 0]
for c in positive:
assert unicodedb.category(ord(c)).startswith(cat)
- assert rsre_char.check_charset(pat_pos, 0, ord(c))
- assert not rsre_char.check_charset(pat_neg, 0, ord(c))
+ assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
+ assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
for c in negative:
assert not unicodedb.category(ord(c)).startswith(cat)
- assert not rsre_char.check_charset(pat_pos, 0, ord(c))
- assert rsre_char.check_charset(pat_neg, 0, ord(c))
+ assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
+ assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
def cat2num(cat):
return ord(cat[0]) | (ord(cat[1]) << 8)
@@ -153,17 +173,17 @@
pat_neg = [70, cat2num(cat) | 0x80, 0]
for c in positive:
assert unicodedb.category(ord(c)) == cat
- assert rsre_char.check_charset(pat_pos, 0, ord(c))
- assert not rsre_char.check_charset(pat_neg, 0, ord(c))
+ assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
+ assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
for c in negative:
assert unicodedb.category(ord(c)) != cat
- assert not rsre_char.check_charset(pat_pos, 0, ord(c))
- assert rsre_char.check_charset(pat_neg, 0, ord(c))
+ assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
+ assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
# test for how the common 'L&' pattern might be compiled
pat = [70, cat2num('Lu'), 70, cat2num('Ll'), 70, cat2num('Lt'), 0]
- assert rsre_char.check_charset(pat, 0, 65) # Lu
- assert rsre_char.check_charset(pat, 0, 99) # Ll
- assert rsre_char.check_charset(pat, 0, 453) # Lt
- assert not rsre_char.check_charset(pat, 0, 688) # Lm
- assert not rsre_char.check_charset(pat, 0, 5870) # Nl
+ assert rsre_char.check_charset(Ctx(pat), 0, 65) # Lu
+ assert rsre_char.check_charset(Ctx(pat), 0, 99) # Ll
+ assert rsre_char.check_charset(Ctx(pat), 0, 453) # Lt
+ assert not rsre_char.check_charset(Ctx(pat), 0, 688) # Lm
+ assert not rsre_char.check_charset(Ctx(pat), 0, 5870) # Nl
diff --git a/rpython/rlib/rsre/test/test_match.py
b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -1,5 +1,5 @@
import re, random, py
-from rpython.rlib.rsre import rsre_core
+from rpython.rlib.rsre import rsre_core, rsre_char
from rpython.rlib.rsre.rpy import get_code, VERSION
@@ -299,3 +299,12 @@
assert rsre_core.fullmatch(r, "ab")
r = get_code(r"(?!a)..")
assert not rsre_core.fullmatch(r, "ab")
+
+ def test_range_ignore(self):
+ from rpython.rlib.unicodedata import unicodedb
+ rsre_char.set_unicode_db(unicodedb)
+ #
+ r = get_code(u"[\U00010428-\U0001044f]", re.I)
+ assert r.count(27) == 1 # OPCODE_RANGE
+ r[r.index(27)] = 32 # => OPCODE_RANGE_IGNORE
+ assert rsre_core.match(r, u"\U00010428")
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -29,7 +29,8 @@
'sys/types.h', 'unistd.h',
'sys/time.h', 'sys/resource.h']
- if not sys.platform.startswith("openbsd"):
+ if not sys.platform.startswith("openbsd") and \
+ not sys.platform.startswith("freebsd"):
includes.append('sys/timeb.h')
need_rusage = True
diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -8,9 +8,15 @@
import os
from subprocess import PIPE, Popen
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
