Author: Matti Picus <matti.pi...@gmail.com>
Branch: cpyext-obj-stealing
Changeset: r91328:92c3ef911221
Date: 2017-05-18 17:35 +0300
http://bitbucket.org/pypy/pypy/changeset/92c3ef911221/
Log: merge default into branch
diff too long, truncating to 2000 out of 4418 lines
diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
This directory contains all the include files needed to build cpython
extensions with PyPy. Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -234,6 +234,9 @@
if ('_abstract_' in cls.__dict__ or cls is Structure
or cls is union.Union):
raise TypeError("abstract class")
+ if hasattr(cls, '_swappedbytes_'):
+ raise NotImplementedError("missing in PyPy: structure/union with "
+ "swapped (non-native) byte ordering")
if hasattr(cls, '_ffistruct_'):
self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
return self
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -79,10 +79,20 @@
BOOL WINAPI CreateProcessA(char *, char *, void *,
void *, BOOL, DWORD, char *,
char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
+ void *, BOOL, DWORD, wchar_t *,
+ wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
BOOL WINAPI TerminateProcess(HANDLE, UINT);
HANDLE WINAPI GetStdHandle(DWORD);
+DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
+
+UINT WINAPI SetErrorMode(UINT);
+#define SEM_FAILCRITICALERRORS 0x0001
+#define SEM_NOGPFAULTERRORBOX 0x0002
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004
+#define SEM_NOOPENFILEERRORBOX 0x8000
""")
# --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
_version = 0x2601,
- _types =
b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x
00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
- _globals =
(b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
- _struct_unions =
((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
- _typenames =
(b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+ _types =
b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x
00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+ _globals =
(b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_set
mode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0),
+ _struct_unions =
((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+ _typenames =
(b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'),
)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -42,8 +42,9 @@
from rpython.jit.backend import detect_cpu
try:
if detect_cpu.autodetect().startswith('x86'):
- working_modules.add('_vmprof')
- working_modules.add('faulthandler')
+ if not sys.platform.startswith('openbsd'):
+ working_modules.add('_vmprof')
+ working_modules.add('faulthandler')
except detect_cpu.ProcessorAutodetectError:
pass
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -501,7 +501,14 @@
the rest is kept. If you return an unexpected string from
``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non
heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+ even for ``dict()`` and ``dict.update()``. CPython 2.7 allows non-string
+ keys in these two cases (and only there, as far as we know). E.g. this
+ code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+ ``dict(**{1: 2})``. (Note that ``dict(**d1)`` is equivalent to
+ ``dict(d1)``.)
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non
heaptypes.
CPython allows that for module subtypes, but not for e.g. ``int``
or ``float`` subtypes. Currently PyPy does not support the
``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,8 @@
.. branch: controller-refactor
Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
To build pypy-c you need a working python environment, and a C compiler.
It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run – depending
+the preferred way, because it will take a lot longer to run � depending
on your architecture, between two and three times as long. So head to
`our downloads`_ and get the latest stable version.
@@ -120,7 +120,7 @@
Download the versions of all the external packages from
https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
(for post-5.7.1 builds) with sha256 checksum
-``f1510452293f22e84d6059464e11f4c62ffd0e2ee97a52be9195bec8a70c6dce`` or
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
(for 2.4 release and later) or
https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -128,9 +128,9 @@
Then expand it into the base directory (base_dir) and modify your environment
to reflect this::
- set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
- set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
- set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+ set PATH=<base_dir>\bin;%PATH%
+ set INCLUDE=<base_dir>\include;%INCLUDE%
+ set LIB=<base_dir>\lib;%LIB%
Now you should be good to go. If you choose this method, you do not need
to download/build anything else.
@@ -236,6 +236,9 @@
copy out32\*.lib <somewhere in LIB>
xcopy /S include\openssl <somewhere in INCLUDE>
+For tests you will also need the dlls::
+ nmake -f ms\ntdll.mak install
+ copy out32dll\*.dll <somewhere in PATH>
TkInter module support
~~~~~~~~~~~~~~~~~~~~~~
@@ -245,18 +248,17 @@
directory found for the release script, create the dlls, libs, headers and
runtime by running::
- svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
- svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
- cd tcl85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0
INSTALLDIR=..\..\tcltk clean all
- nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
- cd ..\..\tk85\win
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1
INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
- nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1
INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
-
-Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready
-for use. The release packaging script will pick up the tcltk runtime in the lib
-directory and put it in the archive.
+ svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
+ svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
+ cd tcl85\win
+ nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0
INSTALLDIR=..\..\tcltk clean all
+ nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
+ cd ..\..\tk85\win
+ nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1
INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
+ nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1
INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
+ copy ..\..\tcltk\bin\* <somewhere in PATH>
+ copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
+ xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
The lzma compression library
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -341,9 +343,9 @@
integer. The simplest fix is to make sure that it is so, but it will
give the following incompatibility between CPython and PyPy on Win64:
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
...and, correspondingly, PyPy supports ints up to the larger value of
sys.maxint before they are converted to ``long``. The first decision
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,9 @@
from rpython.rlib.signature import signature
from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.buffer import StringBuffer
+from pypy.interpreter.buffer import BufferInterfaceNotFound
from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
make_finalizer_queue)
from pypy.interpreter.error import OperationError, new_exception_class, oefmt
@@ -221,7 +223,8 @@
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(flags))
- if space.isinstance_w(w_result, space.w_buffer):
+ if (space.isinstance_w(w_result, space.w_buffer) or
+ space.isinstance_w(w_result, space.w_memoryview)):
return w_result.buffer_w(space, flags)
raise BufferInterfaceNotFound
@@ -233,7 +236,8 @@
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(space.BUF_FULL_RO))
- if space.isinstance_w(w_result, space.w_buffer):
+ if (space.isinstance_w(w_result, space.w_buffer) or
+ space.isinstance_w(w_result, space.w_memoryview)):
return w_result.readbuf_w(space)
raise BufferInterfaceNotFound
@@ -245,7 +249,8 @@
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(space.BUF_FULL))
- if space.isinstance_w(w_result, space.w_buffer):
+ if (space.isinstance_w(w_result, space.w_buffer) or
+ space.isinstance_w(w_result, space.w_memoryview)):
return w_result.writebuf_w(space)
raise BufferInterfaceNotFound
@@ -254,7 +259,8 @@
if w_impl is not None:
w_result = space.get_and_call_function(w_impl, self,
space.newint(space.BUF_FULL_RO))
- if space.isinstance_w(w_result, space.w_buffer):
+ if (space.isinstance_w(w_result, space.w_buffer) or
+ space.isinstance_w(w_result, space.w_memoryview)):
return w_result.charbuf_w(space)
raise BufferInterfaceNotFound
@@ -392,9 +398,6 @@
class DescrMismatch(Exception):
pass
-class BufferInterfaceNotFound(Exception):
- pass
-
@specialize.memo()
def wrappable_class_name(Class):
try:
@@ -1501,18 +1504,28 @@
def readbuf_w(self, w_obj):
# Old buffer interface, returns a readonly buffer
(PyObject_AsReadBuffer)
try:
+ return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+ except OperationError:
+ self._getarg_error("convertible to a buffer", w_obj)
+ except BufferInterfaceNotFound:
+ pass
+ try:
return w_obj.readbuf_w(self)
except BufferInterfaceNotFound:
- raise oefmt(self.w_TypeError,
- "expected a readable buffer object")
+ self._getarg_error("convertible to a buffer", w_obj)
def writebuf_w(self, w_obj):
# Old buffer interface, returns a writeable buffer
(PyObject_AsWriteBuffer)
try:
+ return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+ except OperationError:
+ self._getarg_error("read-write buffer", w_obj)
+ except BufferInterfaceNotFound:
+ pass
+ try:
return w_obj.writebuf_w(self)
except BufferInterfaceNotFound:
- raise oefmt(self.w_TypeError,
- "expected a writeable buffer object")
+ self._getarg_error("read-write buffer", w_obj)
def charbuf_w(self, w_obj):
# Old buffer interface, returns a character buffer
(PyObject_AsCharBuffer)
@@ -1541,12 +1554,10 @@
if self.isinstance_w(w_obj, self.w_unicode):
return self.str(w_obj).readbuf_w(self)
try:
- return w_obj.buffer_w(self, 0)
- except BufferInterfaceNotFound:
- pass
- try:
- return w_obj.readbuf_w(self)
- except BufferInterfaceNotFound:
+ return self.readbuf_w(w_obj)
+ except OperationError as e:
+ if not e.match(self, self.w_TypeError):
+ raise
self._getarg_error("string or buffer", w_obj)
elif code == 's#':
if self.isinstance_w(w_obj, self.w_bytes):
@@ -1558,16 +1569,7 @@
except BufferInterfaceNotFound:
self._getarg_error("string or read-only buffer", w_obj)
elif code == 'w*':
- try:
- return w_obj.buffer_w(self, self.BUF_WRITABLE)
- except OperationError:
- self._getarg_error("read-write buffer", w_obj)
- except BufferInterfaceNotFound:
- pass
- try:
- return w_obj.writebuf_w(self)
- except BufferInterfaceNotFound:
- self._getarg_error("read-write buffer", w_obj)
+ return self.writebuf_w(w_obj)
elif code == 't#':
try:
return w_obj.charbuf_w(self)
@@ -1654,6 +1656,23 @@
def fsencode_or_none_w(self, w_obj):
return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
+ def byte_w(self, w_obj):
+ """
+ Convert an index-like object to an interp-level char
+
+ Used for app-level code like "bytearray(b'abc')[0] = 42".
+ """
+ if self.isinstance_w(w_obj, self.w_bytes):
+ string = self.bytes_w(w_obj)
+ if len(string) != 1:
+ raise oefmt(self.w_ValueError, "string must be of size 1")
+ return string[0]
+ value = self.getindex_w(w_obj, None)
+ if not 0 <= value < 256:
+ # this includes the OverflowError in case the long is too large
+ raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
+ return chr(value)
+
def int_w(self, w_obj, allow_conversion=True):
"""
Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/buffer.py
@@ -0,0 +1,295 @@
+from rpython.rlib.buffer import StringBuffer, SubBuffer
+
+from pypy.interpreter.error import oefmt
+
+class BufferInterfaceNotFound(Exception):
+ pass
+
+
+class BufferView(object):
+ """Abstract base class for buffers."""
+ _attrs_ = ['readonly']
+ _immutable_ = True
+
+ def getlength(self):
+ """Returns the size in bytes (even if getitemsize() > 1)."""
+ raise NotImplementedError
+
+ def as_str(self):
+ "Returns an interp-level string with the whole content of the buffer."
+ return ''.join(self._copy_buffer())
+
+ def getbytes(self, start, size):
+ """Return `size` bytes starting at byte offset `start`.
+
+ This is a low-level operation, it is up to the caller to ensure that
+ the data requested actually correspond to items accessible from the
+ BufferView.
+ Note that `start` may be negative, e.g. if the buffer is reversed.
+ """
+ raise NotImplementedError
+
+ def setbytes(self, start, string):
+ raise NotImplementedError
+
+ def get_raw_address(self):
+ raise ValueError("no raw buffer")
+
+ def as_readbuf(self):
+ # Inefficient. May be overridden.
+ return StringBuffer(self.as_str())
+
+ def as_writebuf(self):
+ """Return a writable Buffer sharing the same data as `self`."""
+ raise BufferInterfaceNotFound
+
+ def getformat(self):
+ raise NotImplementedError
+
+ def getitemsize(self):
+ raise NotImplementedError
+
+ def getndim(self):
+ raise NotImplementedError
+
+ def getshape(self):
+ raise NotImplementedError
+
+ def getstrides(self):
+ raise NotImplementedError
+
+ def releasebuffer(self):
+ pass
+
+ def value_from_bytes(self, space, s):
+ from pypy.module.struct.formatiterator import UnpackFormatIterator
+ buf = StringBuffer(s)
+ fmtiter = UnpackFormatIterator(space, buf)
+ fmtiter.interpret(self.getformat())
+ return fmtiter.result_w[0]
+
+ def _copy_buffer(self):
+ if self.getndim() == 0:
+ itemsize = self.getitemsize()
+ return [self.getbytes(0, itemsize)]
+ data = []
+ self._copy_rec(0, data, 0)
+ return data
+
+ def _copy_rec(self, idim, data, off):
+ shapes = self.getshape()
+ shape = shapes[idim]
+ strides = self.getstrides()
+
+ if self.getndim() - 1 == idim:
+ self._copy_base(data, off)
+ return
+
+ for i in range(shape):
+ self._copy_rec(idim + 1, data, off)
+ off += strides[idim]
+
+ def _copy_base(self, data, off):
+ shapes = self.getshape()
+ step = shapes[0]
+ strides = self.getstrides()
+ itemsize = self.getitemsize()
+ bytesize = self.getlength()
+ copiedbytes = 0
+ for i in range(step):
+ bytes = self.getbytes(off, itemsize)
+ data.append(bytes)
+ copiedbytes += len(bytes)
+ off += strides[0]
+ # do notcopy data if the sub buffer is out of bounds
+ if copiedbytes >= bytesize:
+ break
+
+ def get_offset(self, space, dim, index):
+ "Convert index at dimension `dim` into a byte offset"
+ shape = self.getshape()
+ nitems = shape[dim]
+ if index < 0:
+ index += nitems
+ if index < 0 or index >= nitems:
+ raise oefmt(space.w_IndexError,
+ "index out of bounds on dimension %d", dim + 1)
+ # TODO suboffsets?
+ strides = self.getstrides()
+ return strides[dim] * index
+
+ def w_getitem(self, space, idx):
+ offset = self.get_offset(space, 0, idx)
+ itemsize = self.getitemsize()
+ # TODO: this probably isn't very fast
+ data = self.getbytes(offset, itemsize)
+ return space.newbytes(data)
+
+ def new_slice(self, start, step, slicelength):
+ return BufferSlice(self, start, step, slicelength)
+
+ def w_tolist(self, space):
+ dim = self.getndim()
+ if dim == 0:
+ raise NotImplementedError
+ elif dim == 1:
+ n = self.getshape()[0]
+ values_w = [space.ord(self.w_getitem(space, i)) for i in range(n)]
+ return space.newlist(values_w)
+ else:
+ return self._tolist_rec(space, 0, 0)
+
+ def _tolist_rec(self, space, start, idim):
+ strides = self.getstrides()
+ shape = self.getshape()
+ #
+ dim = idim + 1
+ stride = strides[idim]
+ itemsize = self.getitemsize()
+ dimshape = shape[idim]
+ #
+ if dim >= self.getndim():
+ bytecount = (stride * dimshape)
+ values_w = [
+ self.value_from_bytes(space, self.getbytes(pos, itemsize))
+ for pos in range(start, start + bytecount, stride)]
+ return space.newlist(values_w)
+
+ items = [None] * dimshape
+ for i in range(dimshape):
+ item = self._tolist_rec(space, start, idim + 1)
+ items[i] = item
+ start += stride
+
+ return space.newlist(items)
+
+ def wrap(self, space):
+ return space.newmemoryview(self)
+
+
+class SimpleView(BufferView):
+ _attrs_ = ['readonly', 'data']
+ _immutable_ = True
+
+ def __init__(self, data):
+ self.data = data
+ self.readonly = self.data.readonly
+
+ def getlength(self):
+ return self.data.getlength()
+
+ def as_str(self):
+ return self.data.as_str()
+
+ def getbytes(self, start, size):
+ return self.data[start:start + size]
+
+ def setbytes(self, offset, s):
+ self.data.setslice(offset, s)
+
+ def get_raw_address(self):
+ return self.data.get_raw_address()
+
+ def as_readbuf(self):
+ return self.data
+
+ def as_writebuf(self):
+ assert not self.data.readonly
+ return self.data
+
+ def getformat(self):
+ return 'B'
+
+ def getitemsize(self):
+ return 1
+
+ def getndim(self):
+ return 1
+
+ def getshape(self):
+ return [self.getlength()]
+
+ def getstrides(self):
+ return [1]
+
+ def get_offset(self, space, dim, index):
+ "Convert index at dimension `dim` into a byte offset"
+ assert dim == 0
+ nitems = self.getlength()
+ if index < 0:
+ index += nitems
+ if index < 0 or index >= nitems:
+ raise oefmt(space.w_IndexError,
+ "index out of bounds on dimension %d", dim + 1)
+ return index
+
+ def w_getitem(self, space, idx):
+ idx = self.get_offset(space, 0, idx)
+ ch = self.data[idx]
+ return space.newbytes(ch)
+
+ def new_slice(self, start, step, slicelength):
+ if step == 1:
+ return SimpleView(SubBuffer(self.data, start, slicelength))
+ else:
+ return BufferSlice(self, start, step, slicelength)
+
+
+class BufferSlice(BufferView):
+ _immutable_ = True
+ _attrs_ = ['parent', 'readonly', 'shape', 'strides', 'start', 'step']
+
+ def __init__(self, parent, start, step, length):
+ self.parent = parent
+ self.readonly = self.parent.readonly
+ self.strides = parent.getstrides()[:]
+ self.start = start
+ self.step = step
+ self.strides[0] *= step
+ self.shape = parent.getshape()[:]
+ self.shape[0] = length
+
+ def getlength(self):
+ return self.shape[0] * self.getitemsize()
+
+ def getbytes(self, start, size):
+ offset = self.start * self.parent.getstrides()[0]
+ return self.parent.getbytes(offset + start, size)
+
+ def setbytes(self, start, string):
+ if len(string) == 0:
+ return # otherwise, adding self.offset might make 'start'
+ # out of bounds
+ offset = self.start * self.parent.getstrides()[0]
+ self.parent.setbytes(offset + start, string)
+
+ def get_raw_address(self):
+ from rpython.rtyper.lltypesystem import rffi
+ offset = self.start * self.parent.getstrides()[0]
+ return rffi.ptradd(self.parent.get_raw_address(), offset)
+
+ def getformat(self):
+ return self.parent.getformat()
+
+ def getitemsize(self):
+ return self.parent.getitemsize()
+
+ def getndim(self):
+ return self.parent.getndim()
+
+ def getshape(self):
+ return self.shape
+
+ def getstrides(self):
+ return self.strides
+
+ def parent_index(self, idx):
+ return self.start + self.step * idx
+
+ def w_getitem(self, space, idx):
+ return self.parent.w_getitem(space, self.parent_index(idx))
+
+ def new_slice(self, start, step, slicelength):
+ real_start = start + self.start
+ real_step = self.step * step
+ return BufferSlice(self.parent, real_start, real_step, slicelength)
diff --git a/pypy/module/__builtin__/compiling.py
b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
"compile() arg 3 must be 'exec', 'eval' or 'single'")
if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+ if flags & consts.PyCF_ONLY_AST:
+ return w_source
ast_node = ast.mod.from_object(space, w_source)
return ec.compiler.compile_ast(ast_node, filename, mode, flags)
diff --git a/pypy/module/__builtin__/test/test_compile.py
b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -1,5 +1,6 @@
class AppTestCompile:
def test_simple(self):
+ import sys
co = compile('1+2', '?', 'eval')
assert eval(co) == 3
co = compile(buffer('1+2'), '?', 'eval')
@@ -8,8 +9,10 @@
assert str(exc.value) == "compile() expected string without null bytes"
exc = raises(TypeError, compile, unichr(0), '?', 'eval')
assert str(exc.value) == "compile() expected string without null bytes"
- exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
- assert str(exc.value) == "expected a readable buffer object"
+
+ if '__pypy__' in sys.modules:
+ co = compile(memoryview('1+2'), '?', 'eval')
+ assert eval(co) == 3
compile("from __future__ import with_statement", "<test>", "exec")
raises(SyntaxError, compile, '-', '?', 'eval')
raises(ValueError, compile, '"\\xt"', '?', 'eval')
@@ -50,7 +53,8 @@
co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
raises(TypeError, compile, co1, '<ast>', 'eval')
co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
- compile(co2, '<ast>', 'eval')
+ tree = compile(co2, '<ast>', 'eval')
+ assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
def test_leading_newlines(self):
src = """
diff --git a/pypy/module/_cffi_backend/cbuffer.py
b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -4,6 +4,7 @@
from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
from pypy.module._cffi_backend import ctypestruct
from pypy.objspace.std.bufferobject import W_Buffer
+from pypy.interpreter.buffer import SimpleView
from rpython.rlib.buffer import Buffer
from rpython.rtyper.annlowlevel import llstr
@@ -60,7 +61,7 @@
if space.isinstance_w(w_other, space.w_unicode):
return space.w_NotImplemented
try:
- other_buf = space.buffer_w(w_other, space.BUF_SIMPLE)
+ other_buf = space.readbuf_w(w_other)
except OperationError as e:
if e.async(space):
raise
diff --git a/pypy/module/_cffi_backend/func.py
b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,24 +105,10 @@
# ____________________________________________________________
def _fetch_as_read_buffer(space, w_x):
- # xxx do we really need to implement the same mess as in CPython 2.7
- # w.r.t. buffers and memoryviews??
- try:
- buf = space.readbuf_w(w_x)
- except OperationError as e:
- if not e.match(space, space.w_TypeError):
- raise
- buf = space.buffer_w(w_x, space.BUF_SIMPLE)
- return buf
+ return space.readbuf_w(w_x)
def _fetch_as_write_buffer(space, w_x):
- try:
- buf = space.writebuf_w(w_x)
- except OperationError as e:
- if not e.match(space, space.w_TypeError):
- raise
- buf = space.buffer_w(w_x, space.BUF_WRITABLE)
- return buf
+ return space.writebuf_w(w_x)
@unwrap_spec(w_ctype=ctypeobj.W_CType)
def from_buffer(space, w_ctype, w_x):
diff --git a/pypy/module/_codecs/test/test_codecs.py
b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -292,8 +292,8 @@
assert bytes2.decode("unicode_internal") == u"\U00010098"
assert bytes.decode("unicode_internal") == u"a"
assert _codecs.unicode_internal_decode(array.array('c', bytes))[0] ==
u"a"
- exc = raises(TypeError, _codecs.unicode_internal_decode,
memoryview(bytes))
- assert str(exc.value) == "expected a readable buffer object"
+ if '__pypy__' in sys.modules:
+ assert _codecs.unicode_internal_decode(memoryview(bytes))[0] ==
u"a"
def test_raw_unicode_escape(self):
assert unicode("\u0663", "raw-unicode-escape") == u"\u0663"
diff --git a/pypy/module/_io/interp_bufferedio.py
b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -1,12 +1,15 @@
from __future__ import with_statement
+from rpython.rlib.signature import signature
+from rpython.rlib import types
+
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.typedef import (
TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.rgc import (
- nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import SimpleView
+
+from rpython.rlib.buffer import ByteBuffer, SubBuffer
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rarithmetic import r_longlong, intmask
from rpython.rlib import rposix
@@ -16,7 +19,6 @@
check_readable_w, check_writable_w, check_seekable_w)
from pypy.module._io.interp_io import W_BlockingIOError
from rpython.rlib import rthread
-from rpython.rtyper.lltypesystem import rffi
STATE_ZERO, STATE_OK, STATE_DETACHED = range(3)
@@ -88,12 +90,16 @@
self._unsupportedoperation(space, "detach")
def readinto_w(self, space, w_buffer):
- rwbuffer = space.getarg_w('w*', w_buffer)
+ return self._readinto(space, w_buffer, "read")
+
+ def _readinto(self, space, w_buffer, methodname):
+ rwbuffer = space.writebuf_w(w_buffer)
length = rwbuffer.getlength()
- w_data = space.call_method(self, "read", space.newint(length))
+ w_data = space.call_method(self, methodname, space.newint(length))
if not space.isinstance_w(w_data, space.w_bytes):
- raise oefmt(space.w_TypeError, "read() should return bytes")
+ raise oefmt(space.w_TypeError, "%s() should return bytes",
+ methodname)
data = space.bytes_w(w_data)
rwbuffer.setslice(0, data)
return space.newint(len(data))
@@ -108,25 +114,6 @@
readinto = interp2app(W_BufferedIOBase.readinto_w),
)
-class RawBuffer(Buffer):
- _immutable_ = True
-
- def __init__(self, buf, start, length):
- self.buf = buf
- self.start = start
- self.length = length
- self.readonly = False
-
- def getlength(self):
- return self.length
-
- def setitem(self, index, char):
- self.buf[self.start + index] = char
-
- def get_raw_address(self):
- ptr = nonmoving_raw_ptr_for_resizable_list(self.buf)
- return rffi.ptradd(ptr, self.start)
-
class BufferedMixin:
_mixin_ = True
@@ -165,8 +152,7 @@
raise oefmt(space.w_ValueError,
"buffer size must be strictly positive")
- self.buffer = resizable_list_supporting_raw_ptr(['\0'] *
- self.buffer_size)
+ self.buffer = ByteBuffer(self.buffer_size)
self.lock = TryLock(space)
@@ -238,6 +224,7 @@
# ______________________________________________
+ @signature(types.any(), returns=types.int())
def _readahead(self):
if self.readable and self.read_end != -1:
available = self.read_end - self.pos
@@ -278,7 +265,7 @@
else:
offset = pos
if -self.pos <= offset <= available:
- newpos = self.pos + offset
+ newpos = self.pos + int(offset)
assert newpos >= 0
self.pos = newpos
return space.newint(current - available + offset)
@@ -374,11 +361,7 @@
return written
def _raw_write(self, space, start, end):
- # XXX inefficient
- l = []
- for i in range(start, end):
- l.append(self.buffer[i])
- return self._write(space, ''.join(l))
+ return self._write(space, self.buffer[start:end])
def detach_w(self, space):
self._check_init(space)
@@ -428,6 +411,7 @@
@unwrap_spec(size=int)
def peek_w(self, space, size=0):
self._check_init(space)
+ self._check_closed(space, "peek of closed file")
with self.lock:
if self.writable:
self._flush_and_rewind_unlocked(space)
@@ -439,7 +423,7 @@
# buffer.
have = self._readahead()
if have > 0:
- data = ''.join(self.buffer[self.pos:self.pos+have])
+ data = self.buffer[self.pos:self.pos+have]
return space.newbytes(data)
# Fill the buffer from the raw stream, and copy it to the result
@@ -449,7 +433,7 @@
except BlockingIOError:
size = 0
self.pos = 0
- data = ''.join(self.buffer[:size])
+ data = self.buffer[0:size]
return space.newbytes(data)
@unwrap_spec(size=int)
@@ -486,7 +470,7 @@
if size > have:
size = have
endpos = self.pos + size
- data = ''.join(self.buffer[self.pos:endpos])
+ data = self.buffer[self.pos:endpos]
self.pos = endpos
return space.newbytes(data)
@@ -498,7 +482,7 @@
current_size = self._readahead()
data = None
if current_size:
- data = ''.join(self.buffer[self.pos:self.pos + current_size])
+ data = self.buffer[self.pos:self.pos + current_size]
builder.append(data)
self.pos += current_size
# We're going past the buffer's bounds, flush it
@@ -524,11 +508,13 @@
return space.newbytes(builder.build())
def _raw_read(self, space, buffer, start, length):
+ assert buffer is not None
length = intmask(length)
- w_buf = space.newbuffer(RawBuffer(buffer, start, length))
+ start = intmask(start)
+ w_view = SimpleView(SubBuffer(buffer, start, length)).wrap(space)
while True:
try:
- w_size = space.call_method(self.w_raw, "readinto", w_buf)
+ w_size = space.call_method(self.w_raw, "readinto", w_view)
except OperationError as e:
if trap_eintr(space, e):
continue # try again
@@ -565,12 +551,12 @@
if n <= current_size:
return self._read_fast(n)
- result_buffer = resizable_list_supporting_raw_ptr(['\0'] * n)
+ result_buffer = ByteBuffer(n)
remaining = n
written = 0
if current_size:
- for i in range(current_size):
- result_buffer[written + i] = self.buffer[self.pos + i]
+ result_buffer.setslice(
+ written, self.buffer[self.pos:self.pos + current_size])
remaining -= current_size
written += current_size
self.pos += current_size
@@ -592,7 +578,7 @@
return None
size = 0
if size == 0:
- return ''.join(result_buffer[:written])
+ return result_buffer[0:written]
remaining -= size
written += size
@@ -614,14 +600,13 @@
if remaining > 0:
if size > remaining:
size = remaining
- for i in range(size):
- result_buffer[written + i] = self.buffer[self.pos + i]
+ result_buffer.setslice(
+ written, self.buffer[self.pos:self.pos + size])
self.pos += size
-
written += size
remaining -= size
- return ''.join(result_buffer[:written])
+ return result_buffer[0:written]
def _read_fast(self, n):
"""Read n bytes from the buffer if it can, otherwise return None.
@@ -629,7 +614,7 @@
current_size = self._readahead()
if n <= current_size:
endpos = self.pos + n
- res = ''.join(self.buffer[self.pos:endpos])
+ res = self.buffer[self.pos:endpos]
self.pos = endpos
return res
return None
@@ -652,11 +637,11 @@
else:
pos = -1
if pos >= 0:
- w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1]))
+ w_res = space.newbytes(self.buffer[self.pos:pos+1])
self.pos = pos + 1
return w_res
if have == limit:
- w_res =
space.newbytes(''.join(self.buffer[self.pos:self.pos+have]))
+ w_res = space.newbytes(self.buffer[self.pos:self.pos+have])
self.pos += have
return w_res
@@ -665,7 +650,7 @@
# Now we try to get some more from the raw stream
chunks = []
if have > 0:
- chunks.extend(self.buffer[self.pos:self.pos+have])
+ chunks.append(self.buffer[self.pos:self.pos+have])
written += have
self.pos += have
if limit >= 0:
@@ -683,13 +668,14 @@
pos = 0
found = False
while pos < have:
- c = self.buffer[pos]
+ # 'buffer.data[]' instead of 'buffer[]' because RPython...
+ c = self.buffer.data[pos]
pos += 1
if c == '\n':
self.pos = pos
found = True
break
- chunks.extend(self.buffer[0:pos])
+ chunks.append(self.buffer[0:pos])
if found:
break
if have == limit:
@@ -716,7 +702,6 @@
size = len(data)
with self.lock:
-
if (not (self.readable and self.read_end != -1) and
not (self.writable and self.write_end != -1)):
self.pos = 0
@@ -746,7 +731,8 @@
self._reader_reset_buf()
# Make some place by shifting the buffer
for i in range(self.write_pos, self.write_end):
- self.buffer[i - self.write_pos] = self.buffer[i]
+ # XXX: messing with buffer internals
+ self.buffer.data[i - self.write_pos] = self.buffer.data[i]
self.write_end -= self.write_pos
self.raw_pos -= self.write_pos
newpos = self.pos - self.write_pos
diff --git a/pypy/module/_io/interp_bytesio.py
b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -34,17 +34,17 @@
size = convert_size(space, w_size)
return space.newbytes(self.read(size))
+ def read1_w(self, space, w_size):
+ return self.read_w(space, w_size)
+
def readline_w(self, space, w_limit=None):
self._check_closed(space)
limit = convert_size(space, w_limit)
return space.newbytes(self.readline(limit))
- def read1_w(self, space, w_size):
- return self.read_w(space, w_size)
-
def readinto_w(self, space, w_buffer):
self._check_closed(space)
- rwbuffer = space.getarg_w('w*', w_buffer)
+ rwbuffer = space.writebuf_w(w_buffer)
size = rwbuffer.getlength()
output = self.read(size)
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,5 +1,6 @@
+from rpython.rtyper.lltypesystem import rffi
+
from rpython.rlib.buffer import Buffer
-from rpython.rtyper.lltypesystem import rffi
# XXX not the most efficient implementation
diff --git a/pypy/module/_rawffi/interp_rawffi.py
b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -1,5 +1,6 @@
import sys
from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView
from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import interp_attrproperty
@@ -371,7 +372,7 @@
self._ll_buffer = self.ll_buffer
def buffer_w(self, space, flags):
- return RawFFIBuffer(self)
+ return SimpleView(RawFFIBuffer(self))
def readbuf_w(self, space):
return RawFFIBuffer(self)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -677,7 +677,7 @@
register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
for cpyname in '''PyMethodObject PyListObject PyLongObject
- PyClassObject'''.split():
+ PyClassObject PyBaseExceptionObject'''.split():
FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
% (cpyname, ))
build_exported_objects()
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -257,7 +257,8 @@
if w_dict is None:
return 0
-
+ if not space.isinstance_w(w_dict, space.w_dict):
+ return 0
pos = ppos[0]
py_obj = as_pyobj(space, w_dict)
py_dict = rffi.cast(PyDictObject, py_obj)
@@ -268,6 +269,9 @@
py_dict.c__tmpkeys = create_ref(space, w_keys)
Py_IncRef(space, py_dict.c__tmpkeys)
else:
+ if not py_dict.c__tmpkeys:
+ # pos should have been 0, cannot fail so return 0
+ return 0;
w_keys = from_ref(space, py_dict.c__tmpkeys)
ppos[0] += 1
if pos >= space.len_w(w_keys):
diff --git a/pypy/module/cpyext/memoryobject.py
b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -41,15 +41,15 @@
assert isinstance(w_obj, W_MemoryView)
py_obj = rffi.cast(PyMemoryViewObject, py_obj)
view = py_obj.c_view
- ndim = w_obj.buf.getndim()
+ ndim = w_obj.getndim()
if ndim >= Py_MAX_NDIMS:
# XXX warn?
return
- fill_Py_buffer(space, w_obj.buf, view)
+ fill_Py_buffer(space, w_obj.view, view)
try:
- view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+ view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address())
view.c_obj = make_ref(space, w_userdata)
- rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+ rffi.setintfield(view, 'c_readonly', w_obj.view.readonly)
except ValueError:
w_s = w_obj.descr_tobytes(space)
view.c_obj = make_ref(space, w_s)
@@ -95,7 +95,6 @@
mem_obj.c_view.c_obj = rffi.cast(PyObject, 0)
_dealloc(space, py_obj)
-
def fill_Py_buffer(space, buf, view):
# c_buf, c_obj have been filled in
ndim = buf.getndim()
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -14,13 +14,15 @@
ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj
+from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj, from_ref
from pypy.module.cpyext.pyerrors import PyErr_Occurred
from pypy.module.cpyext.memoryobject import fill_Py_buffer
from pypy.module.cpyext.state import State
from pypy.module.cpyext import userslot
+from pypy.interpreter.buffer import BufferView
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.argument import Arguments
+
from rpython.rlib.buffer import Buffer
from rpython.rlib.unroll import unrolling_iterable
from rpython.rlib.objectmodel import specialize, not_rpython
@@ -263,7 +265,7 @@
check_num_args(space, w_args, 1)
args_w = space.fixedview(w_args)
index = space.int_w(space.index(args_w[0]))
- null = lltype.nullptr(PyObject.TO)
+ null = rffi.cast(PyObject, 0)
res = generic_cpy_call(space, func_target, w_self, index, null)
if rffi.cast(lltype.Signed, res) == -1:
space.fromcache(State).check_and_raise_exception(always=True)
@@ -292,7 +294,8 @@
func_target = rffi.cast(objobjargproc, func)
check_num_args(space, w_args, 1)
w_key, = space.fixedview(w_args)
- res = generic_cpy_call(space, func_target, w_self, w_key, None)
+ null = rffi.cast(PyObject, 0)
+ res = generic_cpy_call(space, func_target, w_self, w_key, null)
if rffi.cast(lltype.Signed, res) == -1:
space.fromcache(State).check_and_raise_exception(always=True)
return space.w_None
@@ -322,7 +325,7 @@
space.fromcache(State).check_and_raise_exception(always=True)
return space.newint(res)
-class CPyBuffer(Buffer):
+class CPyBuffer(BufferView):
# Similar to Py_buffer
_immutable_ = True
@@ -333,9 +336,12 @@
self.space = space
self.ptr = ptr
self.size = size
- self.w_obj = w_obj # kept alive
+ self.w_obj = w_obj # kept alive
self.pyobj = as_pyobj(space, w_obj)
self.format = format
+ self.ndim = ndim
+ self.itemsize = itemsize
+
if not shape:
self.shape = [size]
else:
@@ -344,8 +350,6 @@
self.strides = [1]
else:
self.strides = strides
- self.ndim = ndim
- self.itemsize = itemsize
self.readonly = readonly
self.needs_decref = needs_decref
self.releasebufferproc = releasebufferproc
@@ -378,8 +382,20 @@
def getlength(self):
return self.size
- def getitem(self, index):
- return self.ptr[index]
+ def getbytes(self, start, size):
+ return ''.join([self.ptr[i] for i in range(start, start + size)])
+
+ def setbytes(self, start, string):
+ # absolutely no safety checks, what could go wrong?
+ for i in range(len(string)):
+ self.ptr[start + i] = string[i]
+
+ def as_readbuf(self):
+ return CBuffer(self)
+
+ def as_writebuf(self):
+ assert not self.readonly
+ return CBuffer(self)
def get_raw_address(self):
return rffi.cast(rffi.CCHARP, self.ptr)
@@ -399,10 +415,6 @@
def getndim(self):
return self.ndim
- def setitem(self, index, char):
- # absolutely no safety checks, what could go wrong?
- self.ptr[index] = char
-
class FQ(rgc.FinalizerQueue):
Class = CPyBuffer
def finalizer_trigger(self):
@@ -414,6 +426,37 @@
fq = FQ()
+
+class CBuffer(Buffer):
+ _immutable_ = True
+ def __init__(self, view):
+ self.view = view
+ self.readonly = view.readonly
+
+ def getlength(self):
+ return self.view.getlength()
+
+ def getitem(self, index):
+ return self.view.ptr[index]
+
+ def getslice(self, start, stop, step, size):
+ assert step == 1
+ assert stop - start == size
+ ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), start)
+ return rffi.charpsize2str(ptr, size)
+
+ def setitem(self, index, char):
+ self.view.ptr[index] = char
+
+ def setslice(self, index, s):
+ assert s is not None
+ ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), index)
+ rffi.str2chararray(s, ptr, len(s))
+
+ def get_raw_address(self):
+ return cts.cast('char *', self.view.ptr)
+
+
def wrap_getreadbuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
py_obj = make_ref(space, w_self)
@@ -427,10 +470,10 @@
size = generic_cpy_call(space, func_target, w_self, index, ptr)
if size < 0:
space.fromcache(State).check_and_raise_exception(always=True)
- buf = CPyBuffer(space, ptr[0], size, w_self,
+ view = CPyBuffer(space, ptr[0], size, w_self,
releasebufferproc=rbp)
- fq.register_finalizer(buf)
- return space.newbuffer(buf)
+ fq.register_finalizer(view)
+ return space.newbuffer(CBuffer(view))
def wrap_getwritebuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
@@ -445,10 +488,10 @@
size = generic_cpy_call(space, func_target, w_self, index, ptr)
if size < 0:
space.fromcache(State).check_and_raise_exception(always=True)
- buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
+ view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
releasebufferproc=rbp)
- fq.register_finalizer(buf)
- return space.newbuffer(buf)
+ fq.register_finalizer(view)
+ return space.newbuffer(CBuffer(view))
def wrap_getbuffer(space, w_self, w_args, func):
func_target = rffi.cast(getbufferproc, func)
@@ -490,7 +533,7 @@
needs_decref=True,
releasebufferproc = rbp)
fq.register_finalizer(buf)
- return space.newbuffer(buf)
+ return buf.wrap(space)
def get_richcmp_func(OP_CONST):
def inner(space, w_self, w_args, func):
@@ -570,6 +613,8 @@
handled = True
for tp_name, attr in [('tp_hash', '__hash__'),
+ ('tp_as_sequence.c_sq_length', '__len__'),
+ ('tp_as_mapping.c_mp_length', '__len__'),
]:
if name == tp_name:
slot_fn = w_type.getdictvalue(space, attr)
@@ -595,7 +640,8 @@
('tp_as_number.c_nb_xor', '__xor__'),
('tp_as_number.c_nb_or', '__or__'),
('tp_as_sequence.c_sq_concat', '__add__'),
- ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
+ ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
+ ('tp_as_mapping.c_mp_subscript', '__getitem__'),
]:
if name == tp_name:
slot_fn = w_type.getdictvalue(space, attr)
@@ -609,7 +655,7 @@
handled = True
# binary-with-Py_ssize_t-type
- for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+ for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
('tp_as_sequence.c_sq_repeat', '__mul__'),
('tp_as_sequence.c_sq_repeat', '__mul__'),
('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
@@ -638,7 +684,48 @@
def slot_func(space, w_self, w_arg1, w_arg2):
return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
handled = True
+ # ternary-with-void returning-Py_size_t-type
+ for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
+ ]:
+ if name == tp_name:
+ slot_ass = w_type.getdictvalue(space, attr)
+ if slot_ass is None:
+ return
+ slot_del = w_type.getdictvalue(space, '__delitem__')
+ if slot_del is None:
+ return
+ @slot_function([PyObject, PyObject, PyObject], rffi.INT_real,
error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'),
typedef.name))
+ def slot_func(space, w_self, w_arg1, arg2):
+ if arg2:
+ w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+ space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+ else:
+ space.call_function(slot_del, w_self, w_arg1)
+ return 0
+ handled = True
+ # ternary-Py_size_t-void returning-Py_size_t-type
+ for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
+ ]:
+ if name == tp_name:
+ slot_ass = w_type.getdictvalue(space, attr)
+ if slot_ass is None:
+ return
+ slot_del = w_type.getdictvalue(space, '__delitem__')
+ if slot_del is None:
+ return
+
+ @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real,
error=-1)
+ @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'),
typedef.name))
+ def slot_func(space, w_self, arg1, arg2):
+ if arg2:
+ w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+ space.call_function(slot_ass, w_self, space.newint(arg1),
w_arg2)
+ else:
+ space.call_function(slot_del, w_self, space.newint(arg1))
+ return 0
+ handled = True
if handled:
pass
elif name == 'tp_setattro':
@@ -667,6 +754,7 @@
def slot_tp_getattro(space, w_self, w_name):
return space.call_function(getattr_fn, w_self, w_name)
slot_func = slot_tp_getattro
+
elif name == 'tp_call':
call_fn = w_type.getdictvalue(space, '__call__')
if call_fn is None:
@@ -744,24 +832,24 @@
@slot_function([PyObject, Py_bufferP, rffi.INT_real],
rffi.INT_real, error=-1)
@func_renamer("cpyext_%s_%s" % (name, typedef.name))
- def buff_w(space, w_self, view, flags):
+ def buff_w(space, w_self, c_view, flags):
args = Arguments(space, [space.newint(flags)])
w_obj = space.call_args(space.get(buff_fn, w_self), args)
- if view:
+ if c_view:
#like PyObject_GetBuffer
flags = widen(flags)
buf = space.buffer_w(w_obj, flags)
try:
- view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
- view.c_obj = make_ref(space, w_obj)
+ c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ c_view.c_obj = make_ref(space, w_obj)
except ValueError:
s = buf.as_str()
w_s = space.newbytes(s)
- view.c_obj = make_ref(space, w_s)
- view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+ c_view.c_obj = make_ref(space, w_s)
+ c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
s, track_allocation=False))
- rffi.setintfield(view, 'c_readonly', 1)
- ret = fill_Py_buffer(space, buf, view)
+ rffi.setintfield(c_view, 'c_readonly', 1)
+ ret = fill_Py_buffer(space, buf, c_view)
return ret
return 0
return buff_w
@@ -771,23 +859,23 @@
@slot_function([PyObject, Py_bufferP, rffi.INT_real],
rffi.INT_real, error=-1)
@func_renamer("cpyext_%s_%s" % (name, typedef.name))
- def buff_w(space, w_self, view, flags):
+ def buff_w(space, w_self, c_view, flags):
w_obj = w_self
- if view:
+ if c_view:
#like PyObject_GetBuffer
flags = widen(flags)
buf = space.buffer_w(w_obj, flags)
try:
- view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
- view.c_obj = make_ref(space, w_obj)
+ c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+ c_view.c_obj = make_ref(space, w_obj)
except ValueError:
s = buf.as_str()
w_s = space.newbytes(s)
- view.c_obj = make_ref(space, w_s)
- view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+ c_view.c_obj = make_ref(space, w_s)
+ c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
s, track_allocation=False))
- rffi.setintfield(view, 'c_readonly', 1)
- ret = fill_Py_buffer(space, buf, view)
+ rffi.setintfield(c_view, 'c_readonly', 1)
+ ret = fill_Py_buffer(space, buf, c_view)
return ret
return 0
return buff_w
diff --git a/pypy/module/cpyext/test/buffer_test.c
b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,6 +344,7 @@
#endif
if (m == NULL)
INITERROR;
+ PyMyArrayType.tp_new = PyType_GenericNew;
if (PyType_Ready(&PyMyArrayType) < 0)
INITERROR;
Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/test_dictobject.py
b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -255,4 +255,60 @@
])
d = module.get_type_dict(1)
assert d['real'].__get__(1, 1) == 1
-
+ def test_advanced(self):
+ module = self.import_extension('foo', [
+ ("dict_len", "METH_O",
+ '''
+ int ret = args->ob_type->tp_as_mapping->mp_length(args);
+ return PyLong_FromLong(ret);
+ '''),
+ ("dict_setitem", "METH_VARARGS",
+ '''
+ int ret;
+ PyObject * dict = PyTuple_GetItem(args, 0);
+ if (PyTuple_Size(args) < 3 || !dict ||
+ !dict->ob_type->tp_as_mapping ||
+ !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+ return PyLong_FromLong(-1);
+ ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+ dict, PyTuple_GetItem(args, 1),
+ PyTuple_GetItem(args, 2));
+ return PyLong_FromLong(ret);
+ '''),
+ ("dict_delitem", "METH_VARARGS",
+ '''
+ int ret;
+ PyObject * dict = PyTuple_GetItem(args, 0);
+ if (PyTuple_Size(args) < 2 || !dict ||
+ !dict->ob_type->tp_as_mapping ||
+ !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+ return PyLong_FromLong(-1);
+ ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+ dict, PyTuple_GetItem(args, 1), NULL);
+ return PyLong_FromLong(ret);
+ '''),
+ ("dict_next", "METH_VARARGS",
+ '''
+ PyObject *key, *value;
+ PyObject *arg = NULL;
+ Py_ssize_t pos = 0;
+ int ret = 0;
+ if ((PyArg_ParseTuple(args, "|O", &arg))) {
+ if (arg && PyDict_Check(arg)) {
+ while (PyDict_Next(arg, &pos, &key, &value))
+ ret ++;
+ /* test no crash if pos is not reset to 0*/
+ while (PyDict_Next(arg, &pos, &key, &value))
+ ret ++;
+ }
+ }
+ return PyLong_FromLong(ret);
+ '''),
+ ])
+ d = {'a': 1, 'b':2}
+ assert module.dict_len(d) == 2
+ assert module.dict_setitem(d, 'a', 'c') == 0
+ assert d['a'] == 'c'
+ assert module.dict_delitem(d, 'a') == 0
+ r = module.dict_next({'a': 1, 'b': 2})
+ assert r == 2
diff --git a/pypy/module/cpyext/test/test_memoryobject.py
b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -111,7 +111,7 @@
PyObject* obj = PyTuple_GetItem(args, 0);
PyObject* memoryview = PyMemoryView_FromObject(obj);
if (memoryview == NULL)
- return PyLong_FromLong(-1);
+ return NULL;
view = PyMemoryView_GET_BUFFER(memoryview);
Py_DECREF(memoryview);
return PyLong_FromLong(view->len / view->itemsize);
diff --git a/pypy/module/cpyext/test/test_userslots.py
b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
w_year = space.getattr(w_obj, space.newtext('year'))
assert space.int_w(w_year) == 1
+ def test_descr_slots(self, space, api):
+ w_descr = space.appexec([], """():
+ class Descr(object):
+ def __get__(self, obj, type):
+ return 42
+ def __set__(self, obj, value):
+ obj.append('set')
+ def __delete__(self, obj):
+ obj.append('del')
+ return Descr()
+ """)
+ w_descrtype = space.type(w_descr)
+ py_descr = make_ref(space, w_descr)
+ py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+ w_obj = space.newlist([])
+ py_obj = make_ref(space, w_obj)
+ w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+ py_descr, py_obj, py_obj)
+ assert space.int_w(w_res) == 42
+ assert generic_cpy_call(
+ space, py_descrtype.c_tp_descr_set,
+ py_descr, py_obj, make_ref(space, space.w_None)) == 0
+ assert generic_cpy_call(
+ space, py_descrtype.c_tp_descr_set,
+ py_descr, py_obj, None) == 0
+ assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
class AppTestUserSlots(AppTestCpythonExtensionBase):
def test_tp_hash_from_python(self):
# to see that the functions are being used,
diff --git a/pypy/module/cpyext/tupleobject.py
b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -34,7 +34,7 @@
cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct)
@bootstrap_function
-def init_stringobject(space):
+def init_tupleobject(space):
"Type description of PyTupleObject"
make_typedescr(space.w_tuple.layout.typedef,
basestruct=PyTupleObject.TO,
diff --git a/pypy/module/cpyext/unicodeobject.py
b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -520,7 +520,7 @@
if sys.platform == 'win32':
make_conversion_functions('MBCS', 'mbcs')
-@cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
+@cpython_api([rffi.CCHARP, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
"""Decode length bytes from a UTF-16 encoded buffer string and return the
corresponding Unicode object. errors (if non-NULL) defines the error
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
def slot_tp_getattr(space, w_obj1, w_obj2):
return space.getattr(w_obj1, w_obj2)
+@slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+ return space.get(w_self, w_obj, w_type)
+@slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+ if w_value is not None:
+ space.set(w_self, w_obj, w_value)
+ else:
+ space.delete(w_self, w_obj)
+ return 0
diff --git a/pypy/module/itertools/interp_itertools.py
b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -920,90 +920,42 @@
class W_GroupBy(W_Root):
def __init__(self, space, w_iterable, w_fun):
self.space = space
- self.w_iterable = self.space.iter(w_iterable)
- if space.is_none(w_fun):
- self.w_fun = None
- else:
- self.w_fun = w_fun
- self.index = 0
- self.lookahead = False
- self.exhausted = False
- self.started = False
- # new_group - new group not started yet, next should not skip any items
- self.new_group = True
- self.w_lookahead = self.space.w_None
- self.w_key = self.space.w_None
+ self.w_iterator = self.space.iter(w_iterable)
+ if w_fun is None:
+ w_fun = space.w_None
+ self.w_keyfunc = w_fun
+ self.w_tgtkey = None
+ self.w_currkey = None
+ self.w_currvalue = None
def iter_w(self):
return self
def next_w(self):
- if self.exhausted:
- raise OperationError(self.space.w_StopIteration, self.space.w_None)
+ self._skip_to_next_iteration_group()
+ w_key = self.w_tgtkey = self.w_currkey
+ w_grouper = W_GroupByIterator(self, w_key)
+ return self.space.newtuple([w_key, w_grouper])
- if not self.new_group:
- self._consume_unwanted_input()
+ def _skip_to_next_iteration_group(self):
+ space = self.space
+ while True:
+ if self.w_currkey is None:
+ pass
+ elif self.w_tgtkey is None:
+ break
+ else:
+ if not space.eq_w(self.w_tgtkey, self.w_currkey):
+ break
- if not self.started:
- self.started = True
- try:
- w_obj = self.space.next(self.w_iterable)
- except OperationError as e:
- if e.match(self.space, self.space.w_StopIteration):
- self.exhausted = True
- raise
+ w_newvalue = space.next(self.w_iterator)
+ if space.is_w(self.w_keyfunc, space.w_None):
+ w_newkey = w_newvalue
else:
- self.w_lookahead = w_obj
- if self.w_fun is None:
- self.w_key = w_obj
- else:
- self.w_key = self.space.call_function(self.w_fun, w_obj)
- self.lookahead = True
+ w_newkey = space.call_function(self.w_keyfunc, w_newvalue)
- self.new_group = False
- w_iterator = W_GroupByIterator(self.space, self.index, self)
- return self.space.newtuple([self.w_key, w_iterator])
-
- def _consume_unwanted_input(self):
- # Consume unwanted input until we reach the next group
- try:
- while True:
- self.group_next(self.index)
- except StopIteration:
- pass
- if self.exhausted:
- raise OperationError(self.space.w_StopIteration, self.space.w_None)
-
- def group_next(self, group_index):
- if group_index < self.index:
- raise StopIteration
- else:
- if self.lookahead:
- self.lookahead = False
- return self.w_lookahead
-
- try:
- w_obj = self.space.next(self.w_iterable)
- except OperationError as e:
- if e.match(self.space, self.space.w_StopIteration):
- self.exhausted = True
- raise StopIteration
- else:
- raise
- else:
- if self.w_fun is None:
- w_new_key = w_obj
- else:
- w_new_key = self.space.call_function(self.w_fun, w_obj)
- if self.space.eq_w(self.w_key, w_new_key):
- return w_obj
- else:
- self.index += 1
- self.w_lookahead = w_obj
- self.w_key = w_new_key
- self.lookahead = True
- self.new_group = True #new group
- raise StopIteration
+ self.w_currkey = w_newkey
+ self.w_currvalue = w_newvalue
def W_GroupBy___new__(space, w_subtype, w_iterable, w_key=None):
r = space.allocate_instance(W_GroupBy, w_subtype)
@@ -1036,26 +988,34 @@
class W_GroupByIterator(W_Root):
- def __init__(self, space, index, groupby):
- self.space = space
- self.index = index
+ def __init__(self, groupby, w_tgtkey):
self.groupby = groupby
- self.exhausted = False
+ self.w_tgtkey = w_tgtkey
def iter_w(self):
return self
def next_w(self):
- if self.exhausted:
- raise OperationError(self.space.w_StopIteration, self.space.w_None)
+ groupby = self.groupby
+ space = groupby.space
+ if groupby.w_currvalue is None:
+ w_newvalue = space.next(groupby.w_iterator)
+ if space.is_w(groupby.w_keyfunc, space.w_None):
+ w_newkey = w_newvalue
+ else:
+ w_newkey = space.call_function(groupby.w_keyfunc, w_newvalue)
+ #assert groupby.w_currvalue is None
+ # ^^^ check disabled, see http://bugs.python.org/issue30347
+ groupby.w_currkey = w_newkey
+ groupby.w_currvalue = w_newvalue
- try:
- w_obj = self.groupby.group_next(self.index)
- except StopIteration:
- self.exhausted = True
- raise OperationError(self.space.w_StopIteration, self.space.w_None)
- else:
- return w_obj
+ assert groupby.w_currkey is not None
+ if not space.eq_w(self.w_tgtkey, groupby.w_currkey):
+ raise OperationError(space.w_StopIteration, space.w_None)
+ w_result = groupby.w_currvalue
+ groupby.w_currvalue = None
+ groupby.w_currkey = None
+ return w_result
W_GroupByIterator.typedef = TypeDef(
'itertools._groupby',
diff --git a/pypy/module/itertools/test/test_itertools.py
b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -634,6 +634,17 @@
it = itertools.groupby([0], 1)
raises(TypeError, it.next)
+ def test_groupby_question_43905804(self):
+ # http://stackoverflow.com/questions/43905804/
+ import itertools
+
+ inputs = ((x > 5, x) for x in range(10))
+ (_, a), (_, b) = itertools.groupby(inputs, key=lambda x: x[0])
+ a = list(a)
+ b = list(b)
+ assert a == []
+ assert b == [(True, 9)]
+
def test_iterables(self):
import itertools
diff --git a/pypy/module/micronumpy/concrete.py
b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,3 +1,4 @@
+from pypy.interpreter.buffer import BufferView
from pypy.interpreter.error import oefmt
from rpython.rlib import jit, rgc
from rpython.rlib.rarithmetic import ovfcheck
@@ -21,7 +22,7 @@
TimSort = make_timsort_class()
class StrideSort(TimSort):
- '''
+ '''
argsort (return the indices to sort) a list of strides
'''
def __init__(self, rangelist, strides, order):
@@ -380,14 +381,14 @@
def get_buffer(self, space, flags):
errtype = space.w_ValueError # should be BufferError, numpy does this
instead
- if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
+ if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
not self.flags & NPY.ARRAY_C_CONTIGUOUS):
raise oefmt(errtype, "ndarray is not C-contiguous")
- if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
+ if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
not self.flags & NPY.ARRAY_F_CONTIGUOUS):
raise oefmt(errtype, "ndarray is not Fortran contiguous")
if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
- not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
+ not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
self.flags & NPY.ARRAY_C_CONTIGUOUS)):
raise oefmt(errtype, "ndarray is not contiguous")
if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
@@ -397,7 +398,7 @@
not self.flags & NPY.ARRAY_WRITEABLE):
raise oefmt(errtype, "buffer source array is read-only")
readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
- return ArrayBuffer(self, readonly)
+ return ArrayView(self, readonly)
def astype(self, space, dtype, order, copy=True):
# copy the general pattern of the strides
@@ -527,7 +528,7 @@
try:
length = support.product_check(shape)
self.size = ovfcheck(length * dtype.elsize)
- except OverflowError:
+ except OverflowError:
raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
if storage == lltype.nullptr(RAW_STORAGE):
if dtype.num == NPY.OBJECT:
@@ -701,10 +702,8 @@
def __del__(self):
free_raw_storage(self.storage)
-
-class ArrayBuffer(Buffer):
+class ArrayData(Buffer):
_immutable_ = True
-
def __init__(self, impl, readonly):
self.impl = impl
self.readonly = readonly
@@ -725,6 +724,28 @@
from rpython.rtyper.lltypesystem import rffi
return rffi.ptradd(self.impl.storage, self.impl.start)
+
+class ArrayView(BufferView):
+ _immutable_ = True
+
+ def __init__(self, impl, readonly):
+ self.impl = impl
+ self.readonly = readonly
+ self.data = ArrayData(impl, readonly)
+
+ def getlength(self):
+ return self.data.getlength()
+
+ def getbytes(self, start, size):
+ return self.data[start:start + size]
+
+ def as_readbuf(self):
+ return ArrayData(self.impl, readonly=True)
+
+ def as_writebuf(self):
+ assert not self.readonly
+ return ArrayData(self.impl, readonly=False)
+
def getformat(self):
sb = StringBuilder()
self.impl.dtype.getformat(sb)
@@ -742,4 +763,5 @@
def getstrides(self):
return self.impl.strides
-
+ def get_raw_address(self):
+ return self.data.get_raw_address()
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -91,7 +91,7 @@
w_base = w_object
if read_only:
w_base = None
- return W_NDimArray.from_shape_and_storage(space, shape, w_data,
+ return W_NDimArray.from_shape_and_storage(space, shape, w_data,
dtype, w_base=w_base, strides=strides,
start=offset), read_only
if w_data is None:
@@ -104,11 +104,11 @@
#print 'create view from shape',shape,'dtype',dtype,'data',data
if strides is not None:
raise oefmt(space.w_NotImplementedError,
- "__array_interface__ strides not fully supported yet")
+ "__array_interface__ strides not fully supported yet")
arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
new_impl = arr.implementation.reshape(arr, shape)
return W_NDimArray(new_impl), False
-
+
except OperationError as e:
if e.match(space, space.w_AttributeError):
return None, False
@@ -120,7 +120,7 @@
return descr
msg = "invalid PEP 3118 format string: '%s'" % c_format
space.warn(space.newtext(msg), space.w_RuntimeWarning)
- return None
+ return None
def _array_from_buffer_3118(space, w_object, dtype):
try:
@@ -139,12 +139,12 @@
raise oefmt(space.w_NotImplementedError,
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
