Author: Amaury Forgeot d'Arc <amaur...@gmail.com> Branch: py3.6 Changeset: r93931:eb5abc38e4b2 Date: 2018-02-25 23:09 +0100 http://bitbucket.org/pypy/pypy/changeset/eb5abc38e4b2/
Log: hg merge py3.5 diff too long, truncating to 2000 out of 5407 lines diff --git a/extra_tests/test_pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/__init__.py @@ -0,0 +1,1 @@ + diff --git a/extra_tests/test_pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/infrastructure.py @@ -0,0 +1,87 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <mica...@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import print_function +from contextlib import contextmanager +import os + +from pyrepl.reader import Reader +from pyrepl.console import Console, Event + + +class EqualsAnything(object): + def __eq__(self, other): + return True + + +EA = EqualsAnything() + + +class TestConsole(Console): + height = 24 + width = 80 + encoding = 'utf-8' + + def __init__(self, events, verbose=False): + self.events = events + self.next_screen = None + self.verbose = verbose + + def refresh(self, screen, xy): + if self.next_screen is not None: + assert screen == self.next_screen, "[ %s != %s after %r ]" % ( + screen, self.next_screen, self.last_event_name) + + def get_event(self, block=1): + ev, sc = self.events.pop(0) + self.next_screen = sc + if not isinstance(ev, tuple): + ev = (ev, None) + self.last_event_name = ev[0] + if self.verbose: + print("event", ev) + return Event(*ev) + + +class BaseTestReader(Reader): + + def get_prompt(self, lineno, cursor_on_line): + return '' + + def refresh(self): + Reader.refresh(self) + self.dirty = True + + +def read_spec(test_spec, reader_class=BaseTestReader): + # remember to finish your test_spec with 'accept' or similar! + con = TestConsole(test_spec, verbose=True) + reader = reader_class(con) + reader.readline() + + +@contextmanager +def sane_term(): + """Ensure a TERM that supports clear""" + old_term, os.environ['TERM'] = os.environ.get('TERM'), 'xterm' + yield + if old_term is not None: + os.environ['TERM'] = old_term + else: + del os.environ['TERM'] diff --git a/extra_tests/test_pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_basic.py @@ -0,0 +1,116 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <mica...@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import pytest +from .infrastructure import read_spec + + +def test_basic(): + read_spec([(('self-insert', 'a'), ['a']), + ( 'accept', ['a'])]) + + +def test_repeat(): + read_spec([(('digit-arg', '3'), ['']), + (('self-insert', 'a'), ['aaa']), + ( 'accept', ['aaa'])]) + + +def test_kill_line(): + read_spec([(('self-insert', 'abc'), ['abc']), + ( 'left', None), + ( 'kill-line', ['ab']), + ( 'accept', ['ab'])]) + + +def test_unix_line_discard(): + read_spec([(('self-insert', 'abc'), ['abc']), + ( 'left', None), + ( 'unix-word-rubout', ['c']), + ( 'accept', ['c'])]) + + +def test_kill_word(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'beginning-of-line', ['ab cd']), + ( 'kill-word', [' cd']), + ( 'accept', [' cd'])]) + + +def test_backward_kill_word(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'backward-kill-word', ['ab ']), + ( 'accept', ['ab '])]) + + +def test_yank(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'backward-kill-word', ['ab ']), + ( 'beginning-of-line', ['ab ']), + ( 'yank', ['cdab ']), + ( 'accept', ['cdab '])]) + + +def test_yank_pop(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'backward-kill-word', ['ab ']), + ( 'left', ['ab ']), + ( 'backward-kill-word', [' ']), + ( 'yank', ['ab ']), + ( 'yank-pop', ['cd ']), + ( 'accept', ['cd '])]) + + +# interrupt uses os.kill which doesn't go through signal handlers on windows +@pytest.mark.skipif("os.name == 'nt'") +def test_interrupt(): + with pytest.raises(KeyboardInterrupt): + read_spec([('interrupt', [''])]) + + +# test_suspend -- hah +def test_up(): + read_spec([(('self-insert', 'ab\ncd'), ['ab', 'cd']), + ( 'up', ['ab', 'cd']), + (('self-insert', 'e'), ['abe', 'cd']), + ( 'accept', ['abe', 'cd'])]) + + +def test_down(): + read_spec([(('self-insert', 'ab\ncd'), ['ab', 'cd']), + ( 'up', ['ab', 'cd']), + (('self-insert', 'e'), ['abe', 'cd']), + ( 'down', ['abe', 'cd']), + (('self-insert', 'f'), ['abe', 'cdf']), + ( 'accept', ['abe', 'cdf'])]) + + +def test_left(): + read_spec([(('self-insert', 'ab'), ['ab']), + ( 'left', ['ab']), + (('self-insert', 'c'), ['acb']), + ( 'accept', ['acb'])]) + + +def test_right(): + read_spec([(('self-insert', 'ab'), ['ab']), + ( 'left', ['ab']), + (('self-insert', 'c'), ['acb']), + ( 'right', ['acb']), + (('self-insert', 'd'), ['acbd']), + ( 'accept', ['acbd'])]) diff --git a/extra_tests/test_pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_bugs.py @@ -0,0 +1,75 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <mica...@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from pyrepl.historical_reader import HistoricalReader +from .infrastructure import EA, BaseTestReader, sane_term, read_spec + +# this test case should contain as-verbatim-as-possible versions of +# (applicable) bug reports + +import pytest + + +class HistoricalTestReader(HistoricalReader, BaseTestReader): + pass + + +@pytest.mark.xfail(reason='event missing', run=False) +def test_transpose_at_start(): + read_spec([ + ('transpose', [EA, '']), + ('accept', [''])]) + + +def test_cmd_instantiation_crash(): + spec = [ + ('reverse-history-isearch', ["(r-search `') "]), + (('key', 'left'), ['']), + ('accept', ['']) + ] + read_spec(spec, HistoricalTestReader) + + +@pytest.mark.skipif("os.name != 'posix' or 'darwin' in sys.platform or " + "'kfreebsd' in sys.platform") +def test_signal_failure(monkeypatch): + import os + import pty + import signal + from pyrepl.unix_console import UnixConsole + + def failing_signal(a, b): + raise ValueError + + def really_failing_signal(a, b): + raise AssertionError + + mfd, sfd = pty.openpty() + try: + with sane_term(): + c = UnixConsole(sfd, sfd) + c.prepare() + c.restore() + monkeypatch.setattr(signal, 'signal', failing_signal) + c.prepare() + monkeypatch.setattr(signal, 'signal', really_failing_signal) + c.restore() + finally: + os.close(mfd) + os.close(sfd) diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_functional.py @@ -0,0 +1,28 @@ +# Copyright 2000-2007 Michael Hudson-Doyle <mica...@gmail.com> +# Maciek Fijalkowski +# License: MIT +# some functional tests, to see if this is really working + +import pytest +import sys + + +@pytest.fixture() +def child(): + try: + import pexpect + except ImportError: + pytest.skip("no pexpect module") + except SyntaxError: + pytest.skip('pexpect wont work on py3k') + child = pexpect.spawn(sys.executable, ['-S'], timeout=10) + child.logfile = sys.stdout + child.sendline('from pyrepl.python_reader import main') + child.sendline('main()') + return child + + +def test_basic(child): + child.sendline('a = 3') + child.sendline('a') + child.expect('3') diff --git a/extra_tests/test_pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_keymap.py @@ -0,0 +1,10 @@ +from pyrepl.keymap import compile_keymap + + +def test_compile_keymap(): + k = compile_keymap({ + b'a': 'test', + b'bc': 'test2', + }) + + assert k == {b'a': 'test', b'b': {b'c': 'test2'}} diff --git a/extra_tests/test_pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_reader.py @@ -0,0 +1,9 @@ + +def test_process_prompt(): + from pyrepl.reader import Reader + r = Reader(None) + assert r.process_prompt("hi!") == ("hi!", 3) + assert r.process_prompt("h\x01i\x02!") == ("hi!", 2) + assert r.process_prompt("hi\033[11m!") == ("hi\033[11m!", 3) + assert r.process_prompt("h\x01i\033[11m!\x02") == ("hi\033[11m!", 1) + assert r.process_prompt("h\033[11m\x01i\x02!") == ("h\033[11mi!", 2) diff --git a/extra_tests/test_pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_readline.py @@ -0,0 +1,22 @@ +import pytest + +from .infrastructure import sane_term + + +@pytest.mark.skipif("os.name != 'posix' or 'darwin' in sys.platform or " + "'freebsd' in sys.platform") +def test_raw_input(): + import os + import pty + from pyrepl.readline import _ReadlineWrapper + + master, slave = pty.openpty() + readline_wrapper = _ReadlineWrapper(slave, slave) + os.write(master, b'input\n') + + with sane_term(): + result = readline_wrapper.get_reader().readline() + #result = readline_wrapper.raw_input('prompt:') + assert result == 'input' + # A bytes string on python2, a unicode string on python3. + assert isinstance(result, str) diff --git a/extra_tests/test_pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_wishes.py @@ -0,0 +1,31 @@ +# Copyright 2000-2004 Michael Hudson-Doyle <mica...@gmail.com> +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from .infrastructure import read_spec + +# this test case should contain as-verbatim-as-possible versions of +# (applicable) feature requests + + +def test_quoted_insert_repeat(): + read_spec([ + (('digit-arg', '3'), ['']), + (('quoted-insert', None), ['']), + (('self-insert', '\033'), ['^[^[^[']), + (('accept', None), None)]) diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -255,7 +255,7 @@ """Launch vcvarsall.bat and read the settings from its environment """ vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) + interesting = set(("include", "lib", "path")) result = {} if vcvarsall is None: diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py --- a/lib-python/3/distutils/unixccompiler.py +++ b/lib-python/3/distutils/unixccompiler.py @@ -222,6 +222,10 @@ return "-L" + dir def _is_gcc(self, compiler_name): + if "__pypy__" in sys.builtin_module_names: # issue #2747 + if (compiler_name.startswith('cc') or + compiler_name.startswith('c++')): + return True return "gcc" in compiler_name or "g++" in compiler_name def runtime_library_dir_option(self, dir): diff --git a/lib_pypy/_libmpdec/vccompat.h b/lib_pypy/_libmpdec/vccompat.h --- a/lib_pypy/_libmpdec/vccompat.h +++ b/lib_pypy/_libmpdec/vccompat.h @@ -32,7 +32,11 @@ /* Visual C fixes: no stdint.h, no snprintf ... */ #ifdef _MSC_VER + #if _MSC_VER < 1900 #include "vcstdint.h" + #else + #include "stdint.h" + #endif #undef inline #define inline __inline #undef random diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -63,6 +63,12 @@ HANDLE hStdError; } STARTUPINFO, *LPSTARTUPINFO; +typedef struct _SECURITY_ATTRIBUTES { + DWORD nLength; + LPVOID lpSecurityDescriptor; + BOOL bInheritHandle; +} SECURITY_ATTRIBUTES, *PSECURITY_ATTRIBUTES, *LPSECURITY_ATTRIBUTES; + typedef struct { HANDLE hProcess; HANDLE hThread; @@ -70,9 +76,41 @@ DWORD dwThreadId; } PROCESS_INFORMATION, *LPPROCESS_INFORMATION; +typedef struct _OVERLAPPED { + ULONG_PTR Internal; + ULONG_PTR InternalHigh; + union { + struct { + DWORD Offset; + DWORD OffsetHigh; + } DUMMYSTRUCTNAME; + PVOID Pointer; + } DUMMYUNIONNAME; + + HANDLE hEvent; +} OVERLAPPED, *LPOVERLAPPED; + + DWORD WINAPI GetVersion(void); BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD); +HANDLE WINAPI CreateNamedPipeA(LPCSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateNamedPipeW(LPWSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateFileA(LPCSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD); +BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED); +HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR); +HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR); +VOID WINAPI SetEvent(HANDLE); +BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED); BOOL WINAPI CloseHandle(HANDLE); +DWORD WINAPI GetLastError(VOID); +BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL); + HANDLE WINAPI GetCurrentProcess(void); BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x 00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_set mode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0), - _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')), - _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xAB\x03\x00\x00\x13\x11\x00\x00\xB0\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\xAA\x03\x00\x00\xA8\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\xA7\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x29\x11\x00\x00\x18\x03\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x 00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x6B\x03\x00\x00\x49\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x33\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x66\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\xA9\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xAB\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x6E\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6B\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0 A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x77\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\xB0\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\x05\x09\x00\x00\x03\x09\x00\x00\x02\x01\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xAF\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x27\x23CancelIoEx',0,b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x27\x23ConnectNamedPipe',0,b'\x00\x00\x6D\x23CreateEventA',0,b'\x00\x00\x73\x23CreateEventW',0,b'\x00\x00\x79\x23CreateFileA',0,b'\x00\x00\x9B\x23CreateFileW',0,b'\x00\x00\x82\x23CreateNamedPipeA',0,b'\x00\x00\x91\x23CreateNamedPipeW',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x48\x23CreateProcessW',0,b'\x00\x00\x3F\x23DuplicateHandle',0,b'\x00\x00\x8F\x23GetCurrentProcess',0,b'\x00\x00\x35\x23GetExitCodeProcess',0,b'\x00\x00\x63\x23GetLastError',0,b'\x00\x00\x5E\x23GetModuleFileNameW',0,b'\x00\x00\x2B\x23GetOverlappedResult',0,b'\x00\x00\x8C\x23GetStdHandle',0,b'\x00\x00\x63\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x57\x23SetErrorMode',0,b'\x00\x00\xA4\x23SetEvent',0,b'\x00\x00\x39\x23Se tNamedPipeHandleState',0,b'\x00\x00\x31\x23TerminateProcess',0,b'\x00\x00\x5A\x23WaitForSingleObject',0,b'\x00\x00\x54\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x68\x23_getwch',0,b'\x00\x00\x68\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x6A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x65\x23_ungetwch',0), + _struct_unions = ((b'\x00\x00\x00\xAD\x00\x00\x00\x03$1',b'\x00\x00\xAC\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x00\xAC\x00\x00\x00\x02$2',b'\x00\x00\x18\x11Offset',b'\x00\x00\x18\x11OffsetHigh'),(b'\x00\x00\x00\xA8\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\xAA\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x66\x11wShowWindow',b'\x00\x00\x66\x11cbReserved2',b'\x00\x00\xAE\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x00\xA7\x00\x00\x00\x02_ OVERLAPPED',b'\x00\x00\x18\x11Internal',b'\x00\x00\x18\x11InternalHigh',b'\x00\x00\xAD\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x00\xA9\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x18\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle')), + _typenames = (b'\x00\x00\x00\x29LPOVERLAPPED',b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x6ELPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\xA7OVERLAPPED',b'\x00\x00\x00\xA8PROCESS_INFORMATION',b'\x00\x00\x00\x6EPSECURITY_ATTRIBUTES',b'\x00\x00\x00\xA9SECURITY_ATTRIBUTES',b'\x00\x00\x00\xAASTARTUPINFO',b'\x00\x00\x00\x66wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -1,12 +1,12 @@ """ -Support routines for subprocess module. +Support routines for subprocess and multiprocess module. Currently, this extension module is only required when using the -subprocess module on Windows. +modules on Windows. """ import sys if sys.platform != 'win32': - raise ImportError("The '_subprocess' module is only available on Windows") + raise ImportError("The '_winapi' module is only available on Windows") # Declare external Win32 functions @@ -14,7 +14,7 @@ _kernel32 = _ffi.dlopen('kernel32') GetVersion = _kernel32.GetVersion - +NULL = _ffi.NULL # Now the _subprocess module implementation @@ -33,13 +33,116 @@ def CreatePipe(attributes, size): handles = _ffi.new("HANDLE[2]") - res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size) + res = _kernel32.CreatePipe(handles, handles + 1, NULL, size) if not res: raise _WinError() return _handle2int(handles[0]), _handle2int(handles[1]) +def CreateNamedPipe(*args): + handle = _kernel32.CreateNamedPipeW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def CreateFile(*args): + handle = _kernel32.CreateFileW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def SetNamedPipeHandleState(namedpipe, mode, max_collection_count, collect_data_timeout): + d0 = _ffi.new('DWORD[1]', [mode]) + if max_collection_count is None: + d1 = NULL + else: + d1 = _ffi.new('DWORD[1]', [max_collection_count]) + if collect_data_timeout is None: + d2 = NULL + else: + d2 = _ffi.new('DWORD[1]', [collect_data_timeout]) + ret = _kernel32.SetNamedPipeHandleState(namedpipe, d0, d1, d2) + if not ret: + raise _WinError() + +class Overlapped(object): + def __init__(self, handle): + self.overlapped = _ffi.new('OVERLAPPED[1]') + self.handle = handle + self.readbuffer = None + self.pending = 0 + self.completed = 0 + self.writebuffer = None + self.overlapped[0].hEvent = \ + _kernel32.CreateEventW(NULL, True, False, NULL) + + def __del__(self): + # do this somehow else + xxx + err = _kernel32.GetLastError() + bytes = _ffi.new('DWORD[1]') + o = overlapped[0] + if overlapped[0].pending: + if _kernel32.CancelIoEx(o.handle, o.overlapped) & \ + self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True): + # The operation is no longer pending, nothing to do + pass + else: + raise RuntimeError('deleting an overlapped strucwith a pending operation not supported') + + @property + def event(self): + return None + + def GetOverlappedResult(self, wait): + transferred = _ffi.new('DWORD[1]', [0]) + res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0) + if not res: + res = GetLastError() + if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED): + self.completed = 1 + self.pending = 0 + elif res == ERROR_IO_INCOMPLETE: + pass + else: + self.pending = 0 + raise _WinError() + if self.completed and self.read_buffer: + if transferred != len(self.read_buffer): + raise _WinError() + return transferred[0], err + + def getbuffer(self): + xxx + return None + + def cancel(self): + xxx + return None + + +def ConnectNamedPipe(handle, overlapped=False): + if overlapped: + ov = Overlapped(handle) + else: + ov = Overlapped(None) + success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) + if overlapped: + # Overlapped ConnectNamedPipe never returns a success code + assert success == 0 + err = _kernel32.GetLastError() + if err == ERROR_IO_PENDING: + overlapped[0].pending = 1 + elif err == ERROR_PIPE_CONNECTED: + _kernel32.SetEvent(ov.overlapped[0].hEvent) + else: + del ov + raise _WinError() + return ov + elif not success: + raise _WinError() + def GetCurrentProcess(): return _handle2int(_kernel32.GetCurrentProcess()) @@ -155,6 +258,7 @@ raise _WinError() return _ffi.string(buf) +# #define macros from WinBase.h and elsewhere STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -171,3 +275,52 @@ CREATE_UNICODE_ENVIRONMENT = 0x400 STILL_ACTIVE = 259 _MAX_PATH = 260 + +ERROR_SUCCESS = 0 +ERROR_NETNAME_DELETED = 64 +ERROR_BROKEN_PIPE = 109 +ERROR_MORE_DATA = 234 +ERROR_PIPE_CONNECTED = 535 +ERROR_OPERATION_ABORTED = 995 +ERROR_IO_INCOMPLETE = 996 +ERROR_IO_PENDING = 997 + +PIPE_ACCESS_INBOUND = 0x00000001 +PIPE_ACCESS_OUTBOUND = 0x00000002 +PIPE_ACCESS_DUPLEX = 0x00000003 +PIPE_WAIT = 0x00000000 +PIPE_NOWAIT = 0x00000001 +PIPE_READMODE_BYTE = 0x00000000 +PIPE_READMODE_MESSAGE = 0x00000002 +PIPE_TYPE_BYTE = 0x00000000 +PIPE_TYPE_MESSAGE = 0x00000004 +PIPE_ACCEPT_REMOTE_CLIENTS = 0x00000000 +PIPE_REJECT_REMOTE_CLIENTS = 0x00000008 + +GENERIC_READ = 0x80000000 +GENERIC_WRITE = 0x40000000 +GENERIC_EXECUTE= 0x20000000 +GENERIC_ALL = 0x10000000 +INVALID_HANDLE_VALUE = -1 +FILE_FLAG_WRITE_THROUGH = 0x80000000 +FILE_FLAG_OVERLAPPED = 0x40000000 +FILE_FLAG_NO_BUFFERING = 0x20000000 +FILE_FLAG_RANDOM_ACCESS = 0x10000000 +FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 +FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 +FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 +FILE_FLAG_POSIX_SEMANTICS = 0x01000000 +FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 +FILE_FLAG_OPEN_NO_RECALL = 0x00100000 +FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000 + +NMPWAIT_WAIT_FOREVER = 0xffffffff +NMPWAIT_NOWAIT = 0x00000001 +NMPWAIT_USE_DEFAULT_WAIT = 0x00000000 + +CREATE_NEW = 1 +CREATE_ALWAYS = 2 +OPEN_EXISTING = 3 +OPEN_ALWAYS = 4 +TRUNCATE_EXISTING = 5 + diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -40,3 +40,11 @@ .. branch: memory-accounting Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: winapi + +Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=<base_dir>\bin;%PATH% - set INCLUDE=<base_dir>\include;%INCLUDE% - set LIB=<base_dir>\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll <somewhere in the PATH> - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib <somewhere in LIB> - copy zlib.h zconf.h <somewhere in INCLUDE> - copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib <somewhere in LIB> - copy bzlib.h <somewhere in INCLUDE> - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib <somewhere in LIB> - xcopy /S include\openssl <somewhere in INCLUDE> - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll <somewhere in PATH> - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* <somewhere in PATH> - copy ..\..\tcltk\lib\*.lib <somewhere in LIB> - xcopy /S ..\..\tcltk\include <somewhere in INCLUDE> - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,6 +1,7 @@ import py import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be) from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp @@ -51,7 +52,10 @@ py.test.raises(Hit, decode_utf8, space, "\xed\xb0\x80") py.test.raises(Hit, decode_utf8, space, "\xed\xa0\x80\xed\xb0\x80") got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] def test_decode_utf8_allow_surrogates(): sp = FakeSpace() diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -148,6 +148,7 @@ # which never raises UnicodeEncodeError. Surrogate pairs are then # allowed, either paired or lone. A paired surrogate is considered # like the non-BMP character it stands for. See also *_utf8sp(). + assert isinstance(uni, unicode) return runicode.unicode_encode_utf_8( uni, len(uni), "strict", errorhandler=encode_error_handler(space), diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -105,14 +105,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - -@unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - @unwrap_spec(sizehint=int) def resizelist_hint(space, w_list, sizehint): """ Reallocate the underlying storage of the argument list to sizehint """ diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -288,6 +288,15 @@ ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + def test_from_buffer_BytesIO(self): + from _cffi_backend import FFI + import _io + ffi = FFI() + a = _io.BytesIO(b"Hello, world!") + buf = a.getbuffer() + # used to segfault + raises(TypeError, ffi.from_buffer, buf) + def test_memmove(self): import sys import _cffi_backend as _cffi1_backend diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -589,12 +589,17 @@ assert b'\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys import codecs sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' assert b'\x00\x60\x80'.decode( diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -48,6 +48,9 @@ finally: w_bytesio.seek(tell) + def get_raw_address(self): + raise ValueError("BytesIOBuffer does not have a raw address") + class W_BytesIO(W_BufferedIOBase): import_from_mixin(RStringIO) diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1202,12 +1202,16 @@ start, stop, step, size = self.space.decode_index4(w_idx, self.len) assert step != 0 if w_item.len != size or self is w_item: - # XXX this is a giant slow hack - w_lst = self.descr_tolist(space) - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.fromsequence(w_lst) + if start == self.len and step > 0: + # we actually want simply extend() + self.extend(w_item) + else: + # XXX this is a giant slow hack + w_lst = self.descr_tolist(space) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) else: j = 0 buf = self.get_buffer() diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -300,6 +300,12 @@ b = self.array('u', u'hi') assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_setslice_to_extend(self): + a = self.array('i') + a[0:1] = self.array('i', [9]) + a[1:5] = self.array('i', [99]) + assert list(a) == [9, 99] + def test_sequence(self): a = self.array('i', [1, 2, 3, 4]) assert len(a) == 4 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -31,7 +31,7 @@ from pypy.module.__builtin__.descriptor import W_Property #from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -97,25 +97,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -2,9 +2,10 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, - decref) + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields, cts, parse_dir, bootstrap_function, slot_function) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError @@ -31,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -128,6 +133,7 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, + attach=type_attach, dealloc=date_dealloc, ) @@ -138,8 +144,10 @@ def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj - Can be called with a datetime, or a time ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -131,7 +131,6 @@ def wrap_inquirypred(space, w_self, w_args, func): func_inquiry = rffi.cast(inquiry, func) check_num_args(space, w_args, 0) - args_w = space.fixedview(w_args) res = generic_cpy_call(space, func_inquiry, w_self) res = rffi.cast(lltype.Signed, res) if res == -1: @@ -411,286 +410,334 @@ return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) -from rpython.rlib.nonconst import NonConstant +SLOT_FACTORIES = {} +def slot_factory(tp_name): + def decorate(func): + SLOT_FACTORIES[tp_name] = func + return func + return decorate -def build_slot_tp_function(space, typedef, name): + +SLOTS = {} +@specialize.memo() +def get_slot_tp_function(space, typedef, name, method_name): + """Return a description of the slot C function to use for the built-in + type for 'typedef'. The 'name' is the slot name. This is a memo + function that, after translation, returns one of a built-in finite set. + """ + key = (typedef, name) + try: + return SLOTS[key] + except KeyError: + slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name) + api_func = slot_func.api_func if slot_func else None + SLOTS[key] = api_func + return api_func + + +def make_unary_slot(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - - handled = False - # unary functions - for tp_name, attr in [('tp_as_async.c_am_await', '__await__'), - ('tp_as_async.c_am_anext', '__anext__'), - ('tp_as_async.c_am_aiter', '__aiter__'), - ('tp_as_number.c_nb_int', '__int__'), - ('tp_as_number.c_nb_long', '__long__'), - ('tp_as_number.c_nb_float', '__float__'), - ('tp_as_number.c_nb_negative', '__neg__'), - ('tp_as_number.c_nb_positive', '__pos__'), - ('tp_as_number.c_nb_absolute', '__abs__'), - ('tp_as_number.c_nb_invert', '__invert__'), - ('tp_as_number.c_nb_index', '__index__'), - ('tp_str', '__str__'), - ('tp_repr', '__repr__'), - ('tp_iter', '__iter__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self): - return space.call_function(slot_fn, w_self) - handled = True - - for tp_name, attr in [('tp_hash', '__hash__'), - ('tp_as_sequence.c_sq_length', '__len__'), - ('tp_as_mapping.c_mp_length', '__len__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - @slot_function([PyObject], lltype.Signed, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_obj): - return space.int_w(space.call_function(slot_fn, w_obj)) - handled = True - - - # binary functions - for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__sub__'), - ('tp_as_number.c_nb_multiply', '__mul__'), - ('tp_as_number.c_nb_divide', '__div__'), - ('tp_as_number.c_nb_remainder', '__mod__'), - ('tp_as_number.c_nb_divmod', '__divmod__'), - ('tp_as_number.c_nb_lshift', '__lshift__'), - ('tp_as_number.c_nb_rshift', '__rshift__'), - ('tp_as_number.c_nb_and', '__and__'), - ('tp_as_number.c_nb_xor', '__xor__'), - ('tp_as_number.c_nb_or', '__or__'), - ('tp_as_sequence.c_sq_concat', '__add__'), - ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'), - ('tp_as_mapping.c_mp_subscript', '__getitem__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg): - return space.call_function(slot_fn, w_self, w_arg) - handled = True - - # binary-with-Py_ssize_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, Py_ssize_t], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg): - return space.call_function(slot_fn, w_self, space.newint(arg)) - handled = True - - # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, w_arg2): - return space.call_function(slot_fn, w_self, w_arg1, w_arg2) - handled = True - # ternary-with-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, w_arg1, w_arg2) - else: - space.call_function(slot_del, w_self, w_arg1) - return 0 - handled = True - # ternary-Py_size_t-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) - else: - space.call_function(slot_del, w_self, space.newint(arg1)) - return 0 - handled = True - if handled: - pass - elif name == 'tp_setattro': - setattr_fn = w_type.lookup('__setattr__') - delattr_fn = w_type.lookup('__delattr__') - if setattr_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1) - @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) - def slot_tp_setattro(space, w_self, w_name, w_value): - if w_value is not None: - space.call_function(setattr_fn, w_self, w_name, w_value) - else: - space.call_function(delattr_fn, w_self, w_name) - return 0 - slot_func = slot_tp_setattro - elif name == 'tp_getattro': - getattr_fn = w_type.lookup('__getattribute__') - if getattr_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) - def slot_tp_getattro(space, w_self, w_name): - return space.call_function(getattr_fn, w_self, w_name) - slot_func = slot_tp_getattro - - elif name == 'tp_call': - call_fn = w_type.lookup('__call__') - if call_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_call(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(call_fn, args) - slot_func = slot_tp_call - - elif name == 'tp_iternext': - iternext_fn = w_type.lookup('__next__') - if iternext_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_iternext(space, w_self): - try: - return space.call_function(iternext_fn, w_self) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - return None - slot_func = slot_tp_iternext - - elif name == 'tp_init': - init_fn = w_type.lookup('__init__') - if init_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_init(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - space.call_args(init_fn, args) - return 0 - slot_func = slot_tp_init - elif name == 'tp_new': - new_fn = w_type.lookup('__new__') - if new_fn is None: - return - - @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_new(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(space.get(new_fn, w_self), args) - slot_func = slot_tp_new - elif name == 'tp_as_buffer.c_bf_getbuffer': - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - buff_w = slot_from___buffer__(space, typedef, buff_fn) - elif typedef.buffer: - buff_w = slot_from_buffer_w(space, typedef, buff_fn) - else: - return - slot_func = buff_w - elif name == 'tp_descr_get': - get_fn = w_type.lookup('__get__') - if get_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_get(space, w_self, w_obj, w_value): - if w_obj is None: - w_obj = space.w_None - return space.call_function(get_fn, w_self, w_obj, w_value) - slot_func = slot_tp_descr_get - elif name == 'tp_descr_set': - set_fn = w_type.lookup('__set__') - delete_fn = w_type.lookup('__delete__') - if set_fn is None and delete_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_set(space, w_self, w_obj, w_value): - if w_value is not None: - if set_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __set__", typedef.name) - space.call_function(set_fn, w_self, w_obj, w_value) - else: - if delete_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __delete__", typedef.name) - space.call_function(delete_fn, w_self, w_obj) - return 0 - slot_func = slot_tp_descr_set - else: - # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce - # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length - # richcmpfunc(s) + slot_fn = w_type.lookup(attr) + if slot_fn is None: return + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self): + return space.call_function(slot_fn, w_self) return slot_func +UNARY_SLOTS = [ + 'tp_as_async.c_am_await', + 'tp_as_async.c_am_anext', + 'tp_as_async.c_am_aiter', + 'tp_as_number.c_nb_int', + 'tp_as_number.c_nb_long', + 'tp_as_number.c_nb_float', + 'tp_as_number.c_nb_negative', + 'tp_as_number.c_nb_positive', + 'tp_as_number.c_nb_absolute', + 'tp_as_number.c_nb_invert', + 'tp_as_number.c_nb_index', + 'tp_as_number.c_nb_hex', + 'tp_as_number.c_nb_oct', + 'tp_str', + 'tp_repr', + 'tp_iter'] +for name in UNARY_SLOTS: + slot_factory(name)(make_unary_slot) + +def make_unary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + @slot_function([PyObject], lltype.Signed, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_obj): + return space.int_w(space.call_function(slot_fn, w_obj)) + return slot_func + +UNARY_SLOTS_INT = [ + 'tp_hash', + 'tp_as_sequence.c_sq_length', + 'tp_as_mapping.c_mp_length',] +for name in UNARY_SLOTS_INT: + slot_factory(name)(make_unary_slot_int) + + +def make_binary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg): + return space.call_function(slot_fn, w_self, w_arg) + return slot_func + +BINARY_SLOTS = [ + 'tp_as_number.c_nb_add', + 'tp_as_number.c_nb_subtract', + 'tp_as_number.c_nb_multiply', + 'tp_as_number.c_nb_divide', + 'tp_as_number.c_nb_remainder', + 'tp_as_number.c_nb_divmod', + 'tp_as_number.c_nb_lshift', + 'tp_as_number.c_nb_rshift', + 'tp_as_number.c_nb_and', + 'tp_as_number.c_nb_xor', + 'tp_as_number.c_nb_or', + 'tp_as_sequence.c_sq_concat', + 'tp_as_sequence.c_sq_inplace_concat', + 'tp_as_mapping.c_mp_subscript',] +for name in BINARY_SLOTS: + slot_factory(name)(make_binary_slot) + + +def make_binary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, Py_ssize_t], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.newint(arg)) + return slot_func + +BINARY_SLOTS_INT = [ + 'tp_as_sequence.c_sq_item', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_inplace_repeat',] +for name in BINARY_SLOTS_INT: + slot_factory(name)(make_binary_slot_int) + +@slot_factory('tp_as_number.c_nb_power') +def make_nb_power(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, w_arg2): + return space.call_function(slot_fn, w_self, w_arg1, w_arg2) + return slot_func + _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit