Author: Ronan Lamy <ronan.l...@gmail.com> Branch: py3.5 Changeset: r90903:49c746636960 Date: 2017-04-01 18:47 +0100 http://bitbucket.org/pypy/pypy/changeset/49c746636960/
Log: hg merge default diff too long, truncating to 2000 out of 49367 lines diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -32,10 +32,11 @@ import threading try: - from __pypy__ import newlist_hint + from __pypy__ import newlist_hint, add_memory_pressure except ImportError: assert '__pypy__' not in sys.builtin_module_names newlist_hint = lambda sizehint: [] + add_memory_pressure = lambda size: None if sys.version_info[0] >= 3: StandardError = Exception @@ -152,6 +153,9 @@ check_same_thread=True, factory=None, cached_statements=100, uri=0): factory = Connection if not factory else factory + # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if + # backed by :memory: or a file) + add_memory_pressure(100 * 1024) return factory(database, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -79,6 +79,9 @@ _ssl libssl +_vmprof + libunwind (optional, loaded dynamically at runtime) + Make sure to have these libraries (with development headers) installed before building PyPy, otherwise the resulting binary will not contain these modules. Furthermore, the following libraries should be present @@ -185,7 +188,7 @@ :: cd pypy/tool/release - ./package.py pypy-VER-PLATFORM + ./package.py --archive-name=pypy-VER-PLATFORM This creates a clean and prepared hierarchy, as well as a ``.tar.bz2`` with the same content; both are found by default in diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -364,6 +364,24 @@ .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of +Performance Differences +------------------------- + +CPython has an optimization that can make repeated string concatenation not +quadratic. For example, this kind of code runs in O(n) time:: + + s = '' + for string in mylist: + s += string + +In PyPy, this code will always have quadratic complexity. Note also, that the +CPython optimization is brittle and can break by having slight variations in +your code anyway. So you should anyway replace the code with:: + + parts = [] + for string in mylist: + parts.append(string) + s = "".join(parts) Miscellaneous ------------- diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -57,6 +57,7 @@ .. code-block:: console $ ./pypy-xxx/bin/pypy -m ensurepip + $ ./pypy-xxx/bin/pip install -U pip wheel # to upgrade to the latest versions $ ./pypy-xxx/bin/pip install pygments # for example Third party libraries will be installed in ``pypy-xxx/site-packages``, and @@ -77,7 +78,17 @@ # from the mercurial checkout $ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env -Note that bin/python is now a symlink to bin/pypy. + # in any case activate it + $ source my-pypy-env/bin/activate + +Note that my-pypy-env/bin/python is now a symlink to my-pypy-env/bin/pypy +so you should be able to run pypy simply by typing:: + + $ python + +You should still upgrade pip and wheel to the latest versions via:: + + $ my-pypy-env/bin/pip install -U pip wheel .. _pip: http://pypi.python.org/pypi/pip .. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,7 +2,22 @@ What's new in PyPy2.7 5.8+ ========================== -.. this is a revision shortly after release-pypy2.7-v5.7 +.. this is a revision shortly after release-pypy2.7-v5.7.0 .. startrev: 44f31f6dd39f +Add cpyext interfaces for ``PyModule_New`` +Correctly handle `dict.pop`` where the ``pop`` +key is not the same type as the ``dict``'s and ``pop`` +is called with a default (will be part of release 5.7.1) + +.. branch: issue2522 + +Fix missing tp_new on w_object called through multiple inheritance +(will be part of release 5.7.1) + +.. branch: lstrip_to_empty_string + +.. branch: vmprof-native + +PyPy support to profile native frames in vmprof. diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py --- a/pypy/interpreter/pyparser/future.py +++ b/pypy/interpreter/pyparser/future.py @@ -80,6 +80,7 @@ from pypy.interpreter.pyparser import pygram it = TokenIterator(tokens) result = 0 + last_position = (0, 0) # # The only things that can precede a future statement are another # future statement and a doc string (only one). This is a very @@ -94,6 +95,11 @@ it.skip_name("__future__") and it.skip_name("import")): it.skip(pygram.tokens.LPAR) # optionally + # return in 'last_position' any line-column pair that points + # somewhere inside the last __future__ import statement + # (at the start would be fine too, but it's easier to grab a + # random position inside) + last_position = (it.tok[2], it.tok[3]) result |= future_flags.get_compiler_feature(it.next_feature_name()) while it.skip(pygram.tokens.COMMA): result |= future_flags.get_compiler_feature(it.next_feature_name()) @@ -104,5 +110,4 @@ # remove the flags that were specified but are anyway mandatory result &= ~future_flags.mandatory_flags - position = (it.tok[2], it.tok[3]) - return result, position + return result, last_position diff --git a/pypy/interpreter/pyparser/test/test_future.py b/pypy/interpreter/pyparser/test/test_future.py --- a/pypy/interpreter/pyparser/test/test_future.py +++ b/pypy/interpreter/pyparser/test/test_future.py @@ -2,10 +2,9 @@ from pypy.interpreter.pyparser import future, pytokenizer from pypy.tool import stdlib___future__ as fut -def run(s, expected_last_future=None): +def run(s, expected_last_future=(0, 0)): source_lines = s.splitlines(True) tokens = pytokenizer.generate_tokens(source_lines, 0) - expected_last_future = expected_last_future or tokens[-1][2:4] # flags, last_future_import = future.add_future_flags( future.futureFlags_3_5, tokens) @@ -14,7 +13,7 @@ def test_docstring(): s = '"Docstring\\" "\nfrom __future__ import division\n' - f = run(s) + f = run(s, (2, 24)) assert f == 0 def test_comment(): @@ -45,152 +44,152 @@ def test_from(): s = 'from __future__ import division\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_froms(): s = 'from __future__ import division, generators, with_statement\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_from_as(): s = 'from __future__ import division as b\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_froms_as(): s = 'from __future__ import division as b, generators as c\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_from_paren(): s = 'from __future__ import (division)\n' - f = run(s) + f = run(s, (1, 25)) assert f == 0 def test_froms_paren(): s = 'from __future__ import (division, generators)\n' - f = run(s) + f = run(s, (1, 25)) assert f == 0 def test_froms_paren_as(): s = 'from __future__ import (division as b, generators,)\n' - f = run(s) + f = run(s, (1, 25)) assert f == 0 def test_paren_with_newline(): s = 'from __future__ import (division,\nabsolute_import)\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_paren_with_newline_2(): s = 'from __future__ import (\ndivision,\nabsolute_import)\n' - f = run(s) + f = run(s, (2, 0)) assert f == 0 def test_multiline(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n' - f = run(s) + f = run(s, (4, 23)) assert f == 0 def test_windows_style_lineendings(): s = '"abc" #def\r\n #ghi\r\nfrom __future__ import (division as b, generators,)\r\nfrom __future__ import with_statement\r\n' - f = run(s) + f = run(s, (4, 23)) assert f == 0 def test_mac_style_lineendings(): s = '"abc" #def\r #ghi\rfrom __future__ import (division as b, generators,)\rfrom __future__ import with_statement\r' - f = run(s) + f = run(s, (4, 23)) assert f == 0 def test_semicolon(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' - f = run(s) + f = run(s, (3, 78)) assert f == 0 def test_semicolon_2(): s = 'from __future__ import division; from foo import bar' - f = run(s, expected_last_future=(1, 39)) + f = run(s, expected_last_future=(1, 24)) assert f == 0 def test_full_chain(): s = '"abc" #def\n #ghi\nfrom __future__ import (division as b, generators,); from __future__ import with_statement\n' - f = run(s) + f = run(s, (3, 78)) assert f == 0 def test_intervening_code(): s = 'from __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n' - f = run(s, expected_last_future=(2, 5)) + f = run(s, expected_last_future=(1, 25)) assert f == 0 def test_nonexisting(): s = 'from __future__ import non_existing_feature\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_nonexisting_2(): s = 'from __future__ import non_existing_feature, with_statement\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_from_import_abs_import(): s = 'from __future__ import absolute_import\n' - f = run(s) + f = run(s, (1, 24)) assert f == 0 def test_raw_doc(): s = 'r"Doc"\nfrom __future__ import with_statement\n' - f = run(s) + f = run(s, (2, 23)) assert f == 0 def test_unicode_doc(): s = 'u"Doc"\nfrom __future__ import with_statement\n' - f = run(s) + f = run(s, (2, 23)) assert f == 0 def test_raw_unicode_doc(): s = 'ru"Doc"\nfrom __future__ import with_statement\n' - f = run(s) + f = run(s, (2, 23)) assert f == 0 def test_continuation_line(): s = "\\\nfrom __future__ import with_statement\n" - f = run(s) + f = run(s, (2, 23)) assert f == 0 def test_continuation_lines(): s = "\\\n \t\\\nfrom __future__ import with_statement\n" - f = run(s) + f = run(s, (3, 23)) assert f == 0 def test_lots_of_continuation_lines(): s = "\\\n\\\n\\\n\\\n\\\n\\\n\nfrom __future__ import with_statement\n" - f = run(s) + f = run(s, (8, 23)) assert f == 0 def test_continuation_lines_raise(): s = " \\\n \t\\\nfrom __future__ import with_statement\n" - f = run(s, expected_last_future=(1, 0)) + f = run(s) assert f == 0 # because of the INDENT def test_continuation_lines_in_docstring_single_quoted(): s = '"\\\n\\\n\\\n\\\n\\\n\\\n"\nfrom __future__ import division\n' - f = run(s) + f = run(s, (8, 24)) assert f == 0 def test_continuation_lines_in_docstring_triple_quoted(): s = '"""\\\n\\\n\\\n\\\n\\\n\\\n"""\nfrom __future__ import division\n' - f = run(s) + f = run(s, (8, 24)) assert f == 0 def test_blank_lines(): s = ('\n\t\n\nfrom __future__ import with_statement' ' \n \n \nfrom __future__ import division') - f = run(s) + f = run(s, (7, 23)) assert f == 0 def test_dummy_semicolons(): s = ('from __future__ import division;\n' 'from __future__ import with_statement;') - f = run(s) + f = run(s, (2, 23)) assert f == 0 diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -308,6 +308,15 @@ assert isinstance(ns["c"], bytes) assert isinstance(ns["d"], bytes) + def test_both_futures_with_semicolon(self): + # Issue #2526: a corner case which crashes only if the file + # contains *nothing more* than two __future__ imports separated + # by a semicolon. + s = """ +from __future__ import unicode_literals; from __future__ import print_function +""" + exec s in {} + class AppTestComprehensions: diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -6,6 +6,8 @@ class BuildersModule(MixedModule): + """ Module containing string and unicode builders """ + appleveldefs = {} interpleveldefs = { @@ -34,6 +36,8 @@ class IntOpModule(MixedModule): + """ Module for integer operations that have two-complement overflow + behaviour instead of overflowing to longs """ appleveldefs = {} interpleveldefs = { 'int_add': 'interp_intop.int_add', @@ -55,6 +59,8 @@ class Module(MixedModule): + """ PyPy specific "magic" functions. A lot of them are experimental and + subject to change, many are internal. """ appleveldefs = { } diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -106,13 +106,15 @@ raise wrap_oserror(space, e) @unwrap_spec(sizehint=int) -def resizelist_hint(space, w_iterable, sizehint): - if not isinstance(w_iterable, W_ListObject): +def resizelist_hint(space, w_list, sizehint): + """ Reallocate the underlying storage of the argument list to sizehint """ + if not isinstance(w_list, W_ListObject): raise oefmt(space.w_TypeError, "arg 1 must be a 'list'") - w_iterable._resize_hint(sizehint) + w_list._resize_hint(sizehint) @unwrap_spec(sizehint=int) def newlist_hint(space, sizehint): + """ Create a new empty list that has an underlying storage of length sizehint """ return space.newlist_hint(sizehint) @unwrap_spec(debug=int) @@ -125,6 +127,9 @@ @unwrap_spec(estimate=int) def add_memory_pressure(estimate): + """ Add memory pressure of estimate bytes. Useful when calling a C function + that internally allocates a big chunk of memory. This instructs the GC to + garbage collect sooner than it would otherwise.""" rgc.add_memory_pressure(estimate) @unwrap_spec(w_frame=PyFrame) diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py --- a/pypy/module/_cffi_backend/test/test_recompiler.py +++ b/pypy/module/_cffi_backend/test/test_recompiler.py @@ -1471,8 +1471,13 @@ with self.StdErrCapture(fd=True) as f: res = lib.bar(4, 5) assert res == 0 - assert f.getvalue() == ( + assert f.getvalue() in ( + # If the underlying cffi is <= 1.9 "extern \"Python\": function bar() called, but no code was attached " + "to it yet with @ffi.def_extern(). Returning 0.\n", + # If the underlying cffi is >= 1.10 + "extern \"Python\": function _CFFI_test_extern_python_1.bar() " + "called, but no code was attached " "to it yet with @ffi.def_extern(). Returning 0.\n") @ffi.def_extern("bar") diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -50,8 +50,8 @@ return OperationError(w_VMProfError, space.newtext(e.msg)) -@unwrap_spec(fileno=int, period=float) -def enable(space, fileno, period): +@unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int) +def enable(space, fileno, period, memory, lines, native): """Enable vmprof. Writes go to the given 'fileno', a file descriptor opened for writing. *The file descriptor must remain open at least until disable() is called.* @@ -65,7 +65,7 @@ # "with vmprof will crash"), # space.w_RuntimeWarning) try: - rvmprof.enable(fileno, period) + rvmprof.enable(fileno, period, memory, native) except rvmprof.VMProfError as e: raise VMProfError(space, e) diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -24,10 +24,11 @@ i += 5 * WORD # header assert s[i ] == 5 # MARKER_HEADER assert s[i + 1] == 0 # 0 - assert s[i + 2] == 2 # VERSION_THREAD_ID - assert s[i + 3] == 4 # len('pypy') - assert s[i + 4: i + 8] == b'pypy' - i += 8 + assert s[i + 2] == 6 # VERSION_TIMESTAMP + assert s[i + 3] == 8 # PROFILE_RPYTHON + assert s[i + 4] == 4 # len('pypy') + assert s[i + 5: i + 9] == b'pypy' + i += 9 while i < len(s): if s[i] == 3: break @@ -41,6 +42,17 @@ _, size = struct.unpack("ll", s[i:i + 2 * WORD]) count += 1 i += 2 * WORD + size + elif s[i] == '\x06': + print(s[i:i+24]) + i += 1+8+8+8 + elif s[i] == '\x07': + i += 1 + # skip string + size, = struct.unpack("l", s[i:i + WORD]) + i += WORD+size + # skip string + size, = struct.unpack("l", s[i:i + WORD]) + i += WORD+size else: raise AssertionError(s[i]) return count @@ -48,7 +60,7 @@ import _vmprof gc.collect() # try to make the weakref list deterministic gc.collect() # by freeing all dead code objects - _vmprof.enable(tmpfileno, 0.01) + _vmprof.enable(tmpfileno, 0.01, 0, 0, 0) _vmprof.disable() s = open(self.tmpfilename, 'rb').read() no_of_codes = count(s) @@ -64,7 +76,7 @@ gc.collect() gc.collect() - _vmprof.enable(tmpfileno2, 0.01) + _vmprof.enable(tmpfileno2, 0.01, 0, 0, 0) exec_("""def foo2(): pass @@ -79,9 +91,9 @@ def test_enable_ovf(self): import _vmprof - raises(_vmprof.VMProfError, _vmprof.enable, 2, 0) - raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5) - raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300) - raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300) + raises(_vmprof.VMProfError, _vmprof.enable, 2, 0, 0, 0, 0) + raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5, 0, 0, 0) + raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300, 0, 0, 0) + raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300, 0, 0, 0) NaN = (1e300*1e300) / (1e300*1e300) - raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN) + raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN, 0, 0, 0) diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py --- a/pypy/module/_vmprof/test/test_direct.py +++ b/pypy/module/_vmprof/test/test_direct.py @@ -43,7 +43,7 @@ } -""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read(), include_dirs=[str(srcdir)]) +""" + open(str(srcdir.join("shared/vmprof_get_custom_offset.h"))).read(), include_dirs=[str(srcdir)]) class TestDirect(object): def test_infrastructure(self): diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -4,10 +4,10 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib.rarithmetic import widen -from rpython.rlib import rgc # Force registration of gc.collect +from rpython.rlib import rgc # Force registration of gc.collect from pypy.module.cpyext.api import ( slot_function, generic_cpy_call, PyObject, Py_ssize_t, - pypy_decl, Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) + Py_buffer, Py_bufferP, PyTypeObjectPtr, cts) from pypy.module.cpyext.typeobjectdefs import ( unaryfunc, ternaryfunc, binaryfunc, getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry, @@ -793,6 +793,16 @@ return 0 return buff_w +missing_wrappers = ['wrap_indexargfunc', 'wrap_del'] +for name in missing_wrappers: + assert name not in globals() + def missing_wrapper(space, w_self, w_args, func, w_kwds): + print "cpyext: missing slot wrapper " + name + raise NotImplementedError("Slot wrapper " + name) + missing_wrapper.__name__ = name + globals()[name] = missing_wrapper + + PyWrapperFlag_KEYWORDS = 1 class TypeSlot: @@ -810,7 +820,7 @@ if WRAPPER is None: wrapper = None else: - wrapper = globals().get(WRAPPER, Ellipsis) + wrapper = globals()[WRAPPER] # irregular interface, because of tp_getattr/tp_getattro confusion if NAME == "__getattr__": @@ -824,17 +834,9 @@ function = getattr(userslot, FUNCTION or '!missing', None) assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS if FLAGS: - if wrapper is Ellipsis: - @func_renamer(WRAPPER) - def wrapper(space, w_self, w_args, func, w_kwds): - raise NotImplementedError("Wrapper for slot " + NAME) wrapper1 = None wrapper2 = wrapper else: - if wrapper is Ellipsis: - @func_renamer(WRAPPER) - def wrapper(space, w_self, w_args, func): - raise NotImplementedError("Wrapper for slot " + NAME) wrapper1 = wrapper wrapper2 = None return TypeSlot(NAME, SLOT, function, wrapper1, wrapper2, DOC) diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c --- a/pypy/module/cpyext/test/buffer_test.c +++ b/pypy/module/cpyext/test/buffer_test.c @@ -344,7 +344,6 @@ #endif if (m == NULL) INITERROR; - PyMyArrayType.tp_new = PyType_GenericNew; if (PyType_Ready(&PyMyArrayType) < 0) INITERROR; Py_INCREF(&PyMyArrayType); diff --git a/pypy/module/cpyext/test/comparisons.c b/pypy/module/cpyext/test/comparisons.c --- a/pypy/module/cpyext/test/comparisons.c +++ b/pypy/module/cpyext/test/comparisons.c @@ -68,7 +68,7 @@ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + PyType_GenericNew, /* tp_new */ 0 /* tp_free */ }; @@ -87,7 +87,6 @@ { PyObject *m, *d; - CmpType.tp_new = PyType_GenericNew; if (PyType_Ready(&CmpType) < 0) return NULL; m = PyModule_Create(&moduledef); diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -36,14 +36,14 @@ Py_ssize_t foo_ssizet; } fooobject; -static PyTypeObject footype; +static PyTypeObject fooType; static fooobject * newfooobject(void) { fooobject *foop; - foop = PyObject_New(fooobject, &footype); + foop = PyObject_New(fooobject, &fooType); if (foop == NULL) return NULL; @@ -160,6 +160,28 @@ return PyObject_GenericSetAttr((PyObject *)self, name, value); } +static PyObject * +new_fooType(PyTypeObject * t, PyObject *args, PyObject *kwds) +{ + PyObject * o; + /* copied from numpy scalartypes.c for inherited classes */ + if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1)) + { + PyTypeObject *sup; + /* We are inheriting from a Python type as well so + give it first dibs on conversion */ + sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1); + /* Prevent recursion */ + if (new_fooType != sup->tp_new) + { + o = sup->tp_new(t, args, kwds); + return o; + } + } + o = t->tp_alloc(t, 0); + return o; +}; + static PyMemberDef foo_members[] = { {"int_member", T_INT, offsetof(fooobject, foo), 0, "A helpful docstring."}, @@ -194,7 +216,7 @@ PyDoc_STRVAR(foo_doc, "foo is for testing."); -static PyTypeObject footype = { +static PyTypeObject fooType = { PyVarObject_HEAD_INIT(NULL, 0) "foo.foo", /*tp_name*/ sizeof(fooobject), /*tp_size*/ @@ -592,7 +614,7 @@ 0, /*tp_init*/ 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ - 0, /*tp_new*/ + PyType_GenericNew, /*tp_new*/ 0, /*tp_free Low-level free-memory routine */ 0, /*tp_is_gc For PyObject_IS_GC */ 0, /*tp_bases*/ @@ -648,6 +670,37 @@ return PyInt_FromLong(tf); } +static PyTypeObject GetType1 = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.GetType1", /*tp_name*/ + sizeof(PyObject), /*tp_size*/ +}; +static PyTypeObject GetType2 = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.GetType2", /*tp_name*/ + sizeof(PyObject), /*tp_size*/ +}; +static PyObject *gettype1, *gettype2; + +static PyObject *gettype1_getattr(PyObject *self, char *name) +{ + char buf[200]; + strcpy(buf, "getattr:"); + strcat(buf, name); + return PyBytes_FromString(buf); +} +static PyObject *gettype2_getattro(PyObject *self, PyObject *name) +{ + char buf[200]; + PyObject* temp; + temp = PyUnicode_AsASCIIString(name); + if (temp == NULL) return NULL; + strcpy(buf, "getattro:"); + strcat(buf, PyBytes_AS_STRING(temp)); + return PyBytes_FromString(buf); +} + + /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { @@ -706,13 +759,14 @@ if (module == NULL) INITERROR; - footype.tp_new = PyType_GenericNew; - UnicodeSubtype.tp_base = &PyUnicode_Type; UnicodeSubtype2.tp_base = &UnicodeSubtype; MetaType.tp_base = &PyType_Type; - if (PyType_Ready(&footype) < 0) + fooType.tp_new = &new_fooType; + InitErrType.tp_new = PyType_GenericNew; + + if (PyType_Ready(&fooType) < 0) INITERROR; if (PyType_Ready(&UnicodeSubtype) < 0) INITERROR; @@ -725,8 +779,6 @@ if (PyType_Ready(&SimplePropertyType) < 0) INITERROR; - SimplePropertyType.tp_new = PyType_GenericNew; - InitErrType.tp_new = PyType_GenericNew; Py_TYPE(&CustomType) = &MetaType; if (PyType_Ready(&CustomType) < 0) @@ -744,11 +796,23 @@ if (PyType_Ready(&TupleLike) < 0) INITERROR; + GetType1.tp_flags = Py_TPFLAGS_DEFAULT; + GetType1.tp_getattr = &gettype1_getattr; + if (PyType_Ready(&GetType1) < 0) + INITERROR; + gettype1 = PyObject_New(PyObject, &GetType1); + + GetType2.tp_flags = Py_TPFLAGS_DEFAULT; + GetType2.tp_getattro = &gettype2_getattro; + if (PyType_Ready(&GetType2) < 0) + INITERROR; + gettype2 = PyObject_New(PyObject, &GetType2); + d = PyModule_GetDict(module); if (d == NULL) INITERROR; - if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0) + if (PyDict_SetItemString(d, "fooType", (PyObject *)&fooType) < 0) INITERROR; if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0) INITERROR; @@ -766,6 +830,10 @@ INITERROR; if (PyDict_SetItemString(d, "TupleLike", (PyObject *) &TupleLike) < 0) INITERROR; + if (PyDict_SetItemString(d, "gettype1", gettype1) < 0) + INITERROR; + if (PyDict_SetItemString(d, "gettype2", gettype2) < 0) + INITERROR; #if PY_MAJOR_VERSION >=3 return module; #endif diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -187,6 +187,10 @@ " on too long format string" finally: warnings.resetwarnings() + # calling get_buffer_info on x creates a memory leak, + # which is detected as an error at test teardown: + # Exception TypeError: "'NoneType' object is not callable" + # in <bound method ConcreteArray.__del__ ...> ignored def test_releasebuffer(self): if not self.runappdirect: diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -16,12 +16,12 @@ assert 'foo' in sys.modules assert "copy" in dir(module.fooType) obj = module.new() - print(obj.foo) + #print(obj.foo) assert obj.foo == 42 - print("Obj has type", type(obj)) + #print("Obj has type", type(obj)) assert type(obj) is module.fooType - print("type of obj has type", type(type(obj))) - print("type of type of obj has type", type(type(type(obj)))) + #print("type of obj has type", type(type(obj))) + #print("type of type of obj has type", type(type(type(obj)))) assert module.fooType.__doc__ == "foo is for testing." def test_typeobject_method_descriptor(self): @@ -949,6 +949,8 @@ pass class foo(f2, f1): pass + + x = foo() assert bar.__base__ is f2 # On cpython, the size changes. if '__pypy__' in sys.builtin_module_names: @@ -1159,8 +1161,8 @@ ((PyHeapTypeObject*)Base2)->ht_name = dummyname; ((PyHeapTypeObject*)Base12)->ht_name = dummyname; } - #endif - #endif + #endif + #endif Base1->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; Base2->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; Base12->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE; @@ -1197,3 +1199,14 @@ pass bases = module.foo(C) assert bases == (A, B) + + def test_getattr_getattro(self): + module = self.import_module(name='foo') + assert module.gettype2.dcba == b'getattro:dcba' + assert (type(module.gettype2).__getattribute__(module.gettype2, 'dcBA') + == b'getattro:dcBA') + assert module.gettype1.abcd == b'getattr:abcd' + # GetType1 objects have a __getattribute__ method, but this + # doesn't call tp_getattr at all, also on CPython + raises(AttributeError, type(module.gettype1).__getattribute__, + module.gettype1, 'dcBA') diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -682,10 +682,6 @@ if pto.c_tp_itemsize < pto.c_tp_base.c_tp_itemsize: pto.c_tp_itemsize = pto.c_tp_base.c_tp_itemsize - if space.is_w(w_type, space.w_object): - # will be filled later on with the correct value - # may not be 0 - pto.c_tp_new = cts.cast('newfunc', 1) update_all_slots(space, w_type, pto) if not pto.c_tp_new: base_object_pyo = make_ref(space, space.w_object) diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py --- a/pypy/module/faulthandler/cintf.py +++ b/pypy/module/faulthandler/cintf.py @@ -5,10 +5,14 @@ cwd = py.path.local(__file__).dirpath() +rvmp = cwd.join('../../..') +rvmp = rvmp.join('rpython/rlib/rvmprof/src') + eci = ExternalCompilationInfo( includes=[cwd.join('faulthandler.h')], - include_dirs=[str(cwd), cdir], - separate_module_files=[cwd.join('faulthandler.c')]) + include_dirs=[str(cwd), cdir, rvmp], + separate_module_files=[cwd.join('faulthandler.c')], + compile_extra=['-DRPYTHON_VMPROF=1']) eci_later = eci.merge(ExternalCompilationInfo( pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n'])) diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py --- a/pypy/module/struct/interp_struct.py +++ b/pypy/module/struct/interp_struct.py @@ -42,10 +42,12 @@ def calcsize(space, w_format): """Return size of C struct described by format string fmt.""" format = text_or_bytes_w(space, w_format) + """Return size of C struct described by format string fmt.""" return space.newint(_calcsize(space, format)) def _pack(space, format, args_w): + """Return string containing values v1, v2, ... packed according to fmt.""" if jit.isconstant(format): size = _calcsize(space, format) else: @@ -79,6 +81,9 @@ # XXX inefficient def do_pack_into(space, format, w_buffer, offset, args_w): + """ Pack the values v1, v2, ... according to fmt. +Write the packed bytes into the writable buffer buf starting at offset + """ res = _pack(space, format, args_w) buf = space.getarg_w('w*', w_buffer) if offset < 0: @@ -119,6 +124,8 @@ return do_unpack_from(space, format, w_buffer, offset) def do_unpack_from(space, format, w_buffer, offset=0): + """Unpack the buffer, containing packed C structure data, according to +fmt, starting at offset. Requires len(buffer[offset:]) >= calcsize(fmt).""" size = _calcsize(space, format) buf = space.buffer_w(w_buffer, space.BUF_SIMPLE) if offset < 0: diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,7 +1,7 @@ # Edit these appropriately before running this script maj=5 min=7 -rev=0 +rev=1 branchname=release-pypy2.7-5.x # ==OR== release-$maj.x # ==OR== release-$maj.$min.x tagname=release-pypy2.7-v$maj.$min.$rev # ==OR== release-$maj.$min @@ -50,8 +50,8 @@ rm -rf $rel-src # Print out the md5, sha1, sha256 -md5sum *.bz2 *.zip -sha1sum *.bz2 *.zip +#md5sum *.bz2 *.zip +#sha1sum *.bz2 *.zip sha256sum *.bz2 *.zip # Now upload all the bz2 and zip diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py --- a/rpython/jit/backend/x86/detect_feature.py +++ b/rpython/jit/backend/x86/detect_feature.py @@ -21,7 +21,11 @@ return bool(code & (1<<25)) and bool(code & (1<<26)) def cpu_id(eax = 1, ret_edx = True, ret_ecx = False): - asm = ["\xB8", chr(eax), "\x00\x00\x00", # MOV EAX, $eax + asm = ["\xB8", # MOV EAX, $eax + chr(eax & 0xff), + chr((eax >> 8) & 0xff), + chr((eax >> 16) & 0xff), + chr((eax >> 24) & 0xff), "\x53", # PUSH EBX "\x0F\xA2", # CPUID "\x5B", # POP EBX diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py --- a/rpython/jit/backend/x86/vector_ext.py +++ b/rpython/jit/backend/x86/vector_ext.py @@ -60,6 +60,7 @@ class VectorAssemblerMixin(object): _mixin_ = True + element_ones = [] # overridden in assembler.py def setup_once_vector(self): pass @@ -342,6 +343,7 @@ else: assert lhsloc is xmm0 maskloc = X86_64_XMM_SCRATCH_REG + assert len(self.element_ones) > 0 self.mc.MOVAPD(maskloc, heap(self.element_ones[get_scale(size)])) self.mc.PXOR(resloc, resloc) # note that resloc contains true false for each element by the last compare operation diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4637,8 +4637,8 @@ self.meta_interp(f, [10]) - @py.test.skip("loops!") def test_finalizer_bug(self): + py.test.skip("loops!") from rpython.rlib import rgc driver = JitDriver(greens=[], reds=[]) class Fin(object): diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -244,6 +244,10 @@ return inner def oopspec(spec): + """ The JIT compiler won't look inside this decorated function, + but instead during translation, rewrites it according to the handler in + rpython/jit/codewriter/jtransform.py. + """ def decorator(func): func.oopspec = spec return func diff --git a/rpython/rlib/rstrategies/rstrategies.py b/rpython/rlib/rstrategies/rstrategies.py --- a/rpython/rlib/rstrategies/rstrategies.py +++ b/rpython/rlib/rstrategies/rstrategies.py @@ -1,7 +1,7 @@ import weakref, sys from rpython.rlib.rstrategies import logger -from rpython.rlib import jit, objectmodel, rerased +from rpython.rlib import jit, objectmodel, rerased, rarithmetic from rpython.rlib.objectmodel import specialize, not_rpython def make_accessors(strategy='strategy', storage='storage'): @@ -443,6 +443,7 @@ def store(self, w_self, index0, wrapped_value): self.check_index_store(w_self, index0) + assert index0 >= 0 if self._check_can_handle(wrapped_value): unwrapped = self._unwrap(wrapped_value) self.get_storage(w_self)[index0] = unwrapped @@ -451,6 +452,7 @@ def fetch(self, w_self, index0): self.check_index_fetch(w_self, index0) + assert index0 >= 0 unwrapped = self.get_storage(w_self)[index0] return self._wrap(unwrapped) @@ -517,7 +519,7 @@ self.check_index(w_self, start) self.check_index(w_self, end) def check_index(self, w_self, index0): - if index0 < 0 or index0 >= self.size(w_self): + if not rarithmetic.int_between(0, index0, self.size(w_self)): raise IndexError class UnsafeIndexingMixin(object): diff --git a/rpython/rlib/rstrategies/test/test_rstrategies.py b/rpython/rlib/rstrategies/test/test_rstrategies.py --- a/rpython/rlib/rstrategies/test/test_rstrategies.py +++ b/rpython/rlib/rstrategies/test/test_rstrategies.py @@ -204,7 +204,7 @@ if is_safe: py.test.raises(IndexError, s.fetch, l, -1) else: - assert s.fetch(l, -1) == s.fetch(l, size - 1) + py.test.raises(AssertionError, s.fetch, l, -1) def test_init_Empty(): l = W_List(EmptyStrategy, 0) @@ -249,7 +249,7 @@ if is_safe: py.test.raises(IndexError, s.store, l, -1, stored_value) else: - store_test(-1) + py.test.raises(AssertionError, s.store, l, -1, stored_value) def test_store_Nil(): do_test_store(NilStrategy, stored_value=w_nil) diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -32,8 +32,8 @@ return code._vmprof_unique_id return 0 -def enable(fileno, interval): - _get_vmprof().enable(fileno, interval) +def enable(fileno, interval, memory=0, native=0): + _get_vmprof().enable(fileno, interval, memory, native) def disable(): _get_vmprof().disable() diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -14,32 +14,70 @@ ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof') SRC = ROOT.join('src') +SHARED = SRC.join('shared') +BACKTRACE = SHARED.join('libbacktrace') +compile_extra = ['-DRPYTHON_VMPROF', '-O3'] +separate_module_files = [ + SHARED.join('symboltable.c') +] if sys.platform.startswith('linux'): + if sys.maxint > 2**32: # doesn't seem to compile on 32-bit Linux + separate_module_files += [ + BACKTRACE.join('backtrace.c'), + BACKTRACE.join('state.c'), + BACKTRACE.join('elf.c'), + BACKTRACE.join('dwarf.c'), + BACKTRACE.join('fileline.c'), + BACKTRACE.join('mmap.c'), + BACKTRACE.join('mmapio.c'), + BACKTRACE.join('posix.c'), + BACKTRACE.join('sort.c'), + ] + compile_extra += ['-DVMPROF_BACKTRACE'] _libs = ['dl'] + compile_extra += ['-DVMPROF_UNIX'] + compile_extra += ['-DVMPROF_LINUX'] +elif sys.platform == 'win32': + compile_extra = ['-DRPYTHON_VMPROF', '-DVMPROF_WINDOWS'] + separate_module_files = [SHARED.join('vmprof_main_win32.c')] + _libs = [] else: + # Guessing a BSD-like Unix platform + compile_extra += ['-DVMPROF_UNIX'] + compile_extra += ['-DVMPROF_MAC'] _libs = [] + + eci_kwds = dict( - include_dirs = [SRC], - includes = ['rvmprof.h', 'vmprof_stack.h'], + include_dirs = [SRC, SHARED, BACKTRACE], + includes = ['rvmprof.h','vmprof_stack.h'], libraries = _libs, - separate_module_files = [SRC.join('rvmprof.c')], - post_include_bits=['#define RPYTHON_VMPROF\n'], + separate_module_files = [ + SRC.join('rvmprof.c'), + SHARED.join('compat.c'), + SHARED.join('machine.c'), + SHARED.join('vmp_stack.c'), + # symbol table already in separate_module_files + ] + separate_module_files, + post_include_bits=[], + compile_extra=compile_extra ) global_eci = ExternalCompilationInfo(**eci_kwds) def setup(): - compile_extra = ['-DRPYTHON_LL2CTYPES'] + eci_kwds['compile_extra'].append('-DRPYTHON_LL2CTYPES') platform.verify_eci(ExternalCompilationInfo( - compile_extra=compile_extra, - **eci_kwds)) + **eci_kwds)) eci = global_eci vmprof_init = rffi.llexternal("vmprof_init", - [rffi.INT, rffi.DOUBLE, rffi.CCHARP], + [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT, + rffi.CCHARP, rffi.INT], rffi.CCHARP, compilation_info=eci) - vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT, + vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT, rffi.INT], + rffi.INT, compilation_info=eci, save_err=rffi.RFFI_SAVE_ERRNO) vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, @@ -62,6 +100,7 @@ return CInterface(locals()) + class CInterface(object): def __init__(self, namespace): for k, v in namespace.iteritems(): diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -10,6 +10,8 @@ MAX_FUNC_NAME = 1023 +PLAT_WINDOWS = sys.platform == 'win32' + # ____________________________________________________________ # keep in sync with vmprof_stack.h @@ -122,7 +124,7 @@ self._gather_all_code_objs = gather_all_code_objs @jit.dont_look_inside - def enable(self, fileno, interval): + def enable(self, fileno, interval, memory=0, native=0): """Enable vmprof. Writes go to the given 'fileno'. The sampling interval is given by 'interval' as a number of seconds, as a float which must be smaller than 1.0. @@ -132,12 +134,16 @@ if self.is_enabled: raise VMProfError("vmprof is already enabled") - p_error = self.cintf.vmprof_init(fileno, interval, "pypy") + if PLAT_WINDOWS: + native = 0 # force disabled on Windows + lines = 0 # not supported on PyPy currently + + p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native) if p_error: raise VMProfError(rffi.charp2str(p_error)) self._gather_all_code_objs() - res = self.cintf.vmprof_enable() + res = self.cintf.vmprof_enable(memory, native) if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) self.is_enabled = True @@ -154,6 +160,7 @@ if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) + def _write_code_registration(self, uid, name): assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, ( "the name must be 'class:func_name:func_line:filename' " @@ -171,6 +178,23 @@ arguments given to the decorated function. 'result_class' is ignored (backward compatibility). + + ==================================== + TRANSLATION NOTE CALL THIS ONLY ONCE + ==================================== + + This function can only be called once during translation. + It generates a C function called __vmprof_eval_vmprof which is used by + the vmprof C source code and is bound as an extern function. + This is necessary while walking the native stack. + If you see __vmprof_eval_vmprof defined twice during + translation, read on: + + To remove this restriction do the following: + + *) Extend the macro IS_VMPROF_EVAL in the vmprof source repo to check several + sybmols. + *) Give each function provided to this decorator a unique symbol name in C """ if _hack_update_stack_untranslated: from rpython.rtyper.annlowlevel import llhelper @@ -198,16 +222,27 @@ unique_id = get_code_fn(*args)._vmprof_unique_id unique_id = rffi.cast(lltype.Signed, unique_id) # ^^^ removes the "known non-negative" hint for annotation + # + # Signals can occur at the two places (1) and (2), that will + # have added a stack entry, but the function __vmprof_eval_vmprof + # is not entered. This behaviour will swallow one Python stack frame + # + # Current fix: vmprof will discard this sample. (happens + # very infrequently) + # if not jit.we_are_jitted(): x = enter_code(unique_id) + # (1) signal here try: return func(*args) finally: + # (2) signal here leave_code(x) else: return decorated_jitted_function(unique_id, *args) decorated_function.__name__ = func.__name__ + '_rvmprof' + decorated_function.c_name = '__vmprof_eval_vmprof' return decorated_function return decorate @@ -216,7 +251,6 @@ def _was_registered(CodeClass): return hasattr(CodeClass, '_vmprof_unique_id') - _vmprof_instance = None @specialize.memo() diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -2,25 +2,30 @@ #ifdef RPYTHON_LL2CTYPES /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */ -#ifndef RPY_EXTERN -#define RPY_EXTERN RPY_EXPORTED -#endif -#ifdef _WIN32 -#define RPY_EXPORTED __declspec(dllexport) -#else -#define RPY_EXPORTED extern __attribute__((visibility("default"))) -#endif #else # include "common_header.h" # include "structdef.h" # include "src/threadlocal.h" # include "rvmprof.h" +# include "forwarddecl.h" #endif -#if defined(__unix__) || defined(__APPLE__) -#include "vmprof_main.h" + +#include "shared/vmprof_get_custom_offset.h" +#ifdef VMPROF_UNIX +#include "shared/vmprof_main.h" #else -#include "vmprof_main_win32.h" +#include "shared/vmprof_main_win32.h" #endif + + +#ifdef RPYTHON_LL2CTYPES +int IS_VMPROF_EVAL(void * ptr) { return 0; } +#else +int IS_VMPROF_EVAL(void * ptr) +{ + return ptr == __vmprof_eval_vmprof; +} +#endif diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,18 +1,41 @@ -#ifdef _WIN32 -typedef long intptr_t; +#pragma once + +#include "shared/vmprof.h" + +#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int)) + +#ifdef VMPROF_WINDOWS +#include "msiinttypes/inttypes.h" +#include "msiinttypes/stdint.h" #else -# include <stdint.h> +#include <inttypes.h> +#include <stdint.h> #endif -RPY_EXTERN char *vmprof_init(int, double, char *); +#ifndef RPY_EXTERN +#define RPY_EXTERN RPY_EXPORTED +#endif +#ifdef _WIN32 +#ifndef RPY_EXPORTED +#define RPY_EXPORTED __declspec(dllexport) +#endif +#else +#define RPY_EXPORTED extern __attribute__((visibility("default"))) +#endif + +RPY_EXTERN char *vmprof_init(int fd, double interval, int memory, + int lines, const char *interp_name, int native); RPY_EXTERN void vmprof_ignore_signals(int); -RPY_EXTERN int vmprof_enable(void); +RPY_EXTERN int vmprof_enable(int memory, int native); RPY_EXTERN int vmprof_disable(void); -RPY_EXTERN int vmprof_register_virtual_function(char *, long, int); +RPY_EXTERN int vmprof_register_virtual_function(char *, intptr_t, int); RPY_EXTERN void* vmprof_stack_new(void); RPY_EXTERN int vmprof_stack_append(void*, long); RPY_EXTERN long vmprof_stack_pop(void*); RPY_EXTERN void vmprof_stack_free(void*); RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t); +long vmprof_write_header_for_jit_addr(intptr_t *result, long n, + intptr_t addr, int max_depth); + #define RVMPROF_TRACEBACK_ESTIMATE_N(num_entries) (2 * (num_entries) + 4) diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -0,0 +1,354 @@ +/*[clinic input] +module _vmprof +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b443489e38f2be7d]*/ + +#define _GNU_SOURCE 1 + +#include <Python.h> +#include <frameobject.h> +#include <signal.h> + +#include "_vmprof.h" + +static volatile int is_enabled = 0; +static destructor Original_code_dealloc = 0; +static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0; +void dump_native_symbols(int fileno); + +#if VMPROF_UNIX +#include "trampoline.h" +#include "machine.h" +#include "symboltable.h" +#include "vmprof_main.h" +#else +#include "vmprof_main_win32.h" +#endif +#include "vmp_stack.h" + +#ifdef VMPROF_UNIX +#ifdef __clang__ +__attribute__((optnone)) +#elif defined(__GNUC__) +__attribute__((optimize("O1"))) +#endif +PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag) +{ +#ifdef X86_64 + register PY_STACK_FRAME_T * callee_saved asm("rbx"); +#elif defined(X86_32) + register PY_STACK_FRAME_T * callee_saved asm("edi"); +#else +# error "platform not supported" +#endif + + asm volatile( +#ifdef X86_64 + "movq %1, %0\t\n" +#elif defined(X86_32) + "mov %1, %0\t\n" +#else +# error "platform not supported" +#endif + : "=r" (callee_saved) + : "r" (f) ); + return _default_eval_loop(f, throwflag); +} +#endif + +static int emit_code_object(PyCodeObject *co) +{ + char buf[MAX_FUNC_NAME + 1]; + char *co_name, *co_filename; + int co_firstlineno; + int sz; +#if PY_MAJOR_VERSION >= 3 + co_name = PyUnicode_AsUTF8(co->co_name); + if (co_name == NULL) + return -1; + co_filename = PyUnicode_AsUTF8(co->co_filename); + if (co_filename == NULL) + return -1; +#else + co_name = PyString_AS_STRING(co->co_name); + co_filename = PyString_AS_STRING(co->co_filename); +#endif + co_firstlineno = co->co_firstlineno; + + sz = snprintf(buf, MAX_FUNC_NAME / 2, "py:%s", co_name); + if (sz < 0) sz = 0; + if (sz > MAX_FUNC_NAME / 2) sz = MAX_FUNC_NAME / 2; + snprintf(buf + sz, MAX_FUNC_NAME / 2, ":%d:%s", co_firstlineno, + co_filename); + return vmprof_register_virtual_function(buf, CODE_ADDR_TO_UID(co), 500000); +} + +static int _look_for_code_object(PyObject *o, void *all_codes) +{ + if (PyCode_Check(o) && !PySet_Contains((PyObject *)all_codes, o)) { + Py_ssize_t i; + PyCodeObject *co = (PyCodeObject *)o; + if (emit_code_object(co) < 0) + return -1; + if (PySet_Add((PyObject *)all_codes, o) < 0) + return -1; + + /* as a special case, recursively look for and add code + objects found in the co_consts. The problem is that code + objects are not created as GC-aware in CPython, so we need + to hack like this to hope to find most of them. + */ + i = PyTuple_Size(co->co_consts); + while (i > 0) { + --i; + if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i), + all_codes) < 0) + return -1; + } + } + return 0; +} + +static void emit_all_code_objects(void) +{ + PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL; + Py_ssize_t i, size; + + gc_module = PyImport_ImportModuleNoBlock("gc"); + if (gc_module == NULL) + goto error; + + lst = PyObject_CallMethod(gc_module, "get_objects", ""); + if (lst == NULL || !PyList_Check(lst)) + goto error; + + all_codes = PySet_New(NULL); + if (all_codes == NULL) + goto error; + + size = PyList_GET_SIZE(lst); + for (i = 0; i < size; i++) { + PyObject *o = PyList_GET_ITEM(lst, i); + if (o->ob_type->tp_traverse && + o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes) + < 0) + goto error; + } + + error: + Py_XDECREF(all_codes); + Py_XDECREF(lst); + Py_XDECREF(gc_module); +} + +static void cpyprof_code_dealloc(PyObject *co) +{ + if (is_enabled) { + emit_code_object((PyCodeObject *)co); + /* xxx error return values are ignored */ + } + Original_code_dealloc(co); +} + +static PyObject *enable_vmprof(PyObject* self, PyObject *args) +{ + int fd; + int memory = 0; + int lines = 0; + int native = 0; + double interval; + char *p_error; + + if (!PyArg_ParseTuple(args, "id|iii", &fd, &interval, &memory, &lines, &native)) { + return NULL; + } + assert(fd >= 0 && "file descripter provided to vmprof must not" \ + " be less then zero."); + + if (is_enabled) { + PyErr_SetString(PyExc_ValueError, "vmprof is already enabled"); + return NULL; + } + + vmp_profile_lines(lines); + + if (!Original_code_dealloc) { + Original_code_dealloc = PyCode_Type.tp_dealloc; + PyCode_Type.tp_dealloc = &cpyprof_code_dealloc; + } + + p_error = vmprof_init(fd, interval, memory, lines, "cpython", native); + if (p_error) { + PyErr_SetString(PyExc_ValueError, p_error); + return NULL; + } + + if (vmprof_enable(memory, native) < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + is_enabled = 1; + + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject * +disable_vmprof(PyObject *module, PyObject *noarg) +{ + if (!is_enabled) { + PyErr_SetString(PyExc_ValueError, "vmprof is not enabled"); + return NULL; + } + is_enabled = 0; + vmprof_ignore_signals(1); + emit_all_code_objects(); + + if (vmprof_disable() < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + if (PyErr_Occurred()) + return NULL; + Py_INCREF(Py_None); + return Py_None; +} + +static PyObject * +write_all_code_objects(PyObject *module, PyObject *noargs) +{ + if (!is_enabled) { + PyErr_SetString(PyExc_ValueError, "vmprof is not enabled"); + return NULL; + } + emit_all_code_objects(); + if (PyErr_Occurred()) + return NULL; + Py_INCREF(Py_None); + return Py_None; +} + + +static PyObject * +sample_stack_now(PyObject *module, PyObject * args) +{ + PyThreadState * tstate = NULL; + PyObject * list = NULL; + int i; + int entry_count; + void ** m; + void * routine_ip; + long skip = 0; + + // stop any signal to occur + vmprof_ignore_signals(1); + + list = PyList_New(0); + if (list == NULL) { + goto error; + } + + if (!PyArg_ParseTuple(args, "l", &skip)) { + goto error; + } + + tstate = PyGILState_GetThisThreadState(); + m = (void**)malloc(SINGLE_BUF_SIZE); + if (m == NULL) { + PyErr_SetString(PyExc_MemoryError, "could not allocate buffer for stack trace"); + vmprof_ignore_signals(0); + return NULL; + } + entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, skip, 0); + + for (i = 0; i < entry_count; i++) { + routine_ip = m[i]; + PyList_Append(list, PyLong_NEW((ssize_t)routine_ip)); + } + + free(m); + + Py_INCREF(list); + + vmprof_ignore_signals(0); + return list; +error: + Py_DECREF(list); + Py_INCREF(Py_None); + + vmprof_ignore_signals(0); + return Py_None; +} + +#ifdef VMP_SUPPORTS_NATIVE_PROFILING +static PyObject * +resolve_addr(PyObject *module, PyObject *args) { + long long addr; + PyObject * o_name = NULL; + PyObject * o_lineno = NULL; + PyObject * o_srcfile = NULL; + char name[128]; + int lineno = 0; + char srcfile[256]; + + if (!PyArg_ParseTuple(args, "L", &addr)) { + return NULL; + } + name[0] = '\x00'; + srcfile[0] = '-'; + srcfile[1] = '\x00'; + if (vmp_resolve_addr((void*)addr, name, 128, &lineno, srcfile, 256) != 0) { + goto error; + } + + o_name = PyStr_NEW(name); + if (o_name == NULL) goto error; + o_lineno = PyLong_NEW(lineno); + if (o_lineno == NULL) goto error; + o_srcfile = PyStr_NEW(srcfile); + if (o_srcfile == NULL) goto error; + // + return PyTuple_Pack(3, o_name, o_lineno, o_srcfile); +error: + Py_XDECREF(o_name); + Py_XDECREF(o_lineno); + Py_XDECREF(o_srcfile); + + Py_INCREF(Py_None); + return Py_None; +} +#endif + +static PyMethodDef VMProfMethods[] = { + {"enable", enable_vmprof, METH_VARARGS, "Enable profiling."}, + {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."}, + {"write_all_code_objects", write_all_code_objects, METH_NOARGS, + "Write eagerly all the IDs of code objects"}, + {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"}, +#ifdef VMP_SUPPORTS_NATIVE_PROFILING + {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"}, +#endif + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef VmprofModule = { + PyModuleDef_HEAD_INIT, + "_vmprof", + "", // doc + -1, // size + VMProfMethods +}; + +PyMODINIT_FUNC PyInit__vmprof(void) +{ + return PyModule_Create(&VmprofModule); +} +#else +PyMODINIT_FUNC init_vmprof(void) +{ + Py_InitModule("_vmprof", VMProfMethods); +} +#endif diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.h b/rpython/rlib/rvmprof/src/shared/_vmprof.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.h @@ -0,0 +1,53 @@ +#pragma once + +#include "vmprof.h" + +#ifdef VMPROF_WINDOWS + +#include <Python.h> +// CPython 3.6 defines all the inttypes for us, we do not need the msiinttypes +// library for that version or any newer! +#if (PY_VERSION_HEX < 0x3060000) +#include "msiinttypes/inttypes.h" +#include "msiinttypes/stdint.h" +#endif + +#else +#include <inttypes.h> +#include <stdint.h> +#include <stddef.h> +#endif + +/** + * This whole setup is very strange. There was just one C file called + * _vmprof.c which included all *.h files to copy code. Unsure what + * the goal was with this design, but I assume it just 'GREW' + * + * Thus I'm (plan_rich) slowly trying to separate this. *.h files + * should not have complex implementations (all of them currently have them) + */ + + +#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int)) + +#define ROUTINE_IS_PYTHON(RIP) ((unsigned long long)RIP & 0x1) == 0 +#define ROUTINE_IS_C(RIP) ((unsigned long long)RIP & 0x1) == 1 + +/* This returns the address of the code object + as the identifier. The mapping from identifiers to string + representations of the code object is done elsewhere, namely: + + * If the code object dies while vmprof is enabled, + PyCode_Type.tp_dealloc will emit it. (We don't handle nicely + for now the case where several code objects are created and die + at the same memory address.) + + * When _vmprof.disable() is called, then we look around the + process for code objects and emit all the ones that we can + find (which we hope is very close to 100% of them). +*/ +#define CODE_ADDR_TO_UID(co) (((intptr_t)(co))) + +#define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0 + +int vmp_write_all(const char *buf, size_t bufsize); diff --git a/rpython/rlib/rvmprof/src/shared/compat.c b/rpython/rlib/rvmprof/src/shared/compat.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/shared/compat.c @@ -0,0 +1,140 @@ +#include "compat.h" + +#include <string.h> +#include <assert.h> +#if VMPROF_WINDOWS +#define WIN32_LEAN_AND_MEAN +#include <Windows.h> +#else +#include <time.h> +#include <sys/time.h> +#endif + +static int _vmp_profile_fileno = -1; + +int vmp_profile_fileno(void) { + return _vmp_profile_fileno; +} +void vmp_set_profile_fileno(int fileno) { + _vmp_profile_fileno = fileno; +} + +#ifndef VMPROF_WINDOWS +int vmp_write_all(const char *buf, size_t bufsize) +{ + ssize_t count; + if (_vmp_profile_fileno == -1) { + return -1; + } + while (bufsize > 0) { + count = write(_vmp_profile_fileno, buf, bufsize); + if (count <= 0) + return -1; /* failed */ + buf += count; + bufsize -= count; + } + return 0; +} +#endif + +int vmp_write_meta(const char * key, const char * value) +{ + char marker = MARKER_META; + long x = (long)strlen(key); + vmp_write_all(&marker, 1); + vmp_write_all((char*)&x, sizeof(long)); + vmp_write_all(key, x); + x = (long)strlen(value); + vmp_write_all((char*)&x, sizeof(long)); + vmp_write_all(value, x); + return 0; +} + +/** + * Write the time and zone now. + */ + +struct timezone_buf { + int64_t tv_sec; + int64_t tv_usec; +}; +#define __SIZE (1+sizeof(struct timezone_buf)+8) + +#ifdef VMPROF_UNIX +int vmp_write_time_now(int marker) { + char buffer[__SIZE]; + struct timezone_buf buf; + + (void)memset(&buffer, 0, __SIZE); + + assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \ + "marker must be either a trailer or time_n_zone!"); + + struct timeval tv; + time_t now; + struct tm tm; + + + /* copy over to the struct */ + if (gettimeofday(&tv, NULL) != 0) { + return -1; + } + if (time(&now) == (time_t)-1) { + return -1; + } + if (localtime_r(&now, &tm) == NULL) { + return -1; + } + buf.tv_sec = tv.tv_sec; + buf.tv_usec = tv.tv_usec; + strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8); + + buffer[0] = marker; + (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf)); + vmp_write_all(buffer, __SIZE); + return 0; +} +#endif + +#ifdef VMPROF_WINDOWS +int vmp_write_time_now(int marker) { + char buffer[__SIZE]; + struct timezone_buf buf; + + /** + * http://stackoverflow.com/questions/10905892/equivalent-of-gettimeday-for-windows + */ + + // Note: some broken versions only have 8 trailing zero's, the correct + // epoch has 9 trailing zero's + static const uint64_t EPOCH = ((uint64_t) 116444736000000000ULL); + + SYSTEMTIME system_time; + FILETIME file_time; + uint64_t time; + + (void)memset(&buffer, 0, __SIZE); + + assert((marker == MARKER_TRAILER || marker == MARKER_TIME_N_ZONE) && \ + "marker must be either a trailer or time_n_zone!"); + + + GetSystemTime( &system_time ); + SystemTimeToFileTime( &system_time, &file_time ); + time = ((uint64_t)file_time.dwLowDateTime ) ; + time += ((uint64_t)file_time.dwHighDateTime) << 32; + + buf.tv_sec = ((time - EPOCH) / 10000000L); + buf.tv_usec = (system_time.wMilliseconds * 1000); + + // time zone not implemented on windows + memset(((char*)buffer)+__SIZE-8, 0, 8); + (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3); + + buffer[0] = marker; + (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf)); + vmp_write_all(buffer, __SIZE); + return 0; +} +#endif +#undef __SIZE diff --git a/rpython/rlib/rvmprof/src/shared/compat.h b/rpython/rlib/rvmprof/src/shared/compat.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/shared/compat.h @@ -0,0 +1,25 @@ +#pragma once + +#include "vmprof.h" + +#ifndef RPYTHON_VMPROF +# if PY_MAJOR_VERSION >= 3 + #define PyStr_AS_STRING PyBytes_AS_STRING + #define PyStr_GET_SIZE PyBytes_GET_SIZE + #define PyStr_NEW PyUnicode_FromString + #define PyLong_NEW PyLong_FromSsize_t _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit