Author: Matti Picus <matti.pi...@gmail.com> Branch: cpyext-obj-stealing Changeset: r91133:6b3b57cdab23 Date: 2017-04-26 06:45 +0300 http://bitbucket.org/pypy/pypy/changeset/6b3b57cdab23/
Log: merge default into branch diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -49,6 +49,11 @@ ^rpython/translator/goal/target.+-c$ ^rpython/translator/goal/.+\.exe$ ^rpython/translator/goal/.+\.dll$ +^rpython/rlib/rvmprof/src/shared/libbacktrace/Makefile$ +^rpython/rlib/rvmprof/src/shared/libbacktrace/config.guess$ +^rpython/rlib/rvmprof/src/shared/libbacktrace/config.h$ +^rpython/rlib/rvmprof/src/shared/libbacktrace/config.log$ +^rpython/rlib/rvmprof/src/shared/libbacktrace/config.status$ ^pypy/goal/pypy-translation-snapshot$ ^pypy/goal/pypy-c ^pypy/goal/.+\.exe$ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -28,3 +28,5 @@ Performance tweaks in the x86 JIT-generated machine code: rarely taken blocks are moved off-line. Also, the temporary register used to contain large constants is reused across instructions. + +.. branch: vmprof-0.4.4 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -27,7 +27,7 @@ made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link was checked in Nov 2016). Note that the compiler suite will be installed in ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``. -Using a current version of ``setuptools`` will be able to find it there. For +A current version of ``setuptools`` will be able to find it there. For Windows 10, you must right-click the download, and under ``Properties`` -> ``Compatibility`` mark it as ``Run run this program in comatibility mode for`` ``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``, @@ -40,8 +40,9 @@ Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using Visual Studio 2008, Express -Edition. Other configurations may work as well. +We routinely test translation using v9, also known as Visual Studio 2008. +Our buildbot is still using the Express Edition, not the compiler noted above. +Other configurations may work as well. The translation scripts will set up the appropriate environment variables for the compiler, so you do not need to run vcvars before translation. @@ -117,6 +118,9 @@ ----------------------------------------------------------- Download the versions of all the external packages from +https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip +(for post-5.7.1 builds) with sha256 checksum +``f1510452293f22e84d6059464e11f4c62ffd0e2ee97a52be9195bec8a70c6dce`` or https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip (for 2.4 release and later) or https://bitbucket.org/pypy/pypy/downloads/local.zip @@ -173,13 +177,14 @@ The zlib compression library ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Download http://www.gzip.org/zlib/zlib-1.2.3.tar.gz and extract it in -the base directory. Then compile as a static library:: +Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in +the base directory. Then compile:: - cd zlib-1.2.3 + cd zlib-1.2.11 nmake -f win32\Makefile.msc copy zlib.lib <somewhere in LIB> copy zlib.h zconf.h <somewhere in INCLUDE> + copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes) The bz2 compression library @@ -198,22 +203,23 @@ PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility. +The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. The expat XML parser ~~~~~~~~~~~~~~~~~~~~ Download the source code of expat on sourceforge: -http://sourceforge.net/projects/expat/ and extract it in the base directory. -Version 2.1.0 is known to pass tests. Then open the project file ``expat.dsw`` +https://github.com/libexpat/libexpat/releases and extract it in the base directory. +Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw`` with Visual Studio; follow the instruction for converting the project files, switch to the "Release" configuration, use the ``expat_static`` project, -reconfigure the runtime for Multi-threaded DLL (/MD) and build. +reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for +the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes) -Then, copy the file ``win32\bin\release\libexpat.lib`` somewhere in somewhere -in LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` somewhere in -INCLUDE. +Then, copy the file ``win32\bin\release\libexpat.lib`` into +LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in +INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH. The OpenSSL library @@ -222,15 +228,13 @@ OpenSSL needs a Perl interpreter to configure its makefile. You may use the one distributed by ActiveState, or the one from cygwin.:: - svn export http://svn.python.org/projects/external/openssl-1.0.1i - cd openssl-1.0.1i + svn export http://svn.python.org/projects/external/openssl-1.0.2k + cd openssl-1.0.2k perl Configure VC-WIN32 no-idea no-mdc2 ms\do_ms.bat nmake -f ms\nt.mak install - -Then, copy the files ``out32\*.lib`` somewhere in -somewhere in LIB, and the entire ``include\openssl`` directory as-is somewhere -in INCLUDE. + copy out32\*.lib <somewhere in LIB> + xcopy /S include\openssl <somewhere in INCLUDE> TkInter module support @@ -263,6 +267,9 @@ http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature http://tukaani.org/xz/xz-5.0.5-windows.zip.sig +Then copy the headers to the include directory, rename ``liblzma.a`` to +``lzma.lib`` and copy it to the lib directory + Using the mingw compiler ------------------------ diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -307,6 +307,9 @@ config.translation.jit = True if config.translation.sandbox: + assert 0, ("--sandbox is not tested nor maintained. If you " + "really want to try it anyway, remove this line in " + "pypy/goal/targetpypystandalone.py.") config.objspace.lonepycfiles = False if config.objspace.usemodules.cpyext: diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py --- a/pypy/module/__builtin__/app_functional.py +++ b/pypy/module/__builtin__/app_functional.py @@ -2,7 +2,6 @@ Plain Python definition of the builtin functions oriented towards functional programming. """ -from __future__ import with_statement import operator from __pypy__ import resizelist_hint, newlist_hint from __pypy__ import specialized_zip_2_lists diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -311,11 +311,7 @@ def repr(self): space = self.space - ec = space.getexecutioncontext() - w_currently_in_repr = ec._py_repr - if w_currently_in_repr is None: - w_currently_in_repr = ec._py_repr = space.newdict() - return dequerepr(space, w_currently_in_repr, self) + return dequerepr(space, space.get_objects_in_repr(), self) @specialize.arg(2) def compare(self, w_other, op): @@ -428,16 +424,15 @@ app = gateway.applevel(""" def dequerepr(currently_in_repr, d): 'The app-level part of repr().' - deque_id = id(d) - if deque_id in currently_in_repr: + if d in currently_in_repr: listrepr = '[...]' else: - currently_in_repr[deque_id] = 1 + currently_in_repr[d] = 1 try: listrepr = "[" + ", ".join([repr(x) for x in d]) + ']' finally: try: - del currently_in_repr[deque_id] + del currently_in_repr[d] except: pass if d.maxlen is None: diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -12,6 +12,8 @@ 'enable': 'interp_vmprof.enable', 'disable': 'interp_vmprof.disable', 'write_all_code_objects': 'interp_vmprof.write_all_code_objects', + 'is_enabled': 'interp_vmprof.is_enabled', + 'get_profile_path': 'interp_vmprof.get_profile_path', 'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError', } diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -4,6 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.interpreter.baseobjspace import W_Root from rpython.rlib import rvmprof, jit +from pypy.interpreter.error import oefmt # ____________________________________________________________ @@ -82,3 +83,16 @@ rvmprof.disable() except rvmprof.VMProfError as e: raise VMProfError(space, e) + +def is_enabled(space): + return space.newbool(rvmprof.is_enabled()) + +def get_profile_path(space): + path = rvmprof.get_profile_path(space) + if path is None: + # profiling is not enabled + return space.w_None + if path == "": + # Indicates an error! Assume platform does not implement the function call + raise oefmt(space.w_NotImplementedError, "platform not implemented") + return space.newtext(path) diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -94,3 +94,24 @@ raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300, 0, 0, 0) NaN = (1e300*1e300) / (1e300*1e300) raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN, 0, 0, 0) + + def test_is_enabled(self): + import _vmprof + tmpfile = open(self.tmpfilename, 'wb') + assert _vmprof.is_enabled() is False + _vmprof.enable(tmpfile.fileno(), 0.01, 0, 0, 0) + assert _vmprof.is_enabled() is True + _vmprof.disable() + assert _vmprof.is_enabled() is False + + def test_get_profile_path(self): + import _vmprof + tmpfile = open(self.tmpfilename, 'wb') + assert _vmprof.get_profile_path() is None + _vmprof.enable(tmpfile.fileno(), 0.01, 0, 0, 0) + path = _vmprof.get_profile_path() + if path != tmpfile.name: + with open(path, "rb") as fd1: + assert fd1.read() == tmpfile.read() + _vmprof.disable() + assert _vmprof.get_profile_path() is None diff --git a/pypy/module/cpyext/test/issue2482.c b/pypy/module/cpyext/test/issue2482.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/issue2482.c @@ -0,0 +1,107 @@ + +#include "Python.h" +//#define ISSUE_2482 + +typedef struct { + PyObject_HEAD + // Some extra storage: + char blank[500]; +} instance; + +static PyObject * get_basicsize(PyObject *self, PyObject * arg) +{ + return PyLong_FromLong(((PyTypeObject*)arg)->tp_basicsize); +} + +const char *name = "issue2482_object"; +static +PyObject *make_object_base_type(void) { + + PyHeapTypeObject *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) return NULL; + + PyTypeObject *type = &heap_type->ht_type; + type->tp_name = name; +#ifdef ISSUE_2482 + type->tp_base = &PyBaseObject_Type; /*fails */ +#else + type->tp_base = &PyType_Type; +#endif + type->tp_basicsize = sizeof(instance); + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; + + if (PyType_Ready(type) < 0) + return NULL; + + return (PyObject *) heap_type; +}; + +static PyMethodDef issue2482_functions[] = { + {"get_basicsize", (PyCFunction)get_basicsize, METH_O, NULL}, + {NULL, NULL} /* Sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "issue2482", + "Module Doc", + -1, + issue2482_functions, + NULL, + NULL, + NULL, + NULL, +}; +#define INITERROR return NULL + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +PyInit_issue2482(void) + +#else + +#define INITERROR return + +/* Initialize this module. */ +#ifdef __GNUC__ +extern __attribute__((visibility("default"))) +#else +extern __declspec(dllexport) +#endif + +PyMODINIT_FUNC +initissue2482(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("issue2482", issue2482_functions); +#endif + if (module == NULL) + INITERROR; + + PyHeapTypeObject *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) INITERROR; + + PyTypeObject *type = &heap_type->ht_type; + type->tp_name = name; + + PyObject *base = make_object_base_type(); + if (! base) INITERROR; + Py_INCREF(base); + type->tp_base = (PyTypeObject *) base; + type->tp_basicsize = ((PyTypeObject *) base)->tp_basicsize; + type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_CHECKTYPES; + + if (PyType_Ready(type) < 0) INITERROR; + + PyModule_AddObject(module, name, (PyObject *) type); +}; diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1256,3 +1256,27 @@ # doesn't call tp_getattr at all, also on CPython raises(AttributeError, type(module.gettype1).__getattribute__, module.gettype1, 'dcBA') + + def test_multiple_inheritance_tp_basicsize(self): + module = self.import_module(name='issue2482') + + class PyBase(object): + pass + + basesize = module.get_basicsize(PyBase) + + CBase = module.issue2482_object + class A(CBase, PyBase): + def __init__(self, i): + CBase.__init__(self) + PyBase.__init__(self) + + class B(PyBase, CBase): + def __init__(self, i): + PyBase.__init__(self) + CBase.__init__(self) + + Asize = module.get_basicsize(A) + Bsize = module.get_basicsize(B) + assert Asize == Bsize + assert Asize > basesize diff --git a/pypy/module/test_lib_pypy/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py --- a/pypy/module/test_lib_pypy/test_md5_extra.py +++ b/pypy/module/test_lib_pypy/test_md5_extra.py @@ -68,9 +68,7 @@ w_m2c = space.call_method(w_m2, 'copy') # Update and compare... - for i in range(len(cases)): - message = cases[i][0] - + for message in cases: m1c.update(message) d1 = m1c.hexdigest() @@ -147,13 +145,10 @@ "57edf4a22be3c955ac49da2e2107b67a"), ) - for i in range(len(cases)): - res = self.compare(cases[i][0]) + for message, expectedResult in cases: + res = self.compare(message) if res is not None: d1, d2 = res - message, expectedResult = cases[i][0], None - if len(cases[i]) == 2: - expectedResult = cases[i][1] self.print_diff(message, d1, d2, expectedResult) assert res is None @@ -182,27 +177,25 @@ "123456789 123456789 123456789 12345678", ) - for i in range(len(cases)): - res = self.compare(cases[i][0]) + for message in cases: + res = self.compare(message) if res is not None: d1, d2 = res - message = cases[i][0] self.print_diff(message, d1, d2) assert res is None def test3(self): """Test cases with long messages (can take a while).""" cases = ( - (2**10*'a',), - (2**10*'abcd',), - #(2**20*'a',), # 1 MB, takes about 160 sec. on a 233 Mhz Pentium. + 2**10*'a', + 2**10*'abcd', + #2**20*'a', # 1 MB, takes about 160 sec. on a 233 Mhz Pentium. ) - for i in range(len(cases)): - res = self.compare(cases[i][0]) + for message in cases: + res = self.compare(message) if res is not None: d1, d2 = res - message = cases[i][0] self.print_diff(message, d1, d2) assert res is None diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -139,11 +139,7 @@ init_or_update(space, self, __args__, 'dict') def descr_repr(self, space): - ec = space.getexecutioncontext() - w_currently_in_repr = ec._py_repr - if w_currently_in_repr is None: - w_currently_in_repr = ec._py_repr = space.newdict() - return dictrepr(space, w_currently_in_repr, self) + return dictrepr(space, space.get_objects_in_repr(), self) def descr_eq(self, space, w_other): if space.is_w(self, w_other): @@ -468,10 +464,9 @@ def dictrepr(currently_in_repr, d): if len(d) == 0: return "{}" - dict_id = id(d) - if dict_id in currently_in_repr: + if d in currently_in_repr: return '{...}' - currently_in_repr[dict_id] = 1 + currently_in_repr[d] = 1 try: items = [] # XXX for now, we cannot use iteritems() at app-level because @@ -482,7 +477,7 @@ return "{" + ', '.join(items) + "}" finally: try: - del currently_in_repr[dict_id] + del currently_in_repr[d] except: pass ''', filename=__file__) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -442,11 +442,7 @@ def descr_repr(self, space): if self.length() == 0: return space.newtext('[]') - ec = space.getexecutioncontext() - w_currently_in_repr = ec._py_repr - if w_currently_in_repr is None: - w_currently_in_repr = ec._py_repr = space.newdict() - return listrepr(space, w_currently_in_repr, self) + return listrepr(space, space.get_objects_in_repr(), self) def descr_eq(self, space, w_other): if not isinstance(w_other, W_ListObject): @@ -2038,15 +2034,14 @@ app = applevel(""" def listrepr(currently_in_repr, l): 'The app-level part of repr().' - list_id = id(l) - if list_id in currently_in_repr: + if l in currently_in_repr: return '[...]' - currently_in_repr[list_id] = 1 + currently_in_repr[l] = 1 try: return "[" + ", ".join([repr(x) for x in l]) + ']' finally: try: - del currently_in_repr[list_id] + del currently_in_repr[l] except: pass """, filename=__file__) diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -127,6 +127,14 @@ ec._py_repr = None return ec + def get_objects_in_repr(self): + from pypy.module.__pypy__.interp_identitydict import W_IdentityDict + ec = self.getexecutioncontext() + w_currently_in_repr = ec._py_repr + if w_currently_in_repr is None: + w_currently_in_repr = ec._py_repr = W_IdentityDict(self) + return w_currently_in_repr + def gettypefor(self, cls): return self.gettypeobject(cls.typedef) diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -165,11 +165,7 @@ _initialize_set(space, self, w_iterable) def descr_repr(self, space): - ec = space.getexecutioncontext() - w_currently_in_repr = ec._py_repr - if w_currently_in_repr is None: - w_currently_in_repr = ec._py_repr = space.newdict() - return setrepr(space, w_currently_in_repr, self) + return setrepr(space, space.get_objects_in_repr(), self) def descr_cmp(self, space, w_other): if space.is_w(space.type(self), space.type(w_other)): @@ -1706,15 +1702,14 @@ app = gateway.applevel(""" def setrepr(currently_in_repr, s): 'The app-level part of repr().' - set_id = id(s) - if set_id in currently_in_repr: + if s in currently_in_repr: return '%s(...)' % (s.__class__.__name__,) - currently_in_repr[set_id] = 1 + currently_in_repr[s] = 1 try: return '%s(%s)' % (s.__class__.__name__, [x for x in s]) finally: try: - del currently_in_repr[set_id] + del currently_in_repr[s] except: pass """, filename=__file__) diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -75,6 +75,7 @@ BaseRootWalker.__init__(self, gctransformer) # NB. 'self' is frozen, but we can use self.gcdata to store state gcdata = self.gcdata + gcdata.can_look_at_partial_stack = True def incr_stack(n): top = gcdata.root_stack_top @@ -109,7 +110,19 @@ BaseRootWalker.setup_root_walker(self) def walk_stack_roots(self, collect_stack_root, is_minor=False): + # Note that if we're the first minor collection after a thread + # switch, then we also need to disable the 'is_minor' + # optimization. The reason is subtle: we need to walk the whole + # stack because otherwise, we can be in the middle of an + # incremental major collection, and the new stack was just moved + # off a ShadowStackRef object (gctransform/shadowstack.py) which + # was not seen yet. We might completely miss some old objects + # from the parts of that stack that are skipped by this is_minor + # optimization. gcdata = self.gcdata + if is_minor and not gcdata.can_look_at_partial_stack: + is_minor = False + gcdata.can_look_at_partial_stack = True walk_stack_root(self.invoke_collect_stack_root, collect_stack_root, None, gcdata.root_stack_base, gcdata.root_stack_top, is_minor=is_minor) @@ -298,6 +311,7 @@ "restore_state_from: broken shadowstack") self.gcdata.root_stack_base = shadowstackref.base self.gcdata.root_stack_top = shadowstackref.top + self.gcdata.can_look_at_partial_stack = False self._cleanup(shadowstackref) def start_fresh_new_state(self): diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -3,6 +3,7 @@ from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code, MAX_FUNC_NAME from rpython.rlib.rvmprof.rvmprof import _was_registered from rpython.rlib.rvmprof.cintf import VMProfPlatformUnsupported +from rpython.rtyper.lltypesystem import rffi # # See README.txt. @@ -37,3 +38,21 @@ def disable(): _get_vmprof().disable() + +def is_enabled(): + vmp = _get_vmprof() + return vmp.is_enabled + +def get_profile_path(space): + vmp = _get_vmprof() + if not vmp.is_enabled: + return None + + with rffi.scoped_alloc_buffer(4096) as buf: + length = vmp.cintf.vmprof_get_profile_path(buf.raw, buf.size) + if length == -1: + return "" + return buf.str(length) + + return None + diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -107,6 +107,10 @@ lltype.Signed, compilation_info=eci, _nowrapper=True) + vmprof_get_profile_path = rffi.llexternal("vmprof_get_profile_path", [rffi.CCHARP, lltype.Signed], + lltype.Signed, compilation_info=eci, + _nowrapper=True) + return CInterface(locals()) diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -178,23 +178,6 @@ arguments given to the decorated function. 'result_class' is ignored (backward compatibility). - - ==================================== - TRANSLATION NOTE CALL THIS ONLY ONCE - ==================================== - - This function can only be called once during translation. - It generates a C function called __vmprof_eval_vmprof which is used by - the vmprof C source code and is bound as an extern function. - This is necessary while walking the native stack. - If you see __vmprof_eval_vmprof defined twice during - translation, read on: - - To remove this restriction do the following: - - *) Extend the macro IS_VMPROF_EVAL in the vmprof source repo to check several - sybmols. - *) Give each function provided to this decorator a unique symbol name in C """ if _hack_update_stack_untranslated: from rpython.rtyper.annlowlevel import llhelper diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -29,3 +29,9 @@ return ptr == __vmprof_eval_vmprof; } #endif + + +long vmprof_get_profile_path(const char * buffer, long size) +{ + return vmp_fd_to_path(vmp_profile_fileno(), buffer, size); +} diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -35,6 +35,7 @@ RPY_EXTERN long vmprof_stack_pop(void*); RPY_EXTERN void vmprof_stack_free(void*); RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t); +RPY_EXTERN long vmprof_get_profile_path(const char *, long); long vmprof_write_header_for_jit_addr(intptr_t *result, long n, intptr_t addr, int max_depth); diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c --- a/rpython/rlib/rvmprof/src/shared/_vmprof.c +++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c @@ -1,8 +1,7 @@ -/*[clinic input] -module _vmprof -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b443489e38f2be7d]*/ - +/** + * This file is the CPython module _vmprof. It does not share code + * with PyPy. PyPy's _vmprof module is included in the main repo. + */ #define _GNU_SOURCE 1 #include <Python.h> @@ -14,7 +13,7 @@ static volatile int is_enabled = 0; static destructor Original_code_dealloc = 0; static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0; -void dump_native_symbols(int fileno); +void vmp_scan_profile(int fileno, int dump_native, void *all_code_uids); #if VMPROF_UNIX #include "trampoline.h" @@ -109,6 +108,33 @@ return 0; } +static int _look_for_code_object_seen(PyObject *o, void *all_codes) +{ + if (PyCode_Check(o) && PySet_GET_SIZE(all_codes)) { + Py_ssize_t i; + PyCodeObject *co = (PyCodeObject *)o; + PyObject *uid_co = PyLong_FromVoidPtr((void*)CODE_ADDR_TO_UID(co)); + int check = PySet_Discard(all_codes, uid_co); + + Py_CLEAR(uid_co); + + if (check < 0) + return -1; + + if (check && emit_code_object(co) < 0) + return -1; + + i = PyTuple_Size(co->co_consts); + while (i > 0) { + --i; + if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i), + all_codes) < 0) + return -1; + } + } + return 0; +} + static void emit_all_code_objects(void) { PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL; @@ -118,6 +144,48 @@ if (gc_module == NULL) goto error; + // lst contains all objects that are known by the gc + lst = PyObject_CallMethod(gc_module, "get_objects", ""); + if (lst == NULL || !PyList_Check(lst)) + goto error; + + // the set only includes the code objects found in the profile + all_codes = PySet_New(NULL); + if (all_codes == NULL) + goto error; + + size = PyList_GET_SIZE(lst); + for (i = 0; i < size; i++) { + PyObject *o = PyList_GET_ITEM(lst, i); + if (o->ob_type->tp_traverse && + o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes) + < 0) + goto error; + } + + error: + Py_XDECREF(all_codes); + Py_XDECREF(lst); + Py_XDECREF(gc_module); +} + +static int add_code_addr(void *all_code_uids, void *addr) +{ + PyObject *co_uid = PyLong_FromVoidPtr(addr); + int check = PySet_Add((PyObject*) all_code_uids, co_uid); + Py_CLEAR(co_uid); + return check; +} + +static void emit_all_code_objects_seen(int fileno) +{ + PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL; + Py_ssize_t i, size; + + gc_module = PyImport_ImportModuleNoBlock("gc"); + if (gc_module == NULL) + goto error; + lst = PyObject_CallMethod(gc_module, "get_objects", ""); if (lst == NULL || !PyList_Check(lst)) goto error; @@ -126,16 +194,21 @@ if (all_codes == NULL) goto error; + // fill up all_codes with every code object found in the profile + vmp_scan_profile(fileno, 0, all_codes); + + // intersect the list with the set and dump only the code objects + // found in the set! size = PyList_GET_SIZE(lst); for (i = 0; i < size; i++) { PyObject *o = PyList_GET_ITEM(lst, i); if (o->ob_type->tp_traverse && - o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes) - < 0) + o->ob_type->tp_traverse(o, _look_for_code_object_seen, (void *) all_codes) + < 0) goto error; } - error: + error: Py_XDECREF(all_codes); Py_XDECREF(lst); Py_XDECREF(gc_module); @@ -162,8 +235,16 @@ if (!PyArg_ParseTuple(args, "id|iii", &fd, &interval, &memory, &lines, &native)) { return NULL; } - assert(fd >= 0 && "file descripter provided to vmprof must not" \ - " be less then zero."); + + if (write(fd, NULL, 0) != 0) { + PyErr_SetString(PyExc_ValueError, "file descriptor must be writeable"); + return NULL; + } + + if ((read(fd, NULL, 0) != 0) && (native != 0)) { + PyErr_SetString(PyExc_ValueError, "file descriptor must be readable for native profiling"); + return NULL; + } if (is_enabled) { PyErr_SetString(PyExc_ValueError, "vmprof is already enabled"); @@ -194,16 +275,53 @@ return Py_None; } +static PyObject * vmp_is_enabled(PyObject *module, PyObject *noargs) { + if (is_enabled) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * -disable_vmprof(PyObject *module, PyObject *noarg) +disable_vmprof(PyObject *module, PyObject *args) { + int fd = vmp_profile_fileno(); + int only_needed = 0; + + if (!PyArg_ParseTuple(args, "|i", &only_needed)) { + return NULL; + } + +#if VMPROF_UNIX + if ((read(fd, NULL, 0) != 0) && (only_needed != 0)) { + PyErr_SetString(PyExc_ValueError, + "file descriptor must be readable to save only needed code objects"); + return NULL; + } +#else + if (only_needed) { + PyErr_SetString(PyExc_ValueError, + "saving only needed code objects is not supported for windows"); + return NULL; + } +#endif + if (!is_enabled) { PyErr_SetString(PyExc_ValueError, "vmprof is not enabled"); return NULL; } + is_enabled = 0; vmprof_ignore_signals(1); + +#if VMPROF_UNIX + if (only_needed) + emit_all_code_objects_seen(fd); + else + emit_all_code_objects(); +#else emit_all_code_objects(); +#endif if (vmprof_disable() < 0) { PyErr_SetFromErrno(PyExc_OSError); @@ -320,15 +438,36 @@ } #endif +#ifdef VMPROF_UNIX +static PyObject * vmp_get_profile_path(PyObject *module, PyObject *noargs) { + PyObject * o; + if (is_enabled) { + char buffer[4096]; + buffer[0] = 0; + ssize_t buffer_len = vmp_fd_to_path(vmp_profile_fileno(), buffer, 4096); + if (buffer_len == -1) { + PyErr_Format(PyExc_NotImplementedError, "not implemented platform %s", vmp_machine_os_name()); + return NULL; + } + return PyStr_n_NEW(buffer, buffer_len); + } + Py_RETURN_NONE; +} +#endif + static PyMethodDef VMProfMethods[] = { {"enable", enable_vmprof, METH_VARARGS, "Enable profiling."}, - {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."}, + {"disable", disable_vmprof, METH_VARARGS, "Disable profiling."}, {"write_all_code_objects", write_all_code_objects, METH_NOARGS, "Write eagerly all the IDs of code objects"}, {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"}, #ifdef VMP_SUPPORTS_NATIVE_PROFILING {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"}, #endif + {"is_enabled", vmp_is_enabled, METH_NOARGS, "Indicates if vmprof is currently sampling."}, +#ifdef VMPROF_UNIX + {"get_profile_path", vmp_get_profile_path, METH_NOARGS, "Profile path the profiler logs to."}, +#endif {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/rpython/rlib/rvmprof/src/shared/compat.h b/rpython/rlib/rvmprof/src/shared/compat.h --- a/rpython/rlib/rvmprof/src/shared/compat.h +++ b/rpython/rlib/rvmprof/src/shared/compat.h @@ -7,11 +7,13 @@ #define PyStr_AS_STRING PyBytes_AS_STRING #define PyStr_GET_SIZE PyBytes_GET_SIZE #define PyStr_NEW PyUnicode_FromString + #define PyStr_n_NEW PyUnicode_FromStringAndSize #define PyLong_NEW PyLong_FromSsize_t # else #define PyStr_AS_STRING PyString_AS_STRING #define PyStr_GET_SIZE PyString_GET_SIZE #define PyStr_NEW PyString_FromString + #define PyStr_n_NEW PyString_FromStringAndSize #define PyLong_NEW PyInt_FromSsize_t #define PyLong_AsLong PyInt_AsLong # endif diff --git a/rpython/rlib/rvmprof/src/shared/khash.h b/rpython/rlib/rvmprof/src/shared/khash.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/shared/khash.h @@ -0,0 +1,627 @@ +/* The MIT License + + Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attrac...@live.co.uk> + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* + An example: + +#include "khash.h" +KHASH_MAP_INIT_INT(32, char) +int main() { + int ret, is_missing; + khiter_t k; + khash_t(32) *h = kh_init(32); + k = kh_put(32, h, 5, &ret); + kh_value(h, k) = 10; + k = kh_get(32, h, 10); + is_missing = (k == kh_end(h)); + k = kh_get(32, h, 5); + kh_del(32, h, k); + for (k = kh_begin(h); k != kh_end(h); ++k) + if (kh_exist(h, k)) kh_value(h, k) = 1; + kh_destroy(32, h); + return 0; +} +*/ + +/* + 2013-05-02 (0.2.8): + + * Use quadratic probing. When the capacity is power of 2, stepping function + i*(i+1)/2 guarantees to traverse each bucket. It is better than double + hashing on cache performance and is more robust than linear probing. + + In theory, double hashing should be more robust than quadratic probing. + However, my implementation is probably not for large hash tables, because + the second hash function is closely tied to the first hash function, + which reduce the effectiveness of double hashing. + + Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php + + 2011-12-29 (0.2.7): + + * Minor code clean up; no actual effect. + + 2011-09-16 (0.2.6): + + * The capacity is a power of 2. This seems to dramatically improve the + speed for simple keys. Thank Zilong Tan for the suggestion. Reference: + + - http://code.google.com/p/ulib/ + - http://nothings.org/computer/judy/ + + * Allow to optionally use linear probing which usually has better + performance for random input. Double hashing is still the default as it + is more robust to certain non-random input. + + * Added Wang's integer hash function (not used by default). This hash + function is more robust to certain non-random input. + + 2011-02-14 (0.2.5): + + * Allow to declare global functions. + + 2009-09-26 (0.2.4): + + * Improve portability + + 2008-09-19 (0.2.3): + + * Corrected the example + * Improved interfaces + + 2008-09-11 (0.2.2): + + * Improved speed a little in kh_put() + + 2008-09-10 (0.2.1): + + * Added kh_clear() + * Fixed a compiling error + + 2008-09-02 (0.2.0): + + * Changed to token concatenation which increases flexibility. + + 2008-08-31 (0.1.2): + + * Fixed a bug in kh_get(), which has not been tested previously. + + 2008-08-31 (0.1.1): + + * Added destructor +*/ + + +#ifndef __AC_KHASH_H +#define __AC_KHASH_H + +/*! + @header + + Generic hash table library. + */ + +#define AC_VERSION_KHASH_H "0.2.8" + +#include <stdlib.h> +#include <string.h> +#include <limits.h> + +/* compiler specific configuration */ + +#if UINT_MAX == 0xffffffffu +typedef unsigned int khint32_t; +#elif ULONG_MAX == 0xffffffffu +typedef unsigned long khint32_t; +#endif + +#if ULONG_MAX == ULLONG_MAX +typedef unsigned long khint64_t; +#else +typedef unsigned long long khint64_t; +#endif + +#ifndef kh_inline +#ifdef _MSC_VER +#define kh_inline __inline +#else +#define kh_inline inline +#endif +#endif /* kh_inline */ + +#ifndef klib_unused +#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3) +#define klib_unused __attribute__ ((__unused__)) +#else +#define klib_unused +#endif +#endif /* klib_unused */ + +typedef khint32_t khint_t; +typedef khint_t khiter_t; + +#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2) +#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1) +#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3) +#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1))) +#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1))) +#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) +#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) + +#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4) + +#ifndef kroundup32 +#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) +#endif + +#ifndef kcalloc +#define kcalloc(N,Z) calloc(N,Z) +#endif +#ifndef kmalloc +#define kmalloc(Z) malloc(Z) +#endif +#ifndef krealloc +#define krealloc(P,Z) realloc(P,Z) +#endif +#ifndef kfree +#define kfree(P) free(P) +#endif + +static const double __ac_HASH_UPPER = 0.77; + +#define __KHASH_TYPE(name, khkey_t, khval_t) \ + typedef struct kh_##name##_s { \ + khint_t n_buckets, size, n_occupied, upper_bound; \ + khint32_t *flags; \ + khkey_t *keys; \ + khval_t *vals; \ + } kh_##name##_t; + +#define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \ + extern kh_##name##_t *kh_init_##name(void); \ + extern void kh_destroy_##name(kh_##name##_t *h); \ + extern void kh_clear_##name(kh_##name##_t *h); \ + extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \ + extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ + extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ + extern void kh_del_##name(kh_##name##_t *h, khint_t x); + +#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + SCOPE kh_##name##_t *kh_init_##name(void) { \ + return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \ + } \ + SCOPE void kh_destroy_##name(kh_##name##_t *h) \ + { \ + if (h) { \ + kfree((void *)h->keys); kfree(h->flags); \ + kfree((void *)h->vals); \ + kfree(h); \ + } \ + } \ + SCOPE void kh_clear_##name(kh_##name##_t *h) \ + { \ + if (h && h->flags) { \ + memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \ + h->size = h->n_occupied = 0; \ + } \ + } \ + SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ + { \ + if (h->n_buckets) { \ + khint_t k, i, last, mask, step = 0; \ + mask = h->n_buckets - 1; \ + k = __hash_func(key); i = k & mask; \ + last = i; \ + while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ + i = (i + (++step)) & mask; \ + if (i == last) return h->n_buckets; \ + } \ + return __ac_iseither(h->flags, i)? h->n_buckets : i; \ + } else return 0; \ + } \ + SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ + { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \ + khint32_t *new_flags = 0; \ + khint_t j = 1; \ + { \ + kroundup32(new_n_buckets); \ + if (new_n_buckets < 4) new_n_buckets = 4; \ + if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \ + else { /* hash table size to be changed (shrink or expand); rehash */ \ + new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ + if (!new_flags) return -1; \ + memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ + if (h->n_buckets < new_n_buckets) { /* expand */ \ + khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ + if (!new_keys) { kfree(new_flags); return -1; } \ + h->keys = new_keys; \ + if (kh_is_map) { \ + khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ + if (!new_vals) { kfree(new_flags); return -1; } \ + h->vals = new_vals; \ + } \ + } /* otherwise shrink */ \ + } \ + } \ + if (j) { /* rehashing is needed */ \ + for (j = 0; j != h->n_buckets; ++j) { \ + if (__ac_iseither(h->flags, j) == 0) { \ + khkey_t key = h->keys[j]; \ + khval_t val; \ + khint_t new_mask; \ + new_mask = new_n_buckets - 1; \ + if (kh_is_map) val = h->vals[j]; \ + __ac_set_isdel_true(h->flags, j); \ + while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \ + khint_t k, i, step = 0; \ + k = __hash_func(key); \ + i = k & new_mask; \ + while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \ + __ac_set_isempty_false(new_flags, i); \ + if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \ + { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ + if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \ + __ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \ + } else { /* write the element and jump out of the loop */ \ + h->keys[i] = key; \ + if (kh_is_map) h->vals[i] = val; \ + break; \ + } \ + } \ + } \ + } \ + if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \ + h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ + if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ + } \ + kfree(h->flags); /* free the working space */ \ + h->flags = new_flags; \ + h->n_buckets = new_n_buckets; \ + h->n_occupied = h->size; \ + h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \ + } \ + return 0; \ + } \ + SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \ + { \ + khint_t x; \ + if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \ + if (h->n_buckets > (h->size<<1)) { \ + if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \ + *ret = -1; return h->n_buckets; \ + } \ + } else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \ + *ret = -1; return h->n_buckets; \ + } \ + } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \ + { \ + khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \ + x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \ + if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \ + else { \ + last = i; \ + while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ + if (__ac_isdel(h->flags, i)) site = i; \ + i = (i + (++step)) & mask; \ + if (i == last) { x = site; break; } \ + } \ + if (x == h->n_buckets) { \ + if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \ + else x = i; \ + } \ + } \ + } \ + if (__ac_isempty(h->flags, x)) { /* not present at all */ \ + h->keys[x] = key; \ + __ac_set_isboth_false(h->flags, x); \ + ++h->size; ++h->n_occupied; \ + *ret = 1; \ + } else if (__ac_isdel(h->flags, x)) { /* deleted */ \ + h->keys[x] = key; \ + __ac_set_isboth_false(h->flags, x); \ + ++h->size; \ + *ret = 2; \ + } else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \ + return x; \ + } \ + SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \ + { \ + if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \ + __ac_set_isdel_true(h->flags, x); \ + --h->size; \ + } \ + } + +#define KHASH_DECLARE(name, khkey_t, khval_t) \ + __KHASH_TYPE(name, khkey_t, khval_t) \ + __KHASH_PROTOTYPES(name, khkey_t, khval_t) + +#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + __KHASH_TYPE(name, khkey_t, khval_t) \ + __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) + +#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + KHASH_INIT2(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) + +/* --- BEGIN OF HASH FUNCTIONS --- */ + +/*! @function + @abstract Integer hash function + @param key The integer [khint32_t] + @return The hash value [khint_t] + */ +#define kh_int_hash_func(key) (khint32_t)(key) +/*! @function + @abstract Integer comparison function + */ +#define kh_int_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract 64-bit integer hash function + @param key The integer [khint64_t] + @return The hash value [khint_t] + */ +#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11) +/*! @function + @abstract 64-bit integer comparison function + */ +#define kh_int64_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract const char* hash function + @param s Pointer to a null terminated string + @return The hash value + */ +static kh_inline khint_t __ac_X31_hash_string(const char *s) +{ + khint_t h = (khint_t)*s; + if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s; + return h; +} +/*! @function + @abstract Another interface to const char* hash function + @param key Pointer to a null terminated string [const char*] + @return The hash value [khint_t] + */ +#define kh_str_hash_func(key) __ac_X31_hash_string(key) +/*! @function + @abstract Const char* comparison function + */ +#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0) + +static kh_inline khint_t __ac_Wang_hash(khint_t key) +{ + key += ~(key << 15); + key ^= (key >> 10); + key += (key << 3); + key ^= (key >> 6); + key += ~(key << 11); + key ^= (key >> 16); + return key; +} +#define kh_int_hash_func2(key) __ac_Wang_hash((khint_t)key) + +/* --- END OF HASH FUNCTIONS --- */ + +/* Other convenient macros... */ + +/*! + @abstract Type of the hash table. + @param name Name of the hash table [symbol] + */ +#define khash_t(name) kh_##name##_t + +/*! @function + @abstract Initiate a hash table. + @param name Name of the hash table [symbol] + @return Pointer to the hash table [khash_t(name)*] + */ +#define kh_init(name) kh_init_##name() + +/*! @function + @abstract Destroy a hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + */ +#define kh_destroy(name, h) kh_destroy_##name(h) + +/*! @function + @abstract Reset a hash table without deallocating memory. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + */ +#define kh_clear(name, h) kh_clear_##name(h) + +/*! @function + @abstract Resize a hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param s New size [khint_t] + */ +#define kh_resize(name, h, s) kh_resize_##name(h, s) + +/*! @function + @abstract Insert a key to the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Key [type of keys] + @param r Extra return code: -1 if the operation failed; + 0 if the key is present in the hash table; + 1 if the bucket is empty (never used); 2 if the element in + the bucket has been deleted [int*] + @return Iterator to the inserted element [khint_t] + */ +#define kh_put(name, h, k, r) kh_put_##name(h, k, r) + +/*! @function + @abstract Retrieve a key from the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Key [type of keys] + @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t] + */ +#define kh_get(name, h, k) kh_get_##name(h, k) + +/*! @function + @abstract Remove a key from the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Iterator to the element to be deleted [khint_t] + */ +#define kh_del(name, h, k) kh_del_##name(h, k) + +/*! @function + @abstract Test whether a bucket contains data. + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return 1 if containing data; 0 otherwise [int] + */ +#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x))) + +/*! @function + @abstract Get key given an iterator + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return Key [type of keys] + */ +#define kh_key(h, x) ((h)->keys[x]) + +/*! @function + @abstract Get value given an iterator + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return Value [type of values] + @discussion For hash sets, calling this results in segfault. + */ +#define kh_val(h, x) ((h)->vals[x]) + +/*! @function + @abstract Alias of kh_val() + */ +#define kh_value(h, x) ((h)->vals[x]) + +/*! @function + @abstract Get the start iterator + @param h Pointer to the hash table [khash_t(name)*] + @return The start iterator [khint_t] + */ +#define kh_begin(h) (khint_t)(0) + +/*! @function + @abstract Get the end iterator + @param h Pointer to the hash table [khash_t(name)*] + @return The end iterator [khint_t] + */ +#define kh_end(h) ((h)->n_buckets) + +/*! @function + @abstract Get the number of elements in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @return Number of elements in the hash table [khint_t] + */ +#define kh_size(h) ((h)->size) + +/*! @function + @abstract Get the number of buckets in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @return Number of buckets in the hash table [khint_t] + */ +#define kh_n_buckets(h) ((h)->n_buckets) + +/*! @function + @abstract Iterate over the entries in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @param kvar Variable to which key will be assigned + @param vvar Variable to which value will be assigned + @param code Block of code to execute + */ +#define kh_foreach(h, kvar, vvar, code) { khint_t __i; \ + for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ + if (!kh_exist(h,__i)) continue; \ + (kvar) = kh_key(h,__i); \ + (vvar) = kh_val(h,__i); \ + code; \ + } } + +/*! @function + @abstract Iterate over the values in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @param vvar Variable to which value will be assigned + @param code Block of code to execute + */ +#define kh_foreach_value(h, vvar, code) { khint_t __i; \ + for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ + if (!kh_exist(h,__i)) continue; \ + (vvar) = kh_val(h,__i); \ + code; \ + } } + +/* More conenient interfaces */ + +/*! @function + @abstract Instantiate a hash set containing integer keys + @param name Name of the hash table [symbol] + */ +#define KHASH_SET_INIT_INT(name) \ + KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing integer keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] + */ +#define KHASH_MAP_INIT_INT(name, khval_t) \ + KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing 64-bit integer keys + @param name Name of the hash table [symbol] + */ +#define KHASH_SET_INIT_INT64(name) \ + KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing 64-bit integer keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] + */ +#define KHASH_MAP_INIT_INT64(name, khval_t) \ + KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal) + +typedef const char *kh_cstr_t; +/*! @function + @abstract Instantiate a hash map containing const char* keys + @param name Name of the hash table [symbol] + */ +#define KHASH_SET_INIT_STR(name) \ + KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing const char* keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] + */ +#define KHASH_MAP_INIT_STR(name, khval_t) \ + KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) + +#endif /* __AC_KHASH_H */ diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c --- a/rpython/rlib/rvmprof/src/shared/machine.c +++ b/rpython/rlib/rvmprof/src/shared/machine.c @@ -3,6 +3,11 @@ #include "vmprof.h" #include <stdio.h> +#ifdef VMPROF_UNIX +#include <unistd.h> +#include <fcntl.h> +#endif + int vmp_machine_bits(void) { return sizeof(void*)*8; @@ -27,3 +32,15 @@ #endif } +long vmp_fd_to_path(int fd, char * buffer, long buffer_len) +{ +#ifdef VMPROF_LINUX + char proffs[24]; + (void)snprintf(proffs, 24, "/proc/self/fd/%d", fd); + return readlink(proffs, buffer, buffer_len); +#elif defined(VMPROF_UNIX) + fcntl(fd, F_GETPATH, buffer); + return strlen(buffer); +#endif + return -1; +} diff --git a/rpython/rlib/rvmprof/src/shared/machine.h b/rpython/rlib/rvmprof/src/shared/machine.h --- a/rpython/rlib/rvmprof/src/shared/machine.h +++ b/rpython/rlib/rvmprof/src/shared/machine.h @@ -10,3 +10,9 @@ */ const char * vmp_machine_os_name(void); +/** + * Writes the filename into buffer. Returns -1 if the platform is not + * implemented. + */ +long vmp_fd_to_path(int fd, char * buffer, long buffer_len); + diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c --- a/rpython/rlib/rvmprof/src/shared/symboltable.c +++ b/rpython/rlib/rvmprof/src/shared/symboltable.c @@ -3,11 +3,15 @@ #include "vmprof.h" #include "machine.h" +#include "khash.h" + #include <string.h> #include <stdio.h> #include <stdlib.h> +#include <assert.h> #include <dlfcn.h> + #if defined(VMPROF_LINUX) #include <link.h> #endif @@ -206,15 +210,20 @@ struct backtrace_state * bstate = NULL; int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno, char * srcfile, int srcfile_len) { #ifdef __APPLE__ - Dl_info info; - if (dladdr((const void*)addr, &info) == 0) { + Dl_info dlinfo; + if (dladdr((const void*)addr, &dlinfo) == 0) { return 1; } - if (info.dli_sname != NULL) { - (void)strncpy(name, info.dli_sname, name_len-1); + if (dlinfo.dli_sname != NULL) { + (void)strncpy(name, dlinfo.dli_sname, name_len-1); name[name_len-1] = 0; } - lookup_vmprof_debug_info(name, info.dli_fbase, srcfile, srcfile_len, lineno); + lookup_vmprof_debug_info(name, dlinfo.dli_fbase, srcfile, srcfile_len, lineno); + // copy the shared object name to the source file name if source cannot be determined + if (srcfile[0] == 0 && dlinfo.dli_fname != NULL) { + (void)strncpy(srcfile, dlinfo.dli_fname, srcfile_len-1); + srcfile[srcfile_len-1] = 0; + } #elif defined(VMPROF_LINUX) if (bstate == NULL) { bstate = backtrace_create_state (NULL, 1, backtrace_error_cb, NULL); @@ -238,6 +247,18 @@ (void)strncpy(info.name, dlinfo.dli_sname, info.name_len-1); name[name_len-1] = 0; } + + } + + // copy the shared object name to the source file name if source cannot be determined + if (srcfile[0] == 0) { + Dl_info dlinfo; + dlinfo.dli_fname = NULL; + (void)dladdr((const void*)addr, &dlinfo); + if (dlinfo.dli_fname != NULL) { + (void)strncpy(srcfile, dlinfo.dli_fname, srcfile_len-1); + srcfile[srcfile_len-1] = 0; + } } #endif return 0; @@ -338,8 +359,9 @@ return 0; } +KHASH_MAP_INIT_INT(ptr, intptr_t) -void dump_native_symbols(int fileno) +void vmp_scan_profile(int fileno, int dump_nat_sym, void *all_code_uids) { off_t orig_pos, cur_pos; char marker; @@ -350,6 +372,9 @@ fsync(fileno); orig_pos = lseek(fileno, 0, SEEK_CUR); + khash_t(ptr) * nat_syms = kh_init(ptr); + khiter_t it; + lseek(fileno, 5*WORD_SIZE, SEEK_SET); while (1) { @@ -363,11 +388,17 @@ case MARKER_HEADER: { LOG("header 0x%llx\n", cur_pos); if (_skip_header(fileno, &version, &flags) != 0) { + kh_destroy(ptr, nat_syms); return; } memory = (flags & PROFILE_MEMORY) != 0; native = (flags & PROFILE_NATIVE) != 0; lines = (flags & PROFILE_LINES) != 0; + if (!native && dump_nat_sym) { + lseek(fileno, 0, SEEK_END); + kh_destroy(ptr, nat_syms); + return; + } break; } case MARKER_META: { LOG("meta 0x%llx\n", cur_pos); @@ -400,19 +431,46 @@ #else for (i = 0; i < depth; i++) { void * addr = _read_addr(fileno); + if (lines && i % 2 == 0) { + continue; + } if (((intptr_t)addr & 0x1) == 1) { #endif - LOG("found kind %p\n", addr); - char name[MAXLEN]; - char srcfile[MAXLEN]; - name[0] = 0; - srcfile[0] = 0; - int lineno = 0; - if (vmp_resolve_addr(addr, name, MAXLEN, &lineno, srcfile, MAXLEN) == 0) { - LOG("dumping add %p, name %s, %s:%d\n", addr, name, srcfile, lineno); - _dump_native_symbol(fileno, addr, name, lineno, srcfile); + /* dump the native symbol to disk */ + if (dump_nat_sym) { + LOG("found kind %p\n", addr); + + // if the address has already been dumped, + // do not log it again! + it = kh_get(ptr, nat_syms, (intptr_t)addr); + if (it == kh_end(nat_syms)) { + char name[MAXLEN]; + char srcfile[MAXLEN]; + name[0] = 0; + srcfile[0] = 0; + int lineno = 0; + if (vmp_resolve_addr(addr, name, MAXLEN, &lineno, srcfile, MAXLEN) == 0) { + LOG("dumping add %p, name %s, %s:%d\n", addr, name, srcfile, lineno); + _dump_native_symbol(fileno, addr, name, lineno, srcfile); + int ret; + it = kh_put(ptr, nat_syms, (intptr_t)addr, &ret); + kh_value(nat_syms, it) = 1; + } + } + } +#ifdef RPYTHON_VMPROF + } +#else + } else { + // cpython adds all addresses into a set to get the intersection + // of all gc known code addresses + if (all_code_uids != NULL) { + PyObject *co_uid = PyLong_FromVoidPtr(addr); + int check = PySet_Add(all_code_uids, co_uid); + Py_CLEAR(co_uid); } } +#endif } LOG("passed memory %d \n", memory); @@ -427,6 +485,7 @@ } default: { fprintf(stderr, "unknown marker 0x%x\n", marker); lseek(fileno, 0, SEEK_END); + kh_destroy(ptr, nat_syms); return; } } @@ -437,5 +496,11 @@ } } + kh_destroy(ptr, nat_syms); lseek(fileno, 0, SEEK_END); } + +void dump_native_symbols(int fileno) +{ + vmp_scan_profile(fileno, 1, NULL); +} diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.h b/rpython/rlib/rvmprof/src/shared/symboltable.h --- a/rpython/rlib/rvmprof/src/shared/symboltable.h +++ b/rpython/rlib/rvmprof/src/shared/symboltable.h @@ -17,5 +17,7 @@ */ void dump_all_known_symbols(int fd); +void dump_native_symbols(int fd); + int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno, char * srcfile, int srcfile_len); diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h --- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h +++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h @@ -218,7 +218,7 @@ if (p == NULL) { /* ignore this signal: there are no free buffers right now */ } else { -#ifdef RPYTHON_VMPORF +#ifdef RPYTHON_VMPROF commit = _vmprof_sample_stack(p, NULL, (ucontext_t*)ucontext); #else commit = _vmprof_sample_stack(p, tstate, (ucontext_t*)ucontext); _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
