Author: Antonio Cuni <anto.c...@gmail.com> Branch: Changeset: r93465:c30916ebe15f Date: 2017-12-18 11:03 +0100 http://bitbucket.org/pypy/pypy/changeset/c30916ebe15f/
Log: merge the fix-vmprof-stacklet-switch-2 branch, which fixes vmprof+greenlet: before, vmprof did not take any sample inside greenlets as soon as you do a switch(). diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt --- a/extra_tests/requirements.txt +++ b/extra_tests/requirements.txt @@ -1,2 +1,3 @@ pytest hypothesis +vmprof diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +import vmprof + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -31,7 +31,7 @@ Upgrade the _vmprof backend to vmprof 0.4.10 .. branch: fix-vmprof-stacklet-switch - +.. branch: fix-vmprof-stacklet-switch-2 Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) .. branch: win32-vcvars @@ -39,3 +39,4 @@ .. branch: rdict-fast-hash Make it possible to declare that the hash function of an r_dict is fast in RPython. + diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -1,5 +1,6 @@ from rpython.rlib.rstacklet import StackletThread from rpython.rlib import jit +from rpython.rlib import rvmprof from pypy.interpreter.error import OperationError, get_cleared_operation_error from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.baseobjspace import W_Root @@ -222,12 +223,15 @@ self.h = h global_state.clear() try: + rvmprof.start_sampling() frame = self.bottomframe w_result = frame.execute_frame() except Exception as e: global_state.propagate_exception = e else: global_state.w_value = w_result + finally: + rvmprof.stop_sampling() self.sthread.ec.topframeref = jit.vref_None global_state.origin = self global_state.destination = self diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py --- a/pypy/module/_continuation/test/test_stacklet.py +++ b/pypy/module/_continuation/test/test_stacklet.py @@ -1,7 +1,10 @@ +import pytest import os +from rpython.rlib.rvmprof.test.support import fakevmprof +from pypy.interpreter.gateway import interp2app from pypy.module._continuation.test.support import BaseAppTest - +@pytest.mark.usefixtures('app_fakevmprof') class AppTestStacklet(BaseAppTest): def setup_class(cls): BaseAppTest.setup_class.im_func(cls) @@ -34,10 +37,34 @@ return res return stack """) + cls.w_appdirect = cls.space.wrap(cls.runappdirect) if cls.runappdirect: # make sure that "self.stack" does not pass the self cls.w_stack = staticmethod(cls.w_stack.im_func) + + @pytest.fixture + def app_fakevmprof(self, fakevmprof): + """ + This is automaticaly re-initialized for every method: thanks to + fakevmprof's finalizer, it checks that we called {start,stop}_sampling + the in pairs + """ + w = self.space.wrap + i2a = interp2app + def is_sampling_enabled(space): + return space.wrap(fakevmprof.is_sampling_enabled) + self.w_is_sampling_enabled = w(i2a(is_sampling_enabled)) + # + def start_sampling(space): + fakevmprof.start_sampling() + self.w_start_sampling = w(i2a(start_sampling)) + # + def stop_sampling(space): + fakevmprof.stop_sampling() + self.w_stop_sampling = w(i2a(stop_sampling)) + + def test_new_empty(self): from _continuation import continulet # @@ -770,3 +797,25 @@ continulet.switch(c1, to=c2) raises(error, continulet.switch, c1, to=c2) + + def test_sampling_inside_callback(self): + if self.appdirect: + # see also + # extra_tests.test_vmprof_greenlet.test_sampling_inside_callback + # for a "translated" version of this test + skip("we can't run this until we have _vmprof.is_sampling_enabled") + from _continuation import continulet + # + def my_callback(c1): + assert self.is_sampling_enabled() + return 42 + # + try: + self.start_sampling() + assert self.is_sampling_enabled() + c = continulet(my_callback) + res = c.switch() + assert res == 42 + assert self.is_sampling_enabled() + finally: + self.stop_sampling() diff --git a/pypy/module/_continuation/test/test_translated.py b/pypy/module/_continuation/test/test_translated.py --- a/pypy/module/_continuation/test/test_translated.py +++ b/pypy/module/_continuation/test/test_translated.py @@ -1,4 +1,5 @@ import py +import pytest try: import _continuation except ImportError: @@ -101,11 +102,7 @@ particular, we need to ensure that vmprof does not sample the stack in the middle of a switch, else we read nonsense. """ - try: - import _vmprof - except ImportError: - py.test.skip("no _vmprof") - # + _vmprof = pytest.importorskip('_vmprof') def switch_forever(c): while True: c.switch() diff --git a/rpython/rlib/rstacklet.py b/rpython/rlib/rstacklet.py --- a/rpython/rlib/rstacklet.py +++ b/rpython/rlib/rstacklet.py @@ -3,7 +3,7 @@ from rpython.rlib import jit from rpython.rlib.objectmodel import fetch_translated_config from rpython.rtyper.lltypesystem import lltype, llmemory -from rpython.rlib.rvmprof import cintf +from rpython.rlib import rvmprof DEBUG = False @@ -25,12 +25,12 @@ def new(self, callback, arg=llmemory.NULL): if DEBUG: callback = _debug_wrapper(callback) - x = cintf.save_rvmprof_stack() + x = rvmprof.save_stack() try: - cintf.empty_rvmprof_stack() + rvmprof.empty_stack() h = self._gcrootfinder.new(self, callback, arg) finally: - cintf.restore_rvmprof_stack(x) + rvmprof.restore_stack(x) if DEBUG: debug.add(h) return h @@ -40,11 +40,11 @@ def switch(self, stacklet): if DEBUG: debug.remove(stacklet) - x = cintf.save_rvmprof_stack() + x = rvmprof.save_stack() try: h = self._gcrootfinder.switch(stacklet) finally: - cintf.restore_rvmprof_stack(x) + rvmprof.restore_stack(x) if DEBUG: debug.add(h) return h diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -56,10 +56,27 @@ return None def stop_sampling(): - from rpython.rlib.rvmprof.cintf import vmprof_stop_sampling - fd = vmprof_stop_sampling() - return rffi.cast(lltype.Signed, fd) + return _get_vmprof().stop_sampling() def start_sampling(): - from rpython.rlib.rvmprof.cintf import vmprof_start_sampling - vmprof_start_sampling() + return _get_vmprof().start_sampling() + +# ---------------- +# stacklet support +# ---------------- +# +# Ideally, vmprof_tl_stack, VMPROFSTACK etc. should be part of "self.cintf": +# not sure why they are a global. Eventually, we should probably fix all this +# mess. +from rpython.rlib.rvmprof.cintf import vmprof_tl_stack, VMPROFSTACK + +def save_stack(): + stop_sampling() + return vmprof_tl_stack.get_or_make_raw() + +def empty_stack(): + vmprof_tl_stack.setraw(lltype.nullptr(VMPROFSTACK)) + +def restore_stack(x): + vmprof_tl_stack.setraw(x) + start_sampling() diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -122,32 +122,16 @@ lltype.Signed, compilation_info=eci, _nowrapper=True) + vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [], + rffi.INT, compilation_info=eci, + _nowrapper=True) + vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [], + lltype.Void, compilation_info=eci, + _nowrapper=True) + return CInterface(locals()) -# this is always present, but compiles to no-op if RPYTHON_VMPROF is not -# defined (i.e. if we don't actually use vmprof in the generated C) -auto_eci = ExternalCompilationInfo(post_include_bits=[""" -#ifndef RPYTHON_VMPROF -# define vmprof_stop_sampling() (-1) -# define vmprof_start_sampling() ((void)0) -#endif -"""]) - -if get_translation_config() is None: - # tests need the full eci here - _eci = global_eci -else: - _eci = auto_eci - -vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [], - rffi.INT, compilation_info=_eci, - _nowrapper=True) -vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [], - lltype.Void, compilation_info=_eci, - _nowrapper=True) - - class CInterface(object): def __init__(self, namespace): for k, v in namespace.iteritems(): @@ -232,20 +216,6 @@ leave_code(s) # -# stacklet support - -def save_rvmprof_stack(): - vmprof_stop_sampling() - return vmprof_tl_stack.get_or_make_raw() - -def empty_rvmprof_stack(): - vmprof_tl_stack.setraw(lltype.nullptr(VMPROFSTACK)) - -def restore_rvmprof_stack(x): - vmprof_tl_stack.setraw(x) - vmprof_start_sampling() - -# # traceback support def get_rvmprof_stack(): diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -168,6 +168,21 @@ if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0: raise VMProfError("vmprof buffers full! disk full or too slow") + def stop_sampling(self): + """ + Temporarily stop the sampling of stack frames. Signals are still + delivered, but are ignored. + """ + fd = self.cintf.vmprof_stop_sampling() + return rffi.cast(lltype.Signed, fd) + + def start_sampling(self): + """ + Undo the effect of stop_sampling + """ + self.cintf.vmprof_start_sampling() + + def vmprof_execute_code(name, get_code_fn, result_class=None, _hack_update_stack_untranslated=False): """Decorator to be used on the function that interprets a code object. diff --git a/rpython/rlib/rvmprof/test/support.py b/rpython/rlib/rvmprof/test/support.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/test/support.py @@ -0,0 +1,45 @@ +import pytest +from rpython.rlib import rvmprof + +class FakeVMProf(object): + + def __init__(self): + self._enabled = False + self._ignore_signals = 1 + + # --- VMProf official API --- + # add fake methods as needed by the tests + + def stop_sampling(self): + self._ignore_signals += 1 + + def start_sampling(self): + assert self._ignore_signals > 0, ('calling start_sampling() without ' + 'the corresponding stop_sampling()?') + self._ignore_signals -= 1 + + # --- FakeVMProf specific API --- + # this API is not part of rvmprof, but available only inside tests using + # fakevmprof + + @property + def is_sampling_enabled(self): + return self._ignore_signals == 0 + + def check_status(self): + """ + To be called during test teardown + """ + if self._ignore_signals != 1: + msg = ('Invalid value for fakevmprof._ignore_signals: expected 1, ' + 'got %d. This probably means that you called ' + '{start,stop}_sampling() a wrong number of times') + raise ValueError, msg % self._ignore_signals + + +@pytest.fixture +def fakevmprof(request, monkeypatch): + fake = FakeVMProf() + monkeypatch.setattr(rvmprof.rvmprof, '_vmprof_instance', fake) + request.addfinalizer(fake.check_status) + return fake diff --git a/rpython/rlib/rvmprof/test/test_support.py b/rpython/rlib/rvmprof/test/test_support.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/test/test_support.py @@ -0,0 +1,42 @@ +import pytest +from rpython.rlib import rvmprof +from rpython.rlib.rvmprof.test.support import FakeVMProf, fakevmprof + +class TestFakeVMProf(object): + + def test_sampling(self): + fake = FakeVMProf() + assert not fake.is_sampling_enabled + # + fake.start_sampling() + assert fake.is_sampling_enabled + # + fake.stop_sampling() + fake.stop_sampling() + assert not fake.is_sampling_enabled + # + fake.start_sampling() + assert not fake.is_sampling_enabled + fake.start_sampling() + assert fake.is_sampling_enabled + # + pytest.raises(AssertionError, "fake.start_sampling()") + + def test_check_status(self): + fake = FakeVMProf() + fake.stop_sampling() + pytest.raises(ValueError, "fake.check_status()") + + +class TestFixture(object): + + def test_fixture(self, fakevmprof): + assert isinstance(fakevmprof, FakeVMProf) + assert rvmprof._get_vmprof() is fakevmprof + # + # tweak sampling using the "real" API, and check that we actually used + # the fake + rvmprof.start_sampling() + assert fakevmprof.is_sampling_enabled + rvmprof.stop_sampling() + assert not fakevmprof.is_sampling_enabled _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit