Author: Antonio Cuni <anto.c...@gmail.com>
Branch:
Changeset: r93465:c30916ebe15f
Date: 2017-12-18 11:03 +0100
http://bitbucket.org/pypy/pypy/changeset/c30916ebe15f/
Log: merge the fix-vmprof-stacklet-switch-2 branch, which fixes
vmprof+greenlet: before, vmprof did not take any sample inside
greenlets as soon as you do a switch().
diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt
--- a/extra_tests/requirements.txt
+++ b/extra_tests/requirements.txt
@@ -1,2 +1,3 @@
pytest
hypothesis
+vmprof
diff --git a/extra_tests/test_vmprof_greenlet.py
b/extra_tests/test_vmprof_greenlet.py
new file mode 100644
--- /dev/null
+++ b/extra_tests/test_vmprof_greenlet.py
@@ -0,0 +1,28 @@
+import time
+import pytest
+import greenlet
+import vmprof
+
+def count_samples(filename):
+ stats = vmprof.read_profile(filename)
+ return len(stats.profiles)
+
+def cpuburn(duration):
+ end = time.time() + duration
+ while time.time() < end:
+ pass
+
+def test_sampling_inside_callback(tmpdir):
+ # see also test_sampling_inside_callback inside
+ # pypy/module/_continuation/test/test_stacklet.py
+ #
+ G = greenlet.greenlet(cpuburn)
+ fname = tmpdir.join('log.vmprof')
+ with fname.open('w+b') as f:
+ vmprof.enable(f.fileno(), 1/250.0)
+ G.switch(0.1)
+ vmprof.disable()
+
+ samples = count_samples(str(fname))
+ # 0.1 seconds at 250Hz should be 25 samples
+ assert 23 < samples < 27
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -31,7 +31,7 @@
Upgrade the _vmprof backend to vmprof 0.4.10
.. branch: fix-vmprof-stacklet-switch
-
+.. branch: fix-vmprof-stacklet-switch-2
Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...)
.. branch: win32-vcvars
@@ -39,3 +39,4 @@
.. branch: rdict-fast-hash
Make it possible to declare that the hash function of an r_dict is fast in
RPython.
+
diff --git a/pypy/module/_continuation/interp_continuation.py
b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -1,5 +1,6 @@
from rpython.rlib.rstacklet import StackletThread
from rpython.rlib import jit
+from rpython.rlib import rvmprof
from pypy.interpreter.error import OperationError, get_cleared_operation_error
from pypy.interpreter.executioncontext import ExecutionContext
from pypy.interpreter.baseobjspace import W_Root
@@ -222,12 +223,15 @@
self.h = h
global_state.clear()
try:
+ rvmprof.start_sampling()
frame = self.bottomframe
w_result = frame.execute_frame()
except Exception as e:
global_state.propagate_exception = e
else:
global_state.w_value = w_result
+ finally:
+ rvmprof.stop_sampling()
self.sthread.ec.topframeref = jit.vref_None
global_state.origin = self
global_state.destination = self
diff --git a/pypy/module/_continuation/test/test_stacklet.py
b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -1,7 +1,10 @@
+import pytest
import os
+from rpython.rlib.rvmprof.test.support import fakevmprof
+from pypy.interpreter.gateway import interp2app
from pypy.module._continuation.test.support import BaseAppTest
-
+@pytest.mark.usefixtures('app_fakevmprof')
class AppTestStacklet(BaseAppTest):
def setup_class(cls):
BaseAppTest.setup_class.im_func(cls)
@@ -34,10 +37,34 @@
return res
return stack
""")
+ cls.w_appdirect = cls.space.wrap(cls.runappdirect)
if cls.runappdirect:
# make sure that "self.stack" does not pass the self
cls.w_stack = staticmethod(cls.w_stack.im_func)
+
+ @pytest.fixture
+ def app_fakevmprof(self, fakevmprof):
+ """
+ This is automaticaly re-initialized for every method: thanks to
+ fakevmprof's finalizer, it checks that we called {start,stop}_sampling
+ the in pairs
+ """
+ w = self.space.wrap
+ i2a = interp2app
+ def is_sampling_enabled(space):
+ return space.wrap(fakevmprof.is_sampling_enabled)
+ self.w_is_sampling_enabled = w(i2a(is_sampling_enabled))
+ #
+ def start_sampling(space):
+ fakevmprof.start_sampling()
+ self.w_start_sampling = w(i2a(start_sampling))
+ #
+ def stop_sampling(space):
+ fakevmprof.stop_sampling()
+ self.w_stop_sampling = w(i2a(stop_sampling))
+
+
def test_new_empty(self):
from _continuation import continulet
#
@@ -770,3 +797,25 @@
continulet.switch(c1, to=c2)
raises(error, continulet.switch, c1, to=c2)
+
+ def test_sampling_inside_callback(self):
+ if self.appdirect:
+ # see also
+ # extra_tests.test_vmprof_greenlet.test_sampling_inside_callback
+ # for a "translated" version of this test
+ skip("we can't run this until we have _vmprof.is_sampling_enabled")
+ from _continuation import continulet
+ #
+ def my_callback(c1):
+ assert self.is_sampling_enabled()
+ return 42
+ #
+ try:
+ self.start_sampling()
+ assert self.is_sampling_enabled()
+ c = continulet(my_callback)
+ res = c.switch()
+ assert res == 42
+ assert self.is_sampling_enabled()
+ finally:
+ self.stop_sampling()
diff --git a/pypy/module/_continuation/test/test_translated.py
b/pypy/module/_continuation/test/test_translated.py
--- a/pypy/module/_continuation/test/test_translated.py
+++ b/pypy/module/_continuation/test/test_translated.py
@@ -1,4 +1,5 @@
import py
+import pytest
try:
import _continuation
except ImportError:
@@ -101,11 +102,7 @@
particular, we need to ensure that vmprof does not sample the stack in
the middle of a switch, else we read nonsense.
"""
- try:
- import _vmprof
- except ImportError:
- py.test.skip("no _vmprof")
- #
+ _vmprof = pytest.importorskip('_vmprof')
def switch_forever(c):
while True:
c.switch()
diff --git a/rpython/rlib/rstacklet.py b/rpython/rlib/rstacklet.py
--- a/rpython/rlib/rstacklet.py
+++ b/rpython/rlib/rstacklet.py
@@ -3,7 +3,7 @@
from rpython.rlib import jit
from rpython.rlib.objectmodel import fetch_translated_config
from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rlib.rvmprof import cintf
+from rpython.rlib import rvmprof
DEBUG = False
@@ -25,12 +25,12 @@
def new(self, callback, arg=llmemory.NULL):
if DEBUG:
callback = _debug_wrapper(callback)
- x = cintf.save_rvmprof_stack()
+ x = rvmprof.save_stack()
try:
- cintf.empty_rvmprof_stack()
+ rvmprof.empty_stack()
h = self._gcrootfinder.new(self, callback, arg)
finally:
- cintf.restore_rvmprof_stack(x)
+ rvmprof.restore_stack(x)
if DEBUG:
debug.add(h)
return h
@@ -40,11 +40,11 @@
def switch(self, stacklet):
if DEBUG:
debug.remove(stacklet)
- x = cintf.save_rvmprof_stack()
+ x = rvmprof.save_stack()
try:
h = self._gcrootfinder.switch(stacklet)
finally:
- cintf.restore_rvmprof_stack(x)
+ rvmprof.restore_stack(x)
if DEBUG:
debug.add(h)
return h
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -56,10 +56,27 @@
return None
def stop_sampling():
- from rpython.rlib.rvmprof.cintf import vmprof_stop_sampling
- fd = vmprof_stop_sampling()
- return rffi.cast(lltype.Signed, fd)
+ return _get_vmprof().stop_sampling()
def start_sampling():
- from rpython.rlib.rvmprof.cintf import vmprof_start_sampling
- vmprof_start_sampling()
+ return _get_vmprof().start_sampling()
+
+# ----------------
+# stacklet support
+# ----------------
+#
+# Ideally, vmprof_tl_stack, VMPROFSTACK etc. should be part of "self.cintf":
+# not sure why they are a global. Eventually, we should probably fix all this
+# mess.
+from rpython.rlib.rvmprof.cintf import vmprof_tl_stack, VMPROFSTACK
+
+def save_stack():
+ stop_sampling()
+ return vmprof_tl_stack.get_or_make_raw()
+
+def empty_stack():
+ vmprof_tl_stack.setraw(lltype.nullptr(VMPROFSTACK))
+
+def restore_stack(x):
+ vmprof_tl_stack.setraw(x)
+ start_sampling()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -122,32 +122,16 @@
lltype.Signed,
compilation_info=eci,
_nowrapper=True)
+ vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [],
+ rffi.INT, compilation_info=eci,
+ _nowrapper=True)
+ vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [],
+ lltype.Void, compilation_info=eci,
+ _nowrapper=True)
+
return CInterface(locals())
-# this is always present, but compiles to no-op if RPYTHON_VMPROF is not
-# defined (i.e. if we don't actually use vmprof in the generated C)
-auto_eci = ExternalCompilationInfo(post_include_bits=["""
-#ifndef RPYTHON_VMPROF
-# define vmprof_stop_sampling() (-1)
-# define vmprof_start_sampling() ((void)0)
-#endif
-"""])
-
-if get_translation_config() is None:
- # tests need the full eci here
- _eci = global_eci
-else:
- _eci = auto_eci
-
-vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [],
- rffi.INT, compilation_info=_eci,
- _nowrapper=True)
-vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [],
- lltype.Void, compilation_info=_eci,
- _nowrapper=True)
-
-
class CInterface(object):
def __init__(self, namespace):
for k, v in namespace.iteritems():
@@ -232,20 +216,6 @@
leave_code(s)
#
-# stacklet support
-
-def save_rvmprof_stack():
- vmprof_stop_sampling()
- return vmprof_tl_stack.get_or_make_raw()
-
-def empty_rvmprof_stack():
- vmprof_tl_stack.setraw(lltype.nullptr(VMPROFSTACK))
-
-def restore_rvmprof_stack(x):
- vmprof_tl_stack.setraw(x)
- vmprof_start_sampling()
-
-#
# traceback support
def get_rvmprof_stack():
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -168,6 +168,21 @@
if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
raise VMProfError("vmprof buffers full! disk full or too slow")
+ def stop_sampling(self):
+ """
+ Temporarily stop the sampling of stack frames. Signals are still
+ delivered, but are ignored.
+ """
+ fd = self.cintf.vmprof_stop_sampling()
+ return rffi.cast(lltype.Signed, fd)
+
+ def start_sampling(self):
+ """
+ Undo the effect of stop_sampling
+ """
+ self.cintf.vmprof_start_sampling()
+
+
def vmprof_execute_code(name, get_code_fn, result_class=None,
_hack_update_stack_untranslated=False):
"""Decorator to be used on the function that interprets a code object.
diff --git a/rpython/rlib/rvmprof/test/support.py
b/rpython/rlib/rvmprof/test/support.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/test/support.py
@@ -0,0 +1,45 @@
+import pytest
+from rpython.rlib import rvmprof
+
+class FakeVMProf(object):
+
+ def __init__(self):
+ self._enabled = False
+ self._ignore_signals = 1
+
+ # --- VMProf official API ---
+ # add fake methods as needed by the tests
+
+ def stop_sampling(self):
+ self._ignore_signals += 1
+
+ def start_sampling(self):
+ assert self._ignore_signals > 0, ('calling start_sampling() without '
+ 'the corresponding stop_sampling()?')
+ self._ignore_signals -= 1
+
+ # --- FakeVMProf specific API ---
+ # this API is not part of rvmprof, but available only inside tests using
+ # fakevmprof
+
+ @property
+ def is_sampling_enabled(self):
+ return self._ignore_signals == 0
+
+ def check_status(self):
+ """
+ To be called during test teardown
+ """
+ if self._ignore_signals != 1:
+ msg = ('Invalid value for fakevmprof._ignore_signals: expected 1, '
+ 'got %d. This probably means that you called '
+ '{start,stop}_sampling() a wrong number of times')
+ raise ValueError, msg % self._ignore_signals
+
+
+@pytest.fixture
+def fakevmprof(request, monkeypatch):
+ fake = FakeVMProf()
+ monkeypatch.setattr(rvmprof.rvmprof, '_vmprof_instance', fake)
+ request.addfinalizer(fake.check_status)
+ return fake
diff --git a/rpython/rlib/rvmprof/test/test_support.py
b/rpython/rlib/rvmprof/test/test_support.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/test/test_support.py
@@ -0,0 +1,42 @@
+import pytest
+from rpython.rlib import rvmprof
+from rpython.rlib.rvmprof.test.support import FakeVMProf, fakevmprof
+
+class TestFakeVMProf(object):
+
+ def test_sampling(self):
+ fake = FakeVMProf()
+ assert not fake.is_sampling_enabled
+ #
+ fake.start_sampling()
+ assert fake.is_sampling_enabled
+ #
+ fake.stop_sampling()
+ fake.stop_sampling()
+ assert not fake.is_sampling_enabled
+ #
+ fake.start_sampling()
+ assert not fake.is_sampling_enabled
+ fake.start_sampling()
+ assert fake.is_sampling_enabled
+ #
+ pytest.raises(AssertionError, "fake.start_sampling()")
+
+ def test_check_status(self):
+ fake = FakeVMProf()
+ fake.stop_sampling()
+ pytest.raises(ValueError, "fake.check_status()")
+
+
+class TestFixture(object):
+
+ def test_fixture(self, fakevmprof):
+ assert isinstance(fakevmprof, FakeVMProf)
+ assert rvmprof._get_vmprof() is fakevmprof
+ #
+ # tweak sampling using the "real" API, and check that we actually used
+ # the fake
+ rvmprof.start_sampling()
+ assert fakevmprof.is_sampling_enabled
+ rvmprof.stop_sampling()
+ assert not fakevmprof.is_sampling_enabled
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
