Author: mattip <matti.pi...@gmail.com> Branch: numpy-fixes Changeset: r77156:be47257b1b03 Date: 2015-05-06 17:06 +0300 http://bitbucket.org/pypy/pypy/changeset/be47257b1b03/
Log: merge default into branch diff too long, truncating to 2000 out of 5994 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -420,3 +420,10 @@ the terms of the GPL license version 2 or any later version. Thus the gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed under the terms of the GPL license as well. + +License for 'pypy/module/_vmprof/src' +-------------------------------------- + +The code is based on gperftools. You may see a copy of the License for it at + + https://code.google.com/p/gperftools/source/browse/COPYING diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -276,7 +276,11 @@ if argtypes: args = [argtype._CData_retval(argtype.from_address(arg)._buffer) for argtype, arg in zip(argtypes, args)] - return to_call(*args) + try: + return to_call(*args) + except SystemExit, e: + handle_system_exit(e) + raise return f def __call__(self, *args, **kwargs): @@ -305,7 +309,11 @@ except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) try: - res = self.callable(*newargs) + try: + res = self.callable(*newargs) + except SystemExit, e: + handle_system_exit(e) + raise except: exc_info = sys.exc_info() traceback.print_tb(exc_info[2], file=sys.stderr) @@ -715,3 +723,22 @@ make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast return CFuncPtrFast make_fastpath_subclass.memo = {} + + +def handle_system_exit(e): + # issue #1194: if we get SystemExit here, then exit the interpreter. + # Highly obscure imho but some people seem to depend on it. + if sys.flags.inspect: + return # Don't exit if -i flag was given. + else: + code = e.code + if isinstance(code, int): + exitcode = code + else: + f = getattr(sys, 'stderr', None) + if f is None: + f = sys.__stderr__ + print >> f, code + exitcode = 1 + + _rawffi.exit(exitcode) diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.5 +Version: 0.4.6 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.5" +__version__ = "0.4.6" # ____________________________________________________________ # Exceptions diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -38,6 +38,10 @@ "_csv", "cppyy", "_pypyjson" ]) +if sys.platform.startswith('linux') and sys.maxint > 2147483647: + if 0: # XXX disabled until we fix the absurd .so mess + working_modules.add('_vmprof') + translation_modules = default_modules.copy() translation_modules.update([ "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5", @@ -99,6 +103,7 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -67,3 +67,10 @@ .. branch: object-dtype2 Extend numpy dtypes to allow using objects with associated garbage collection hook + +.. branch: vmprof2 +Add backend support for vmprof - a lightweight statistical profiler - +to linux64, see client at https://vmprof.readthedocs.org + +.. branch: jit_hint_docs +Add more detail to @jit.elidable and @jit.promote in rpython/rlib/jit.py diff --git a/pypy/goal/pypy.ico b/pypy/goal/pypy.ico new file mode 100644 index 0000000000000000000000000000000000000000..09d07dcc5a783200f440c68c0987926a80d6b667 GIT binary patch [cut] diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -238,6 +238,7 @@ config.translation.suggest(check_str_without_nul=True) config.translation.suggest(shared=True) + config.translation.suggest(icon=os.path.join(this_dir, 'pypy.ico')) if config.translation.shared: if config.translation.output is not None: raise Exception("Cannot use the --output option with PyPy " diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -11,7 +11,7 @@ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction) + UserDelAction, CodeUniqueIds) from pypy.interpreter.error import OperationError, new_exception_class, oefmt from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary @@ -388,6 +388,7 @@ self.actionflag = ActionFlag() # changed by the signal module self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) + self.code_unique_ids = CodeUniqueIds() self._code_of_sys_exc_info = None # can be overridden to a subclass @@ -666,6 +667,16 @@ assert ec is not None return ec + def register_code_callback(self, callback): + cui = self.code_unique_ids + cui.code_callback = callback + + def register_code_object(self, pycode): + cui = self.code_unique_ids + if cui.code_callback is None: + return + cui.code_callback(self, pycode) + def _freeze_(self): return True @@ -1080,7 +1091,7 @@ def call_valuestack(self, w_func, nargs, frame): from pypy.interpreter.function import Function, Method, is_builtin_code - if frame.is_being_profiled and is_builtin_code(w_func): + if frame.get_is_being_profiled() and is_builtin_code(w_func): # XXX: this code is copied&pasted :-( from the slow path below # call_valuestack(). args = frame.make_arguments(nargs) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -96,7 +96,7 @@ def _c_call_return_trace(self, frame, w_func, args, event): if self.profilefunc is None: - frame.is_being_profiled = False + frame.getorcreatedebug().is_being_profiled = False else: # undo the effect of the CALL_METHOD bytecode, which would be # that even on a built-in method call like '[].append()', @@ -114,7 +114,7 @@ def c_exception_trace(self, frame, w_exc): "Profile function called upon OperationError." if self.profilefunc is None: - frame.is_being_profiled = False + frame.getorcreatedebug().is_being_profiled = False else: self._trace(frame, 'c_exception', w_exc) @@ -123,7 +123,7 @@ if self.gettrace() is not None or self.profilefunc is not None: self._trace(frame, 'call', self.space.w_None) if self.profilefunc: - frame.is_being_profiled = True + frame.getorcreatedebug().is_being_profiled = True def return_trace(self, frame, w_retval): "Trace the return from a function" @@ -145,7 +145,7 @@ Like bytecode_trace() but doesn't invoke any other events besides the trace function. """ - if (frame.w_f_trace is None or self.is_tracing or + if (frame.get_w_f_trace() is None or self.is_tracing or self.gettrace() is None): return self.run_trace_func(frame) @@ -154,8 +154,9 @@ @jit.unroll_safe def run_trace_func(self, frame): code = frame.pycode - if frame.instr_lb <= frame.last_instr < frame.instr_ub: - if frame.last_instr < frame.instr_prev_plus_one: + d = frame.getorcreatedebug() + if d.instr_lb <= frame.last_instr < d.instr_ub: + if frame.last_instr < d.instr_prev_plus_one: # We jumped backwards in the same line. self._trace(frame, 'line', self.space.w_None) else: @@ -170,7 +171,7 @@ break addr += c if c: - frame.instr_lb = addr + d.instr_lb = addr line += ord(lineno[p + 1]) p += 2 @@ -185,15 +186,15 @@ if ord(lineno[p + 1]): break p += 2 - frame.instr_ub = addr + d.instr_ub = addr else: - frame.instr_ub = sys.maxint + d.instr_ub = sys.maxint - if frame.instr_lb == frame.last_instr: # At start of line! - frame.f_lineno = line + if d.instr_lb == frame.last_instr: # At start of line! + d.f_lineno = line self._trace(frame, 'line', self.space.w_None) - frame.instr_prev_plus_one = frame.last_instr + 1 + d.instr_prev_plus_one = frame.last_instr + 1 def bytecode_trace_after_exception(self, frame): "Like bytecode_trace(), but without increasing the ticker." @@ -287,8 +288,9 @@ # field of all frames, during the loop below.) frame = self.gettopframe_nohidden() while frame: + frame.getorcreatedebug().f_lineno = frame.get_last_lineno() if is_being_profiled: - frame.is_being_profiled = True + frame.getorcreatedebug().is_being_profiled = True frame = self.getnextframe_nohidden(frame) def call_tracing(self, w_func, w_args): @@ -309,7 +311,7 @@ if event == 'call': w_callback = self.gettrace() else: - w_callback = frame.w_f_trace + w_callback = frame.get_w_f_trace() if w_callback is not None and event != "leaveframe": if operr is not None: @@ -320,15 +322,16 @@ frame.fast2locals() self.is_tracing += 1 try: + d = frame.getorcreatedebug() try: w_result = space.call_function(w_callback, space.wrap(frame), space.wrap(event), w_arg) if space.is_w(w_result, space.w_None): - frame.w_f_trace = None + d.w_f_trace = None else: - frame.w_f_trace = w_result + d.w_f_trace = w_result except: self.settrace(space.w_None) - frame.w_f_trace = None + d.w_f_trace = None raise finally: self.is_tracing -= 1 @@ -579,3 +582,11 @@ # there is no list of length n: if n is large, then the GC # will run several times while walking the list, but it will # see lower and lower memory usage, with no lower bound of n. + +class CodeUniqueIds(object): + def __init__(self): + if sys.maxint == 2147483647: + self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit + else: + self.code_unique_id = 0x7000000000000000 + self.code_callback = None diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -14,9 +14,10 @@ CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY) from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, r_longlong from rpython.rlib.objectmodel import compute_hash from rpython.rlib import jit +from rpython.rlib.debug import debug_start, debug_stop, debug_print class BytecodeCorruption(Exception): @@ -54,8 +55,9 @@ "CPython-style code objects." _immutable_ = True _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]", - "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"] - + "co_freevars[*]", "co_cellvars[*]", + "_args_as_cellvars[*]"] + def __init__(self, space, argcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars, @@ -83,6 +85,7 @@ self.magic = magic self._signature = cpython_code_signature(self) self._initialize() + space.register_code_object(self) def _initialize(self): if self.co_cellvars: @@ -124,6 +127,15 @@ from pypy.objspace.std.mapdict import init_mapdict_cache init_mapdict_cache(self) + cui = self.space.code_unique_ids + self._unique_id = cui.code_unique_id + cui.code_unique_id += 4 # so we have two bits that we can mark stuff + # with + + def _get_full_name(self): + return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno, + self.co_filename) + def _cleanup_(self): if (self.magic == cpython_magic and '__pypy__' not in sys.builtin_module_names): diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -23,6 +23,16 @@ globals()[op] = stdlib_opcode.opmap[op] HAVE_ARGUMENT = stdlib_opcode.HAVE_ARGUMENT +class FrameDebugData(object): + """ A small object that holds debug data for tracing + """ + w_f_trace = None + instr_lb = 0 + instr_ub = 0 + instr_prev_plus_one = 0 + f_lineno = 0 # current lineno for tracing + is_being_profiled = False + w_locals = None class PyFrame(W_Root): """Represents a frame for a regular Python function @@ -31,7 +41,8 @@ Public fields: * 'space' is the object space this frame is running in * 'code' is the PyCode object this frame runs - * 'w_locals' is the locals dictionary to use + * 'w_locals' is the locals dictionary to use, if needed, stored on a + debug object * 'w_globals' is the attached globals dictionary * 'builtin' is the attached built-in module * 'valuestack_w', 'blockstack', control the interpretation @@ -49,13 +60,26 @@ last_instr = -1 last_exception = None f_backref = jit.vref_None - w_f_trace = None - # For tracing - instr_lb = 0 - instr_ub = 0 - instr_prev_plus_one = 0 - is_being_profiled = False + escaped = False # see mark_as_escaped() + debugdata = None + + w_globals = None + pycode = None # code object executed by that frame + locals_stack_w = None # the list of all locals and valuestack + valuestackdepth = 0 # number of items on valuestack + lastblock = None + cells = None # cells + + # other fields: + + # builtin - builtin cache, only if honor__builtins__ is True + # defaults to False + + # there is also self.space which is removed by the annotator + + # additionally JIT uses vable_token field that is representing + # frame current virtualizable state as seen by the JIT def __init__(self, space, code, w_globals, outer_func): if not we_are_translated(): @@ -65,11 +89,9 @@ assert isinstance(code, pycode.PyCode) self.space = space self.w_globals = w_globals - self.w_locals = None self.pycode = code self.locals_stack_w = [None] * (code.co_nlocals + code.co_stacksize) self.valuestackdepth = code.co_nlocals - self.lastblock = None make_sure_not_resized(self.locals_stack_w) check_nonneg(self.valuestackdepth) # @@ -78,7 +100,32 @@ # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS. # class bodies only have CO_NEWLOCALS. self.initialize_frame_scopes(outer_func, code) - self.f_lineno = code.co_firstlineno + + def getdebug(self): + return self.debugdata + + def getorcreatedebug(self): + if self.debugdata is None: + self.debugdata = FrameDebugData() + return self.debugdata + + def get_w_f_trace(self): + d = self.getdebug() + if d is None: + return None + return d.w_f_trace + + def get_is_being_profiled(self): + d = self.getdebug() + if d is None: + return False + return d.is_being_profiled + + def get_w_locals(self): + d = self.getdebug() + if d is None: + return None + return d.w_locals def __repr__(self): # NOT_RPYTHON: useful in tracebacks @@ -142,10 +189,10 @@ flags = code.co_flags if not (flags & pycode.CO_OPTIMIZED): if flags & pycode.CO_NEWLOCALS: - self.w_locals = self.space.newdict(module=True) + self.getorcreatedebug().w_locals = self.space.newdict(module=True) else: assert self.w_globals is not None - self.w_locals = self.w_globals + self.getorcreatedebug().w_locals = self.w_globals ncellvars = len(code.co_cellvars) nfreevars = len(code.co_freevars) @@ -367,10 +414,10 @@ else: w_cells = space.newlist([space.wrap(cell) for cell in cells]) - if self.w_f_trace is None: + if self.get_w_f_trace() is None: f_lineno = self.get_last_lineno() else: - f_lineno = self.f_lineno + f_lineno = self.getorcreatedebug().f_lineno nlocals = self.pycode.co_nlocals values_w = self.locals_stack_w[nlocals:self.valuestackdepth] @@ -386,6 +433,7 @@ w_exc_value = self.last_exception.get_w_value(space) w_tb = w(self.last_exception.get_traceback()) + d = self.getorcreatedebug() tup_state = [ w(self.f_backref()), w(self.get_builtin()), @@ -402,11 +450,11 @@ space.w_None, #XXX placeholder for f_locals #f_restricted requires no additional data! - space.w_None, ## self.w_f_trace, ignore for now + space.w_None, - w(self.instr_lb), #do we need these three (that are for tracing) - w(self.instr_ub), - w(self.instr_prev_plus_one), + w(d.instr_lb), + w(d.instr_ub), + w(d.instr_prev_plus_one), w_cells, ] return nt(tup_state) @@ -464,18 +512,19 @@ ) new_frame.last_instr = space.int_w(w_last_instr) new_frame.frame_finished_execution = space.is_true(w_finished) - new_frame.f_lineno = space.int_w(w_f_lineno) + d = new_frame.getorcreatedebug() + d.f_lineno = space.int_w(w_f_lineno) fastlocals_w = maker.slp_from_tuple_with_nulls(space, w_fastlocals) new_frame.locals_stack_w[:len(fastlocals_w)] = fastlocals_w if space.is_w(w_f_trace, space.w_None): - new_frame.w_f_trace = None + d.w_f_trace = None else: - new_frame.w_f_trace = w_f_trace + d.w_f_trace = w_f_trace - new_frame.instr_lb = space.int_w(w_instr_lb) #the three for tracing - new_frame.instr_ub = space.int_w(w_instr_ub) - new_frame.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one) + d.instr_lb = space.int_w(w_instr_lb) #the three for tracing + d.instr_ub = space.int_w(w_instr_ub) + d.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one) self._setcellvars(cellvars) @@ -503,30 +552,31 @@ Get the locals as a dictionary """ self.fast2locals() - return self.w_locals + return self.debugdata.w_locals def setdictscope(self, w_locals): """ Initialize the locals from a dictionary. """ - self.w_locals = w_locals + self.getorcreatedebug().w_locals = w_locals self.locals2fast() @jit.unroll_safe def fast2locals(self): # Copy values from the fastlocals to self.w_locals - if self.w_locals is None: - self.w_locals = self.space.newdict() + d = self.getorcreatedebug() + if d.w_locals is None: + d.w_locals = self.space.newdict() varnames = self.getcode().getvarnames() for i in range(min(len(varnames), self.getcode().co_nlocals)): name = varnames[i] w_value = self.locals_stack_w[i] if w_value is not None: - self.space.setitem_str(self.w_locals, name, w_value) + self.space.setitem_str(d.w_locals, name, w_value) else: w_name = self.space.wrap(name) try: - self.space.delitem(self.w_locals, w_name) + self.space.delitem(d.w_locals, w_name) except OperationError as e: if not e.match(self.space, self.space.w_KeyError): raise @@ -545,13 +595,14 @@ except ValueError: pass else: - self.space.setitem_str(self.w_locals, name, w_value) + self.space.setitem_str(d.w_locals, name, w_value) @jit.unroll_safe def locals2fast(self): # Copy values from self.w_locals to the fastlocals - assert self.w_locals is not None + w_locals = self.getorcreatedebug().w_locals + assert w_locals is not None varnames = self.getcode().getvarnames() numlocals = self.getcode().co_nlocals @@ -559,7 +610,7 @@ for i in range(min(len(varnames), numlocals)): name = varnames[i] - w_value = self.space.finditem_str(self.w_locals, name) + w_value = self.space.finditem_str(w_locals, name) if w_value is not None: new_fastlocals_w[i] = w_value @@ -578,7 +629,7 @@ for i in range(len(freevarnames)): name = freevarnames[i] cell = self.cells[i] - w_value = self.space.finditem_str(self.w_locals, name) + w_value = self.space.finditem_str(w_locals, name) if w_value is not None: cell.set(w_value) @@ -613,10 +664,10 @@ def fget_f_lineno(self, space): "Returns the line number of the instruction currently being executed." - if self.w_f_trace is None: + if self.get_w_f_trace() is None: return space.wrap(self.get_last_lineno()) else: - return space.wrap(self.f_lineno) + return space.wrap(self.getorcreatedebug().f_lineno) def fset_f_lineno(self, space, w_new_lineno): "Returns the line number of the instruction currently being executed." @@ -626,7 +677,7 @@ raise OperationError(space.w_ValueError, space.wrap("lineno must be an integer")) - if self.w_f_trace is None: + if self.get_w_f_trace() is None: raise OperationError(space.w_ValueError, space.wrap("f_lineno can only be set by a trace function.")) @@ -745,7 +796,7 @@ block.cleanup(self) f_iblock -= 1 - self.f_lineno = new_lineno + self.getorcreatedebug().f_lineno = new_lineno self.last_instr = new_lasti def get_last_lineno(self): @@ -763,17 +814,18 @@ return self.space.wrap(self.last_instr) def fget_f_trace(self, space): - return self.w_f_trace + return self.get_w_f_trace() def fset_f_trace(self, space, w_trace): if space.is_w(w_trace, space.w_None): - self.w_f_trace = None + self.getorcreatedebug().w_f_trace = None else: - self.w_f_trace = w_trace - self.f_lineno = self.get_last_lineno() + d = self.getorcreatedebug() + d.w_f_trace = w_trace + d = self.get_last_lineno() def fdel_f_trace(self, space): - self.w_f_trace = None + self.getorcreatedebug().w_f_trace = None def fget_f_exc_type(self, space): if self.last_exception is not None: diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -109,14 +109,14 @@ # dispatch_bytecode(), causing the real exception to be # raised after the exception handler block was popped. try: - trace = self.w_f_trace + trace = self.get_w_f_trace() if trace is not None: - self.w_f_trace = None + self.getorcreatedebug().w_f_trace = None try: ec.bytecode_trace_after_exception(self) finally: if trace is not None: - self.w_f_trace = trace + self.getorcreatedebug().w_f_trace = trace except OperationError, e: operr = e pytraceback.record_application_traceback( @@ -773,7 +773,7 @@ raise RaiseWithExplicitTraceback(operror) def LOAD_LOCALS(self, oparg, next_instr): - self.pushvalue(self.w_locals) + self.pushvalue(self.getorcreatedebug().w_locals) def EXEC_STMT(self, oparg, next_instr): w_locals = self.popvalue() @@ -789,8 +789,8 @@ self.space.gettypeobject(PyCode.typedef)) w_prog, w_globals, w_locals = self.space.fixedview(w_resulttuple, 3) - plain = (self.w_locals is not None and - self.space.is_w(w_locals, self.w_locals)) + plain = (self.get_w_locals() is not None and + self.space.is_w(w_locals, self.get_w_locals())) if plain: w_locals = self.getdictscope() co = self.space.interp_w(eval.Code, w_prog) @@ -840,12 +840,13 @@ def STORE_NAME(self, varindex, next_instr): varname = self.getname_u(varindex) w_newvalue = self.popvalue() - self.space.setitem_str(self.w_locals, varname, w_newvalue) + self.space.setitem_str(self.getorcreatedebug().w_locals, varname, + w_newvalue) def DELETE_NAME(self, varindex, next_instr): w_varname = self.getname_w(varindex) try: - self.space.delitem(self.w_locals, w_varname) + self.space.delitem(self.getorcreatedebug().w_locals, w_varname) except OperationError, e: # catch KeyErrors and turn them into NameErrors if not e.match(self.space, self.space.w_KeyError): @@ -881,9 +882,10 @@ self.space.delitem(self.w_globals, w_varname) def LOAD_NAME(self, nameindex, next_instr): - if self.w_locals is not self.w_globals: + if self.getorcreatedebug().w_locals is not self.w_globals: varname = self.getname_u(nameindex) - w_value = self.space.finditem_str(self.w_locals, varname) + w_value = self.space.finditem_str(self.getorcreatedebug().w_locals, + varname) if w_value is not None: self.pushvalue(w_value) return @@ -1013,7 +1015,7 @@ if w_import is None: raise OperationError(space.w_ImportError, space.wrap("__import__ not found")) - w_locals = self.w_locals + w_locals = self.getorcreatedebug().w_locals if w_locals is None: # CPython does this w_locals = space.w_None w_modulename = space.wrap(modulename) @@ -1185,7 +1187,7 @@ args = self.argument_factory(arguments, keywords, keywords_w, w_star, w_starstar) w_function = self.popvalue() - if self.is_being_profiled and function.is_builtin_code(w_function): + if self.get_is_being_profiled() and function.is_builtin_code(w_function): w_result = self.space.call_args_and_c_profile(self, w_function, args) else: diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -64,6 +64,8 @@ f.f_lineno += 1 return x + open # force fetching of this name now + def function(): xyz with open(self.tempfile1, 'w') as f: diff --git a/pypy/module/_rawffi/__init__.py b/pypy/module/_rawffi/__init__.py --- a/pypy/module/_rawffi/__init__.py +++ b/pypy/module/_rawffi/__init__.py @@ -29,6 +29,7 @@ 'get_last_error' : 'interp_rawffi.get_last_error', 'set_last_error' : 'interp_rawffi.set_last_error', 'SegfaultException' : 'space.new_exception_class("_rawffi.SegfaultException")', + 'exit' : 'interp_exit.exit', } appleveldefs = { diff --git a/pypy/module/_rawffi/interp_exit.py b/pypy/module/_rawffi/interp_exit.py new file mode 100644 --- /dev/null +++ b/pypy/module/_rawffi/interp_exit.py @@ -0,0 +1,9 @@ +from pypy.interpreter.gateway import unwrap_spec +from rpython.rtyper.lltypesystem import lltype, rffi + + +ll_exit = rffi.llexternal('exit', [rffi.INT], lltype.Void, _nowrapper=True) + +@unwrap_spec(status="c_int") +def exit(space, status): + ll_exit(rffi.cast(rffi.INT, status)) diff --git a/pypy/module/_rawffi/test/test_exit.py b/pypy/module/_rawffi/test/test_exit.py new file mode 100644 --- /dev/null +++ b/pypy/module/_rawffi/test/test_exit.py @@ -0,0 +1,16 @@ + +class AppTestFfi: + spaceconfig = dict(usemodules=['_rawffi', 'posix']) + + def test_exit(self): + try: + import posix, _rawffi + except ImportError: + skip("requires posix.fork() to test") + # + pid = posix.fork() + if pid == 0: + _rawffi.exit(5) # in the child + pid, status = posix.waitpid(pid, 0) + assert posix.WIFEXITED(status) + assert posix.WEXITSTATUS(status) == 5 diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/__init__.py @@ -0,0 +1,18 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + """ + Write me :) + """ + appleveldefs = { + } + + interpleveldefs = { + 'enable': 'interp_vmprof.enable', + 'disable': 'interp_vmprof.disable', + } + + def setup_after_space_initialization(self): + # force the __extend__ hacks to occur early + from pypy.module._vmprof.interp_vmprof import VMProf + self.vmprof = VMProf() diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -0,0 +1,240 @@ +import py, os, sys +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.rarithmetic import ovfcheck_float_to_int +from rpython.rtyper.tool import rffi_platform as platform +from rpython.rlib.rstring import StringBuilder +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import oefmt, wrap_oserror, OperationError +from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.pyframe import PyFrame +from pypy.interpreter.pycode import PyCode + +ROOT = py.path.local(__file__).join('..') +SRC = ROOT.join('src') + +# by default, we statically link vmprof.c into pypy; however, if you set +# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so +# which is expected to be inside pypy/module/_vmprof/src: this is very useful +# during development. Note that you have to manually build libvmprof by +# running make inside the src dir +DYNAMIC_VMPROF = False + +eci_kwds = dict( + include_dirs = [SRC], + includes = ['vmprof.h', 'trampoline.h'], + separate_module_files = [SRC.join('trampoline.asmgcc.s')], + libraries = ['unwind'], + + post_include_bits=[""" + void pypy_vmprof_init(void); + """], + + separate_module_sources=[""" + void pypy_vmprof_init(void) { + vmprof_set_mainloop(pypy_execute_frame_trampoline, 0, + NULL); + } + """], + ) + + +if DYNAMIC_VMPROF: + eci_kwds['libraries'] += ['vmprof'] + eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC] +else: + eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')] + +eci = ExternalCompilationInfo(**eci_kwds) + +check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[ + SRC.join('fake_pypy_api.c')])) + +platform.verify_eci(check_eci) + +pypy_execute_frame_trampoline = rffi.llexternal( + "pypy_execute_frame_trampoline", + [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed], + llmemory.GCREF, + compilation_info=eci, + _nowrapper=True, sandboxsafe=True, + random_effects_on_gcobjs=True) + +pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void, + compilation_info=eci) +vmprof_enable = rffi.llexternal("vmprof_enable", + [rffi.INT, rffi.LONG, rffi.INT, + rffi.CCHARP, rffi.INT], + rffi.INT, compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) +vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, + compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) + +vmprof_register_virtual_function = rffi.llexternal( + "vmprof_register_virtual_function", + [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, + compilation_info=eci, _nowrapper=True) + +original_execute_frame = PyFrame.execute_frame.im_func +original_execute_frame.c_name = 'pypy_pyframe_execute_frame' +original_execute_frame._dont_inline_ = True + +class __extend__(PyFrame): + def execute_frame(frame, w_inputvalue=None, operr=None): + # go through the asm trampoline ONLY if we are translated but not being JITted. + # + # If we are not translated, we obviously don't want to go through the + # trampoline because there is no C function it can call. + # + # If we are being JITted, we want to skip the trampoline, else the JIT + # cannot see throug it + if we_are_translated() and not jit.we_are_jitted(): + # if we are translated, call the trampoline + gc_frame = cast_instance_to_gcref(frame) + gc_inputvalue = cast_instance_to_gcref(w_inputvalue) + gc_operr = cast_instance_to_gcref(operr) + unique_id = frame.pycode._unique_id + gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, + gc_operr, unique_id) + return cast_base_ptr_to_instance(W_Root, gc_result) + else: + return original_execute_frame(frame, w_inputvalue, operr) + + + +def write_long_to_string_builder(l, b): + if sys.maxint == 2147483647: + b.append(chr(l & 0xff)) + b.append(chr((l >> 8) & 0xff)) + b.append(chr((l >> 16) & 0xff)) + b.append(chr((l >> 24) & 0xff)) + else: + b.append(chr(l & 0xff)) + b.append(chr((l >> 8) & 0xff)) + b.append(chr((l >> 16) & 0xff)) + b.append(chr((l >> 24) & 0xff)) + b.append(chr((l >> 32) & 0xff)) + b.append(chr((l >> 40) & 0xff)) + b.append(chr((l >> 48) & 0xff)) + b.append(chr((l >> 56) & 0xff)) + +def try_cast_to_pycode(gcref): + return rgc.try_cast_gcref_to_instance(PyCode, gcref) + +MAX_CODES = 1000 + +class VMProf(object): + def __init__(self): + self.is_enabled = False + self.ever_enabled = False + self.fileno = -1 + self.current_codes = [] + + def enable(self, space, fileno, period_usec): + if self.is_enabled: + raise oefmt(space.w_ValueError, "_vmprof already enabled") + self.fileno = fileno + self.is_enabled = True + self.write_header(fileno, period_usec) + if not self.ever_enabled: + if we_are_translated(): + pypy_vmprof_init() + self.ever_enabled = True + self.gather_all_code_objs(space) + space.register_code_callback(vmprof_register_code) + if we_are_translated(): + # does not work untranslated + res = vmprof_enable(fileno, period_usec, 0, + lltype.nullptr(rffi.CCHARP.TO), 0) + else: + res = 0 + if res == -1: + raise wrap_oserror(space, OSError(rposix.get_saved_errno(), + "_vmprof.enable")) + + def gather_all_code_objs(self, space): + all_code_objs = rgc.do_get_objects(try_cast_to_pycode) + for code in all_code_objs: + self.register_code(space, code) + + def write_header(self, fileno, period_usec): + assert period_usec > 0 + b = StringBuilder() + write_long_to_string_builder(0, b) + write_long_to_string_builder(3, b) + write_long_to_string_builder(0, b) + write_long_to_string_builder(period_usec, b) + write_long_to_string_builder(0, b) + b.append('\x04') # interp name + b.append(chr(len('pypy'))) + b.append('pypy') + os.write(fileno, b.build()) + + def register_code(self, space, code): + if self.fileno == -1: + raise OperationError(space.w_RuntimeError, + space.wrap("vmprof not running")) + self.current_codes.append(code) + if len(self.current_codes) >= MAX_CODES: + self._flush_codes(space) + + def _flush_codes(self, space): + b = StringBuilder() + for code in self.current_codes: + name = code._get_full_name() + b.append('\x02') + write_long_to_string_builder(code._unique_id, b) + write_long_to_string_builder(len(name), b) + b.append(name) + os.write(self.fileno, b.build()) + self.current_codes = [] + + def disable(self, space): + if not self.is_enabled: + raise oefmt(space.w_ValueError, "_vmprof not enabled") + self.is_enabled = False + space.register_code_callback(None) + self._flush_codes(space) + self.fileno = -1 + if we_are_translated(): + # does not work untranslated + res = vmprof_disable() + else: + res = 0 + if res == -1: + raise wrap_oserror(space, OSError(rposix.get_saved_errno(), + "_vmprof.disable")) + +def vmprof_register_code(space, code): + from pypy.module._vmprof import Module + mod_vmprof = space.getbuiltinmodule('_vmprof') + assert isinstance(mod_vmprof, Module) + mod_vmprof.vmprof.register_code(space, code) + +@unwrap_spec(fileno=int, period=float) +def enable(space, fileno, period=0.01): # default 100 Hz + from pypy.module._vmprof import Module + mod_vmprof = space.getbuiltinmodule('_vmprof') + assert isinstance(mod_vmprof, Module) + # + try: + period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) + if period_usec <= 0 or period_usec >= 1e6: + # we don't want seconds here at all + raise ValueError + except (ValueError, OverflowError): + raise OperationError(space.w_ValueError, + space.wrap("'period' too large or non positive")) + # + mod_vmprof.vmprof.enable(space, fileno, period_usec) + +def disable(space): + from pypy.module._vmprof import Module + mod_vmprof = space.getbuiltinmodule('_vmprof') + assert isinstance(mod_vmprof, Module) + mod_vmprof.vmprof.disable(space) + diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/config.h @@ -0,0 +1,2 @@ +#define HAVE_SYS_UCONTEXT_H +#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP] diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/fake_pypy_api.c @@ -0,0 +1,21 @@ + +long pypy_jit_stack_depth_at_loc(long x) +{ + return 0; +} + +void *pypy_find_codemap_at_addr(long x) +{ + return (void *)0; +} + +long pypy_yield_codemap_at_addr(void *x, long y, long *a) +{ + return 0; +} + +void pypy_pyframe_execute_frame(void) +{ +} + +volatile int pypy_codemap_currently_invalid = 0; diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/get_custom_offset.c @@ -0,0 +1,66 @@ + +extern volatile int pypy_codemap_currently_invalid; + +void *pypy_find_codemap_at_addr(long addr, long *start_addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long pypy_jit_stack_depth_at_loc(long loc); + + +void vmprof_set_tramp_range(void* start, void* end) +{ +} + +int custom_sanity_check() +{ + return !pypy_codemap_currently_invalid; +} + +static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) { + intptr_t ip_l = (intptr_t)ip; + return pypy_jit_stack_depth_at_loc(ip_l); +} + +static long vmprof_write_header_for_jit_addr(void **result, long n, + void *ip, int max_depth) +{ + void *codemap; + long current_pos = 0; + intptr_t id; + long start_addr = 0; + intptr_t addr = (intptr_t)ip; + int start, k; + void *tmp; + + codemap = pypy_find_codemap_at_addr(addr, &start_addr); + if (codemap == NULL) + // not a jit code at all + return n; + + // modify the last entry to point to start address and not the random one + // in the middle + result[n - 1] = (void*)start_addr; + result[n] = (void*)2; + n++; + start = n; + while (n < max_depth) { + id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); + if (id == -1) + // finish + break; + if (id == 0) + continue; // not main codemap + result[n++] = (void *)id; + } + k = 0; + while (k < (n - start) / 2) { + tmp = result[start + k]; + result[start + k] = result[n - k - 1]; + result[n - k - 1] = tmp; + k++; + } + if (n < max_depth) { + result[n++] = (void*)3; + } + return n; +} diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/getpc.h @@ -0,0 +1,187 @@ +// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Craig Silverstein +// +// This is an internal header file used by profiler.cc. It defines +// the single (inline) function GetPC. GetPC is used in a signal +// handler to figure out the instruction that was being executed when +// the signal-handler was triggered. +// +// To get this, we use the ucontext_t argument to the signal-handler +// callback, which holds the full context of what was going on when +// the signal triggered. How to get from a ucontext_t to a Program +// Counter is OS-dependent. + +#ifndef BASE_GETPC_H_ +#define BASE_GETPC_H_ + +#include "config.h" + +// On many linux systems, we may need _GNU_SOURCE to get access to +// the defined constants that define the register we want to see (eg +// REG_EIP). Note this #define must come first! +#define _GNU_SOURCE 1 +// If #define _GNU_SOURCE causes problems, this might work instead. +// It will cause problems for FreeBSD though!, because it turns off +// the needed __BSD_VISIBLE. +//#define _XOPEN_SOURCE 500 + +#include <string.h> // for memcmp +#if defined(HAVE_SYS_UCONTEXT_H) +#include <sys/ucontext.h> +#elif defined(HAVE_UCONTEXT_H) +#include <ucontext.h> // for ucontext_t (and also mcontext_t) +#elif defined(HAVE_CYGWIN_SIGNAL_H) +#include <cygwin/signal.h> +typedef ucontext ucontext_t; +#endif + + +// Take the example where function Foo() calls function Bar(). For +// many architectures, Bar() is responsible for setting up and tearing +// down its own stack frame. In that case, it's possible for the +// interrupt to happen when execution is in Bar(), but the stack frame +// is not properly set up (either before it's done being set up, or +// after it's been torn down but before Bar() returns). In those +// cases, the stack trace cannot see the caller function anymore. +// +// GetPC can try to identify this situation, on architectures where it +// might occur, and unwind the current function call in that case to +// avoid false edges in the profile graph (that is, edges that appear +// to show a call skipping over a function). To do this, we hard-code +// in the asm instructions we might see when setting up or tearing +// down a stack frame. +// +// This is difficult to get right: the instructions depend on the +// processor, the compiler ABI, and even the optimization level. This +// is a best effort patch -- if we fail to detect such a situation, or +// mess up the PC, nothing happens; the returned PC is not used for +// any further processing. +struct CallUnrollInfo { + // Offset from (e)ip register where this instruction sequence + // should be matched. Interpreted as bytes. Offset 0 is the next + // instruction to execute. Be extra careful with negative offsets in + // architectures of variable instruction length (like x86) - it is + // not that easy as taking an offset to step one instruction back! + int pc_offset; + // The actual instruction bytes. Feel free to make it larger if you + // need a longer sequence. + unsigned char ins[16]; + // How many bytes to match from ins array? + int ins_size; + // The offset from the stack pointer (e)sp where to look for the + // call return address. Interpreted as bytes. + int return_sp_offset; +}; + + +// The dereferences needed to get the PC from a struct ucontext were +// determined at configure time, and stored in the macro +// PC_FROM_UCONTEXT in config.h. The only thing we need to do here, +// then, is to do the magic call-unrolling for systems that support it. + +// -- Special case 1: linux x86, for which we have CallUnrollInfo +#if defined(__linux) && defined(__i386) && defined(__GNUC__) +static const CallUnrollInfo callunrollinfo[] = { + // Entry to a function: push %ebp; mov %esp,%ebp + // Top-of-stack contains the caller IP. + { 0, + {0x55, 0x89, 0xe5}, 3, + 0 + }, + // Entry to a function, second instruction: push %ebp; mov %esp,%ebp + // Top-of-stack contains the old frame, caller IP is +4. + { -1, + {0x55, 0x89, 0xe5}, 3, + 4 + }, + // Return from a function: RET. + // Top-of-stack contains the caller IP. + { 0, + {0xc3}, 1, + 0 + } +}; + +inline void* GetPC(ucontext_t *signal_ucontext) { + // See comment above struct CallUnrollInfo. Only try instruction + // flow matching if both eip and esp looks reasonable. + const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP]; + const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP]; + if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 && + (esp & 0xffff0000) != 0) { + char* eip_char = reinterpret_cast<char*>(eip); + for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) { + if (!memcmp(eip_char + callunrollinfo[i].pc_offset, + callunrollinfo[i].ins, callunrollinfo[i].ins_size)) { + // We have a match. + void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset); + return *retaddr; + } + } + } + return (void*)eip; +} + +// Special case #2: Windows, which has to do something totally different. +#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__) +// If this is ever implemented, probably the way to do it is to have +// profiler.cc use a high-precision timer via timeSetEvent: +// http://msdn2.microsoft.com/en-us/library/ms712713.aspx +// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC. +// The callback function would be something like prof_handler, but +// alas the arguments are different: no ucontext_t! I don't know +// how we'd get the PC (using StackWalk64?) +// http://msdn2.microsoft.com/en-us/library/ms680650.aspx + +#include "base/logging.h" // for RAW_LOG +#ifndef HAVE_CYGWIN_SIGNAL_H +typedef int ucontext_t; +#endif + +inline void* GetPC(ucontext_t *signal_ucontext) { + RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n"); + return NULL; +} + +// Normal cases. If this doesn't compile, it's probably because +// PC_FROM_UCONTEXT is the empty string. You need to figure out +// the right value for your system, and add it to the list in +// configure.ac (or set it manually in your config.h). +#else +inline void* GetPC(ucontext_t *signal_ucontext) { + return (void*)signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h +} + +#endif + +#endif // BASE_GETPC_H_ diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s @@ -0,0 +1,16 @@ +// NOTE: you need to use TABs, not spaces! + + .text + .p2align 4,,-1 + .globl pypy_execute_frame_trampoline + .type pypy_execute_frame_trampoline, @function +pypy_execute_frame_trampoline: + .cfi_startproc + pushq %rcx + .cfi_def_cfa_offset 16 + call pypy_pyframe_execute_frame@PLT + popq %rcx + .cfi_def_cfa_offset 8 + ret + .cfi_endproc + .size pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/trampoline.h @@ -0,0 +1,1 @@ +void* pypy_execute_frame_trampoline(void*, void*, void*, long); diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/vmprof.c @@ -0,0 +1,398 @@ +/* VMPROF + * + * statistical sampling profiler specifically designed to profile programs + * which run on a Virtual Machine and/or bytecode interpreter, such as Python, + * etc. + * + * The logic to dump the C stack traces is partly stolen from the code in gperftools. + * The file "getpc.h" has been entirely copied from gperftools. + * + * Tested only on gcc, linux, x86_64. + * + * Copyright (C) 2014-2015 + * Antonio Cuni - anto.c...@gmail.com + * Maciej Fijalkowski - fij...@gmail.com + * + */ + + +#include "getpc.h" // should be first to get the _GNU_SOURCE dfn +#include <signal.h> +#include <stdio.h> +#include <string.h> +#include <stddef.h> +#include <assert.h> +#include <unistd.h> +#include <sys/time.h> +#include <sys/types.h> +#include <errno.h> +#include <pthread.h> + +#define UNW_LOCAL_ONLY +#include <libunwind.h> + +#include "vmprof.h" + +#define _unused(x) ((void)x) + +#define MAX_FUNC_NAME 128 +#define MAX_STACK_DEPTH 1024 +#define BUFFER_SIZE 8192 + + +static int profile_file = 0; +static char profile_write_buffer[BUFFER_SIZE]; +static int profile_buffer_position = 0; +void* vmprof_mainloop_func; +static ptrdiff_t mainloop_sp_offset; +static vmprof_get_virtual_ip_t mainloop_get_virtual_ip; +static long last_period_usec = 0; +static int atfork_hook_installed = 0; + + +/* ************************************************************* + * functions to write a profile file compatible with gperftools + * ************************************************************* + */ + +#define MARKER_STACKTRACE '\x01' +#define MARKER_VIRTUAL_IP '\x02' +#define MARKER_TRAILER '\x03' + +static void prof_word(long x) { + ((long*)(profile_write_buffer + profile_buffer_position))[0] = x; + profile_buffer_position += sizeof(long); +} + +static void prof_header(long period_usec) { + // XXX never used here? + prof_word(0); + prof_word(3); + prof_word(0); + prof_word(period_usec); + prof_word(0); + write(profile_file, profile_write_buffer, profile_buffer_position); + profile_buffer_position = 0; +} + +static void prof_write_stacktrace(void** stack, int depth, int count) { + int i; + char marker = MARKER_STACKTRACE; + + profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE; + prof_word(count); + prof_word(depth); + for(i=0; i<depth; i++) + prof_word((long)stack[i]); + write(profile_file, profile_write_buffer, profile_buffer_position); + profile_buffer_position = 0; +} + + +/* ****************************************************** + * libunwind workaround for process JIT frames correctly + * ****************************************************** + */ + +#include "get_custom_offset.c" + +typedef struct { + void* _unused1; + void* _unused2; + void* sp; + void* ip; + void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4]; +} vmprof_hacked_unw_cursor_t; + +static int vmprof_unw_step(unw_cursor_t *cp, int first_run) { + void* ip; + void* sp; + ptrdiff_t sp_offset; + unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip); + unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp); + if (!first_run) + // make sure we're pointing to the CALL and not to the first + // instruction after. If the callee adjusts the stack for us + // it's not safe to be at the instruction after + ip -= 1; + sp_offset = vmprof_unw_get_custom_offset(ip, cp); + + if (sp_offset == -1) { + // it means that the ip is NOT in JITted code, so we can use the + // stardard unw_step + return unw_step(cp); + } + else { + // this is a horrible hack to manually walk the stack frame, by + // setting the IP and SP in the cursor + vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp; + void* bp = (void*)sp + sp_offset; + cp2->sp = bp; + bp -= sizeof(void*); + cp2->ip = ((void**)bp)[0]; + // the ret is on the top of the stack minus WORD + return 1; + } +} + + +/* ************************************************************* + * functions to dump the stack trace + * ************************************************************* + */ + +// The original code here has a comment, "stolen from pprof", +// about a "__thread int recursive". But general __thread +// variables are not really supposed to be accessed from a +// signal handler. Moreover, we are using SIGPROF, which +// should not be recursively called on the same thread. +//static __thread int recursive; + +int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) { + void *ip; + int n = 0; + unw_cursor_t cursor; + unw_context_t uc = *ucontext; + //if (recursive) { + // return 0; + //} + if (!custom_sanity_check()) { + return 0; + } + //++recursive; + + int ret = unw_init_local(&cursor, &uc); + assert(ret >= 0); + _unused(ret); + int first_run = 1; + + while (n < max_depth) { + if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { + break; + } + + unw_proc_info_t pip; + unw_get_proc_info(&cursor, &pip); + + /* char funcname[4096]; */ + /* unw_word_t offset; */ + /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */ + /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */ + + /* if n==0, it means that the signal handler interrupted us while we + were in the trampoline, so we are not executing (yet) the real main + loop function; just skip it */ + if (vmprof_mainloop_func && + (void*)pip.start_ip == (void*)vmprof_mainloop_func && + n > 0) { + // found main loop stack frame + void* sp; + unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp); + void *arg_addr = (char*)sp + mainloop_sp_offset; + void **arg_ptr = (void**)arg_addr; + // fprintf(stderr, "stacktrace mainloop: rsp %p &f2 %p offset %ld\n", + // sp, arg_addr, mainloop_sp_offset); + if (mainloop_get_virtual_ip) { + ip = mainloop_get_virtual_ip(*arg_ptr); + } else { + ip = *arg_ptr; + } + } + + result[n++] = ip; + n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth); + if (vmprof_unw_step(&cursor, first_run) <= 0) { + break; + } + first_run = 0; + } + //--recursive; + return n; +} + + +static int __attribute__((noinline)) frame_forcer(int rv) { + return rv; +} + +static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { + void* stack[MAX_STACK_DEPTH]; + int saved_errno = errno; + stack[0] = GetPC((ucontext_t*)ucontext); + int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); + depth++; // To account for pc value in stack[0]; + prof_write_stacktrace(stack, depth, 1); + errno = saved_errno; +} + +/* ************************************************************* + * functions to enable/disable the profiler + * ************************************************************* + */ + +static int open_profile(int fd, long period_usec, int write_header, char *s, + int slen) { + if ((fd = dup(fd)) == -1) { + return -1; + } + profile_buffer_position = 0; + profile_file = fd; + if (write_header) + prof_header(period_usec); + if (s) + write(profile_file, s, slen); + return 0; +} + +static int close_profile(void) { + // XXX all of this can happily fail + FILE* src; + char buf[BUFSIZ]; + size_t size; + int marker = MARKER_TRAILER; + write(profile_file, &marker, 1); + + // copy /proc/PID/maps to the end of the profile file + sprintf(buf, "/proc/%d/maps", getpid()); + src = fopen(buf, "r"); + while ((size = fread(buf, 1, BUFSIZ, src))) { + write(profile_file, buf, size); + } + fclose(src); + close(profile_file); + return 0; +} + + +static int install_sigprof_handler(void) { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = sigprof_handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + if (sigemptyset(&sa.sa_mask) == -1 || + sigaction(SIGPROF, &sa, NULL) == -1) { + return -1; + } + return 0; +} + +static int remove_sigprof_handler(void) { + sighandler_t res = signal(SIGPROF, SIG_DFL); + if (res == SIG_ERR) { + return -1; + } + return 0; +}; + +static int install_sigprof_timer(long period_usec) { + static struct itimerval timer; + last_period_usec = period_usec; + timer.it_interval.tv_sec = 0; + timer.it_interval.tv_usec = period_usec; + timer.it_value = timer.it_interval; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { + return -1; + } + return 0; +} + +static int remove_sigprof_timer(void) { + static struct itimerval timer; + last_period_usec = 0; + timer.it_interval.tv_sec = 0; + timer.it_interval.tv_usec = 0; + timer.it_value.tv_sec = 0; + timer.it_value.tv_usec = 0; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { + return -1; + } + return 0; +} + +static void atfork_disable_timer(void) { + remove_sigprof_timer(); +} + +static void atfork_enable_timer(void) { + install_sigprof_timer(last_period_usec); +} + +static int install_pthread_atfork_hooks(void) { + /* this is needed to prevent the problems described there: + - http://code.google.com/p/gperftools/issues/detail?id=278 + - http://lists.debian.org/debian-glibc/2010/03/msg00161.html + + TL;DR: if the RSS of the process is large enough, the clone() syscall + will be interrupted by the SIGPROF before it can complete, then + retried, interrupted again and so on, in an endless loop. The + solution is to disable the timer around the fork, and re-enable it + only inside the parent. + */ + if (atfork_hook_installed) + return 0; + int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); + if (ret != 0) + return -1; + atfork_hook_installed = 1; + return 0; +} + +/* ************************************************************* + * public API + * ************************************************************* + */ + +void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, + vmprof_get_virtual_ip_t get_virtual_ip) { + mainloop_sp_offset = sp_offset; + mainloop_get_virtual_ip = get_virtual_ip; + vmprof_mainloop_func = func; +} + +int vmprof_enable(int fd, long period_usec, int write_header, char *s, + int slen) +{ + assert(period_usec > 0); + if (open_profile(fd, period_usec, write_header, s, slen) == -1) { + return -1; + } + if (install_sigprof_handler() == -1) { + return -1; + } + if (install_sigprof_timer(period_usec) == -1) { + return -1; + } + if (install_pthread_atfork_hooks() == -1) { + return -1; + } + return 0; +} + +int vmprof_disable(void) { + if (remove_sigprof_timer() == -1) { + return -1; + } + if (remove_sigprof_handler() == -1) { + return -1; + } + if (close_profile() == -1) { + return -1; + } + return 0; +} + +void vmprof_register_virtual_function(const char* name, void* start, void* end) { + // XXX unused by pypy + // for now *end is simply ignored + char buf[1024]; + int lgt = strlen(name) + 2 * sizeof(long) + 1; + + if (lgt > 1024) { + lgt = 1024; + } + buf[0] = MARKER_VIRTUAL_IP; + ((void **)(((void*)buf) + 1))[0] = start; + ((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1; + strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1); + write(profile_file, buf, lgt); +} diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/src/vmprof.h @@ -0,0 +1,22 @@ +#ifndef VMPROF_VMPROF_H_ +#define VMPROF_VMPROF_H_ + +#include <stddef.h> + +typedef void* (*vmprof_get_virtual_ip_t)(void*); + +extern void* vmprof_mainloop_func; +void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, + vmprof_get_virtual_ip_t get_virtual_ip); + +void vmprof_register_virtual_function(const char* name, void* start, void* end); + + +int vmprof_enable(int fd, long period_usec, int write_header, char* vips, + int vips_len); +int vmprof_disable(void); + +// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library) +void vmprof_set_tramp_range(void* start, void* end); + +#endif diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py new file mode 100644 diff --git a/rpython/jit/backend/x86/test/conftest.py b/pypy/module/_vmprof/test/conftest.py copy from rpython/jit/backend/x86/test/conftest.py copy to pypy/module/_vmprof/test/conftest.py --- a/rpython/jit/backend/x86/test/conftest.py +++ b/pypy/module/_vmprof/test/conftest.py @@ -1,12 +1,7 @@ -import py, os +import py from rpython.jit.backend import detect_cpu cpu = detect_cpu.autodetect() def pytest_runtest_setup(item): - if not cpu.startswith('x86'): - py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,)) - if cpu == 'x86_64': - if os.name == "nt": - py.test.skip("Windows cannot allocate non-reserved memory") - from rpython.rtyper.lltypesystem import ll2ctypes - ll2ctypes.do_allocation_in_far_regions() + if cpu != detect_cpu.MODEL_X86_64: + py.test.skip("x86_64 tests only") diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -0,0 +1,72 @@ + +import tempfile +from pypy.tool.pytest.objspace import gettestobjspace + +class AppTestVMProf(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) + cls.tmpfile = tempfile.NamedTemporaryFile() + cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno()) + cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name) + cls.tmpfile2 = tempfile.NamedTemporaryFile() + cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno()) + cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name) + + def test_import_vmprof(self): + import struct, sys + + WORD = struct.calcsize('l') + + def count(s): + i = 0 + count = 0 + i += 5 * WORD # header + assert s[i] == '\x04' + i += 1 # marker + assert s[i] == '\x04' + i += 1 # length + i += len('pypy') + while i < len(s): + if s[i] == '\x03': + break + if s[i] == '\x01': + xxx + assert s[i] == '\x02' + i += 1 + _, size = struct.unpack("ll", s[i:i + 2 * WORD]) + count += 1 + i += 2 * WORD + size + return count + + import _vmprof + _vmprof.enable(self.tmpfileno) + _vmprof.disable() + s = open(self.tmpfilename).read() + no_of_codes = count(s) + assert no_of_codes > 10 + d = {} + + exec """def foo(): + pass + """ in d + + _vmprof.enable(self.tmpfileno2) + + exec """def foo2(): + pass + """ in d + + _vmprof.disable() + s = open(self.tmpfilename2).read() + no_of_codes2 = count(s) + assert "py:foo:" in s + assert "py:foo2:" in s + assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests + + def test_enable_ovf(self): + import _vmprof + raises(ValueError, _vmprof.enable, 999, 0) + raises(ValueError, _vmprof.enable, 999, -2.5) + raises(ValueError, _vmprof.enable, 999, 1e300) + raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300) + raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300)) diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py new file mode 100644 --- /dev/null +++ b/pypy/module/_vmprof/test/test_direct.py @@ -0,0 +1,71 @@ + +import py +try: + import cffi +except ImportError: + py.test.skip('cffi required') + +srcdir = py.path.local(__file__).join("..", "..", "src") + +ffi = cffi.FFI() +ffi.cdef(""" +long vmprof_write_header_for_jit_addr(void **, long, void*, int); +void *pypy_find_codemap_at_addr(long addr, long *start_addr); +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr); +long buffer[]; +""") + +lib = ffi.verify(""" +volatile int pypy_codemap_currently_invalid = 0; + +long buffer[] = {0, 0, 0, 0, 0}; + + + +void *pypy_find_codemap_at_addr(long addr, long *start_addr) +{ + return (void*)buffer; +} + +long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, + long *current_pos_addr) +{ + long c = *current_pos_addr; + if (c >= 5) + return -1; + *current_pos_addr = c + 1; + return *((long*)codemap_raw + c); +} + + +""" + open(str(srcdir.join("get_custom_offset.c"))).read()) + _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
