Author: Richard Plangger <planri...@gmail.com>
Branch: vmprof-native
Changeset: r90010:d671b63850ea
Date: 2017-02-08 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/d671b63850ea/
Log: some more changes to the native test
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -46,7 +46,7 @@
eci_kwds = dict(
include_dirs = [SRC, SHARED, BACKTRACE, UDIS86],
- includes = ['rvmprof.h', 'vmprof_stack.h'],
+ includes = ['rvmprof.h','vmprof_stack.h'],
libraries = _libs,
separate_module_files = [
SRC.join('rvmprof.c'),
@@ -99,6 +99,7 @@
return CInterface(locals())
+
class CInterface(object):
def __init__(self, namespace):
for k, v in namespace.iteritems():
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c
b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -16,16 +16,3 @@
#include "shared/vmprof_main_win32.h"
#endif
-void dump_native_symbols(int fileno)
-{
-// TODO PyObject * mod = NULL;
-// TODO
-// TODO mod = PyImport_ImportModuleNoBlock("vmprof");
-// TODO if (mod == NULL)
-// TODO goto error;
-// TODO
-// TODO PyObject_CallMethod(mod, "dump_native_symbols", "(l)", fileno);
-// TODO
-// TODOerror:
-// TODO Py_XDECREF(mod);
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h
b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -21,7 +21,6 @@
#define RPY_EXPORTED extern __attribute__((visibility("default")))
#endif
-
RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
int lines, const char *interp_name, int native);
RPY_EXTERN void vmprof_ignore_signals(int);
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c
b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -170,11 +170,6 @@
}
}
-#else
-// other platforms than linux & mac os x
-void dump_all_known_symbols(int fd) {
- // oh, nothing to do!! a not supported platform
-}
#endif
#ifdef __unix__
@@ -247,3 +242,193 @@
#endif
return 0;
}
+
+#ifdef RPYTHON_VMPROF
+
+#define WORD_SIZE sizeof(long)
+#define ADDR_SIZE sizeof(void*)
+#define MAXLEN 1024
+
+void _dump_native_symbol(int fileno, void * addr, char * sym, int linenumber,
char * filename) {
+ char natsym[64];
+ off_t pos_before;
+ struct str {
+ void * addr;
+ // NOTE windows 64, not supported yet
+ long size;
+ char str[1024];
+ } s;
+ pos_before = lseek(fileno, 0, SEEK_CUR);
+ lseek(fileno, 0, SEEK_END);
+
+ s.addr = addr;
+ /* must mach '<lang>:<name>:<line>:<file>'
+ * 'n' has been chosen as lang here, because the symbol
+ * can be generated from several languages (e.g. C, C++, ...)
+ */
+ // MARKER_NATIVE_SYMBOLS is \x08
+ write(fileno, "\x08", 1);
+ if (sym == NULL || sym[0] == '\x00') {
+ snprintf(natsym, 64, "<native symbol %p>", addr);
+ sym = natsym;
+ }
+ if (filename != NULL) {
+ s.size = snprintf(s.str, 1024, "n:%s:%d:%s", sym, linenumber,
filename);
+ } else {
+ s.size = snprintf(s.str, 1024, "n:%s:%d:-", sym, linenumber);
+ }
+ write(fileno, &s, sizeof(void*)+sizeof(long)+s.size);
+
+ lseek(fileno, pos_before, SEEK_SET);
+}
+
+int _skip_string(int fileno)
+{
+ long chars;
+ int count = read(fileno, &chars, sizeof(long));
+ LOG("reading string of %d chars\n", chars);
+ if (count <= 0) {
+ return 1;
+ }
+ lseek(fileno, chars, SEEK_CUR);
+
+ return 0;
+}
+
+int _skip_header(int fileno, int * version, int * flags)
+{
+ unsigned char r[4];
+ (void)read(fileno, r, 4);
+ unsigned char count = r[3];
+ *version = (r[0] & 0xff) << 8 | (r[1] & 0xff);
+ *flags = r[2];
+ lseek(fileno, (int)count, SEEK_CUR);
+ return 0;
+}
+
+long _read_word(int fileno)
+{
+ long w;
+ read(fileno, &w, WORD_SIZE);
+ return w;
+}
+
+void * _read_addr(int fileno)
+{
+ void * a;
+ read(fileno, &a, ADDR_SIZE);
+ return a;
+}
+
+int _skip_word(int fileno)
+{
+ lseek(fileno, WORD_SIZE, SEEK_CUR);
+ return 0;
+}
+
+int _skip_addr(int fileno)
+{
+ lseek(fileno, ADDR_SIZE, SEEK_CUR);
+ return 0;
+}
+
+int _skip_time_and_zone(int fileno)
+{
+ lseek(fileno, sizeof(int64_t)*2 + 8, SEEK_CUR);
+ return 0;
+}
+
+
+void dump_native_symbols(int fileno)
+{
+ // only call this function
+ off_t orig_pos, cur_pos;
+ char marker;
+ ssize_t count;
+ int version;
+ int flags;
+ int memory, lines, native;
+ orig_pos = lseek(fileno, 0, SEEK_CUR);
+
+ lseek(fileno, 5*WORD_SIZE, SEEK_SET);
+
+ while (1) {
+ LOG("pre read\n");
+ count = read(fileno, &marker, 1);
+ LOG("post read\n");
+ if (count <= 0) {
+ break;
+ }
+ cur_pos = lseek(fileno, 0, SEEK_CUR);
+ LOG("posss 0x%llx %d\n", cur_pos-1, cur_pos-1);
+ switch (marker) {
+ case MARKER_HEADER: {
+ LOG("header 0x%llx\n", cur_pos);
+ if (_skip_header(fileno, &version, &flags) != 0) {
+ return;
+ }
+ memory = (flags & PROFILE_MEMORY) != 0;
+ native = (flags & PROFILE_NATIVE) != 0;
+ lines = (flags & PROFILE_LINES) != 0;
+ break;
+ } case MARKER_META: {
+ LOG("meta 0x%llx\n", cur_pos);
+ if (_skip_string(fileno) != 0) { return; }
+ if (_skip_string(fileno) != 0) { return; }
+ break;
+ } case MARKER_TIME_N_ZONE:
+ case MARKER_TRAILER: {
+ LOG("tnz or trailer 0x%llx\n", cur_pos);
+ if (_skip_time_and_zone(fileno) != 0) { return; }
+ break;
+ } case MARKER_VIRTUAL_IP:
+ case MARKER_NATIVE_SYMBOLS: {
+ LOG("virtip 0x%llx\n", cur_pos);
+ if (_skip_addr(fileno) != 0) { return; }
+ if (_skip_string(fileno) != 0) { return; }
+ break;
+ } case MARKER_STACKTRACE: {
+ long trace_count = _read_word(fileno);
+ long depth = _read_word(fileno);
+
+ LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
+
+ for (long i = depth/2-1; i >= 0; i--) {
+ long kind = (long)_read_addr(fileno);
+ void * addr = _read_addr(fileno);
+ if (kind == VMPROF_NATIVE_TAG) {
+ LOG("found kind %p\n", addr);
+ char name[MAXLEN];
+ char srcfile[MAXLEN];
+ name[0] = 0;
+ srcfile[0] = 0;
+ int lineno = 0;
+ if (vmp_resolve_addr(addr, name, MAXLEN, &lineno,
srcfile, MAXLEN) == 0) {
+ LOG("dumping add %p, name %s, %s:%d\n", addr,
name, srcfile, lineno);
+ _dump_native_symbol(fileno, addr, name, lineno,
srcfile);
+ }
+ }
+ }
+ LOG("passed memory %d \n", memory);
+
+ if (_skip_addr(fileno) != 0) { return; } // thread id
+ if (memory) {
+ if (_skip_addr(fileno) != 0) { return; } // profile memory
+ }
+
+ break;
+ } default: {
+ fprintf(stderr, "unknown marker 0x%x\n", marker);
+ return;
+ }
+ }
+
+ cur_pos = lseek(fileno, 0, SEEK_CUR);
+ if (cur_pos >= orig_pos) {
+ break;
+ }
+ }
+
+ lseek(fileno, 0, SEEK_END);
+}
+#endif
diff --git a/rpython/rlib/rvmprof/src/shared/trampoline.c
b/rpython/rlib/rvmprof/src/shared/trampoline.c
--- a/rpython/rlib/rvmprof/src/shared/trampoline.c
+++ b/rpython/rlib/rvmprof/src/shared/trampoline.c
@@ -183,7 +183,7 @@
int pagesize;
if (g_trampoline != NULL) {
- fprintf(stderr, "trampoline already patched\n");
+ //fprintf(stderr, "trampoline already patched\n");
return 0; // already patched
}
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -98,8 +98,6 @@
result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
*depth = *depth + 1;
#else
- //result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame));
- //*depth = *depth + 1;
if (frame->kind == VMPROF_CODE_TAG) {
int n = *depth;
@@ -107,12 +105,7 @@
result[n++] = (void*)frame->value;
*depth = n;
}
-#ifdef PYPY_JIT_CODEMAP
- else if (frame->kind == VMPROF_JITTED_TAG) {
- intptr_t pc = ((intptr_t*)(frame->value - sizeof(intptr_t)))[0];
- n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
- }
-#endif
+
#endif
@@ -130,11 +123,15 @@
#ifdef VMP_SUPPORTS_NATIVE_PROFILING
int _write_native_stack(void* addr, void ** result, int depth) {
+#ifdef RPYTHON_VMPROF
+ result[depth++] = (void*)VMPROF_NATIVE_TAG;
+#else
if (vmp_profiles_python_lines()) {
// even if we do not log a python stack frame,
// we must keep the profile readable
result[depth++] = 0;
}
+#endif
result[depth++] = addr;
return depth;
}
@@ -143,16 +140,6 @@
int vmp_walk_and_record_stack(PY_STACK_FRAME_T *frame, void ** result,
int max_depth, int native_skip, intptr_t pc) {
-//#ifdef PYPY_JIT_CODEMAP
-// intptr_t codemap_addr;
-// if (pypy_find_codemap_at_addr((intptr_t)pc, &codemap_addr)) {
-// // the bottom part is jitted, means we can fill up the first part
-// // from the JIT
-// depth = vmprof_write_header_for_jit_addr(result, depth, pc,
max_depth);
-// frame = FRAME_STEP(frame); // skip the first item as it contains
garbage
-// }
-//#endif
-
// called in signal handler
#ifdef VMP_SUPPORTS_NATIVE_PROFILING
intptr_t func_addr;
@@ -179,12 +166,22 @@
native_skip--;
}
+ //printf("stack trace:\n");
int depth = 0;
PY_STACK_FRAME_T * top_most_frame = frame;
while (depth < max_depth) {
unw_get_proc_info(&cursor, &pip);
func_addr = pip.start_ip;
+
+ //{
+ // char name[64];
+ // unw_word_t x;
+ // unw_get_proc_name(&cursor, name, 64, &x);
+ // printf(" %s %p\n", name, func_addr);
+ //}
+
+
//if (func_addr == 0) {
// unw_word_t rip = 0;
// if (unw_get_reg(&cursor, UNW_REG_IP, &rip) < 0) {
@@ -197,6 +194,7 @@
if ((void*)pip.start_ip == (void*)VMPROF_EVAL()) {
// yes we found one stack entry of the python frames!
+#ifndef RPYTHON_VMPROF
unw_word_t rbx = 0;
if (unw_get_reg(&cursor, REG_RBX, &rbx) < 0) {
break;
@@ -208,6 +206,9 @@
// current top_most_frame
return 0;
} else {
+#else
+ {
+#endif
if (top_most_frame == NULL) {
break;
}
@@ -222,13 +223,27 @@
// mark native routines with the first bit set,
// this is possible because compiler align to 8 bytes.
//
- depth = _write_native_stack((void*)(func_addr | 0x1), result,
depth);
+
+#ifdef PYPY_JIT_CODEMAP
+ if (func_addr == 0 && top_most_frame->kind == VMPROF_JITTED_TAG) {
+ intptr_t pc = ((intptr_t*)(frame->value -
sizeof(intptr_t)))[0];
+ n = vmprof_write_header_for_jit_addr(result, n, pc, max_depth);
+ frame = FRAME_STEP(frame);
+ } else if (func_addr != 0) {
+ depth = _write_native_stack((void*)(func_addr | 0x1), result,
depth);
+ }
+#else
+ if (func_addr != 0) {
+ depth = _write_native_stack((void*)(func_addr | 0x1), result,
depth);
+ }
+#endif
}
int err = unw_step(&cursor);
- if (err <= 0) {
- // on mac this breaks on Py_Main?
+ if (err == 0) {
break;
+ } else if (err < 0) {
+ return 0; // this sample is broken, cannot walk it fully
}
}
@@ -256,15 +271,25 @@
int _ignore_symbols_from_path(const char * name) {
// which symbols should not be considered while walking
// the native stack?
+#ifdef RPYTHON_VMPROF
+ if (strstr(name, "libpypy-c.so") != NULL
+ || strstr(name, "pypy-c") != NULL
+ || strstr(name, "pypy") != NULL) {
+ printf("ignoring %s\n", name);
+ return 1;
+ }
+#else
+ // cpython
if (strstr(name, "python") != NULL &&
-#ifdef __unix__
+# ifdef __unix__
strstr(name, ".so\n") == NULL
-#elif defined(__APPLE__)
+# elif defined(__APPLE__)
strstr(name, ".so") == NULL
-#endif
+# endif
) {
return 1;
}
+#endif
return 0;
}
@@ -440,6 +465,9 @@
}
int vmp_ignore_ip(intptr_t ip) {
+ if (vmp_range_count == 0) {
+ return 0;
+ }
int i = vmp_binary_search_ranges(ip, vmp_ranges, vmp_range_count);
if (i == -1) {
return 0;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h
b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -1,5 +1,7 @@
#pragma once
+#include <unistd.h>
+
// common defines
#define MARKER_STACKTRACE '\x01'
#define MARKER_VIRTUAL_IP '\x02'
@@ -21,6 +23,7 @@
#define PROFILE_MEMORY '\x01'
#define PROFILE_LINES '\x02'
#define PROFILE_NATIVE '\x04'
+#define PROFILE_RPYTHON '\x08'
#ifdef VMPROF_UNIX
#define VMP_SUPPORTS_NATIVE_PROFILING
@@ -41,6 +44,9 @@
#define PY_THREAD_STATE_T void
#define FRAME_STEP(f) f->next
#define FRAME_CODE(f) f->
+// Is there is a way to tell the compiler
+// that this prototype can have ANY return value. Just removing
+// the return type will default to int
typedef long Signed;
RPY_EXTERN Signed __vmprof_eval_vmprof();
#define VMPROF_EVAL() __vmprof_eval_vmprof
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -106,6 +106,9 @@
header.interp_name[2] = VERSION_TIMESTAMP;
header.interp_name[3] = memory*PROFILE_MEMORY + lines*PROFILE_LINES + \
native*PROFILE_NATIVE;
+#ifdef RPYTHON_VMPROF
+ header.interp_name[3] += PROFILE_RPYTHON;
+#endif
header.interp_name[4] = (char)namelen;
memcpy(&header.interp_name[5], interp_name, namelen);
diff --git a/rpython/rlib/rvmprof/src/vmprof_stack.h
b/rpython/rlib/rvmprof/src/vmprof_stack.h
--- a/rpython/rlib/rvmprof/src/vmprof_stack.h
+++ b/rpython/rlib/rvmprof/src/vmprof_stack.h
@@ -12,6 +12,7 @@
#define VMPROF_JITTING_TAG 4
#define VMPROF_GC_TAG 5
#define VMPROF_ASSEMBLER_TAG 6
+#define VMPROF_NATIVE_TAG 7
// whatever we want here
typedef struct vmprof_stack_s {
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py
b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -154,7 +154,11 @@
eci = ExternalCompilationInfo(compile_extra=['-g','-O2'],
separate_module_sources=["""
RPY_EXTERN int native_func(void) {
- return 42;
+ int j = 0;
+ for (int i = 0; i < 420000; i++) {
+ j += 1;
+ }
+ return j;
}
"""])
@@ -176,8 +180,6 @@
if num > 0:
return main(code, num-1)
else:
- for i in range(100):
- native_func()
return native_func()
tmpfilename = str(udir.join('test_rvmprof'))
@@ -188,15 +190,12 @@
os.open('foo', 1, 1)
code = MyCode()
rvmprof.register_code(code, get_name)
- fd = os.open(tmpfilename, os.O_WRONLY | os.O_CREAT, 0666)
- if we_are_translated():
- num = 100000000
- period = 0.0001
- else:
- num = 10000
- period = 0.9
+ fd = os.open(tmpfilename, os.O_RDWR | os.O_CREAT, 0666)
+ num = 10000
+ period = 0.0001
rvmprof.enable(fd, period, native=1)
- res = main(code, num)
+ for i in range(num):
+ res = main(code, 10)
#assert res == 499999500000
rvmprof.disable()
os.close(fd)
@@ -204,10 +203,13 @@
def check_profile(filename):
from vmprof import read_profile
+ from vmprof.show import PrettyPrinter
prof = read_profile(filename)
- assert prof.get_tree().name.startswith("py:")
- assert prof.get_tree().count
+ tree = prof.get_tree()
+ p = PrettyPrinter()
+ p._print_tree(tree)
+ assert tree.name.startswith("n:pypy_g_main")
#assert f() == 0
#assert os.path.exists(tmpfilename)
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
