When running some of the PAPI tests from PAPI git checkout as root we noticed that fmultiplex1 failed sometimes with an invalid free, but this problem did not occur when running the test as a normal user. It looks like the problem is tied to libpfm getting additional tracepoint events. The problem has been reproduced on a Intel Westmere machine running RHEL7 and a Intel Ivy Bridge machine running Fedora 22.
Steps to reproduce: # git clone https://icl.cs.utk.edu/git/papi.git/ # cd papi/src # CFLAGS="-g" ./configure # make # cd ftests # ulimit -c unlimited # ./fmultiplex1 multiplex1: Using *** iterations case1: Does PAPI_multiplex_init() not break regular operation? Event set list 1: PAPI_TOT_INS 2: PAPI_TOT_CYC case1: 6983630740 6892812568 case2: Does setmpx/add work? Event set list 1: PAPI_TOT_INS 2: PAPI_TOT_CYC case2: 6983630766 6892856004 *** Error in `./fmultiplex1': free(): invalid pointer: 0x00000000004e7543 *** ======= Backtrace: ========= /lib64/libc.so.6(+0x7cfe1)[0x7fbe28055fe1] ./fmultiplex1[0x4212cc] ./fmultiplex1[0x41d6e2] ./fmultiplex1[0x42a6f5] ./fmultiplex1[0x41a4c8] ./fmultiplex1[0x40a94e] ./fmultiplex1[0x4033e1] ./fmultiplex1[0x40390e] ./fmultiplex1[0x403a9a] /lib64/libc.so.6(__libc_start_main+0xf5)[0x7fbe27ffab15] ./fmultiplex1[0x402421] ======= Memory map: ======== 00400000-00565000 r-xp 00000000 fd:00 136896780 /root/testing/papi/src/ftests/fmultiplex1 00765000-00766000 r--p 00165000 fd:00 136896780 /root/testing/papi/src/ftests/fmultiplex1 00766000-00780000 rw-p 00166000 fd:00 136896780 /root/testing/papi/src/ftests/fmultiplex1 00780000-04787000 rw-p 00000000 00:00 0 05948000-059f7000 rw-p 00000000 00:00 0 [heap] 7fbe20000000-7fbe20021000 rw-p 00000000 00:00 0 7fbe20021000-7fbe24000000 ---p 00000000 00:00 0 7fbe27fd9000-7fbe2818f000 r-xp 00000000 fd:00 201328876 /usr/lib64/libc-2.17.so 7fbe2818f000-7fbe2838f000 ---p 001b6000 fd:00 201328876 /usr/lib64/libc-2.17.so 7fbe2838f000-7fbe28393000 r--p 001b6000 fd:00 201328876 /usr/lib64/libc-2.17.so 7fbe28393000-7fbe28395000 rw-p 001ba000 fd:00 201328876 /usr/lib64/libc-2.17.so 7fbe28395000-7fbe2839a000 rw-p 00000000 00:00 0 7fbe2839a000-7fbe283d5000 r-xp 00000000 fd:00 201329542 /usr/lib64/libquadmath.so.0.0.0 7fbe283d5000-7fbe285d4000 ---p 0003b000 fd:00 201329542 /usr/lib64/libquadmath.so.0.0.0 7fbe285d4000-7fbe285d5000 r--p 0003a000 fd:00 201329542 /usr/lib64/libquadmath.so.0.0.0 7fbe285d5000-7fbe285d6000 rw-p 0003b000 fd:00 201329542 /usr/lib64/libquadmath.so.0.0.0 7fbe285d6000-7fbe285eb000 r-xp 00000000 fd:00 201326732 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7fbe285eb000-7fbe287ea000 ---p 00015000 fd:00 201326732 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7fbe287ea000-7fbe287eb000 r--p 00014000 fd:00 201326732 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7fbe287eb000-7fbe287ec000 rw-p 00015000 fd:00 201326732 /usr/lib64/libgcc_s-4.8.5-20150702.so.1 7fbe287ec000-7fbe288ed000 r-xp 00000000 fd:00 201328884 /usr/lib64/libm-2.17.so 7fbe288ed000-7fbe28aec000 ---p 00101000 fd:00 201328884 /usr/lib64/libm-2.17.so 7fbe28aec000-7fbe28aed000 r--p 00100000 fd:00 201328884 /usr/lib64/libm-2.17.so 7fbe28aed000-7fbe28aee000 rw-p 00101000 fd:00 201328884 /usr/lib64/libm-2.17.so 7fbe28aee000-7fbe28c0d000 r-xp 00000000 fd:00 201329761 /usr/lib64/libgfortran.so.3.0.0 7fbe28c0d000-7fbe28e0d000 ---p 0011f000 fd:00 201329761 /usr/lib64/libgfortran.so.3.0.0 7fbe28e0d000-7fbe28e0f000 r--p 0011f000 fd:00 201329761 /usr/lib64/libgfortran.so.3.0.0 7fbe28e0f000-7fbe28e11000 rw-p 00121000 fd:00 201329761 /usr/lib64/libgfortran.so.3.0.0 7fbe28e11000-7fbe28e32000 r-xp 00000000 fd:00 201328869 /usr/lib64/ld-2.17.so 7fbe29023000-7fbe29027000 rw-p 00000000 00:00 0 7fbe29030000-7fbe29032000 rw-p 00000000 00:00 0 7fbe29032000-7fbe29033000 r--p 00021000 fd:00 201328869 /usr/lib64/ld-2.17.so 7fbe29033000-7fbe29034000 rw-p 00022000 fd:00 201328869 /usr/lib64/ld-2.17.so 7fbe29034000-7fbe29035000 rw-p 00000000 00:00 0 7ffdb0d6f000-7ffdb0d90000 rw-p 00000000 00:00 0 [stack] 7ffdb0dfb000-7ffdb0dfd000 r-xp 00000000 00:00 0 [vdso] ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] Program received signal SIGABRT: Process abort signal. Backtrace for this error: #0 0x7FBE28B07467 #1 0x7FBE28B07AAE #2 0x7FBE2800E66F #3 0x7FBE2800E5F7 #4 0x7FBE2800FCE7 #5 0x7FBE2804E316 #6 0x7FBE28055FE0 #7 0x4212CB in pfm_perf_terminate at pfmlib_perf_event_pmu.c:793 #8 0x41D6E1 in pfm_terminate at pfmlib_common.c:839 #9 0x42A6F4 in _papi_libpfm4_shutdown at papi_libpfm4_events.c:77 #10 0x41A4C7 in _peu_shutdown_component at perf_event_uncore.c:696 #11 0x40A94D in PAPI_shutdown at papi.c:4551 #12 0x4033E0 in case2_ #13 0x40390D in MAIN__ at fmultiplex1.F:? Aborted (core dumped) Using gdb to see where the program died: # gdb ./fmultiplex1 core.* GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-80.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /root/testing/papi/src/ftests/fmultiplex1...done. [New LWP 16871] Core was generated by `./fmultiplex1'. Program terminated with signal 6, Aborted. #0 0x00007fbe2800e5f7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56 56 return INLINE_SYSCALL (tgkill, 3, pid, selftid, sig); Missing separate debuginfos, use: debuginfo-install libgcc-4.8.5-4.el7.x86_64 libgfortran-4.8.5-4.el7.x86_64 libquadmath-4.8.5-4.el7.x86_64 (gdb) where #0 0x00007fbe2800e5f7 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56 #1 0x00007fbe2800fce8 in __GI_abort () at abort.c:90 #2 0x00007fbe2804e317 in __libc_message (do_abort=do_abort@entry=2, fmt=fmt@entry=0x7fbe28157988 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/unix/sysv/linux/libc_fatal.c:196 #3 0x00007fbe28055fe1 in malloc_printerr (ar_ptr=0x7fbe28393760 <main_arena>, ptr=<optimized out>, str=0x7fbe28155074 "free(): invalid pointer", action=3) at malloc.c:5013 #4 _int_free (av=0x7fbe28393760 <main_arena>, p=<optimized out>, have_lock=0) at malloc.c:3835 #5 0x00000000004212cc in pfm_perf_terminate ( this=0x774960 <perf_event_support>) at pfmlib_perf_event_pmu.c:793 #6 0x000000000041d6e2 in pfm_terminate () at pfmlib_common.c:839 #7 0x000000000042a6f5 in _papi_libpfm4_shutdown () at papi_libpfm4_events.c:77 #8 0x000000000041a4c8 in _peu_shutdown_component () at components/perf_event_uncore/perf_event_uncore.c:696 #9 0x000000000040a94e in PAPI_shutdown () at papi.c:4551 #10 0x00000000004033e1 in case2_ () #11 0x000000000040390e in MAIN__ () #12 0x0000000000403a9a in main () To see what is happinging in pfm_terminate(): (gdb) up #1 0x00007fbe2800fce8 in __GI_abort () at abort.c:90 90 raise (SIGABRT); (gdb) up #2 0x00007fbe2804e317 in __libc_message (do_abort=do_abort@entry=2, fmt=fmt@entry=0x7fbe28157988 "*** Error in `%s': %s: 0x%s ***\n") at ../sysdeps/unix/sysv/linux/libc_fatal.c:196 196 abort (); (gdb) up #3 0x00007fbe28055fe1 in malloc_printerr (ar_ptr=0x7fbe28393760 <main_arena>, ptr=<optimized out>, str=0x7fbe28155074 "free(): invalid pointer", action=3) at malloc.c:5013 5013 __libc_message (action & 2, "*** Error in `%s': %s: 0x%s ***\n", (gdb) up #4 _int_free (av=0x7fbe28393760 <main_arena>, p=<optimized out>, have_lock=0) at malloc.c:3835 3835 malloc_printerr (check_action, errstr, chunk2mem(p), av); (gdb) up #5 0x00000000004212cc in pfm_perf_terminate ( this=0x774960 <perf_event_support>) at pfmlib_perf_event_pmu.c:793 793 free((void *)p->name); (gdb) print p->type $1 = 2 (gdb) print p->name $2 = 0x4e7543 "snbep_unc_ubo" p->type of 2 is a trace point, but that name definitely not a tracepoint name. -Will ------------------------------------------------------------------------------ _______________________________________________ perfmon2-devel mailing list perfmon2-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/perfmon2-devel
