I've managed to narrow down a crash with collectd in curl_json. This is 100% reproducable and isn't related to "old" data (ie: removing rrd directory does nothing).
I'm polling fpm through nginx, which looks something like this: # curl -s "http://foobar/_fpmstatus?json" | json_reformat { "pool": "fpm", "process manager": "dynamic", "accepted conn": 22, "listen queue len": 0, "max listen queue len": -1, "idle processes": 2, "active processes": 0, "total processes": 2, "max children reached": 0 } curl_json set up like this: LoadPlugin curl_json <Plugin curl_json> <URL "http://foobar/_fpmstatus?json"> Instance "fpm" <Key "accepted conn"> Type "http_requests" </Key> <Key "listen queue len"> Type "listen_queue" </Key> <Key "active processes"> Type "active_processes" </Key> <Key "total processes"> Type "total_processes" </Key> </URL> </Plugin> All types are set up as derives (I've tried gauge as well) ..and this is what happens 100% of the time: # /usr/sbin/collectd -P /var/run/collectd/collectd.pid -C /etc/collectd.conf -f *** longjmp causes uninitialized stack frame ***: /usr/sbin/collectd terminated ======= Backtrace: ========= /lib/libc.so.6(__fortify_fail+0x50)[0xb7e48270] /lib/libc.so.6(+0xe61da)[0xb7e481da] /usr/lib/libcurl.so.4(+0xc839)[0xb7b66839] /lib/libpthread.so.0(+0xea68)[0xb7ed6a68] /lib/libc.so.6(__libc_start_main+0xe7)[0xb7d78cd7] /usr/sbin/collectd[0x804d541] ======= Memory map: ======== 08048000-08065000 r-xp 00000000 08:04 89066737 /usr/sbin/collectd 08065000-08066000 r--p 0001c000 08:04 89066737 /usr/sbin/collectd 08066000-08067000 rw-p 0001d000 08:04 89066737 /usr/sbin/collectd 09360000-09478000 rw-p 09360000 00:00 0 [heap] b2c00000-b2c21000 rw-p b2c00000 00:00 0 b2c21000-b2d00000 ---p b2c21000 00:00 0 b2ddd000-b2df8000 r-xp 00000000 08:04 8995185 /usr/lib/gcc/i686-pc-linux-gnu/4.4.5/libgcc_s.so.1 b2df8000-b2df9000 r--p 0001a000 08:04 8995185 /usr/lib/gcc/i686-pc-linux-gnu/4.4.5/libgcc_s.so.1 b2df9000-b2dfa000 rw-p 0001b000 08:04 8995185 /usr/lib/gcc/i686-pc-linux-gnu/4.4.5/libgcc_s.so.1 b2e00000-b2e11000 r-xp 00000000 08:04 88493285 /lib/libresolv-2.12.2.so b2e11000-b2e12000 r--p 00010000 08:04 88493285 /lib/libresolv-2.12.2.so b2e12000-b2e13000 rw-p 00011000 08:04 88493285 /lib/libresolv-2.12.2.sob2e13000-b2e15000 rw-p b2e13000 00:00 0 b2e15000-b2e1a000 r-xp 00000000 08:04 88493233 /lib/libnss_dns-2.12.2.sob2e1a000-b2e1b000 r--p 00004000 08:04 88493233 /lib/libnss_dns-2.12.2.so b2e1b000-b2e1c000 rw-p 00005000 08:04 88493233 /lib/libnss_dns-2.12.2.so b2e1c000-b2e1d000 ---p b2e1c000 00:00 0 b2e1d000-b381d000 rw-p b2e1d000 00:00 0 b381d000-b381e000 ---p b381d000 00:00 0 b381e000-b421e000 rw-p b381e000 00:00 0 b421e000-b421f000 ---p b421e000 00:00 0 b421f000-b4c1f000 rw-p b421f000 00:00 0 b4c1f000-b4c20000 ---p b4c1f000 00:00 0 b4c20000-b5620000 rw-p b4c20000 00:00 0 b5620000-b5621000 ---p b5620000 00:00 0 b5621000-b6021000 rw-p b5621000 00:00 0 b6021000-b6022000 ---p b6021000 00:00 0 b6022000-b6a22000 rw-p b6022000 00:00 0 b6a22000-b6a23000 ---p b6a22000 00:00 0 b6a23000-b7423000 rw-p b6a23000 00:00 0 b7423000-b742d000 r-xp 00000000 08:04 88492866 /lib/libnss_files-2.12.2.so b742d000-b742e000 r--p 00009000 08:04 88492866 /lib/libnss_files-2.12.2.so b742e000-b742f000 rw-p 0000a000 08:04 88492866 /lib/libnss_files-2.12.2.so b742f000-b7453000 r-xp 00000000 08:04 91050978 /usr/lib/libexpat.so.1.5.2 b7453000-b7454000 ---p 00024000 08:04 91050978 /usr/lib/libexpat.so.1.5.2 b7454000-b7456000 r--p 00024000 08:04 91050978 /usr/lib/libexpat.so.1.5.2 b7456000-b7457000 rw-p 00026000 08:04 91050978 /usr/lib/libexpat.so.1.5.2 b7457000-b74da000 r-xp 00000000 08:04 11305059 /usr/lib/libfreetype.so.6.6.2 b74da000-b74de000 r--p 00083000 08:04 11305059 /usr/lib/libfreetype.so.6.6.2 b74de000-b74df000 rw-p 00087000 08:04 11305059 /usr/lib/libfreetype.so.6.6.2 b74df000-b750d000 r-xp 00000000 08:04 26674259 /usr/lib/libfontconfig.so.1.4.4 b750d000-b750e000 r--p 0002d000 08:04 26674259 /usr/lib/libfontconfig.so.1.4.4 b750e000-b750f000 rw-p 0002e000 08:04 26674259 /usr/lib/libfontconfig.so.1.4.4 b750f000-b7512000 r-xp 00000000 08:04 88376473 /usr/lib/libgmodule-2.0.so.0.2800.6 b7512000-b7513000 r--p 00002000 08:04 88376473 /usr/lib/libgmodule-2.0.so.0.2800.6 7513000-b7514000 rw-p 00003000 08:04 88376473 /usr/lib/libgmodule-2.0.so.0.2800.6 b7514000-b7518000 r-xp 00000000 08:04 88376429 /usr/lib/libgthread-2.0.so.0.2800.6b7518000-b7519000 r--p 00003000 08:04 88376429 /usr/lib/libgthread-2.0.so.0.2800.6 b7519000-b751a000 rw-p 00004000 08:04 88376429 /usr/lib/libgthread-2.0.so.0.2800.6b751a000-b753f000 r-xp 00000000 08:04 88359566 /usr/lib/libpangoft2-1.0.so.0.2800.3 b753f000-b7540000 r--p 00024000 08:04 88359566 /usr/lib/libpangoft2-1.0.so.0.2800.3 b7540000-b7541000 rw-p 00025000 08:04 88359566 /usr/lib/libpangoft2-1.0.so.0.2800.3 b7541000-b7597000 r-xp 00000000 08:04 88359907 /usr/lib/libpixman-1.so.0.20.2b7597000-b7599000 r--p 00055000 08:04 88359907 /usr/lib/libpixman-1.so.0.20.2 b7599000-b759a000 rw-p 00057000 08:04 88359907 /usr/lib/libpixman-1.so.0.20.2 b759a000-b769c000 r-xp 00000000 08:04 88376494 /usr/lib/libglib-2.0.so.0.2800.6 b769c000-b769d000 r--p 00101000 08:04 88376494 /usr/lib/libglib-2.0.so.0.2800.6 b769d000-b769e000 rw-p 00102000 08:04 88376494 /usr/lib/libglib-2.0.so.0.2800.6 b769e000-b76e0000 r-xp 00000000 08:04 88376414 /usr/lib/libgobject-2.0.so.0.2800.6 b76e0000-b76e1000 ---p 00042000 08:04 88376414 /usr/lib/libgobject-2.0.so.0.2800.6 b76e1000-b76e2000 r--p 00042000 08:04 88376414 /usr/lib/libgobject-2.0.so.0.2800.6 b76e2000-b76e3000 rw-p 00043000 08:04 88376414 /usr/lib/libgobject-2.0.so.0.2800.6 b76e3000-b7778000 r-xp 00000000 08:04 89081020 /usr/lib/libcairo.so.2.11000.2 b7778000-b7779000 ---p 00095000 08:04 89081020 /usr/lib/libcairo.so.2.11000.2 b7779000-b777a000 r--p 00095000 08:04 89081020 /usr/lib/libcairo.so.2.11000.2 b777a000-b777b000 rw-p 00096000 08:04 89081020 /usr/lib/libcairo.so.2.11000.2 b777b000-b777d000 rw-p b777b000 00:00 0 b777d000-b77be000 r-xp 00000000 08:04 88359724 /usr/lib/libpango-1.0.so.0.2800.3 b77be000-b77bf000 ---p 00041000 08:04 88359724 /usr/lib/libpango-1.0.so.0.2800.3 b77bf000-b77c0000 r--p 00041000 08:04 88359724 /usr/lib/libpango-1.0.so.0.2800.3 b77c0000-b77c1000 rw-p 00042000 08:04 88359724 /usr/lib/libpango-1.0.so.0.2800.3 b77c1000-b77e5000 r-xp 00000000 08:04 88347090 /usr/lib/libpng14.so.14.5.0 b77e5000-b77e6000 r--p 00023000 08:04 88347090 /usr/lib/libpng14.so.14.5.0 b77e6000-b77e7000 rw-p 00024000 08:04 88347090 /usr/lib/libpng14.so.14.5.0 b77e7000-b77f1000 r-xp 00000000 08:04 88359547 /usr/lib/libpangocairo-1.0.so.0.2800.3 b77f1000-b77f2000 r--p 00009000 08:04 88359547 /usr/lib/libpangocairo-1.0.so.0.2800.3 b77f2000-b77f3000 rw-p 0000a000 08:04 88359547 /usr/lib/libpangocairo-1.0.so.0.2800.3 b77f3000-b7918000 r-xp 00000000 08:04 88641889 /usr/lib/libxml2.so.2.7.8 b7918000-b791c000 r--p 00125000 08:04 88641889 /usr/lib/libxml2.so.2.7.8 b791c000-b791d000 rw-p 00129000 08:04 88641889 /usr/lib/libxml2.so.2.7.8 b791d000-b791e000 rw-p b791d000 00:00 0 b791e000-b7952000 r-xp 00000000 08:04 89097100 /usr/lib/librrd_th.so.4.1.4b7952000-b7953000 r--p 00033000 08:04 89097100 /usr/lib/librrd_th.so.4.1.4 b7953000-b7956000 rw-p 00034000 08:04 89097100 /usr/lib/librrd_th.so.4.1.4b7956000-b795b000 r-xp 00000000 08:04 89066743 /usr/lib/collectd/rrdtool.so b795b000-b795c000 r--p 00004000 08:04 89066743 /usr/lib/collectd/rrdtool.so b795c000-b795d000 rw-p 00005000 08:04 89066743 /usr/lib/collectd/rrdtool.so b795d000-b7981000 r-xp 00000000 08:04 88493215 /lib/libm-2.12.2.sob7981000-b7982000 r--p 00023000 08:04 88493215 /lib/libm-2.12.2.so b7982000-b7983000 rw-p 00024000 08:04 88493215 /lib/libm-2.12.2.so b7983000-b7987000 r-xp 00000000 08:04 88346933 /usr/lib/liboping.so.0.2.6 b7987000-b7988000 r--p 00003000 08:04 88346933 /usr/lib/liboping.so.0.2.6 b7988000-b7989000 rw-p 00004000 08:04 88346933 /usr/lib/liboping.so.0.2.6 b798e000-b798f000 rw-p b798e000 00:00 0 b798f000-b7996000 r-xp 00000000 08:04 88359953 /usr/lib/libyajl.so.1.0.11 b7996000-b7997000 r--p 00006000 08:04 88359953 /usr/lib/libyajl.so.1.0.11 b7997000-b7998000 rw-p 00007000 08:04 88359953 /usr/lib/libyajl.so.1.0.11 b7998000-b7999000 rw-p b7998000 00:00 0 b7999000-b799c000 r-xp 00000000 08:04 89066723 /usr/lib/collectd/ping.so b799c000-b799d000 r--p 00002000 08:04 89066723 /usr/lib/collectd/ping.so b799d000-b799e000 rw-p 00003000 08:04 89066723 /usr/lib/collectd/ping.so b799e000-b7aef000 r-xp 00000000 08:04 109461545 /usr/lib/libcrypto.so.1.0.0 b7aef000-b7afe000 r--p 00150000 08:04 109461545 /usr/lib/libcrypto.so.1.0.0 b7afe000-b7b04000 rw-p 0015f000 08:04 109461545 /usr/lib/libcrypto.so.1.0.0 b7b04000-b7b07000 rw-p b7b04000 00:00 0 b7b07000-b7b55000 r-xp 00000000 08:04 109461584 /usr/lib/libssl.so.1.0.0 b7b55000-b7b57000 r--p 0004d000 08:04 109461584 /usr/lib/libssl.so.1.0.0 b7b57000-b7b5a000 rw-p 0004f000 08:04 109461584 /usr/lib/libssl.so.1.0.0 b7b5a000-b7ba8000 r-xp 00000000 08:04 88361959 /usr/lib/libcurl.so.4.2.0 b7ba8000-b7ba9000 ---p 0004e000 08:04 88361959 /usr/lib/libcurl.so.4.2.0 b7ba9000-b7baa000 r--p 0004e000 08:04 88361959 /usr/lib/libcurl.so.4.2.0 b7baa000-b7bab000 rw-p 0004f000 08:04 88361959 /usr/lib/libcurl.so.4.2.0 b7bab000-b7bad000 r-xp 00000000 08:04 89066747 /usr/lib/collectd/nginx.so b7bad000-b7bae000 r--p 00001000 08:04 89066747 /usr/lib/collectd/nginx.so b7bae000-b7baf000 rw-p 00002000 08:04 89066747 /usr/lib/collectd/nginx.so b7baf000-b7bb3000 rw-p b7baf000 00:00 0 b7bb3000-b7bc6000 r-xp 00000000 08:04 52494521 /lib/libz.so.1.2.5b7bc6000-b7bc7000 r--p 00012000 08:04 52494521 /lib/libz.so.1.2.5 b7bc7000-b7bc8000 rw-p 00013000 08:04 52494521 /lib/libz.so.1.2.5b7bc8000-b7bc9000 rw-p b7bc8000 00:00 0 b7bc9000-b7bcc000 r-xp 00000000 08:04 89066733 /usr/lib/collectd/curl_json.so b7bcc000-b7bcd000 r--p 00002000 08:04 89066733 /usr/lib/collectd/curl_json.so b7bcd000-b7bce000 rw-p 00003000 08:04 89066733 /usr/lib/collectd/curl_json.sob7bce000-b7cf0000 r-xp 00000000 08:04 88507051 /usr/lib/mysql/libmysqlclient.so.16.0.0 b7cf0000-b7cf2000 r--p 00121000 08:04 88507051 /usr/lib/mysql/libmysqlclient.so.16.0.0 b7cf2000-b7d37000 rw-p 00123000 08:04 88507051 /usr/lib/mysql/libmysqlclient.so.16.0.0 b7d37000-b7d38000 rw-p b7d37000 00:00 0 b7d38000-b7d3c000 r-xp 00000000 08:04 89066745 /usr/lib/collectd/mysql.so b7d3c000-b7d3d000 r--p 00003000 08:04 89066745 /usr/lib/collectd/mysql.so b7d3d000-b7d3e000 rw-p 00004000 08:04 89066745 /usr/lib/collectd/mysql.so b7d3e000-b7d41000 r-xp 00000000 08:04 89066735 /usr/lib/collectd/memcached.so b7d41000-b7d42000 r--p 00002000 08:04 89066735 /usr/lib/collectd/memcached.so b7d42000-b7d43000 rw-p 00003000 08:04 89066735 /usr/lib/collectd/memcached.so b7d43000-b7d45000 r-xp 00000000 08:04 89066725 /usr/lib/collectd/interface.so b7d45000-b7d46000 r--p 00001000 08:04 89066725 /usr/lib/collectd/interface.so b7d46000-b7d47000 rw-p 00002000 08:04 89066725 /usr/lib/collectd/interface.so b7d47000-b7d4a000 r-xp 00000000 08:04 89066741 /usr/lib/collectd/df.so b7d4a000-b7d4b000 r--p 00002000 08:04 89066741 /usr/lib/collectd/df.so b7d4b000-b7d4c000 rw-p 00003000 08:04 89066741 /usr/lib/collectd/df.so b7d4c000-b7d4d000 r-xp 00000000 08:04 89066729 /usr/lib/collectd/cpu.so b7d4d000-b7d4e000 r--p 00000000 08:04 89066729 /usr/lib/collectd/cpu.so b7d4e000-b7d4f000 rw-p 00001000 08:04 89066729 /usr/lib/collectd/cpu.so b7d4f000-b7d51000 r-xp 00000000 08:04 89066752 /usr/lib/collectd/tail.so b7d51000-b7d52000 r--p 00001000 08:04 89066752 /usr/lib/collectd/tail.so b7d52000-b7d53000 rw-p 00002000 08:04 89066752 /usr/lib/collectd/tail.so b7d53000-b7d57000 r-xp 00000000 08:04 89066710 /usr/lib/collectd/processes.so b7d57000-b7d58000 r--p 00003000 08:04 89066710 /usr/lib/collectd/processes.so b7d58000-b7d59000 rw-p 00004000 08:04 89066710 /usr/lib/collectd/processes.so b7d59000-b7d5a000 r-xp 00000000 08:04 89066739 /usr/lib/collectd/memory.so b7d5a000-b7d5b000 r--p 00000000 08:04 89066739 /usr/lib/collectd/memory.so b7d5b000-b7d5c000 rw-p 00001000 08:04 89066739 /usr/lib/collectd/memory.so b7d5c000-b7d5d000 rw-p b7d5c000 00:00 0 b7d5d000-b7d5f000 r-xp 00000000 08:04 88493356 /lib/libdl-2.12.2.so b7d5f000-b7d60000 r--p 00001000 08:04 88493356 /lib/libdl-2.12.2.sob7d60000-b7d61000 rw-p 00002000 08:04 88493356 /lib/libdl-2.12.2.so b7d61000-b7d62000 rw-p b7d61000 00:00 0 b7d62000-b7eb8000 r-xp 00000000 08:04 88493206 /lib/libc-2.12.2.so b7eb8000-b7eba000 r--p 00156000 08:04 88493206 /lib/libc-2.12.2.sob7eba000-b7ebb000 rw-p 00158000 08:04 88493206 /lib/libc-2.12.2.so b7ebb000-b7ebe000 rw-p b7ebb000 00:00 0 b7ebe000-b7ec6000 r-xp 00000000 08:04 56869659 /usr/lib/libltdl.so.7.2.2 b7ec6000-b7ec7000 r--p 00007000 08:04 56869659 /usr/lib/libltdl.so.7.2.2 b7ec7000-b7ec8000 rw-p 00008000 08:04 56869659 /usr/lib/libltdl.so.7.2.2 b7ec8000-b7edd000 r-xp 00000000 08:04 88493358 /lib/libpthread-2.12.2.so b7edd000-b7ede000 ---p 00015000 08:04 88493358 /lib/libpthread-2.12.2.so b7ede000-b7edf000 r--p 00015000 08:04 88493358 /lib/libpthread-2.12.2.so b7edf000-b7ee0000 rw-p 00016000 08:04 88493358 /lib/libpthread-2.12.2.so b7ee0000-b7ee2000 rw-p b7ee0000 00:00 0 b7ee2000-b7ee9000 r-xp 00000000 08:04 88493485 /lib/librt-2.12.2.so b7ee9000-b7eea000 r--p 00006000 08:04 88493485 /lib/librt-2.12.2.so b7eea000-b7eeb000 rw-p 00007000 08:04 88493485 /lib/librt-2.12.2.so b7eeb000-b7eec000 r-xp 00000000 08:04 89066727 /usr/lib/collectd/load.so b7eec000-b7eed000 r--p 00000000 08:04 89066727 /usr/lib/collectd/load.so b7eed000-b7eee000 rw-p 00001000 08:04 89066727 /usr/lib/collectd/load.so b7eee000-b7eef000 r-xp 00000000 08:04 89066721 /usr/lib/collectd/syslog.so b7eef000-b7ef0000 r--p 00000000 08:04 89066721 /usr/lib/collectd/syslog.so b7ef0000-b7ef1000 rw-p 00001000 08:04 89066721 /usr/lib/collectd/syslog.so b7ef1000-b7f0d000 r-xp 00000000 08:04 88493227 /lib/ld-2.12.2.so b7f0d000-b7f0e000 rw-p b7f0d000 00:00 0 b7f0e000-b7f0f000 r--p 0001c000 08:04 88493227 /lib/ld-2.12.2.so b7f0f000-b7f10000 rw-p 0001d000 08:04 88493227 /lib/ld-2.12.2.so bfaf3000-bfb08000 rw-p 7ffffffe9000 00:00 0 [stack] ..gdb backtrace: (gdb) backtrace full #0 0xb7f898d2 in ?? () from /lib/ld-linux.so.2 No symbol table info available. #1 0xb7e24872 in raise () from /lib/libc.so.6 No symbol table info available. #2 0xb7e25fe0 in abort () from /lib/libc.so.6 No symbol table info available. #3 0xb7e60395 in ?? () from /lib/libc.so.6 No symbol table info available. #4 0xb7ee0270 in __fortify_fail () from /lib/libc.so.6 No symbol table info available. #5 0xb7ee01da in ?? () from /lib/libc.so.6 No symbol table info available. #6 0xb7ee014a in __longjmp_chk () from /lib/libc.so.6 No symbol table info available. #7 0xb7bfd839 in ?? () from /usr/lib/libcurl.so.4 No symbol table info available. #8 <signal handler called> No symbol table info available. #9 0xb7f898d2 in ?? () from /lib/ld-linux.so.2 No symbol table info available. #10 0xb7f6de06 in nanosleep () from /lib/libpthread.so.0 No symbol table info available. #11 0x0804e10a in main () No symbol table info available. This is: - Linux s.nu 2.6.18-194.17.1.el5.028stab070.7 #1 SMP Fri Oct 1 14:17:14 MSD 2010 i686 Intel(R) Xeon(R) CPU E5420 @ 2.50GHz GenuineIntel GNU/Linux - yajl 1.0.11 _______________________________________________ collectd mailing list collectd@verplant.org http://mailman.verplant.org/listinfo/collectd