it is a good report. backtraces are very useful
is there github issue filled for it ? if no, can you please create one ?
I hope, it won't be lost that way
вт, 9 июн. 2020 г. в 15:13, Sander Hoentjen <san...@hoentjen.eu
<mailto:san...@hoentjen.eu>>:
Is there anybody with a clue? If I need to supply more info I can
do so,
of course.
Kind regards,
Sander
On 6/2/20 4:12 PM, Sander Hoentjen wrote:
> Hi list,
>
> Some time ago (around april 21st) we were using 1.8.13 and we
switched
> from nbthread = 1 to nbthread = 4
>
> This seemed stable for us, but 2 weeks ago we started seeing hangs
> (100% CPU haproxy processes)
>
> We then updated to haproxy 2.0.14. The hangs are gone, but
instead we
> see disappearing haproxy processes.
>
> I turned on coredumps, and I have some output from gdb.
>
> Hope this is helpful in figuring out what's wrong.
>
>
=====================================================================
>
> (gdb) bt
> #0 0x00007fb4770854f5 in raise () from /lib64/libc.so.6
> #1 0x00007fb477086cd5 in abort () from /lib64/libc.so.6
> #2 0x000000000054ce3c in ha_panic () at src/debug.c:207
> #3 0x000000000054d674 in wdt_handler (sig=14, si=<value optimized
> out>, arg=<value optimized out>) at src/wdt.c:119
> #4 <signal handler called>
> #5 0x00007fb47713ccd7 in socket () from /lib64/libc.so.6
> #6 0x00000000004fc75e in my_socketat (conn=0x7fb470036490,
flags=8)
> at include/common/namespace.h:28
> #7 create_server_socket (conn=0x7fb470036490, flags=8) at
> src/proto_tcp.c:257
> #8 tcp_connect_server (conn=0x7fb470036490, flags=8) at
> src/proto_tcp.c:323
> #9 0x00000000004e8c94 in si_connect (s=0x7fb4703ad540) at
> include/proto/stream_interface.h:513
> #10 connect_server (s=0x7fb4703ad540) at src/backend.c:1591
> #11 0x000000000044f373 in sess_update_stream_int
(s=0x7fb4703ad540) at
> src/stream.c:1015
> #12 0x000000000045527e in process_stream (t=0x7fb4703b5620,
> context=0x7fb4703ad540, state=<value optimized out>)
> at src/stream.c:2414
> #13 0x000000000052a875 in process_runnable_tasks () at
src/task.c:413
> #14 0x00000000004980f8 in run_poll_loop (data=<value optimized
out>)
> at src/haproxy.c:2627
> #15 run_thread_poll_loop (data=<value optimized out>) at
> src/haproxy.c:2754
> #16 0x00007fb477a25aa1 in start_thread (arg=0x7fb476786700) at
> pthread_create.c:301
> #17 0x00007fb47713bc2d in clone () from /lib64/libc.so.6
> (gdb) bt full
> #0 0x00007fb4770854f5 in raise () from /lib64/libc.so.6
> No symbol table info available.
> #1 0x00007fb477086cd5 in abort () from /lib64/libc.so.6
> No symbol table info available.
> #2 0x000000000054ce3c in ha_panic () at src/debug.c:207
> No locals.
> #3 0x000000000054d674 in wdt_handler (sig=14, si=<value optimized
> out>, arg=<value optimized out>) at src/wdt.c:119
> n = <value optimized out>
> p = <value optimized out>
> thr = 1
> #4 <signal handler called>
> No symbol table info available.
> #5 0x00007fb47713ccd7 in socket () from /lib64/libc.so.6
> No symbol table info available.
> #6 0x00000000004fc75e in my_socketat (conn=0x7fb470036490,
flags=8)
> at include/common/namespace.h:28
> No locals.
> #7 create_server_socket (conn=0x7fb470036490, flags=8) at
> src/proto_tcp.c:257
> ns = 0x0
> #8 tcp_connect_server (conn=0x7fb470036490, flags=8) at
> src/proto_tcp.c:323
> fd = <value optimized out>
> srv = 0x4850bf0
> be = 0x25606b0
> src = <value optimized out>
> use_fastopen = 0
> addr = <value optimized out>
> #9 0x00000000004e8c94 in si_connect (s=0x7fb4703ad540) at
> include/proto/stream_interface.h:513
> ret = 0
> conn_flags = <value optimized out>
> ---Type <return> to continue, or q <return> to quit---
> #10 connect_server (s=0x7fb4703ad540) at src/backend.c:1591
> cli_conn = 0x7fb4703ca8f0
> srv_conn = 0x7fb470036490
> old_conn = <value optimized out>
> srv_cs = 0x7fb4703eb310
> srv = <value optimized out>
> reuse = <value optimized out>
> reuse_orphan = <value optimized out>
> init_mux = 1
> alloced_cs = <value optimized out>
> err = <value optimized out>
> #11 0x000000000044f373 in sess_update_stream_int
(s=0x7fb4703ad540) at
> src/stream.c:1015
> conn_err = <value optimized out>
> srv = 0x4850bf0
> si = 0x7fb4703ad840
> req = 0x7fb4703ad550
> #12 0x000000000045527e in process_stream (t=0x7fb4703b5620,
> context=0x7fb4703ad540, state=<value optimized out>)
> at src/stream.c:2414
> srv = <value optimized out>
> s = 0x7fb4703ad540
> sess = 0x7fb4703ca620
> rqf_last = 9469952
> rpf_last = 2147483648
> rq_prod_last = 8
> rq_cons_last = 0
> rp_cons_last = 8
> rp_prod_last = 0
> req_ana_back = 0
> ---Type <return> to continue, or q <return> to quit---
> req = 0x7fb4703ad550
> res = 0x7fb4703ad5b0
> si_f = 0x7fb4703ad7e8
> si_b = 0x7fb4703ad840
> rate = 151
> #13 0x000000000052a875 in process_runnable_tasks () at
src/task.c:413
> t = 0x7fb4703b5620
> state = <value optimized out>
> ctx = <value optimized out>
> process = 0x453b30 <process_stream>
> lrq = <value optimized out>
> grq = <value optimized out>
> t = <value optimized out>
> max_processed = 46
> #14 0x00000000004980f8 in run_poll_loop (data=<value optimized
out>)
> at src/haproxy.c:2627
> next = <value optimized out>
> wake = <value optimized out>
> #15 run_thread_poll_loop (data=<value optimized out>) at
> src/haproxy.c:2754
> ptaf = <value optimized out>
> ptif = <value optimized out>
> ptdf = <value optimized out>
> ptff = <value optimized out>
> init_left = 0
> init_mutex = {__data = {__lock = 0, __count = 0, __owner
= 0,
> __nusers = 0, __kind = 0, __spins = 0,
> __list = {__prev = 0x0, __next = 0x0}}, __size = '\000'
> <repeats 39 times>, __align = 0}
> init_cond = {__data = {__lock = 0, __futex = 10,
__total_seq =
> 5, __wakeup_seq = 5, __woken_seq = 5,
> __mutex = 0xa42120, __nwaiters = 0, __broadcast_seq
= 3},
> __size =
>
"\000\000\000\000\n\000\000\000\005\000\000\000\000\000\000\000\005\000\000\000\000\000\000\000\005\---Type
> <return> to continue, or q <return> to quit---
> 000\000\000\000\000\000\000
> !\244\000\000\000\000\000\000\000\000\000\003\000\000", __align =
> 42949672960}
> #16 0x00007fb477a25aa1 in start_thread (arg=0x7fb476786700) at
> pthread_create.c:301
> __res = <value optimized out>
> pd = 0x7fb476786700
> now = <value optimized out>
> unwind_buf = {cancel_jmp_buf = {{jmp_buf =
{140413058443008,
> 698637182382513297, 140413080150880,
> 140413058443712, 0, 3, -658965645218931567,
> -658966367837741935}, mask_was_saved = 0}}, priv = {
> pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0,
cleanup =
> 0x0, canceltype = 0}}}
> not_first_call = <value optimized out>
> pagesize_m1 = <value optimized out>
> sp = <value optimized out>
> freesize = <value optimized out>
> #17 0x00007fb47713bc2d in clone () from /lib64/libc.so.6
> No symbol table info available.
>
> ===================================================
>
> haproxy.cfg:
>
> global
> # to have these messages end up in /var/log/haproxy.log you will
> # need to:
> #
> # 1) configure syslog to accept network log events. This is
done
> # by adding the '-r' option to the SYSLOGD_OPTIONS in
> # /etc/sysconfig/syslog
> #
> # 2) configure local2 events to go to the /var/log/haproxy.log
> # file. A line like the following can be added to
> # /etc/sysconfig/syslog
> #
> # local2.* /var/log/haproxy.log
> #
> log 127.0.0.1 local2
>
> chroot /var/lib/haproxy
> pidfile /var/run/haproxy.pid
> maxconn 4000
> user haproxy
> group haproxy
> nbthread 4
> daemon
>
> # turn on stats unix socket
> stats socket /var/lib/haproxy/stats
> ssl-default-bind-options no-tls-tickets ssl-min-ver TLSv1.2
> ssl-default-bind-ciphers
>
ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA
> tune.ssl.default-dh-param 4096
>
>
> defaults
> mode tcp
> log global
> option tcplog
> option dontlognull
> retries 3
> timeout http-request 10s
> timeout queue 1m
> timeout connect 10s
> timeout client 1m
> timeout server 1m
> #tunnel timeout needs to be higher than
> lsapi_backend_max_process_time
> timeout tunnel 16m
> timeout http-keep-alive 10s
> timeout check 10s
> maxconn 3000
>
> <snip>
> =================================================
>
> haproxy -vv
> HA-Proxy version 2.0.14 2020/04/02 - https://haproxy.org/
> Build options :
> TARGET = linux-glibc
> CPU = generic
> CC = gcc
> CFLAGS = -O2 -g -fno-strict-aliasing
-Wdeclaration-after-statement
> -fwrapv -Wno-unused-label -Wno-sign-compare -Wno-unused-parameter
> -Wno-old-style-declaration -Wno-ignored-qualifiers -Wno-clobbered
> -Wno-missing-field-initializers -Wtype-limits
> OPTIONS = USE_PCRE=1 USE_PTHREAD_PSHARED=1 USE_REGPARM=1
> USE_LINUX_TPROXY=1 USE_OPENSSL=1 USE_ZLIB=1 USE_NS=
>
> Feature list : +EPOLL -KQUEUE -MY_EPOLL -MY_SPLICE +NETFILTER +PCRE
> -PCRE_JIT -PCRE2 -PCRE2_JIT +POLL -PRIVATE_CACHE +THREAD
> +PTHREAD_PSHARED +REGPARM -STATIC_PCRE -STATIC_PCRE2 +TPROXY
> +LINUX_TPROXY +LINUX_SPLICE +LIBCRYPT +CRYPT_H -VSYSCALL
+GETADDRINFO
> +OPENSSL -LUA +FUTEX +ACCEPT4 -MY_ACCEPT4 +ZLIB -SLZ +CPU_AFFINITY
> +TFO -NS +DL +RT -DEVICEATLAS -51DEGREES -WURFL -SYSTEMD
> -OBSOLETE_LINKER +PRCTL +THREAD_DUMP -EVPORTS
>
> Default settings :
> bufsize = 16384, maxrewrite = 1024, maxpollevents = 200
>
> Built with multi-threading support (MAX_THREADS=64, default=24).
> Built with OpenSSL version : OpenSSL 1.1.1g 21 Apr 2020
> Running on OpenSSL version : OpenSSL 1.1.1g 21 Apr 2020
> OpenSSL library supports TLS extensions : yes
> OpenSSL library supports SNI : yes
> OpenSSL library supports : TLSv1.0 TLSv1.1 TLSv1.2 TLSv1.3
> Built with transparent proxy support using: IP_TRANSPARENT
> IPV6_TRANSPARENT IP_FREEBIND
> Built with zlib version : 1.2.3
> Running on zlib version : 1.2.3
> Compression algorithms supported : identity("identity"),
> deflate("deflate"), raw-deflate("deflate"), gzip("gzip")
> Built with PCRE version : 7.8 2008-09-05
> Running on PCRE version : 7.8 2008-09-05
> PCRE library supports JIT : no (USE_PCRE_JIT not set)
> Encrypted password support via crypt(3): yes
>
> Available polling systems :
> epoll : pref=300, test result OK
> poll : pref=200, test result OK
> select : pref=150, test result OK
> Total: 3 (3 usable), will use epoll.
>
> Available multiplexer protocols :
> (protocols marked as <default> cannot be specified using 'proto'
keyword)
> h2 : mode=HTX side=FE|BE mux=H2
> h2 : mode=HTTP side=FE mux=H2
> <default> : mode=HTX side=FE|BE mux=H1
> <default> : mode=TCP|HTTP side=FE|BE mux=PASS
>
> Available services : none
>
> Available filters :
> [SPOE] spoe
> [COMP] compression
> [CACHE] cache
> [TRACE] trace
> ==========================================================
>
> Is there anything else I can provide, or try?
>
> Kind regards,
>
> Sander Hoentjen
>
>
