Hi! We accidentally got a spike of client requests to our site and under that heavy load to ssl-protected backends haproxy=1.9.5 had fallen down :(
backtrace Core was generated by `/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy'. Program terminated with signal SIGSEGV, Segmentation fault. #0 wake_srv_chk (cs=cs@entry=0x0) at src/checks.c:1410 1410 src/checks.c: No such file or directory. [Current thread is 1 (Thread 0x7f11cd14f180 (LWP 52133))] (gdb) thread apply all bt Thread 4 (Thread 0x7f11c972d700 (LWP 52135)): #0 0x00007f11cb8a719c in malloc () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f11cc0ffe78 in CRYPTO_malloc () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #2 0x00007f11cc1e7ea9 in ASN1_STRING_type_new () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #3 0x00007f11cc1d19fa in c2i_ASN1_INTEGER () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #4 0x00007f11cc1dd9b4 in asn1_ex_c2i () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #5 0x00007f11cc1dde2b in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #6 0x00007f11cc1dece8 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #7 0x00007f11cc1df036 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #8 0x00007f11cc1df241 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #9 0x00007f11cc1dea3d in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #10 0x00007f11cc1df036 in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #11 0x00007f11cc1df2ad in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #12 0x00007f11cc1dea3d in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #13 0x00007f11cc1df415 in ASN1_item_ex_d2i () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #14 0x00007f11cc1df46b in ASN1_item_d2i () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #15 0x00007f11cc52c83b in d2i_SSL_SESSION () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #16 0x000055df0ca744b8 in ssl_sock_init (conn=0x7f119c438430) at src/ssl_sock.c:5129 #17 0x000055df0cb69300 in conn_xprt_init (conn=0x7f119c438430) at include/proto/connection.h:84 #18 tcp_connect_server (conn=0x7f119c438430, data=<optimized out>, delack=<optimized out>) at src/proto_tcp.c:559 #19 0x000055df0cb551e0 in si_connect (conn=0x7f119c438430, si=<optimized out>) at include/proto/stream_interface.h:492 #20 connect_server (s=s@entry=0x7f119e31d8f0) at src/backend.c:1511 #21 0x000055df0cacbfba in sess_update_stream_int (s=0x7f119e31d8f0) at src/stream.c:929 #22 process_stream (t=<optimized out>, context=0x7f119e31d8f0, state=<optimized out>) at src/stream.c:2306 #23 0x000055df0cb97c4d in process_runnable_tasks () at src/task.c:434 #24 0x000055df0cb0f2a0 in run_poll_loop () at src/haproxy.c:2640 #25 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2705 #26 0x00007f11cc95a6ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #27 0x00007f11cb92a41d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 3 (Thread 0x7f11c8f2c700 (LWP 52136)): #0 0x00007f11cb92b577 in shutdown () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x000055df0cb9bd75 in conn_sock_shutw (clean=1, c=<optimized out>) at include/proto/connection.h:456 #2 mux_pt_shutw (cs=0x7f11c03c34e0, mode=CS_SHW_NORMAL) at src/mux_pt.c:233 #3 0x000055df0cb61aeb in cs_shutw (mode=CS_SHW_NORMAL, cs=<optimized out>) at include/proto/connection.h:493 #4 stream_int_shutw_conn (si=0x7f11c0760f98) at src/stream_interface.c:988 #5 0x000055df0cacc216 in si_shutw (si=0x7f11c0760f98) at include/proto/stream_interface.h:420 #6 process_stream (t=<optimized out>, context=0x7f11c0760d00, state=<optimized out>) at src/stream.c:2442 #7 0x000055df0cb97c4d in process_runnable_tasks () at src/task.c:434 ---Type <return> to continue, or q <return> to quit--- #8 0x000055df0cb0f2a0 in run_poll_loop () at src/haproxy.c:2640 #9 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2705 #10 0x00007f11cc95a6ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #11 0x00007f11cb92a41d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 2 (Thread 0x7f11c9f2e700 (LWP 52134)): #0 0x00007f11cb8a2e4e in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f11cb8a753c in free () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f11cc1000cd in CRYPTO_free () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #3 0x00007f11cc1bc8f1 in EVP_MD_CTX_cleanup () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #4 0x00007f11cc126b26 in HMAC_CTX_cleanup () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #5 0x00007f11cc12700e in ?? () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #6 0x00007f11cc1c990c in EVP_PKEY_CTX_free () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #7 0x00007f11cc1bc882 in EVP_MD_CTX_cleanup () from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0 #8 0x00007f11cc514487 in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #9 0x00007f11cc51521a in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #10 0x00007f11cc508a58 in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #11 0x00007f11cc509603 in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #12 0x00007f11cc50a687 in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #13 0x00007f11cc50a192 in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #14 0x00007f11cc5033ce in ?? () from /lib/x86_64-linux-gnu/libssl.so.1.0.0 #15 0x000055df0ca7b00f in ssl_sock_handshake (conn=conn@entry=0x7f11c70a37b0, flag=flag@entry=33554432) at src/ssl_sock.c:5338 #16 0x000055df0cb83624 in conn_fd_handler (fd=<optimized out>) at src/connection.c:88 #17 0x000055df0cb93373 in fdlist_process_cached_events (fdlist=<optimized out>) at src/fd.c:443 #18 fd_process_cached_events () at src/fd.c:461 #19 0x000055df0cb0f28d in run_poll_loop () at src/haproxy.c:2676 #20 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2705 #21 0x00007f11cc95a6ba in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #22 0x00007f11cb92a41d in clone () from /lib/x86_64-linux-gnu/libc.so.6 Thread 1 (Thread 0x7f11cd14f180 (LWP 52133)): #0 wake_srv_chk (cs=cs@entry=0x0) at src/checks.c:1410 #1 0x000055df0cb077c8 in event_srv_chk_io (t=<optimized out>, ctx=0x55df0e5e6a20, state=<optimized out>) at src/checks.c:724 #2 0x000055df0cb97de6 in process_runnable_tasks () at src/task.c:437 #3 0x000055df0cb0f2a0 in run_poll_loop () at src/haproxy.c:2640 #4 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2705 #5 0x000055df0ca6a68c in main (argc=<optimized out>, argv=<optimized out>) at src/haproxy.c:3341 # haproxy -vv HA-Proxy version 1.9.5-3 2019/03/27 - https://haproxy.org/ Build options : TARGET = linux2628 CPU = generic CC = gcc CFLAGS = -O2 -g -O2 -fPIE -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fno-strict-aliasing -Wdeclaration-after-statement -fwrapv -Wno-unused-label -Wno-sign-compare -Wno-unused-parameter -Wno-old-style-declaration -Wno-ignored-qualifiers -Wno-clobbered -Wno-missing-field-initializers -Wtype-limits OPTIONS = USE_GETADDRINFO=1 USE_ZLIB=1 USE_REGPARM=1 USE_OPENSSL=1 USE_LUA=1 USE_SYSTEMD=1 USE_PCRE2=1 USE_PCRE2_JIT=1 USE_TFO=1 USE_NS=1 Default settings : maxconn = 2000, bufsize = 16384, maxrewrite = 1024, maxpollevents = 200 Built with OpenSSL version : OpenSSL 1.0.2g 1 Mar 2016 Running on OpenSSL version : OpenSSL 1.0.2g 1 Mar 2016 OpenSSL library supports TLS extensions : yes OpenSSL library supports SNI : yes OpenSSL library supports : TLSv1.0 TLSv1.1 TLSv1.2 Built with Lua version : Lua 5.3.1 Built with network namespace support. Built with transparent proxy support using: IP_TRANSPARENT IPV6_TRANSPARENT IP_FREEBIND Built with zlib version : 1.2.8 Running on zlib version : 1.2.8 Compression algorithms supported : identity("identity"), deflate("deflate"), raw-deflate("deflate"), gzip("gzip") Built with PCRE2 version : 10.21 2016-01-12 PCRE2 library supports JIT : yes Encrypted password support via crypt(3): yes Built with multi-threading support. Available polling systems : epoll : pref=300, test result OK poll : pref=200, test result OK select : pref=150, test result OK Total: 3 (3 usable), will use epoll. Available multiplexer protocols : (protocols marked as <default> cannot be specified using 'proto' keyword) h2 : mode=HTX side=FE|BE h2 : mode=HTTP side=FE <default> : mode=HTX side=FE|BE <default> : mode=TCP|HTTP side=FE|BE Available filters : [SPOE] spoe [COMP] compression [CACHE] cache [TRACE] trace -- Best regards Maksim Kupriianov
