Hi Willy,
Am 04.07.2017 um 22:24 schrieb Willy Tarreau:
> Hi Lukas,
>
> On Tue, Jul 04, 2017 at 09:56:09PM +0200, Lukas Tribus wrote:
>> Hi Emeric,
>>
>>
>> since 8d85aa4 ("BUG/MAJOR: map: fix segfault during 'show
>> map/acl' on cli") my setup crashes when a request comes in
>> going through SSL termination.
>>
>> memory corruption, invalid pointers, double free is what haproxy
>> randomly crashes with.
> Hmmm bad! Do you want me to revert it now or can this wait for
> Emeric to try to spot the issue ? William, I've seen you've already
> backported it into 1.7, be careful not to release the next version
> before this one is fixed!
No, I got it wrong, its a different commit that is 1.8-only, sorry.
The commit is 019489 (MAJOR: task: task scheduler rework).
I hope I got the right one this time (I don't know why, but I bisected to the
wrong commit twice now).
root@www:/usr/sbin# haproxy -vv
HA-Proxy version 1.8-dev2-019489-36 2017/06/27
Copyright 2000-2017 Willy Tarreau <[email protected]>
Build options :
TARGET = linux2628
CPU = generic
CC = gcc
CFLAGS = -O0 -g -fno-strict-aliasing -Wdeclaration-after-statement -fwrapv
OPTIONS = USE_GETADDRINFO=1 USE_SLZ=1 USE_OPENSSL=1 USE_PCRE=1 USE_PCRE_JIT=1
Default settings :
maxconn = 2000, bufsize = 16384, maxrewrite = 1024, maxpollevents = 200
Built with OpenSSL version : OpenSSL 1.1.0g-dev xx XXX xxxx
Running on OpenSSL version : OpenSSL 1.1.0g-dev xx XXX xxxx
OpenSSL library supports TLS extensions : yes
OpenSSL library supports SNI : yes
OpenSSL library supports : SSLv3 TLSv1.0 TLSv1.1 TLSv1.2
Built with transparent proxy support using: IP_TRANSPARENT IPV6_TRANSPARENT
IP_FREEBIND
Built with network namespace support.
Built with libslz for stateless compression.
Compression algorithms supported : identity("identity"), deflate("deflate"),
raw-deflate("deflate"), gzip("gzip")
Encrypted password support via crypt(3): yes
Built with PCRE version : 8.38 2015-11-23
Running on PCRE version : 8.38 2015-11-23
PCRE library supports JIT : yes
Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 3 (3 usable), will use epoll.
Available filters :
[SPOE] spoe
[COMP] compression
[TRACE] trace
root@www:/usr/sbin# gdb -ex=r --args haproxy -f /etc/haproxy/haproxy-slim.cfg -d
GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.04) 7.11.1
Copyright (C) 2016 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from haproxy...done.
Starting program: /usr/sbin/haproxy -f /etc/haproxy/haproxy-slim.cfg -d
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
[WARNING] 184/222222 (30237) : config : log format ignored for frontend
'tls-termination' since it has no log address.
[WARNING] 184/222222 (30237) : Proxy 'tls-termination': no-sslv3/no-tlsv1x are
ignored for bind ':443' at [/etc/haproxy/haproxy-slim.cfg:18]. Use only
'ssl-min-ver' and 'ssl-max-ver' to fix.
Note: setting global.maxconn to 2000.
Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result FAILED
Total: 3 (2 usable), will use epoll.
Available filters :
[SPOE] spoe
[COMP] compression
[TRACE] trace
Using epoll() as the polling mechanism.
00000000:tls-termination.accept(0004)=0005 from [10.0.0.4:56156]
00000000:tls-termination.clireq[0005:ffffffff]: GET /robots.txx HTTP/1.1
00000000:tls-termination.clihdr[0005:ffffffff]: Host: temp.lan.ltri.eu
00000000:tls-termination.clihdr[0005:ffffffff]: User-Agent: curl/7.48.0
00000000:tls-termination.clihdr[0005:ffffffff]: Accept: */*
*** Error in `/usr/sbin/haproxy': malloc(): memory corruption:
0x0000000000a419c0 ***
======= Backtrace: =========
/lib/x86_64-linux-gnu/libc.so.6(+0x777e5)[0x7ffff71bb7e5]
/lib/x86_64-linux-gnu/libc.so.6(+0x8213e)[0x7ffff71c613e]
/lib/x86_64-linux-gnu/libc.so.6(__libc_malloc+0x54)[0x7ffff71c8184]
/usr/sbin/haproxy[0x523d66]
/usr/sbin/haproxy[0x522154]
/usr/sbin/haproxy[0x522732]
/usr/sbin/haproxy[0x532480]
/usr/sbin/haproxy[0x4120b6]
/usr/sbin/haproxy[0x4d89dd]
/usr/sbin/haproxy[0x4d9a95]
/usr/sbin/haproxy[0x4fa433]
/usr/sbin/haproxy[0x5116a2]
/usr/sbin/haproxy[0x4d4a75]
/usr/sbin/haproxy[0x4d5d67]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf0)[0x7ffff7164830]
/usr/sbin/haproxy[0x4055a9]
======= Memory map: ========
00400000-007ae000 r-xp 00000000 ca:02 40972
/usr/sbin/haproxy
009ae000-009ce000 r--p 003ae000 ca:02 40972
/usr/sbin/haproxy
009ce000-009ea000 rw-p 003ce000 ca:02 40972
/usr/sbin/haproxy
009ea000-00abf000 rw-p 00000000 00:00 0 [heap]
7ffff0000000-7ffff0021000 rw-p 00000000 00:00 0
7ffff0021000-7ffff4000000 ---p 00000000 00:00 0
7ffff6b0e000-7ffff6b24000 r-xp 00000000 ca:02 24641
/lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6b24000-7ffff6d23000 ---p 00016000 ca:02 24641
/lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d23000-7ffff6d24000 rw-p 00015000 ca:02 24641
/lib/x86_64-linux-gnu/libgcc_s.so.1
7ffff6d24000-7ffff7144000 rw-p 00000000 00:00 0
7ffff7144000-7ffff7304000 r-xp 00000000 ca:02 26350
/lib/x86_64-linux-gnu/libc-2.23.so
7ffff7304000-7ffff7504000 ---p 001c0000 ca:02 26350
/lib/x86_64-linux-gnu/libc-2.23.so
7ffff7504000-7ffff7508000 r--p 001c0000 ca:02 26350
/lib/x86_64-linux-gnu/libc-2.23.so
7ffff7508000-7ffff750a000 rw-p 001c4000 ca:02 26350
/lib/x86_64-linux-gnu/libc-2.23.so
7ffff750a000-7ffff750e000 rw-p 00000000 00:00 0
7ffff750e000-7ffff7526000 r-xp 00000000 ca:02 24805
/lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7526000-7ffff7725000 ---p 00018000 ca:02 24805
/lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7725000-7ffff7726000 r--p 00017000 ca:02 24805
/lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7726000-7ffff7727000 rw-p 00018000 ca:02 24805
/lib/x86_64-linux-gnu/libpthread-2.23.so
7ffff7727000-7ffff772b000 rw-p 00000000 00:00 0
7ffff772b000-7ffff7799000 r-xp 00000000 ca:02 24672
/lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7799000-7ffff7999000 ---p 0006e000 ca:02 24672
/lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff7999000-7ffff799a000 r--p 0006e000 ca:02 24672
/lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799a000-7ffff799b000 rw-p 0006f000 ca:02 24672
/lib/x86_64-linux-gnu/libpcre.so.3.13.2
7ffff799b000-7ffff799e000 r-xp 00000000 ca:02 26330
/lib/x86_64-linux-gnu/libdl-2.23.so
7ffff799e000-7ffff7b9d000 ---p 00003000 ca:02 26330
/lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9d000-7ffff7b9e000 r--p 00002000 ca:02 26330
/lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9e000-7ffff7b9f000 rw-p 00003000 ca:02 26330
/lib/x86_64-linux-gnu/libdl-2.23.so
7ffff7b9f000-7ffff7ba8000 r-xp 00000000 ca:02 24741
/lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7ba8000-7ffff7da7000 ---p 00009000 ca:02 24741
/lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da7000-7ffff7da8000 r--p 00008000 ca:02 24741
/lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da8000-7ffff7da9000 rw-p 00009000 ca:02 24741
/lib/x86_64-linux-gnu/libcrypt-2.23.so
7ffff7da9000-7ffff7dd7000 rw-p 00000000 00:00 0
7ffff7dd7000-7ffff7dfd000 r-xp 00000000 ca:02 24651
/lib/x86_64-linux-gnu/ld-2.23.so
7ffff7fec000-7ffff7ff0000 rw-p 00000000 00:00 0
7ffff7ff5000-7ffff7ff8000 rw-p 00000000 00:00 0
7ffff7ff8000-7ffff7ffa000 r--p 00000000 00:00 0 [vvar]
7ffff7ffa000-7ffff7ffc000 r-xp 00000000 00:00 0 [vdso]
7ffff7ffc000-7ffff7ffd000 r--p 00025000 ca:02 24651
/lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffd000-7ffff7ffe000 rw-p 00026000 ca:02 24651
/lib/x86_64-linux-gnu/ld-2.23.so
7ffff7ffe000-7ffff7fff000 rw-p 00000000 00:00 0
7fffffede000-7ffffffff000 rw-p 00000000 00:00 0 [stack]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0
[vsyscall]
Program received signal SIGABRT, Aborted.
0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at
../sysdeps/unix/sysv/linux/raise.c:54
54 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at
../sysdeps/unix/sysv/linux/raise.c:54
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
#2 0x00007ffff71bb7ea in __libc_message (do_abort=2,
fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at
../sysdeps/posix/libc_fatal.c:175
#3 0x00007ffff71c613e in malloc_printerr (ar_ptr=0x7ffff7508b20 <main_arena>,
ptr=0xa419c0, str=0x7ffff72d1cff "malloc(): memory corruption",
action=<optimized out>) at malloc.c:5006
#4 _int_malloc (av=av@entry=0x7ffff7508b20 <main_arena>,
bytes=bytes@entry=16472) at malloc.c:3474
#5 0x00007ffff71c8184 in __GI___libc_malloc (bytes=16472) at malloc.c:2913
#6 0x0000000000523d66 in ssl3_setup_write_buffer ()
#7 0x0000000000522154 in do_ssl3_write ()
#8 0x0000000000522732 in ssl3_write_bytes ()
#9 0x0000000000532480 in SSL_write ()
#10 0x00000000004120b6 in ssl_sock_from_buf (conn=0xa43ce0, buf=0xa5e240,
flags=1) at src/ssl_sock.c:4974
#11 0x00000000004d89dd in si_conn_send (conn=0xa43ce0) at
src/stream_interface.c:658
#12 0x00000000004d9a95 in si_conn_send_cb (conn=0xa43ce0) at
src/stream_interface.c:1295
#13 0x00000000004fa433 in conn_fd_handler (fd=5) at src/connection.c:118
#14 0x00000000005116a2 in fd_process_cached_events () at src/fd.c:240
#15 0x00000000004d4a75 in run_poll_loop () at src/haproxy.c:2186
#16 0x00000000004d5d67 in main (argc=4, argv=0x7fffffffe658) at
src/haproxy.c:2701
(gdb) bt full
#0 0x00007ffff7179428 in __GI_raise (sig=sig@entry=6) at
../sysdeps/unix/sysv/linux/raise.c:54
resultvar = 0
pid = 30237
selftid = 30237
#1 0x00007ffff717b02a in __GI_abort () at abort.c:89
save_stage = 2
act = {__sigaction_handler = {sa_handler = 0x20302030303a3030,
sa_sigaction = 0x20302030303a3030}, sa_mask = {__val = {2314885530818453536,
2314885530818453536, 7017579609838738208, 4206752516204751980,
3545519503966220848, 2314885530818453536, 2314885530818453536,
7795484802351636512, 3917909816998060649,
3276497845987585332, 7161402270846119527, 3615882721633532274,
7378645557452156467, 3472337303646987878, 3991990709698112816,
8223625903104156004}}, sa_flags = 544222583, sa_restorer = 0x56}
sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007ffff71bb7ea in __libc_message (do_abort=2,
fmt=fmt@entry=0x7ffff72d4e98 "*** Error in `%s': %s: 0x%s ***\n") at
../sysdeps/posix/libc_fatal.c:175
ap = <error reading variable ap (Attempt to dereference a generic
pointer.)>
fd = 6
on_2 = <optimized out>
list = <optimized out>
nlist = <optimized out>
cp = <optimized out>
written = <optimized out>
#3 0x00007ffff71c613e in malloc_printerr (ar_ptr=0x7ffff7508b20 <main_arena>,
ptr=0xa419c0, str=0x7ffff72d1cff "malloc(): memory corruption",
action=<optimized out>) at malloc.c:5006
buf = "0000000000a419c0"
cp = <optimized out>
ar_ptr = 0x7ffff7508b20 <main_arena>
ptr = 0xa419c0
str = 0x7ffff72d1cff "malloc(): memory corruption"
action = <optimized out>
#4 _int_malloc (av=av@entry=0x7ffff7508b20 <main_arena>,
bytes=bytes@entry=16472) at malloc.c:3474
iters = <optimized out>
nb = 16480
idx = 114
bin = <optimized out>
victim = 0xa419b0
size = <optimized out>
victim_index = <optimized out>
remainder = <optimized out>
remainder_size = <optimized out>
block = <optimized out>
bit = <optimized out>
map = <optimized out>
fwd = <optimized out>
bck = 0x7ffff7508b78 <main_arena+88>
errstr = 0x0
__func__ = "_int_malloc"
#5 0x00007ffff71c8184 in __GI___libc_malloc (bytes=16472) at malloc.c:2913
ar_ptr = 0x7ffff7508b20 <main_arena>
victim = <optimized out>
hook = <optimized out>
#6 0x0000000000523d66 in ssl3_setup_write_buffer ()
No symbol table info available.
#7 0x0000000000522154 in do_ssl3_write ()
No symbol table info available.
#8 0x0000000000522732 in ssl3_write_bytes ()
No symbol table info available.
#9 0x0000000000532480 in SSL_write ()
No symbol table info available.
#10 0x00000000004120b6 in ssl_sock_from_buf (conn=0xa43ce0, buf=0xa5e240,
flags=1) at src/ssl_sock.c:4974
ret = 256209204
try = 212
done = 0
#11 0x00000000004d89dd in si_conn_send (conn=0xa43ce0) at
src/stream_interface.c:658
send_flag = 1
si = 0xaa4648
oc = 0xaa4460
ret = 0
#12 0x00000000004d9a95 in si_conn_send_cb (conn=0xa43ce0) at
src/stream_interface.c:1295
si = 0xaa4648
#13 0x00000000004fa433 in conn_fd_handler (fd=5) at src/connection.c:118
conn = 0xa43ce0
flags = 0
#14 0x00000000005116a2 in fd_process_cached_events () at src/fd.c:240
fd = 5
entry = 0
e = 50
#15 0x00000000004d4a75 in run_poll_loop () at src/haproxy.c:2186
next = 256159205
#16 0x00000000004d5d67 in main (argc=4, argv=0x7fffffffe658) at
src/haproxy.c:2701
err = 0
retry = 200
limit = {rlim_cur = 4011, rlim_max = 4011}
errmsg =
"\000\060\243\000\000\000\000\000X\346\377\377\377\177\000\000\004\000\000\000\000\000\000\000ʍ\034\367\377\177\000\000\260-\243\000\000\000\000\000\"\000\000\000\000\000\000\000\000\345\377\377\377\177\000\000\370\344\232\000\000\000\000\000\200\346\377\377\377\177\000\000\327\363L\000\000\000\000\000\001\000\000\000\001\000\000\000\060/\243\000\000\000\000\000\"\000\000"
pidfd = -1
(gdb) quit
A debugging session is active.
Inferior 1 [process 30237] will be killed.
Quit anyway? (y or n) y
root@www:/usr/sbin# cat /etc/haproxy/haproxy-slim.cfg
global
ssl-default-bind-ciphers
ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:!DSS
ssl-default-bind-options no-tls-tickets no-tlsv10 no-tlsv11 force-tlsv12
defaults
log global
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-keep-alive 60s
option http-keep-alive
option forwardfor
frontend tls-termination
mode http
bind :443 ssl crt /etc/ssl/private/temp.example.com crt /etc/ssl/private/ npn
http/1.1 alpn http/1.1 curves X25519:P-256 #strict-sni
use_backend robots if { path /robots.txt }
#use_backend temp if { ssl_fc_sni -i temp.example.com }
backend temp
mode http
server local-nginx 127.0.0.1:80 maxconn 200
backend robots
mode http
errorfile 403 /etc/haproxy/errors/robotstxt.http
http-request deny
root@www:/usr/sbin#
