We are trying to deploy HAProxy into our environment. We have a script that
does some 600k api calls during approximately 24 hours. During that time,
when haproxy is in place, there are a handful (8-12) of responses that come
back as 502 with SH--. We've tested this against Ubuntu's default 2.8
version of HAProxy, along with 3.0.3 and most recently 3.1-dev5-8427c5b. It
is the same in each version. The logs don't seem to tell us much. The
backend server doesn't show any issues, and indeed, when HAProxy is not in
place, there are no errors whatsoever.
I'm afraid that at the moment, I don't know how to reproduce this beyond
what we are doing in our environment, but that's not something we can share
with others. That said, if you have any guidance on how I can discover what
is going on, I'd be more than happy to dig in further. I turned the logging
to debug, but it doesn't seem to show me anything more than what info was
giving us.
Our config file looks like the following:
global
log stdout len 65535 local0 debug
user haproxy
group haproxy
daemon
stats timeout 30
# Default SSL material locations
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
ssl-default-bind-ciphers (list)
ssl-default-bind-ciphersuites (list)
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
tune.bufsize 8388608
tune.maxrewrite 1024
defaults
log global
mode http
option dontlognull
log-format
'{"host":"%H","ident":"haproxy","pid":%pid,"timestamp":"%Tl","haproxy":{"connections":{"active":%ac,"frontend":%fc,"backend":%bc,"server":%sc},"queue":{"backend":%bq,"server":%sq},"timing_ms":{"time_to_end_of_headers":%TR,"client_request_send_time":%Tq,"queue_wait_time":%Tw,"server_wait_time":%Tc,"server_response_send_time":%Tr,"response_time":%Td,"session_duration":%Tt},"termination_state":"%tsc","retries":%rc,"network":{"client_ip":"%ci","client_port":%cp,"frontend_ip":"%fi","frontend_port":%fp},"ssl":{"version":"%sslv","ciphers":"%sslc"},"request":{"method":"%HM","uri":"%[front4.req.uri,json(utf8s)]","protocol":"%HV","header":{"host":"%[front4.req.hdr(0),json(utf8s)]","xforwardfor":"%[front4.req.hdr(1),json(utf8s)]","referer":"%[front4.req.hdr(2),json(utf8s)]"}},"name":{"frontend":"%ft","backend":"%b","server":"%s"},"response":{"status_code":%ST,"header":{"xrequestid":"%[front4.res.hdr(0),json(utf8s)]"}},"bytes":{"uploaded":%U,"read":%B}}}'
timeout connect 86400s
timeout client 86400s
timeout server 86400s
errorfile 400 /etc/haproxy/errors/400.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 408 /etc/haproxy/errors/408.http
errorfile 500 /etc/haproxy/errors/500.http
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
errorfile 504 /etc/haproxy/errors/504.http
#listen prometheus #this doesn't work in 3.1
# bind *:9100
# mode http
# http-request use-service prometheus-exporter if { path /metrics }
# no log
frontend www
http-request front4 req.hdr(Host) len 1000
http-request front4 req.hdr(Referer) len 1000
bind *:80
bind *:443 ssl crt /etc/ssl/private/star.domain.com.pem crt
/etc/ssl/private/star.domain2.com.pem crt
/etc/ssl/private/star.domain3.com.pem alpn h2,http/1.1
option forwardfor
# stats stuff for admin path for web interface
stats enable
stats uri /admin
stats realm HAProxy\ Statistics
stats auth admin:"$AUTH"
stats admin if TRUE
stats show-node
#### data frontends ####
acl front1_acl hdr(host) -i front1.domain.com
use_backend front1 if front1_acl
acl front2_acl hdr(host) -i front2.domain.com
use_backend front2 if front2_acl
acl front3_pgst_acl hdr(host) -i front3.domain.com
use_backend front3 if front3_pgst_acl
acl front3_redx_acl hdr(host) -i front3.domain2.com
use_backend front3 if front3_redx_acl
acl front3_theredx_acl hdr(host) -i front3.domain3.com
use_backend front3 if front3_theredx_acl
acl front4_acl hdr(host) -i front4.domain.com
use_backend front4 if front4_acl
acl front5_acl hdr(host) -i front5.domain.com
use_backend front5 if front5_acl
acl front6_acl hdr(host) -i front6.domain.com
use_backend front6 if front6_acl
acl front7_acl hdr(host) -i front7.domain.com
use_backend front7 if front7_acl
acl front8_acl hdr(host) -i front8.domain.com
use_backend front8 if front8_acl
acl front9_acl hdr(host) -i front9.domain.com
use_backend front9 if front9_acl
acl front10_acl hdr(host) -i front10.domain.com
use_backend front10 if front10_acl
acl front11_acl hdr(host) -i front11.domain.com
use_backend front11 if front11_acl
acl front12_acl hdr(host) -i front12.domain.com
use_backend front12 if front12_acl
acl front12_redx_com_acl hdr(host) -i front12.domain2.com
use_backend front3 if front12_redx_com_acl
acl front12_theredx_com_acl hdr(host) -i front12.domain3.com
use_backend front3 if front12_theredx_com_acl
##### data backends#####
backend front1
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front1.domain.com
http-check expect status 200
server pdafront12 192.168.69.102:443 ssl verify none check fall 1 rise
2 inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front2
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front2.domain.com
http-check expect status 200
server pdafront22 192.168.5.244:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front3
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front3.domain.com
http-check expect status 200
server pdafront32 192.168.6.3:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front4
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front4.domain.com
http-check expect status 200
server pdafront42 192.168.5.244:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front5
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front5.domain.com
http-check expect status 200
server pdafront52 192.168.5.244:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front6
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front6.domain.com
http-check expect status 200
server pdafront62 192.168.69.4:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front7
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front7.domain.com
http-check expect status 200
server pdafront72 192.168.5.244:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front8
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front8.domain.com
http-check expect status 200
server pdafront82 192.168.5.244:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front9
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front9.domain.com
http-check expect status 200
server pdafront92 192.168.5.244:443 ssl verify none check fall 2 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front10
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front10.domain.com
http-check expect status 200
server pdafront102 192.168.6.4:443 ssl verify none check fall 1 rise 2
inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front11
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front11.domain.com
http-check expect status 200
server pdafront112 192.168.5.244:443 ssl verify none check fall 1 rise
2 inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
backend front12
option httpchk
http-check send meth HEAD uri /srvstatus ver HTTP/1.1 hdr Host
front12.domain.com
http-check expect status 200
server pdafront122 192.168.5.244:443 ssl verify none check fall 1 rise
2 inter 5s fastinter 1s downinter 1m alpn h2,http/1.1
Thanks in advance,
BJ
