I'm experiencing a problem that I can't diagnose but I can recreate pretty
consistently. I have a single server that responds for example.com and
api.example.com and it runs haproxy. All the names run through an SSL front
door but an ACL makes it such that requests for example.com get sent to 8443
where Apache runs and requests for api.example.com get sent to 8445 where the
same instance of haproxy runs and does further examination of the request and
sends it to an application server running on localhost.
This configuration works great except when I take a server out of the rotation
by disabling it with disable-on-404. As soon as I take any server out of the
rotation, haproxy completely stops responding to ANY requests for ANY backend
even things that aren't part of the group such as the stats backend and
frontend. If I put the server back in to service haproxy does not recover. I
must restart haproxy on all hosts to recover. Nothing shows up in the logs and
I can't figure out how to debug it such that I can provide more information but
it's very consistently reproducible using the configuration below. I am running
1.8.3 and I have not tried this on 1.7 or earlier versions of 1.8.
Thanks for your help.
-Paul
global
log /dev/log local0
user nobody
group nobody
tune.ssl.default-dh-param 2048
stats socket /var/run/haproxy.sock user nobody group nobody
daemon
defaults
timeout connect 5000ms
timeout client 600000ms
timeout server 600000ms
option httplog
option forwardfor
option http-server-close
option contstats
frontend stats-frontend
bind *:2999
mode http
log global
stats enable
stats uri /haproxy
backend stats-backend
mode http
log global
server stats /var/run/haproxy.sock check
frontend secured
# get the list of certificate options from a list in a file
bind *:443 ssl crt-list /srv/haproxy/certificates.lst
mode http
log global
# tell backend connections what our ssl client cn is
http-request set-header X-SSL-Client-Verify %[ssl_c_verify]
http-request set-header X-SSL-Client-DN %{+Q}[ssl_c_s_dn]
http-request set-header X-SSL-Client-CN %{+Q}[ssl_c_s_dn(cn)]
http-request set-header X-SSL-Issuer-DN %{+Q}[ssl_c_i_dn]
http-request set-header X-SSL-Issuer-CN %{+Q}[ssl_c_i_dn(cn)]
acl server-status path_beg /server-
use_backend bogus-http if server-status
# connection requests for apis go to the api backends
acl request_api hdr_beg(Host) -i api.
use_backend example-api if request_api
default_backend example-http
backend example-http
mode http
log global
balance source
hash-type consistent
option httpchk GET /haproxy/alive.txt
http-check disable-on-404
server myhost myhost.example.com:8443 check ssl ca-file
/usr/local/ssl/certs/cacerts.cert
backend bogus-http
mode http
errorfile 503 /netops/www/haproxy/403.http
backend example-api
mode http
log global
balance roundrobin
option httpchk GET /haproxy/alive.txt
http-check disable-on-404
server myhost myhost.example.com:8445 track example-http/myhost ssl ca-file
/usr/local/ssl/certs/cacerts.cert
frontend localhost-api-frontend
bind *:8445 ssl crt /usr/local/ssl/certs/example.com.pem
mode http
log global
option forwardfor if-none
option dontlog-normal
# the alerts api backend
acl alerts-api_host hdr_beg(Host) -i api.alerts
use_backend localhost-api-backend-alerts if alerts-api_host
default_backend bogus-http
backend localhost-api-backend-alerts
mode http
log global
option forwardfor if-none
option dontlog-normal
server localhost localhost:4002
And the certificates.lst file referenced above looks like this:
# this order is because we need to work with older clients that don't
# speak sni and this works for them in our setup.
/usr/local/ssl/certs/example.com.pem *
/usr/local/ssl/certs/example.com.pem [ca-file
/usr/local/ssl/certs/example-ca.cert verify optional] api.example.com
