Some additional information:
We are using conntrac and iptables for firewalling on this box. Our typical
environment has a pair of load balancing firewalls in front of the
application. We are trying to switch from ipvs to haproxy.
r...@lb1:/proc/sys# cat ./net/nf_conntrack_max
65536
r...@lb1:/proc/sys# cat ./net/netfilter/nf_conntrack_max
65536
r...@lb1:/proc/sys# cat ./net/netfilter/nf_conntrack_count
4468
r...@lb1:/proc/sys# cat ./net/ipv4/netfilter/ip_conntrack_count
4389
r...@lb1:/proc/sys# cat ./net/ipv4/netfilter/ip_conntrack_buckets
16384
Configuration File ------------------------------------------
# vim:ft=haproxy
global
log 127.0.0.1 local0
# log 127.0.0.1 local1 notice
node proxy1
# ulimit-n 65536 this is auto-tuned
daemon
maxconn 32000
stats socket /tmp/haproxy_stats
defaults
log global
mode http
timeout connect 5000ms
timeout client 50000ms
timeout server 1800s
timeout queue 5000ms
retries 5
option httplog
option forwardfor
option redispatch
option httpclose
listen smtp
mode tcp
option smtpchk
bind 76.12.34.30:25
server pn1 172.17.48.31:25 check inter 15s
server pn2 172.17.48.32:25 check inter 15s
frontend internal
maxconn 10000
bind 172.17.48.1:80
acl hdr.host:alarm hdr_beg(host) alarm
acl hdr.host:asset hdr_beg(host) asset
acl hdr.host:config hdr_beg(host) config
use_backend alarm if hdr.host:alarm
use_backend configurator if hdr.host:config
default_backend internal_core
frontend ssbe-alarm
maxconn 10000
bind 76.12.34.26:80
acl hdr.host:alarm hdr_beg(host) alarm
default_backend alarm
frontend ssbe-assets
maxconn 4000
bind 76.12.34.27:80
acl hdr.host:asset hdr_beg(host) asset
default_backend servers
frontend ssbe-config
maxconn 4000
bind 76.12.34.28:80
bind 76.12.34.28:443
acl hdr.host:config hdr_beg(host) config
default_backend configurator
frontend ssbe-dash
maxconn 10000
bind 76.12.34.29:80
acl hdr.host:dash hdr_beg(host) dash
use_backend dash if hdr.host:dash
default_backend dash
frontend ssbe-core
maxconn 20000
bind 76.12.34.30:80
acl hdr.host:core hdr_beg(host) core
acl src:internal src 172.17.48.0/24
use_backend internal_core if hdr.host:core src:internal
default_backend servers
#1-IP#frontend http-in
#1-IP# bind *:80
#1-IP## stats enable
#1-IP## stats auth dev:dev
#1-IP#
#1-IP## ACLs to filter incoming requests
#1-IP# # Host: dash*
#1-IP# acl hdr.host:dash hdr_beg(host) dash
#1-IP#
#1-IP# # Host: core*
#1-IP# acl hdr.host:core hdr_beg(host) core
#1-IP#
#1-IP# # Host: asset*
#1-IP# acl hdr.host:asset hdr_beg(host) asset
#1-IP#
#1-IP# # Host: alarm*
#1-IP# acl hdr.host:alarm hdr_beg(host) alarm
#1-IP#
#1-IP# # Internal machines
#1-IP# acl src:internal src 172.16.101.0/24
#1-IP#
#1-IP# # Map ACLs to backend sections below
#1-IP# use_backend dash if hdr.host:dash
#1-IP# use_backend internal_core if hdr.host:core src:internal
#1-IP# default_backend servers
backend servers
balance roundrobin
#option allbackups
option httpchk GET /service_descriptors/kernel HTTP/1.0\r\nHost:\
core.monitoring.hosting.com
server pn1 172.17.48.31 check inter 5000 rise 2 fall 2 port 80
server pn2 172.17.48.32 check inter 5000 rise 2 fall 2 port 80
backend dash
balance roundrobin
cookie hapnode insert
# option httpchk HEAD / HTTP/1.0\r\nHost:\
dashboard.monitoring.hosting.com
server pn1 172.17.48.31 cookie pn1 check inter 5000 rise 2 fall 2
port 80
server pn2 172.17.48.32 cookie pn2 check inter 5000 rise 2 fall 2
port 80
backend internal_core
balance leastconn
option httpchk GET /service_descriptors/kernel HTTP/1.0\r\nHost:\
core.monitoring.hosting.com
server pn1 172.17.48.31 track servers/pn1 inter 5000 rise 2 fall 2
port 80
server pn2 172.17.48.32 track servers/pn2 inter 5000 rise 2 fall 2
port 80
backend alarm
balance leastconn
option httpchk HEAD / HTTP/1.0\r\nHost:\
alarm.monitoring.hosting.com
server pn1 172.17.48.31 track servers/pn1 inter 5000 rise 2 fall 2
port 80
server pn2 172.17.48.32 track servers/pn2 inter 5000 rise 2 fall 2
port 80
backend configurator
balance leastconn
option httpchk GET / HTTP/1.0\r\nHost:\
config.monitoring.hosting.com
server pn1 172.17.48.31 check inter 5000 rise 2 fall 2 port 80
server pn2 172.17.48.32 check inter 5000 rise 2 fall 2 port 80
End Configuration File -----------------------------------
uname -a: Linux lb1 2.6.32-23-server #37-Ubuntu SMP Fri Jun 11 09:11:11 UTC
2010 x86_64 GNU/Linux
(I was mistaken earlier - 64bit - not 32)
r...@lb1:~# haproxy -vv
HA-Proxy version 1.3.22 2009/10/14
Copyright 2000-2009 Willy Tarreau <[email protected]>
Build options :
TARGET = linux26
CPU = generic
CC = gcc
CFLAGS = -O2 -g
OPTIONS = USE_PCRE=1
Default settings :
maxconn = 2000, maxpollevents = 200
Available polling systems :
sepoll : pref=400, test result OK
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 4 (4 usable), will use sepoll.
---- log file showing connections made successfully while the failure is
occurring - retries is now set to 5 from 10 in other example
Aug 1 16:23:23 localhost haproxy[19005]:
172.17.48.31:54091[01/Aug/2010:16:23:23.315] ssbe-core
internal_core/pn2 0/0/0/18/18 304 330 -
- ---- 62/62/61/28/0 0/0 "GET /hosts/67/statuses HTTP/1.1"
Aug 1 16:23:23 localhost haproxy[19005]:
172.17.48.31:54125[01/Aug/2010:16:23:23.470] ssbe-core
internal_core/pn1 0/0/0/17/17 304 330 -
- ---- 63/63/62/31/0 0/0 "GET /hosts/10/statuses HTTP/1.1"
Aug 1 16:23:23 localhost haproxy[19005]:
204.12.71.93:4990[01/Aug/2010:16:23:22.666] ssbe-core servers/pn2
0/0/0/955/1002 200 161 - -
---- 62/62/0/0/0 0/0 "POST /Importd HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
172.17.48.32:44006[01/Aug/2010:16:23:21.790] ssbe-core
internal_core/pn1 0/0/0/2321/2321 200
473 - - ---- 62/62/61/30/0 0/0 "POST /observations HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
172.17.48.32:44012[01/Aug/2010:16:23:24.122] ssbe-core
internal_core/pn2 0/0/0/27/27 304 386 -
- ---- 63/63/62/29/0 0/0 "GET /clients/H205659/hosts HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
172.17.48.32:44013[01/Aug/2010:16:23:24.294] ssbe-core
internal_core/pn1 0/0/0/16/16 200 448 -
- ---- 63/63/62/31/0 0/0 "POST /observations HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
172.17.48.32:44016[01/Aug/2010:16:23:24.418] ssbe-core
internal_core/pn1 0/0/0/18/18 304 163 -
- ---- 64/64/63/31/0 0/0 "GET /hosts/30/metrics HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
172.17.48.31:54248[01/Aug/2010:16:23:24.447] ssbe-core
internal_core/pn1 0/0/0/16/16 200 448 -
- ---- 64/64/63/31/0 0/0 "POST /observations HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
204.12.72.237:53772[01/Aug/2010:16:23:23.720] ssbe-core servers/pn1
0/0/0/699/747 200 136 - -
---- 63/63/0/0/0 0/0 "POST /Importd HTTP/1.1"
Aug 1 16:23:24 localhost haproxy[19005]:
172.17.48.32:43980[01/Aug/2010:16:23:19.449] ssbe-core
internal_core/pn1 0/5028/-1/-1/5028 503
212 - - SC-- 62/62/62/31/+5 0/0 "GET /hosts/15/statuses HTTP/1.1"
Jerry Champlin
Absolute Performance Inc.
Phone: 303-565-4401
--
Enabling businesses to deliver critical applications at lower cost and
higher value to their customers.
