Hi!
Today, our haproxy CPU grow to 100%. And the machine become terribly slow.
There are some messages below.
model name : Quad-Core AMD Opteron(tm) Processor 2384*2
memory: 8GB
NIC: Intel Corporation 82576 Gigabit Network
Linux haproxy 3.2.0-4-amd64 #1 SMP Debian 3.2.32-1 x86_64 GNU/Linux
root@haproxybackup:/usr/local/etc# haproxy -vv
HA-Proxy version 1.4.22 2012/08/09
Copyright 2000-2012 Willy Tarreau <w...@1wt.eu>
Build options :
TARGET = linux26
CPU = generic
CC = gcc
CFLAGS = -m64 -march=x86-64 -O2 -g -fno-strict-aliasing
OPTIONS = USE_LINUX_SPLICE=1 USE_LINUX_TPROXY=1 USE_EPOLL=1 USE_REGPARM=1
USE_PCRE=1 USE_STATIC_PCRE=1
Default settings :
maxconn = 2000, bufsize = 16384, maxrewrite = 8192, maxpollevents = 200
Encrypted password support via crypt(3): yes
Available polling systems :
sepoll : pref=400, test result OK
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 4 (4 usable), will use sepoll.
and the coredump messages
[1297314.773522] ------------[ cut here ]------------
[1297314.773536] WARNING: at
/build/buildd-linux_3.2.32-1-amd64-bkoeca/linux-3.2.32/net/ipv4/tcp.c:1201
tcp_cleanup_rbuf+0x4a/0xfb()
[1297314.773539] Hardware name: PowerEdge SC1435
[1297314.773541] cleanup rbuf bug: copied DBE7B6DA seq DBE7B3C8 rcvnxt DBE7B6DA
[1297314.773542] Modules linked in: ipt_REDIRECT xt_TPROXY nf_tproxy_core
xt_set xt_mark xt_socket nf_defrag_ipv6 ip6_tables xt_tcpudp ip_set_hash_net
ip_set nfnetlink iptable_nat nf_nat nf_conntrack_ipv4 nf_defrag_ipv4
iptable_raw iptable_mangle iptable_filter ip_tables x_tables ip_vs nf_conntrack
crc32c libcrc32c nfsd nfs nfs_acl auth_rpcgss fscache lockd sunrpc 8021q garp
stp bonding tcp_htcp ext4 crc16 jbd2 loop radeon ttm drm_kms_helper drm
power_supply i2c_algo_bit k10temp mperf i2c_piix4 i2c_core processor shpchp
amd64_edac_mod edac_mce_amd edac_core snd_pcm snd_page_alloc snd_timer snd
soundcore psmouse dcdbas serio_raw pcspkr evdev button thermal_sys ext2 mbcache
microcode sg sr_mod cdrom usbhid hid sd_mod crc_t10dif ata_generic ohci_hcd
igb(O) pata_serverworks dca sata_svw libata ehci_hcd tg3 usbcore libphy
scsi_mod usb_common [last unloaded: scsi_wait_scan]
[1297314.773612] Pid: 23579, comm: haproxy Tainted: G O 3.2.0-4-amd64
#1 Debian 3.2.32-1
[1297314.773614] Call Trace:
[1297314.773625] [<ffffffff81046a75>] ? warn_slowpath_common+0x78/0x8c
[1297314.773629] [<ffffffff81046b21>] ? warn_slowpath_fmt+0x45/0x4a
[1297314.773632] [<ffffffff812c0df1>] ? tcp_cleanup_rbuf+0x4a/0xfb
[1297314.773635] [<ffffffff812c1ed8>] ? tcp_read_sock+0x127/0x138
[1297314.773637] [<ffffffff812c1fba>] ? tcp_splice_read+0xd1/0x21f
[1297314.773643] [<ffffffff8111aab7>] ? sys_splice+0x389/0x404
[1297314.773649] [<ffffffff81351ad2>] ? system_call_fastpath+0x16/0x1b
[1297314.773651] ---[ end trace 54eae6935f54c0f5 ]---
[1297315.006259] ------------[ cut here ]------------
[1297315.006270] WARNING: at
/build/buildd-linux_3.2.32-1-amd64-bkoeca/linux-3.2.32/net/ipv4/tcp.c:1495
tcp_recvmsg+0x24e/0x8f7()
[1297315.006273] Hardware name: PowerEdge SC1435
[1297315.006276] recvmsg bug 2: copied DBE7B6DA seq DBE7A5BE rcvnxt DBE7B6DA fl 0
[1297315.006277] Modules linked in: ipt_REDIRECT xt_TPROXY nf_tproxy_core
xt_set xt_mark xt_socket nf_defrag_ipv6 ip6_tables xt_tcpudp ip_set_hash_net
ip_set nfnetlink iptable_nat nf_nat nf_conntrack_ipv4 nf_defrag_ipv4
iptable_raw iptable_mangle iptable_filter ip_tables x_tables ip_vs nf_conntrack
crc32c libcrc32c nfsd nfs nfs_acl auth_rpcgss fscache lockd sunrpc 8021q garp
stp bonding tcp_htcp ext4 crc16 jbd2 loop radeon ttm drm_kms_helper drm
power_supply i2c_algo_bit k10temp mperf i2c_piix4 i2c_core processor shpchp
amd64_edac_mod edac_mce_amd edac_core snd_pcm snd_page_alloc snd_timer snd
soundcore psmouse dcdbas serio_raw pcspkr evdev button thermal_sys ext2 mbcache
microcode sg sr_mod cdrom usbhid hid sd_mod crc_t10dif ata_generic ohci_hcd
igb(O) pata_serverworks dca sata_svw libata ehci_hcd tg3 usbcore libphy
scsi_mod usb_common [last unloaded: scsi_wait_scan]
[1297315.006357] Pid: 23579, comm: haproxy Tainted: G W O 3.2.0-4-amd64
#1 Debian 3.2.32-1
[1297315.006359] Call Trace:
[1297315.006369] [<ffffffff81046a75>] ? warn_slowpath_common+0x78/0x8c
[1297315.006372] [<ffffffff81046b21>] ? warn_slowpath_fmt+0x45/0x4a
[1297315.006377] [<ffffffff8134cc83>] ? _raw_spin_lock_bh+0xe/0x1c
[1297315.006381] [<ffffffff810363d8>] ? should_resched+0x5/0x23
[1297315.006383] [<ffffffff812c1708>] ? tcp_recvmsg+0x24e/0x8f7
[1297315.006386] [<ffffffff8134cc83>] ? _raw_spin_lock_bh+0xe/0x1c
[1297315.006388] [<ffffffff810363d8>] ? should_resched+0x5/0x23
[1297315.006393] [<ffffffff812daf7e>] ? inet_recvmsg+0x5b/0x6f
[1297315.006397] [<ffffffff8127d060>] ? sock_recvmsg+0xcd/0xec
[1297315.006402] [<ffffffff8128f562>] ? dev_queue_xmit+0x448/0x45b
[1297315.006407] [<ffffffff810ea1a2>] ? virt_to_head_page+0x6/0x29
[1297315.006410] [<ffffffff810ea414>] ? virt_to_slab+0x6/0x16
[1297315.006417] [<ffffffff810ea4f7>] ? __cache_free.isra.41+0x7d/0x198
[1297315.006420] [<ffffffff81282c8d>] ? sock_pipe_buf_release+0x8/0x8
[1297315.006425] [<ffffffff810fad94>] ? fget_light+0x2c/0x73
[1297315.006428] [<ffffffff8127eb1d>] ? sys_recvfrom+0xbf/0x121
[1297315.006434] [<ffffffff8111aab7>] ? sys_splice+0x389/0x404
[1297315.006438] [<ffffffff81351ad2>] ? system_call_fastpath+0x16/0x1b
[1297315.006440] ---[ end trace 54eae6935f54c0f6 ]---
[1297315.006442] ------------[ cut here ]------------
[1297315.006446] WARNING: at
/build/buildd-linux_3.2.32-1-amd64-bkoeca/linux-3.2.32/net/ipv4/tcp.c:1201
tcp_cleanup_rbuf+0x4a/0xfb()
[1297315.006448] Hardware name: PowerEdge SC1435
[1297315.006450] cleanup rbuf bug: copied DBE7B6DA seq DBE7B3C8 rcvnxt DBE7B6DA
[1297315.006451] Modules linked in: ipt_REDIRECT xt_TPROXY nf_tproxy_core
xt_set xt_mark xt_socket nf_defrag_ipv6 ip6_tables xt_tcpudp ip_set_hash_net
ip_set nfnetlink iptable_nat nf_nat nf_conntrack_ipv4 nf_defrag_ipv4
iptable_raw iptable_mangle iptable_filter ip_tables x_tables ip_vs nf_conntrack
crc32c libcrc32c nfsd nfs nfs_acl auth_rpcgss fscache lockd sunrpc 8021q garp
stp bonding tcp_htcp ext4 crc16 jbd2 loop radeon ttm drm_kms_helper drm
power_supply i2c_algo_bit k10temp mperf i2c_piix4 i2c_core processor shpchp
amd64_edac_mod edac_mce_amd edac_core snd_pcm snd_page_alloc snd_timer snd
soundcore psmouse dcdbas serio_raw pcspkr evdev button thermal_sys ext2 mbcache
microcode sg sr_mod cdrom usbhid hid sd_mod crc_t10dif ata_generic ohci_hcd
igb(O) pata_serverworks dca sata_svw libata ehci_hcd tg3 usbcore libphy
scsi_mod usb_common [last unloaded: scsi_wait_scan]
[1297315.006491] Pid: 23579, comm: haproxy Tainted: G W O 3.2.0-4-amd64
#1 Debian 3.2.32-1
[1297315.006492] Call Trace:
[1297315.006495] [<ffffffff81046a75>] ? warn_slowpath_common+0x78/0x8c
[1297315.006498] [<ffffffff81046b21>] ? warn_slowpath_fmt+0x45/0x4a
[1297315.006501] [<ffffffff812c0df1>] ? tcp_cleanup_rbuf+0x4a/0xfb
[1297315.006503] [<ffffffff812c1cc0>] ? tcp_recvmsg+0x806/0x8f7
[1297315.006505] [<ffffffff8134cc83>] ? _raw_spin_lock_bh+0xe/0x1c
[1297315.006508] [<ffffffff810363d8>] ? should_resched+0x5/0x23
[1297315.006529] [<ffffffff812daf7e>] ? inet_recvmsg+0x5b/0x6f
[1297315.006532] [<ffffffff8127d060>] ? sock_recvmsg+0xcd/0xec
[1297315.006535] [<ffffffff8128f562>] ? dev_queue_xmit+0x448/0x45b
[1297315.006537] [<ffffffff810ea1a2>] ? virt_to_head_page+0x6/0x29
[1297315.006540] [<ffffffff810ea414>] ? virt_to_slab+0x6/0x16
[1297315.006543] [<ffffffff810ea4f7>] ? __cache_free.isra.41+0x7d/0x198
[1297315.006545] [<ffffffff81282c8d>] ? sock_pipe_buf_release+0x8/0x8
[1297315.006548] [<ffffffff810fad94>] ? fget_light+0x2c/0x73
[1297315.006551] [<ffffffff8127eb1d>] ? sys_recvfrom+0xbf/0x121
[1297315.006554] [<ffffffff8111aab7>] ? sys_splice+0x389/0x404
[1297315.006557] [<ffffffff81351ad2>] ? system_call_fastpath+0x16/0x1b
[1297315.006559] ---[ end trace 54eae6935f54c0f7 ]---
[1297315.021080] ------------[ cut here ]------------
[1297315.021091] WARNING: at
/build/buildd-linux_3.2.32-1-amd64-bkoeca/linux-3.2.32/net/ipv4/tcp.c:1201
tcp_cleanup_rbuf+0x4a/0xfb()
[1297315.021094] Hardware name: PowerEdge SC1435
[1297315.021096] cleanup rbuf bug: copied DBE7BC8E seq DBE7B3C8 rcvnxt DBE7BC8E
[1297315.021097] Modules linked in: ipt_REDIRECT xt_TPROXY nf_tproxy_core
xt_set xt_mark xt_socket nf_defrag_ipv6 ip6_tables xt_tcpudp ip_set_hash_net
ip_set nfnetlink iptable_nat nf_nat nf_conntrack_ipv4 nf_defrag_ipv4
iptable_raw iptable_mangle iptable_filter ip_tables x_tables ip_vs nf_conntrack
crc32c libcrc32c nfsd nfs nfs_acl auth_rpcgss fscache lockd sunrpc 8021q garp
stp bonding tcp_htcp ext4 crc16 jbd2 loop radeon ttm drm_kms_helper drm
power_supply i2c_algo_bit k10temp mperf i2c_piix4 i2c_core processor shpchp
amd64_edac_mod edac_mce_amd edac_core snd_pcm snd_page_alloc snd_timer snd
soundcore psmouse dcdbas serio_raw pcspkr evdev button thermal_sys ext2 mbcache
microcode sg sr_mod cdrom usbhid hid sd_mod crc_t10dif ata_generic ohci_hcd
igb(O) pata_serverworks dca sata_svw libata ehci_hcd tg3 usbcore libphy
scsi_mod usb_common [last unloaded: scsi_wait_scan]
[1297315.021155] Pid: 23579, comm: haproxy Tainted: G W O 3.2.0-4-amd64
#1 Debian 3.2.32-1
[1297315.021157] Call Trace:
[1297315.021164] [<ffffffff81046a75>] ? warn_slowpath_common+0x78/0x8c
[1297315.021167] [<ffffffff81046b21>] ? warn_slowpath_fmt+0x45/0x4a
[1297315.021171] [<ffffffff812c0df1>] ? tcp_cleanup_rbuf+0x4a/0xfb
[1297315.021173] [<ffffffff812c1ed8>] ? tcp_read_sock+0x127/0x138
[1297315.021176] [<ffffffff812c1fba>] ? tcp_splice_read+0xd1/0x21f
[1297315.021181] [<ffffffff8111aab7>] ? sys_splice+0x389/0x404
[1297315.021185] [<ffffffff81351ad2>] ? system_call_fastpath+0x16/0x1b
[1297315.021187] ---[ end trace 54eae6935f54c0f8 ]---
and my config
###############
global
pidfile /var/run/haproxy.pid
maxconn 200000
maxpipes 50000
daemon
stats socket /tmp/haproxy_stats.sock
nbproc 4
spread-checks 5
tune.rcvbuf.client 16384
tune.rcvbuf.server 16384
tune.sndbuf.client 16384
tune.sndbuf.server 16384
node frontend_node2
# debug
defaults
#TCP SECTION
maxconn 200000
backlog 32768
timeout connect 5s
timeout client 60s
timeout server 60s
timeout check 30s
timeout http-request 15s
timeout http-keep-alive 50s
timeout queue 60s
option tcpka
# option nolinger
### Linux specified
option splice-auto
option splice-request
option splice-response
#HTTP SECTION
hash-type consistent
balance uri whole
option accept-invalid-http-request
option accept-invalid-http-response
option redispatch
retries 2
# option http-no-delay
# option httplog
option http-server-close
# option http-pretend-keepalive
no option checkcache
#SYSTEM SECTION
option dontlog-normal
option dontlognull
option log-separate-errors
# cookie SERVERID
######### frontend ##############
frontend tcp-in
bind 192.168.1.1:2222 transparent
option forwardfor
# option tcp-smart-accept
# option httplog
mode http
capture request header Host len 32
log global
default_backend SquidClusters
backend SquidClusters
mode http
# option tcp-smart-connect
option abortonclose
log global
source 0.0.0.0 usesrc clientip
reqidel Accept-Encoding
option httpchk GET http://192.168.135.102/check.txt
server backend1a 192.168.134.4:3128 weight 20 check inter 5s
maxconn 20000
server backend1b 192.168.134.5:3128 weight 20 check inter 5s
maxconn 20000
Regards
Jinge
