Hi,
We're experiencing strange behaviour of haproxy-1.3.15.8 and haproxy-1.3.16, at
frequent times it will use 100% cpu. It appears that it is wait_time is not
used. I've used both poll and epoll, same behaviour. The kernel is 2.6.26-1-686
#1 SMP.
The box is used, but not using many connections.
Any ideas?
Thanks,
Remco
config:
-----------------------------------------------------
global
maxconn 10000
log 127.0.0.1 local0
uid 200
gid 200
chroot /var/empty
daemon
tune.maxpollevents 200
nbproc 1
nosepoll
noepoll
============================
strace with poll
--------------------------------------------------------
Process 32211 attached - interrupt to quit
^CProcess 32211 detached
% time seconds usecs/call calls errors syscall
------ ----------- ----------- --------- --------- ----------------
76.98 0.144607 0 982302 poll
16.42 0.030855 0 982303 gettimeofday
3.25 0.006101 0 26947 13 send
2.13 0.004000 0 58097 24425 recv
0.37 0.000687 0 1976 1976 connect
0.23 0.000425 0 2502 close
0.22 0.000413 1 526 sendto
0.19 0.000349 0 1976 socket
0.09 0.000173 0 1057 522 accept
0.08 0.000146 0 2511 fcntl64
0.03 0.000065 0 2511 setsockopt
0.02 0.000033 0 523 shutdown
0.00 0.000000 0 1 restart_syscall
0.00 0.000000 0 1 brk
0.00 0.000000 0 1448 getsockopt
------ ----------- ----------- --------- --------- ----------------
100.00 0.187854 2064681 26936 total
==================================
strace with epoll
--------------------------------------------------------------------
Process 31718 attached - interrupt to quit
^CProcess 31718 detached
% time seconds usecs/call calls errors syscall
------ ----------- ----------- --------- --------- ----------------
47.81 0.005649 0 169141 epoll_wait
42.82 0.005059 0 169140 gettimeofday
6.19 0.000731 0 2811 97 send
2.82 0.000333 0 6045 2716 recv
0.19 0.000022 0 93 socket
0.18 0.000021 0 156 getsockopt
0.00 0.000000 0 107 close
0.00 0.000000 0 109 fcntl64
0.00 0.000000 0 221 1 epoll_ctl
0.00 0.000000 0 93 93 connect
0.00 0.000000 0 32 16 accept
0.00 0.000000 0 15 sendto
0.00 0.000000 0 14 shutdown
0.00 0.000000 0 109 setsockopt
------ ----------- ----------- --------- --------- ----------------
100.00 0.011815 348086 2923 total
==================================
strace attached with poll
-----------------------------------------------------------------------
gettimeofday({1238154432, 213206}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 213356}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 213506}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 213683}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 213834}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 213984}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 227138}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 1 ([{fd=13, revents=POLLIN}])
gettimeofday({1238154432, 227385}, NULL) = 0
recv(13, "liE9jcPlUI/1Zw4bj3cjp29q\r\nk+FUJOh"..., 16384, MSG_NOSIGNAL) = 12288
send(16, "liE9jcPlUI/1Zw4bj3cjp29q\r\nk+FUJOh"..., 12288,
MSG_DONTWAIT|MSG_NOSIGNAL) = 12288
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 227770}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 227942}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 228116}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 228291}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 228464}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 228629}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 228802}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 228973}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 229134}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 229294}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 229454}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 229645}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 229797}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 229948}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 230105}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 230257}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 230408}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 230585}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 230742}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 250838}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 1 ([{fd=13, revents=POLLIN}])
gettimeofday({1238154432, 251084}, NULL) = 0
recv(13, "FdF4UstU+xRtlNQt5SJLqzuJAZInkw5IZ"..., 16384, MSG_NOSIGNAL) = 12288
send(16, "FdF4UstU+xRtlNQt5SJLqzuJAZInkw5IZ"..., 12288,
MSG_DONTWAIT|MSG_NOSIGNAL) = 12288
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 251497}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 251685}, NULL) = 0
poll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0) = 0 (Timeout)
gettimeofday({1238154432, 251898}, NULL) = 0
^Cpoll([{fd=0, events=POLLOUT}, {fd=1, events=POLLOUT}, {fd=2, events=POLLOUT},
{fd=3, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=7,
events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLOUT}, {fd=11,
events=POLLOUT}, {fd=13, events=POLLIN}, {fd=15, events=POLLOUT}, {fd=16,
events=POLLIN}, {fd=17, events=POLLIN}], 14, 0 <unfinished ...>
======================================
