Hi Willy, > Note that with all the scheduling issues we've fixed over the last > days, there are multiple candidates which could cause this. Another > one was the lack of effect of the nice parameter which is normally > set on the CLI but the lack of which could result in socat timing > out during the first half second in absence of any response.
we got a similar issue with last v1.9.7+HEAD (last commit http://git.haproxy.org/?p=haproxy-1.9.git;a=commit;h=f3c64c69b1a293ae54db359a2b2a5f9e0c5265dd) Here are the complete threads backtraces: (gdb) bt #0 0x000056153d958570 in fwrr_update_server_weight (srv=0x56157f7a8680) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f7a8680) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f7a8680, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c1a20, context=0x56157f7a8680, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=data@entry=0x5615893fab00) at src/haproxy.c:2726 #7 0x000056153d83b455 in main (argc=<optimized out>, argv=0x7fff630890d8) at src/haproxy.c:3388 (gdb) thread apply all bt Thread 16 (Thread 0x7fe9b6e32700 (LWP 2807)): #0 0x000056153d958459 in fwrr_update_server_weight (srv=0x56157f5b2fc0) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f5b2fc0) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f5b2fc0, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899bf2f0, context=0x56157f5b2fc0, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7fe9b6631700 (LWP 2808)): #0 0x000056153d96d7a0 in __eb_insert_dup (new=0x56157f52f424, sub=0x56157f5640a4) at ebtree/ebtree.h:478 #1 eb_insert_dup (sub=<optimized out>, new=0x56157f52f424) at ebtree/ebtree.c:31 #2 0x000056153d96df10 in __eb32_insert (new=new@entry=0x56157f52f424, root=<optimized out>, root@entry=0x56157deb4140) at ebtree/eb32tree.h:337 #3 eb32_insert (root=root@entry=0x56157deb4140, new=new@entry=0x56157f52f424) at ebtree/eb32tree.c:27 #4 0x000056153d957fcb in fwrr_queue_srv (s=s@entry=0x56157f52f080) at src/lb_fwrr.c:371 #5 0x000056153d9585e8 in fwrr_update_server_weight (srv=0x56157f52f080) at src/lb_fwrr.c:242 #6 0x000056153d8ae8ac in srv_update_status (s=0x56157f52f080) at src/server.c:4923 #7 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f52f080, must_update=must_update@entry=1) at src/server.c:1310 #8 0x000056153d8b6edd in server_warmup (t=0x5615899be8a0, context=0x56157f52f080, state=<optimized out>) at src/checks.c:1492 #9 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #10 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #11 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #12 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #13 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7fe9b5e30700 (LWP 2809)): #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157f625580) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f625580) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f625580, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899bfbe0, context=0x56157f625580, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7fe9b562f700 (LWP 2810)): #0 fwrr_update_server_weight (srv=0x56157f563d00) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f563d00) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f563d00, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899becc0, context=0x56157f563d00, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7fe9a7fff700 (LWP 2811)): #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157f6861c0) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f6861c0) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f6861c0, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c0370, context=0x56157f6861c0, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7fe9a77fe700 (LWP 2812)): ---Type <return> to continue, or q <return> to quit--- #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157f911140) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f911140) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f911140, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c3650, context=0x56157f911140, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7fe9a6ffd700 (LWP 2813)): #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157f7f7940) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f7f7940) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f7f7940, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c2050, context=0x56157f7f7940, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7fe9a67fc700 (LWP 2814)): #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157e48d580) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157e48d580) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157e48d580, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899a9be0, context=0x56157e48d580, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7fe9a5ddc700 (LWP 2815)): #0 fwrr_update_server_weight (srv=0x56157f701440) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f701440) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f701440, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c0d10, context=0x56157f701440, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7fe9a53bc700 (LWP 2816)): #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157f846c00) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f846c00) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f846c00, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c2680, context=0x56157f846c00, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7fe9a4bbb700 (LWP 2817)): #0 fwrr_update_server_weight (srv=0x56157fc28cc0) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157fc28cc0) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157fc28cc0, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c7430, context=0x56157fc28cc0, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 ---Type <return> to continue, or q <return> to quit--- #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7fe98ffff700 (LWP 2818)): #0 0x000056153d958459 in fwrr_update_server_weight (srv=0x56157f6b2180) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f6b2180) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f6b2180, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c06e0, context=0x56157f6b2180, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7fe98f7fe700 (LWP 2819)): #0 0x000056153d958572 in fwrr_update_server_weight (srv=0x56157f922ac0) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f922ac0) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f922ac0, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c37b0, context=0x56157f922ac0, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7fe98effd700 (LWP 2820)): #0 0x000056153d958459 in fwrr_update_server_weight (srv=0x56157fc20000) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157fc20000) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157fc20000, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c7380, context=0x56157fc20000, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7fe98e7fc700 (LWP 2821)): #0 fwrr_update_server_weight (srv=0x56157e9cf440) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157e9cf440) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157e9cf440, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899b0510, context=0x56157e9cf440, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=<optimized out>) at src/haproxy.c:2726 #7 0x00007fe9bd5e7dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fe9bc320ead in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7fe9be059b00 (LWP 2806)): #0 0x000056153d958570 in fwrr_update_server_weight (srv=0x56157f7a8680) at src/lb_fwrr.c:198 #1 0x000056153d8ae8ac in srv_update_status (s=0x56157f7a8680) at src/server.c:4923 #2 0x000056153d8adfc2 in server_recalc_eweight (sv=sv@entry=0x56157f7a8680, must_update=must_update@entry=1) at src/server.c:1310 #3 0x000056153d8b6edd in server_warmup (t=0x5615899c1a20, context=0x56157f7a8680, state=<optimized out>) at src/checks.c:1492 #4 0x000056153d94d97a in process_runnable_tasks () at src/task.c:390 #5 0x000056153d8c5c4f in run_poll_loop () at src/haproxy.c:2661 #6 run_thread_poll_loop (data=data@entry=0x5615893fab00) at src/haproxy.c:2726 #7 0x000056153d83b455 in main (argc=<optimized out>, argv=0x7fff630890d8) at src/haproxy.c:3388 -- William
