Ok, Just happened again here's all the information that I have:-
netstat -LAan Current listen queue sizes (qlen/incqlen/maxqlen) Tcpcb Proto Listen Local Address fffffe04c3ddeb70 tcp4 0/0/512 *.143 fffffe001ede47a0 tcp4 769/0/512 *.110 <----- max queue fills up (and reaches backlog setting) fffffe039c5e37a0 tcp46 0/0/459 *.3306 fffffe04ac4d6b70 tcp4 0/0/1024 127.0.0.1.2812 fffffe05a1b943d0 tcp4 0/0/128 *.28 fffffe00934827a0 tcp4 0/0/16 *.2000 fffffe00931873d0 tcp4 0/0/128 *.22 fffffe00931877a0 tcp6 0/0/128 *.22 fffffe00935317a0 tcp4 0/0/100 *.25 fffffe00933b97a0 tcp4 0/0/128 *.199 [root@redacted ~]# dmesg sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance sonewconn: pcb 0xfffffe001e681188: Listen queue overflow: 769 already in queue awaiting acceptance [root@ redacted ~]# telnet localhost 110 Trying 127.0.0.1... Connected to localhost. Escape character is '^]'. Connection closed by foreign host. [root@redacted ]# top last pid: 86783; load averages: 1.13, 1.10, 0.92 up 5+23:11:40 14:12:24 33 processes: 1 running, 32 sleeping CPU: % user, % nice, % system, % interrupt, % idle Mem: 12G Active, 1608M Inact, 8703M Wired, 33M Cache, 1046M Free ARC: 7592M Total, 2723M MFU, 4327M MRU, 24M Anon, 166M Header, 363M Other PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND 79266 nobody 2 20 0 168M 78248K uwait 0 86:59 100.00% dbmail-pop3d <---- using 100% CPU 78725 mysql 38 20 0 14861M 13127M uwait 1 322:42 9.23% mysqld 86766 nobody 12 20 0 265M 95552K uwait 5 0:01 0.88% dbmail-imapd 35214 nobody 1 20 0 128M 46364K kqread 2 59:31 0.39% dbmail-lmtpd 1016 root 1 20 0 47756K 2480K select 3 63:03 0.10% snmpd 1047 root 1 20 0 12084K 888K select 6 169:26 0.00% powerd 950 root 1 20 0 12084K 1052K select 10 22:44 0.00% syslogd 5300 root 2 20 0 33116K 1728K select 9 4:36 0.00% monit Trace output:- rite(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' write(59,"opYFaM7Zxww5zxx9Kqyb45mRpGgMi4Xn"...,262143) ERR#35 'Resource temporarily unavailable' procstat:- PID TID COMM TDNAME CPU PRI STATE WCHAN 79266 101601 dbmail-pop3d - 1 203 run - 79266 102226 dbmail-pop3d - 0 120 sleep uwait GDB output:- [New Thread 80500a400 (LWP 102226)] [New Thread 805007400 (LWP 101601)] [Switching to Thread 80500a400 (LWP 102226)] 0x0000000803ed64dc in ?? () from /lib/libthr.so.3 [Switching to thread 2 (Thread 805007400 (LWP 101601))] #0 0x00000008041f2afc in write () from /lib/libc.so.7 #0 0x00000008041f2afc in write () from /lib/libc.so.7 No symbol table info available. #1 0x0000000803ed0ce0 in write () from /lib/libthr.so.3 No symbol table info available. #2 0x0000000800864390 in ci_write (client=0x805d7e000, msg=<optimized out>) at clientbase.c:318 ap = {{gp_offset = 1835430759, fp_offset = 2034895373, overflow_arg_area = 0x786f5563366a6a44, reg_save_area = 0x47464b7845797964}} cp = {{gp_offset = 4294942656, fp_offset = 32767, overflow_arg_area = 0x803ed57a4, reg_save_area = 0x193}} t = <optimized out> e = <optimized out> n = 262143 left = 1096120 s = 0x8096261e4 "avpOnW/i/VtPD22raFLIXTzEKrGcn5B64I+U+hHevp61XofCwpI93+Gnhbw1rP\r\nxi17wvbeNZbLTvB2kziHWdPiivxKsFtJLuCv8sqgFs7uysMA18j/ALTniLVvGHwf+Ira1aad/Zf2\r\nWJleCH7KkzRgDeFySmSzEDgewAr0zRbSLR7XXNSjtLuxbTJFVJBCXVV3Oo"... buf = "39jXtyUbpLh4B9cAD9K+jtPkijhKSY3zk7Bn\r\nn" __func__ = "ci_write" #3 0x000000080086446c in ci_write_cb (client=0x805d7e000) at clientbase.c:263 rest = <optimized out> result = <optimized out> #4 0x0000000800863095 in client_session_bailout (session=<optimized out>) at clientsession.c:146 c = 0x805269f00 args = 0x0 from = 0x0 rcpt = 0x0 messagelst = 0x0 client = 0x805d7e000 __func__ = "client_session_bailout" #5 0x0000000000402285 in pop3_cb_time (arg=<optimized out>) at pop3.c:272 session = 0x805269f00 #6 0x0000000802961aee in event_base_loop () from /usr/local/lib/event2/libevent-2.0.so.6 No symbol table info available. #7 0x00000008008628a1 in server_run (conf=0x7fffffffa310) at server.c:842 i = 1 __func__ = "server_run" #8 0x0000000800862d6d in server_mainloop (config=0x7fffffffa310, service=0x403b3a "POP", servicename=<optimized out>) at server.c:976 __func__ = "server_mainloop" #9 0x0000000000403a37 in main (argc=1, argv=<optimized out>) at pop3d.c:52 config = {no_daemonize = 0, log_verbose = 0, pidFile = 0x805017d80 "/var/run/dbmail/dbmail-pop3d.pid", timeout = 120, login_timeout = 30, iplist = 0x8050dd920, port = "110", '\000' <repeats 1020 times>, ssl_port = '\000' <repeats 1023 times>, ipcount = 1, socketcount = 1, ssl_socketcount = 0, listenSockets = 0x80500a800, ssl_listenSockets = 0x80500ac00, service_before_smtp = 0, authlog = 0, ssl = 0, backlog = 512, resolveIP = 0, evh = 0x0, service_name = "POP", '\000' <repeats 1020 times>, process_name = "dbmail-pop3d", '\000' <repeats 1011 times>, serverUser = "nobody", '\000' <repeats 1017 times>, serverGroup = "nogroup", '\000' <repeats 1016 times>, socket = '\000' <repeats 1023 times>, log = "/var/log/dbmail.log", '\000' <repeats 1004 times>, error_log = "/var/log/dbmail.err", '\000' <repeats 1004 times>, pid_dir = "/var/run/dbmail", '\000' <repeats 1008 times>, tls_cafile = '\000' <repeats 1023 times>, tls_cert = '\000' <repeats 1023 times>, tls_key = '\000' <rep eats 1023 times>, tls_ciphers = '\000' <repeats 1023 times>, ClientHandler = 0x402060 <pop3_handle_connection>, cb = 0x0} result = 0 __func__ = "main" Best regards, James Greig -----Original Message----- From: dbmail-boun...@dbmail.org [mailto:dbmail-boun...@dbmail.org] On Behalf Of James Greig Sent: 17 March 2014 10:54 To: DBMail mailinglist Subject: Re: [Dbmail] Iphone HTML emails Just had a short spike on pop3d but it recovered fairly quickly so probably nothing significant here until it happens for a prolonged period again. The main thing I tend to look at is the listen queue which I've set the backlog at 512 now 79266 nobody 2 20 0 164M 85344K uwait 5 46:06 100.00% dbmail-pop3d netstat -LAan Tcpcb Proto Listen Local Address fffffe0061598b70 tcp4 0/0/512 *.143 fffffe001ede47a0 tcp4 157/0/512 *.110 <---- pop3d listen queue jumbs up to 157 fffffe039c5e37a0 tcp46 0/0/459 *.3306 fffffe04ac4d6b70 tcp4 0/0/1024 127.0.0.1.2812 fffffe05a1b943d0 tcp4 0/0/128 *.28 fffffe00934827a0 tcp4 0/0/16 *.2000 fffffe00931873d0 tcp4 0/0/128 *.22 fffffe00931877a0 tcp6 0/0/128 *.22 fffffe00935317a0 tcp4 0/0/100 *.25 fffffe00933b97a0 tcp4 0/0/128 *.199 PID TID COMM TDNAME CPU PRI STATE WCHAN 79266 101601 dbmail-pop3d - 3 203 run - 79266 102226 dbmail-pop3d - 5 120 sleep uwait [New Thread 80500a400 (LWP 102226)] [New Thread 805007400 (LWP 101601)] [Switching to Thread 80500a400 (LWP 102226)] 0x0000000803ed64dc in ?? () from /lib/libthr.so.3 [Switching to thread 2 (Thread 805007400 (LWP 101601))] #0 0x00000008041f2afc in write () from /lib/libc.so.7 #0 0x00000008041f2afc in write () from /lib/libc.so.7 No symbol table info available. #1 0x0000000803ed0ce0 in write () from /lib/libthr.so.3 No symbol table info available. #2 0x0000000800864390 in ci_write (client=0x80bc4d000, msg=<optimized out>) at clientbase.c:318 ap = {{gp_offset = 1648906823, fp_offset = 1499822669, overflow_arg_area = 0x7665307538745955, reg_save_area = 0x6a6f5366550a0d50}} cp = {{gp_offset = 4294942656, fp_offset = 32767, overflow_arg_area = 0x803ed57a4, reg_save_area = 0x7}} t = <optimized out> e = <optimized out> n = 262143 left = 407634 s = 0x80af2a3c8 "iL5WIDKMjpjtT1iZSTGq7m6lhVOQx1uS0joFBOcj1FMmhE6SghWDkZAUD\r\nGP8A9VQ2BlLKb2/aGMo8hb94zHkHtk1N9nkmLyq+ZozgYflyOn1HvVJ6DLkMMaOoLbBnJIHr1oBN\r\ntc7cHY8gAYDO0Dv7VD1JYeSFLCR/MGc7j3yeKSa3Xy8gEHv701G47kEUM6zK4AP"... buf = "717eCpXd2NNJn0jp8DW\r\nmmSCBti7cPuXOQeoFa" __func__ = "ci_write" #3 0x000000080086446c in ci_write_cb (client=0x80bc4d000) at clientbase.c:263 rest = <optimized out> result = <optimized out> #4 0x0000000800863095 in client_session_bailout (session=<optimized out>) at clientsession.c:146 c = 0x805269c00 args = 0x0 from = 0x0 rcpt = 0x0 messagelst = 0x0 client = 0x80bc4d000 __func__ = "client_session_bailout" #5 0x0000000000402285 in pop3_cb_time (arg=<optimized out>) at pop3.c:272 session = 0x805269c00 #6 0x0000000802961aee in event_base_loop () from /usr/local/lib/event2/libevent-2.0.so.6 No symbol table info available. #7 0x00000008008628a1 in server_run (conf=0x7fffffffa310) at server.c:842 i = 1 __func__ = "server_run" #8 0x0000000800862d6d in server_mainloop (config=0x7fffffffa310, service=0x403b3a "POP", servicename=<optimized out>) at server.c:976 __func__ = "server_mainloop" #9 0x0000000000403a37 in main (argc=1, argv=<optimized out>) at pop3d.c:52 config = {no_daemonize = 0, log_verbose = 0, pidFile = 0x805017d80 "/var/run/dbmail/dbmail-pop3d.pid", timeout = 120, login_timeout = 30, iplist = 0x8050dd920, port = "110", '\000' <repeats 1020 times>, ssl_port = '\000' <repeats 1023 times>, ipcount = 1, socketcount = 1, ssl_socketcount = 0, listenSockets = 0x80500a800, ssl_listenSockets = 0x80500ac00, service_before_smtp = 0, authlog = 0, ssl = 0, backlog = 512, resolveIP = 0, evh = 0x0, service_name = "POP", '\000' <repeats 1020 times>, process_name = "dbmail-pop3d", '\000' <repeats 1011 times>, serverUser = "nobody", '\000' <repeats 1017 times>, serverGroup = "nogroup", '\000' <repeats 1016 times>, socket = '\000' <repeats 1023 times>, log = "/var/log/dbmail.log", '\000' <repeats 1004 times>, error_log = "/var/log/dbmail.err", '\000' <repeats 1004 times>, pid_dir = "/var/run/dbmail", '\000' <repeats 1008 times>, tls_cafile = '\000' <repeats 1023 times>, tls_cert = '\000' <repeats 1023 times>, tls_key = '\000' <rep eats 1023 times>, tls_ciphers = '\000' <repeats 1023 times>, ClientHandler = 0x402060 <pop3_handle_connection>, cb = 0x0} result = 0 __func__ = "main" Best regards, James Greig -----Original Message----- From: dbmail-boun...@dbmail.org [mailto:dbmail-boun...@dbmail.org] On Behalf Of Paul J Stevens Sent: 17 March 2014 07:36 To: DBMail mailinglist Subject: Re: [Dbmail] Iphone HTML emails On 16-03-14 15:51, James Greig wrote: > :D > > Getting somewhere now. I think basically attach gdb to the process and then > run 'info threads' and switch to the second thread. > > :- > > Ok the below output was run whilst dbmail-pop3d was actually in an OK state > but does the below actually look more like what is needed? That's it exactly. bt full is probably overkill, but it doesn't hurt either... -- ________________________________________________________________ Paul J Stevens pjstevns @ gmail, twitter, github, linkedin www.nfg.nl/i...@nfg.nl/+31.85.877.99.97 _______________________________________________ DBmail mailing list DBmail@dbmail.org http://mailman.fastxs.nl/cgi-bin/mailman/listinfo/dbmail _______________________________________________ DBmail mailing list DBmail@dbmail.org http://mailman.fastxs.nl/cgi-bin/mailman/listinfo/dbmail _______________________________________________ DBmail mailing list DBmail@dbmail.org http://mailman.fastxs.nl/cgi-bin/mailman/listinfo/dbmail