Hello all,
We have used haproxy for several months. And periodically see the next
error messages in the log:
============================================================================
Sep 27 16:17:06 localhost haproxy[12874]: Connect() failed for backend
https: no free ports.
============================================================================
I've investigated this issue and found that EADDRNOTAVAIL error is
returned sometimes.
Probably it is caused by the fact that we are using one port from
ephemeral range for our internal needs.
According to http://en.wikipedia.org/wiki/Ephemeral_port 'connect'
function usually just uses Round-Robin algorithm to choice the next
ephemeral port, and so when it encountered already used port -- it just
produces the above error.
Solution for this issue is simple -- add a loop around connect. We have
implemented it and tested on our environment. It works for us. May it
will be good enough to include into the core haproxy...
Logic of the solution is simple -- try to connect 3 times in case of
EAGAIN, EADDRINUSE or EADDRNOTAVAIL errors:
============================================================================
*# diff -b ../../haproxy-1.5-dev21-orig/src/proto_tcp.c proto_tcp.c*
277c277
< int fd;
---
int fd, try_count = 0;
437a438
while (1) {
441a443
if (++try_count >= 3) {
453a456
}
467a471,473
} else {
break;
}
============================================================================
Full patch created with "diff -Naur
../../haproxy-1.5-dev21-orig/src/proto_tcp.c proto_tcp.c" is attached to
this letter.
What do you think about it? Does it make sense for anybody else?
--
Best regards,
Denis Malyshkin,
Senior C++ Developer
of ISS Art, Ltd., Omsk, Russia.
Mobile Phone: +7 913 669 2896
Office tel/fax +7 3812 396959
Yahoo Messenger: dmalyshkin
Web: http://www.issart.com
E-mail: [email protected]
--- ../../haproxy-1.5-dev21-orig/src/proto_tcp.c 2013-12-16
23:45:49.000000000 +0000
+++ proto_tcp.c 2014-01-27 06:36:05.956975800 +0000
@@ -274,7 +274,7 @@
int tcp_connect_server(struct connection *conn, int data, int delack)
{
- int fd;
+ int fd, try_count = 0;
struct server *srv;
struct proxy *be;
struct conn_src *src;
@@ -435,35 +435,41 @@
if (global.tune.server_rcvbuf)
setsockopt(fd, SOL_SOCKET, SO_RCVBUF,
&global.tune.server_rcvbuf, sizeof(global.tune.server_rcvbuf));
- if ((connect(fd, (struct sockaddr *)&conn->addr.to,
get_addr_len(&conn->addr.to)) == -1) &&
- (errno != EINPROGRESS) && (errno != EALREADY) && (errno !=
EISCONN)) {
-
- if (errno == EAGAIN || errno == EADDRINUSE || errno ==
EADDRNOTAVAIL) {
- char *msg;
- if (errno == EAGAIN || errno == EADDRNOTAVAIL)
- msg = "no free ports";
- else
- msg = "local address already in use";
-
- qfprintf(stderr,"Connect() failed for backend %s:
%s.\n", be->id, msg);
- port_range_release_port(fdinfo[fd].port_range,
fdinfo[fd].local_port);
- fdinfo[fd].port_range = NULL;
- close(fd);
- send_log(be, LOG_ERR, "Connect() failed for backend %s:
%s.\n", be->id, msg);
- return SN_ERR_RESOURCE;
- } else if (errno == ETIMEDOUT) {
- //qfprintf(stderr,"Connect(): ETIMEDOUT");
- port_range_release_port(fdinfo[fd].port_range,
fdinfo[fd].local_port);
- fdinfo[fd].port_range = NULL;
- close(fd);
- return SN_ERR_SRVTO;
+ while (1) {
+ if ((connect(fd, (struct sockaddr *)&conn->addr.to,
get_addr_len(&conn->addr.to)) == -1) &&
+ (errno != EINPROGRESS) && (errno != EALREADY) && (errno !=
EISCONN)) {
+
+ if (errno == EAGAIN || errno == EADDRINUSE || errno ==
EADDRNOTAVAIL) {
+ if (++try_count >= 3) {
+ char *msg;
+ if (errno == EAGAIN || errno ==
EADDRNOTAVAIL)
+ msg = "no free ports";
+ else
+ msg = "local address already in
use";
+
+ qfprintf(stderr,"Connect() failed for
backend %s: %s.\n", be->id, msg);
+
port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
+ fdinfo[fd].port_range = NULL;
+ close(fd);
+ send_log(be, LOG_ERR, "Connect() failed
for backend %s: %s.\n", be->id, msg);
+ return SN_ERR_RESOURCE;
+ }
+ } else if (errno == ETIMEDOUT) {
+ //qfprintf(stderr,"Connect(): ETIMEDOUT");
+ port_range_release_port(fdinfo[fd].port_range,
fdinfo[fd].local_port);
+ fdinfo[fd].port_range = NULL;
+ close(fd);
+ return SN_ERR_SRVTO;
+ } else {
+ // (errno == ECONNREFUSED || errno ==
ENETUNREACH || errno == EACCES || errno == EPERM)
+ //qfprintf(stderr,"Connect(): %d", errno);
+ port_range_release_port(fdinfo[fd].port_range,
fdinfo[fd].local_port);
+ fdinfo[fd].port_range = NULL;
+ close(fd);
+ return SN_ERR_SRVCL;
+ }
} else {
- // (errno == ECONNREFUSED || errno == ENETUNREACH ||
errno == EACCES || errno == EPERM)
- //qfprintf(stderr,"Connect(): %d", errno);
- port_range_release_port(fdinfo[fd].port_range,
fdinfo[fd].local_port);
- fdinfo[fd].port_range = NULL;
- close(fd);
- return SN_ERR_SRVCL;
+ break;
}
}
