re-sending the patch as an attachment as the formatting is still weird,
and fixed typo I spotted..
On 02/04/2023 18:57, J Carter wrote:
Hello,
I've also attached an example nginx.conf and test script that
simulates the asynchronous close events.
Two different test cases can be found within that, one with path /1
for single peer upstream and /2 for multi-peer.
You should see 2 upstream addresses repeated in a row
per-upstream-server in the access log by default, as it fails
through the cached connections & next performs next upstream tries.
Any feedback would be appreciated.
# HG changeset patch
# User jordanc.car...@outlook.com
# Date 1680457073 -3600
# Sun Apr 02 18:37:53 2023 +0100
# Node ID 9ec4d7a8cdf6cdab00d09dff75fa6045f6f5533f
# Parent 5f1d05a21287ba0290dd3a17ad501595b442a194
Added keepalive_async_fails command to keepalive load balancer module.
This value determines the number suspected keepalive race events
per-upstream-try that will be tolerated before a subsequent network
connection
error is considered a true failure.
diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h Tue Mar 28 18:01:54 2023 +0300
+++ b/src/event/ngx_event_connect.h Sun Apr 02 18:37:53 2023 +0100
@@ -17,6 +17,7 @@
#define NGX_PEER_KEEPALIVE 1
#define NGX_PEER_NEXT 2
#define NGX_PEER_FAILED 4
+#define NGX_PEER_ASYNC_FAILED 8
typedef struct ngx_peer_connection_s ngx_peer_connection_t;
@@ -41,6 +42,7 @@
ngx_str_t *name;
ngx_uint_t tries;
+ ngx_uint_t async_fails;
ngx_msec_t start_time;
ngx_event_get_peer_pt get;
diff -r 5f1d05a21287 -r 9ec4d7a8cdf6
src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Mar
28 18:01:54 2023 +0300
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sun Apr
02 18:37:53 2023 +0100
@@ -13,6 +13,7 @@
typedef struct {
ngx_uint_t max_cached;
ngx_uint_t requests;
+ ngx_uint_t max_async_fails;
ngx_msec_t time;
ngx_msec_t timeout;
@@ -108,6 +109,13 @@
offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
NULL },
+ { ngx_string("keepalive_async_fails"),
+ NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_num_slot,
+ NGX_HTTP_SRV_CONF_OFFSET,
+ offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
+ NULL },
+
ngx_null_command
};
@@ -160,6 +168,7 @@
ngx_conf_init_msec_value(kcf->time, 3600000);
ngx_conf_init_msec_value(kcf->timeout, 60000);
ngx_conf_init_uint_value(kcf->requests, 1000);
+ ngx_conf_init_uint_value(kcf->max_async_fails, 2);
if (kcf->original_init_upstream(cf, us) != NGX_OK) {
return NGX_ERROR;
@@ -320,6 +329,21 @@
u = kp->upstream;
c = pc->connection;
+ if (state & NGX_PEER_ASYNC_FAILED) {
+ pc->async_fails++;
+
+ if (pc->async_fails == 2) {
+ pc->async_fails = 0;
+ state = NGX_PEER_FAILED;
+
+ } else {
+ pc->tries++;
+ }
+ goto invalid;
+ }
+
+ pc->async_fails = 0;
+
if (state & NGX_PEER_FAILED
|| c == NULL
|| c->read->eof
@@ -529,6 +553,8 @@
conf->time = NGX_CONF_UNSET_MSEC;
conf->timeout = NGX_CONF_UNSET_MSEC;
conf->requests = NGX_CONF_UNSET_UINT;
+ conf->max_async_fails = NGX_CONF_UNSET_UINT;
+
return conf;
}
diff -r 5f1d05a21287 -r 9ec4d7a8cdf6 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream.c Sun Apr 02 18:37:53 2023 +0100
@@ -4317,6 +4317,8 @@
{
state = NGX_PEER_NEXT;
+ } else if (u->peer.cached && ft_type ==
NGX_HTTP_UPSTREAM_FT_ERROR) {
+ state = NGX_PEER_ASYNC_FAILED;
} else {
state = NGX_PEER_FAILED;
}
@@ -4330,11 +4332,6 @@
"upstream timed out");
}
- if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
- /* TODO: inform balancer instead */
- u->peer.tries++;
- }
-
switch (ft_type) {
case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
@@ -4421,7 +4418,6 @@
return;
}
#endif
-
ngx_http_upstream_finalize_request(r, u, status);
return;
}
diff -r 5f1d05a21287 -r 9ec4d7a8cdf6
src/http/ngx_http_upstream_round_robin.c
--- a/src/http/ngx_http_upstream_round_robin.c Tue Mar 28 18:01:54
2023 +0300
+++ b/src/http/ngx_http_upstream_round_robin.c Sun Apr 02 18:37:53
2023 +0100
@@ -297,6 +297,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session =
ngx_http_upstream_set_round_robin_peer_session;
@@ -418,6 +419,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
r->upstream->peer.save_session =
ngx_http_upstream_empty_save_session;
@@ -459,7 +461,10 @@
rrp->current = peer;
- } else {
+ } else if (pc->async_fails > 0) {
+ peer = rrp->current;
+ }
+ else {
/* there are several peers */
@@ -615,18 +620,7 @@
ngx_http_upstream_rr_peers_rlock(rrp->peers);
ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
- if (rrp->peers->single) {
-
- peer->conns--;
-
- ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
- ngx_http_upstream_rr_peers_unlock(rrp->peers);
-
- pc->tries = 0;
- return;
- }
-
- if (state & NGX_PEER_FAILED) {
+ if (state & NGX_PEER_FAILED && !rrp->peers->single) {
now = ngx_time();
peer->fails++;
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
# HG changeset patch
# User jordanc.car...@outlook.com
# Date 1680459126 -3600
# Sun Apr 02 19:12:06 2023 +0100
# Node ID 4295bf4613e155e063914ac10f898dbe98ae4a54
# Parent 5f1d05a21287ba0290dd3a17ad501595b442a194
Added keepalive_async_fails command to keepalive load balancer module.
This value determines the number suspected keepalive race events
per-upstream-try that will be tolerated before a subsequent network connection
error is considered a true failure.
diff -r 5f1d05a21287 -r 4295bf4613e1 src/event/ngx_event_connect.h
--- a/src/event/ngx_event_connect.h Tue Mar 28 18:01:54 2023 +0300
+++ b/src/event/ngx_event_connect.h Sun Apr 02 19:12:06 2023 +0100
@@ -17,6 +17,7 @@
#define NGX_PEER_KEEPALIVE 1
#define NGX_PEER_NEXT 2
#define NGX_PEER_FAILED 4
+#define NGX_PEER_ASYNC_FAILED 8
typedef struct ngx_peer_connection_s ngx_peer_connection_t;
@@ -41,6 +42,7 @@
ngx_str_t *name;
ngx_uint_t tries;
+ ngx_uint_t async_fails;
ngx_msec_t start_time;
ngx_event_get_peer_pt get;
diff -r 5f1d05a21287 -r 4295bf4613e1
src/http/modules/ngx_http_upstream_keepalive_module.c
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c Tue Mar 28
18:01:54 2023 +0300
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c Sun Apr 02
19:12:06 2023 +0100
@@ -13,6 +13,7 @@
typedef struct {
ngx_uint_t max_cached;
ngx_uint_t requests;
+ ngx_uint_t max_async_fails;
ngx_msec_t time;
ngx_msec_t timeout;
@@ -108,6 +109,13 @@
offsetof(ngx_http_upstream_keepalive_srv_conf_t, requests),
NULL },
+ { ngx_string("keepalive_async_fails"),
+ NGX_HTTP_UPS_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_num_slot,
+ NGX_HTTP_SRV_CONF_OFFSET,
+ offsetof(ngx_http_upstream_keepalive_srv_conf_t, max_async_fails),
+ NULL },
+
ngx_null_command
};
@@ -160,6 +168,7 @@
ngx_conf_init_msec_value(kcf->time, 3600000);
ngx_conf_init_msec_value(kcf->timeout, 60000);
ngx_conf_init_uint_value(kcf->requests, 1000);
+ ngx_conf_init_uint_value(kcf->max_async_fails, 2);
if (kcf->original_init_upstream(cf, us) != NGX_OK) {
return NGX_ERROR;
@@ -320,6 +329,21 @@
u = kp->upstream;
c = pc->connection;
+ if (state & NGX_PEER_ASYNC_FAILED) {
+ pc->async_fails++;
+
+ if (pc->async_fails == kp->conf->max_async_fails) {
+ pc->async_fails = 0;
+ state = NGX_PEER_FAILED;
+
+ } else {
+ pc->tries++;
+ }
+ goto invalid;
+ }
+
+ pc->async_fails = 0;
+
if (state & NGX_PEER_FAILED
|| c == NULL
|| c->read->eof
@@ -529,6 +553,8 @@
conf->time = NGX_CONF_UNSET_MSEC;
conf->timeout = NGX_CONF_UNSET_MSEC;
conf->requests = NGX_CONF_UNSET_UINT;
+ conf->max_async_fails = NGX_CONF_UNSET_UINT;
+
return conf;
}
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream.c
--- a/src/http/ngx_http_upstream.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream.c Sun Apr 02 19:12:06 2023 +0100
@@ -4317,6 +4317,8 @@
{
state = NGX_PEER_NEXT;
+ } else if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
+ state = NGX_PEER_ASYNC_FAILED;
} else {
state = NGX_PEER_FAILED;
}
@@ -4330,11 +4332,6 @@
"upstream timed out");
}
- if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) {
- /* TODO: inform balancer instead */
- u->peer.tries++;
- }
-
switch (ft_type) {
case NGX_HTTP_UPSTREAM_FT_TIMEOUT:
@@ -4421,7 +4418,6 @@
return;
}
#endif
-
ngx_http_upstream_finalize_request(r, u, status);
return;
}
diff -r 5f1d05a21287 -r 4295bf4613e1 src/http/ngx_http_upstream_round_robin.c
--- a/src/http/ngx_http_upstream_round_robin.c Tue Mar 28 18:01:54 2023 +0300
+++ b/src/http/ngx_http_upstream_round_robin.c Sun Apr 02 19:12:06 2023 +0100
@@ -297,6 +297,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session =
ngx_http_upstream_set_round_robin_peer_session;
@@ -418,6 +419,7 @@
r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer;
r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer;
r->upstream->peer.tries = ngx_http_upstream_tries(rrp->peers);
+ r->upstream->peer.async_fails = 0;
#if (NGX_HTTP_SSL)
r->upstream->peer.set_session = ngx_http_upstream_empty_set_session;
r->upstream->peer.save_session = ngx_http_upstream_empty_save_session;
@@ -459,7 +461,10 @@
rrp->current = peer;
- } else {
+ } else if (pc->async_fails > 0) {
+ peer = rrp->current;
+ }
+ else {
/* there are several peers */
@@ -615,18 +620,7 @@
ngx_http_upstream_rr_peers_rlock(rrp->peers);
ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
- if (rrp->peers->single) {
-
- peer->conns--;
-
- ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
- ngx_http_upstream_rr_peers_unlock(rrp->peers);
-
- pc->tries = 0;
- return;
- }
-
- if (state & NGX_PEER_FAILED) {
+ if (state & NGX_PEER_FAILED && !rrp->peers->single) {
now = ngx_time();
peer->fails++;
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
https://mailman.nginx.org/mailman/listinfo/nginx-devel
