Subject: [RFC] Decrease server health based on http responses / events, version
2
This RFC quality patch implements decreasing server health based on
observing communication between HAProxy and servers.
Changes in this version:
- documentation
- close race between a started check and health analysis event
- don't force fastinter if it is not set
- better names for options
- layer4 support
Possibility to set/change default options is going to be implemented in
a different patch.
diff --git a/doc/configuration.txt b/doc/configuration.txt
index ef768f8..bdb5c78 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -4642,6 +4642,13 @@ cookie <value>
the same cookie value, and it is in fact somewhat common between normal and
backup servers. See also the "cookie" keyword in backend section.
+error_limit <count>
+ If health observing is enabled, the "error_limit" parameter specifies the
number
+ of consecutive errors that triggers event selected by the "on-error" option.
+ By default it is set to 10 consecutive errors.
+
+ See also the "check", "error_limit" and "on-error".
+
fall <count>
The "fall" parameter states that a server will be considered as dead after
<count> consecutive unsuccessful health checks. This value defaults to 3 if
@@ -4707,9 +4714,31 @@ minconn <minconn>
the ramp between both values when the backend has less than <fullconn>
concurrent connections. This makes it possible to limit the load on the
server during normal loads, but push it further for important loads without
- overloading the server during exceptionnal loads. See also the "maxconn"
+ overloading the server during exceptional loads. See also the "maxconn"
and "maxqueue" parameters, as well as the "fullconn" backend keyword.
-
+
+observe <mode>
+ This option enables health adjusting based on observing communication with
+ the server. By default this functionality is disabled and enabling it also
+ requires to enable healt checks. There are two supported modes: "layer4" and
+ "layer7". In layer4 mode, only successful/unsuccessful tcp connections are
+ significant. In layer7, which is only allowed for http proxies, responses
+ received from server are verified, like valid/wrong http code, unparsable
+ headers, a timeout, etc.
+
+ See also the "check", "on-error" and "error_limit".
+
+on-error <mode>
+ Select what should happen when enough consecutive errors are detected.
+ Currently, four modes are available:
+ - fastinter: force fastinter
+ - fail-check: simulate a failed check, also forces fastinter (default)
+ - sudden-death: simulate a pre-fatal failed health check, one more failed
+ check will mark a server down, forces fastinter
+ - mark-down: mark the server immediately down and force fastinter
+
+ See also the "check", "observe" and "error_limit".
+
port <port>
Using the "port" parameter, it becomes possible to use a different port to
send health-checks. On some servers, it may be desirable to dedicate a port
diff --git a/include/common/defaults.h b/include/common/defaults.h
index b0aee86..cada729 100644
--- a/include/common/defaults.h
+++ b/include/common/defaults.h
@@ -120,6 +120,9 @@
#define DEF_CHECK_REQ "OPTIONS / HTTP/1.0\r\n\r\n"
#define DEF_SMTP_CHECK_REQ "HELO localhost\r\n"
+#define DEF_HANA_ONERR HANA_ONERR_FAILCHK
+#define DEF_HANA_ERRLIMIT 10
+
// X-Forwarded-For header default
#define DEF_XFORWARDFOR_HDR "X-Forwarded-For"
diff --git a/include/proto/checks.h b/include/proto/checks.h
index bd70164..e3fe7ac 100644
--- a/include/proto/checks.h
+++ b/include/proto/checks.h
@@ -29,6 +29,7 @@ const char *get_check_status_description(short check_status);
const char *get_check_status_info(short check_status);
struct task *process_chk(struct task *t);
int start_checks();
+void health_adjust(struct server *s, short status);
#endif /* _PROTO_CHECKS_H */
diff --git a/include/types/checks.h b/include/types/checks.h
index 1b04608..ac784dd 100644
--- a/include/types/checks.h
+++ b/include/types/checks.h
@@ -18,6 +18,9 @@ enum {
/* Below we have finished checks */
HCHK_STATUS_CHECKED, /* DUMMY STATUS */
+
+ HCHK_STATUS_HANA, /* Healt analyze detected enough
consecutive errors */
+
HCHK_STATUS_SOCKERR, /* Socket error */
HCHK_STATUS_L4OK, /* L4 check passed, for example tcp
connect */
@@ -41,8 +44,51 @@ enum {
HCHK_STATUS_SIZE
};
+
+/* healt status for response tracking */
+enum {
+ HANA_STATUS_UNKNOWN = 0,
+
+ HANA_STATUS_L4_OK, /* L4 successful connection */
+ HANA_STATUS_L4_ERR, /* L4 unsuccessful connection */
+
+ HANA_STATUS_HTTP_OK, /* Correct http response */
+ HANA_STATUS_HTTP_STS, /* Wrong http response, for example
HTTP 5xx */
+ HANA_STATUS_HTTP_HDRRSP, /* Invalid http response (headers) */
+ HANA_STATUS_HTTP_RSP, /* Invalid http response */
+
+ HANA_STATUS_HTTP_READ_ERROR, /* Read error */
+ HANA_STATUS_HTTP_READ_TIMEOUT, /* Read timeout */
+ HANA_STATUS_HTTP_BROKEN_PIPE, /* Unexpected close from server */
+
+ HANA_STATUS_SIZE
+};
+
+enum {
+ HANA_ONERR_UNKNOWN = 0,
+
+ HANA_ONERR_FASTINTER, /* Force fastinter*/
+ HANA_ONERR_FAILCHK, /* Simulate a failed check */
+ HANA_ONERR_SUDDTH, /* Enters sudden death - one more
failed check will mark this server down */
+ HANA_ONERR_MARKDWN, /* Mark this server down, now! */
+};
+
+enum {
+ HANA_OBS_NONE = 0,
+
+ HANA_OBS_LAYER4, /* Observe L4 - for example tcp */
+ HANA_OBS_LAYER7, /* Observe L7 - for example http */
+
+ HANA_OBS_SIZE
+};
+
struct check_status {
short result; /* one of SRV_CHK_* */
char *info; /* human readable short info */
char *desc; /* long description */
};
+
+struct analyze_status {
+ char *desc; /* description */
+ unsigned char lr[HANA_OBS_SIZE]; /* result for l4/l7: 0 =
ignore, 1 - error, 2 - OK */
+};
diff --git a/include/types/server.h b/include/types/server.h
index b3fe83d..a55a79e 100644
--- a/include/types/server.h
+++ b/include/types/server.h
@@ -115,7 +115,10 @@ struct server {
struct sockaddr_in check_addr; /* the address to check, if
different from <addr> */
short check_port; /* the port to use for the
health checks */
int health; /* 0->rise-1 = bad;
rise->rise+fall-1 = good */
+ int consecutive_errors; /* current number of
consecutive errors */
int rise, fall; /* time in iterations */
+ int consecutive_errors_limit; /* number of consecutive errors
that triggers an event */
+ short observe, onerror; /* observing mode: one of
HANA_OBS_*; what to do on error: on of ANA_ONERR_* */
int inter, fastinter, downinter; /* checks: time in milliseconds
*/
int slowstart; /* slowstart time in seconds
(ms in the conf) */
int result; /* health-check result :
SRV_CHK_* */
@@ -137,9 +140,9 @@ struct server {
unsigned down_time; /* total time the server was
down */
time_t last_change; /* last time, when the state
was changed */
struct timeval check_start; /* last health check start time
*/
- unsigned long check_duration; /* time in ms took to finish
last health check */
+ long check_duration; /* time in ms took to finish
last health check */
short check_status, check_code; /* check result, check code */
- char check_desc[HCHK_DESC_LEN]; /* healt check descritpion */
+ char check_desc[HCHK_DESC_LEN]; /* healt check description */
struct freq_ctr sess_per_sec; /* sessions per second on this
server */
int puid; /* proxy-unique server ID, used
for SNMP */
diff --git a/src/cfgparse.c b/src/cfgparse.c
index 428d7b9..31b367e 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -2541,6 +2541,8 @@ int cfg_parse_listen(const char *file, int linenum, char
**args, int kwm)
newsrv->uweight = newsrv->iweight = 1;
newsrv->maxqueue = 0;
newsrv->slowstart = 0;
+ newsrv->onerror = DEF_HANA_ONERR;
+ newsrv->consecutive_errors_limit = DEF_HANA_ERRLIMIT;
cur_arg = 3;
while (*args[cur_arg]) {
@@ -2746,6 +2748,65 @@ int cfg_parse_listen(const char *file, int linenum, char
**args, int kwm)
do_check = 1;
cur_arg += 1;
}
+ else if (!strcmp(args[cur_arg], "observe")) {
+ if (!strcmp(args[cur_arg + 1], "none"))
+ newsrv->observe = HANA_OBS_NONE;
+ else if (!strcmp(args[cur_arg + 1], "layer4"))
+ newsrv->observe = HANA_OBS_LAYER4;
+ else if (!strcmp(args[cur_arg + 1], "layer7")) {
+ if (curproxy->mode != PR_MODE_HTTP) {
+ Alert("parsing [%s:%d]: '%s'
can only be used in http proxies.\n",
+ file, linenum,
args[cur_arg + 1]);
+ err_code |= ERR_ALERT;
+ }
+ newsrv->observe = HANA_OBS_LAYER7;
+ }
+ else {
+ Alert("parsing [%s:%d]: '%s' expects
one of 'none', "
+ "'l4events', 'http-responses'
but get '%s'\n",
+ file, linenum, args[cur_arg],
args[cur_arg + 1]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+
+ cur_arg += 2;
+ }
+ else if (!strcmp(args[cur_arg], "on-error")) {
+ if (!strcmp(args[cur_arg + 1], "fastinter"))
+ newsrv->onerror = HANA_ONERR_FASTINTER;
+ else if (!strcmp(args[cur_arg + 1],
"fail-check"))
+ newsrv->onerror = HANA_ONERR_FAILCHK;
+ else if (!strcmp(args[cur_arg + 1],
"sudden-death"))
+ newsrv->onerror = HANA_ONERR_SUDDTH;
+ else if (!strcmp(args[cur_arg + 1],
"mark-down"))
+ newsrv->onerror = HANA_ONERR_MARKDWN;
+ else {
+ Alert("parsing [%s:%d]: '%s' expects
one of 'fastinter', "
+ "'fail-check', 'sudden-death'
or 'mark-down' but get '%s'\n",
+ file, linenum, args[cur_arg],
args[cur_arg + 1]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+
+ cur_arg += 2;
+ }
+ else if (!strcmp(args[cur_arg], "error_limit")) {
+ if (!*args[cur_arg + 1]) {
+ Alert("parsing [%s:%d]: '%s' expects an
integer argument.\n",
+ file, linenum, args[cur_arg]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+
+ newsrv->consecutive_errors_limit =
atoi(args[cur_arg + 1]);
+
+ if (newsrv->consecutive_errors_limit <= 0) {
+ Alert("parsing [%s:%d]: %s has to be >
0.\n",
+ file, linenum, args[cur_arg]);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+ }
+ }
else if (!strcmp(args[cur_arg], "source")) { /*
address to which we bind when connecting */
int port_low, port_high;
if (!*args[cur_arg + 1]) {
diff --git a/src/checks.c b/src/checks.c
index 0e9aca5..3bfd791 100644
--- a/src/checks.c
+++ b/src/checks.c
@@ -52,6 +52,8 @@ const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
[HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI",
"Initializing" },
[HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
+ [HCHK_STATUS_HANA] = { SRV_CHK_ERROR, "HANA",
"Health analyze" },
+
[HCHK_STATUS_SOCKERR] = { SRV_CHK_ERROR,
"SOCKERR", "Socket error" },
[HCHK_STATUS_L4OK] = { SRV_CHK_RUNNING, "L4OK",
"Layer4 check passed" },
@@ -72,6 +74,22 @@ const struct check_status check_statuses[HCHK_STATUS_SIZE] =
{
[HCHK_STATUS_L7STS] = { SRV_CHK_ERROR, "L7STS",
"Layer7 wrong status" },
};
+const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = {
/* 0: ignore, 1: error, 2: OK */
+ [HANA_STATUS_UNKNOWN] = { "Unknown",
{ 0, 0 }},
+
+ [HANA_STATUS_L4_OK] = { "L4 successful connection",
{ 2, 0 }},
+ [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection",
{ 1, 1 }},
+
+ [HANA_STATUS_HTTP_OK] = { "Correct http response",
{ 0, 2 }},
+ [HANA_STATUS_HTTP_STS] = { "Wrong http response",
{ 0, 1 }},
+ [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)",
{ 0, 1 }},
+ [HANA_STATUS_HTTP_RSP] = { "Invalid http response",
{ 0, 1 }},
+
+ [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)",
{ 0, 1 }},
+ [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)",
{ 0, 1 }},
+ [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)",
{ 0, 1 }},
+};
+
/*
* Convert check_status code to description
*/
@@ -108,6 +126,21 @@ const char *get_check_status_info(short check_status) {
return check_statuses[HCHK_STATUS_UNKNOWN].info;
}
+const char *get_analyze_status(short analyze_status) {
+
+ const char *desc;
+
+ if (analyze_status < HANA_STATUS_SIZE)
+ desc = analyze_statuses[analyze_status].desc;
+ else
+ desc = NULL;
+
+ if (desc && *desc)
+ return desc;
+ else
+ return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
+}
+
#define SSP_O_VIA 0x0001
#define SSP_O_HCHK 0x0002
#define SSP_O_STATUS 0x0004
@@ -136,7 +169,8 @@ static void server_status_printf(struct chunk *msg, struct
server *s, unsigned o
chunk_printf(msg, "\"");
}
- chunk_printf(msg, ", check duration: %lums", s->check_duration);
+ if (s->check_duration >= 0)
+ chunk_printf(msg, ", check duration: %ldms",
s->check_duration);
}
if (options & SSP_O_STATUS) {
@@ -184,9 +218,11 @@ static void set_server_check_status(struct server *s,
short status, char *desc)
s->check_status = status;
if (check_statuses[status].result)
- s->result |= check_statuses[status].result;
+ s->result = check_statuses[status].result;
- if (!tv_iszero(&s->check_start)) {
+ if (status == HCHK_STATUS_HANA)
+ s->check_duration = -1;
+ else if (!tv_iszero(&s->check_start)) {
/* set_server_check_status() may be called more than once */
s->check_duration = tv_ms_elapsed(&s->check_start, &now);
tv_zero(&s->check_start);
@@ -229,6 +265,10 @@ static void set_server_check_status(struct server *s,
short status, char *desc)
if (health >= rise)
health = rise + fall - 1; /* OK now */
}
+
+ /* clear consecutive_errors if observing is enabled */
+ if (s->onerror)
+ s->consecutive_errors = 0;
}
/* FIXME end: calculate local version of the
health/rise/fall/state */
@@ -505,6 +545,95 @@ static void set_server_enabled(struct server *s) {
set_server_enabled(srv);
}
+void health_adjust(struct server *s, short status) {
+
+ int failed;
+ int expire;
+
+ /* return now if observing nor healt check is not enabled */
+ if (!s->observe || !s->check)
+ return;
+
+ if (s->observe >= HANA_OBS_SIZE)
+ return;
+
+ if (status >= HCHK_STATUS_SIZE || !analyze_statuses[status].desc)
+ return;
+
+ switch (analyze_statuses[status].lr[s->observe - 1]) {
+ case 1:
+ failed = 1;
+ break;
+
+ case 2:
+ failed = 0;
+ break;
+
+ default:
+ return;
+ }
+
+ if (!failed) {
+ /* good: clear consecutive_errors */
+ s->consecutive_errors = 0;
+ return;
+ }
+
+ s->consecutive_errors++;
+
+ if (s->consecutive_errors < s->consecutive_errors_limit)
+ return;
+
+ sprintf(trash, "Detected %d consecutive errors, last one was: %s",
+ s->consecutive_errors, get_analyze_status(status));
+
+ switch (s->onerror) {
+ case HANA_ONERR_FASTINTER:
+ /* force fastinter - nothing to do here as all modes force it */
+ break;
+
+ case HANA_ONERR_SUDDTH:
+ /* simulate a pre-fatal failed health check */
+ if (s->health > s->rise)
+ s->health = s->rise + 1;
+
+ /* no break - fall through */
+
+ case HANA_ONERR_FAILCHK:
+ /* simulate a failed health check */
+ set_server_check_status(s, HCHK_STATUS_HANA, trash);
+
+ if (s->health > s->rise) {
+ s->health--; /* still good */
+ s->counters.failed_checks++;
+ }
+ else
+ set_server_down(s);
+
+ break;
+
+ case HANA_ONERR_MARKDWN:
+ /* mark server down */
+ s->health = s->rise;
+ set_server_check_status(s, HCHK_STATUS_HANA, trash);
+ set_server_down(s);
+
+ break;
+
+ default:
+ /* write a warning? */
+ break;
+ }
+
+ s->consecutive_errors = 0;
+
+ if (s->fastinter) {
+ expire = tick_add(now_ms, MS_TO_TICKS(s->fastinter));
+ if (s->check->expire > expire)
+ s->check->expire = expire;
+ }
+}
+
/*
* This function is used only for server health-checks. It handles
* the connection acknowledgement. If the proxy requires L7 health-checks,
diff --git a/src/dumpstats.c b/src/dumpstats.c
index 522c3f8..d23602a 100644
--- a/src/dumpstats.c
+++ b/src/dumpstats.c
@@ -1545,7 +1545,7 @@ int stats_dump_proxy(struct session *s, struct proxy *px,
struct uri_auth *uri)
if (sv->check_status >=
HCHK_STATUS_L57DATA)
chunk_printf(&msg, "/%d",
sv->check_code);
- if (sv->check_status >=
HCHK_STATUS_CHECKED)
+ if (sv->check_status >=
HCHK_STATUS_CHECKED && sv->check_duration >= 0)
chunk_printf(&msg, " in %lums",
sv->check_duration);
} else {
chunk_printf(&msg, "</td><td>");
diff --git a/src/proto_http.c b/src/proto_http.c
index 2300738..408066b 100644
--- a/src/proto_http.c
+++ b/src/proto_http.c
@@ -41,6 +41,7 @@
#include <proto/acl.h>
#include <proto/backend.h>
#include <proto/buffers.h>
+#include <proto/checks.h>
#include <proto/client.h>
#include <proto/dumpstats.h>
#include <proto/fd.h>
@@ -2857,8 +2858,10 @@ int process_response(struct session *t)
buffer_shutr_now(rep);
buffer_shutw_now(req);
- if (t->srv)
+ if (t->srv) {
t->srv->counters.failed_resp++;
+ health_adjust(t->srv,
HANA_STATUS_HTTP_HDRRSP);
+ }
t->be->counters.failed_resp++;
rep->analysers = 0;
txn->status = 502;
@@ -2880,8 +2883,10 @@ int process_response(struct session *t)
http_capture_bad_message(&t->be->invalid_rep, t, rep, msg, t->fe);
buffer_shutr_now(rep);
buffer_shutw_now(req);
- if (t->srv)
+ if (t->srv) {
t->srv->counters.failed_resp++;
+ health_adjust(t->srv,
HANA_STATUS_HTTP_READ_ERROR);
+ }
t->be->counters.failed_resp++;
rep->analysers = 0;
txn->status = 502;
@@ -2898,8 +2903,10 @@ int process_response(struct session *t)
http_capture_bad_message(&t->be->invalid_rep, t, rep, msg, t->fe);
buffer_shutr_now(rep);
buffer_shutw_now(req);
- if (t->srv)
+ if (t->srv) {
t->srv->counters.failed_resp++;
+ health_adjust(t->srv,
HANA_STATUS_HTTP_READ_TIMEOUT);
+ }
t->be->counters.failed_resp++;
rep->analysers = 0;
txn->status = 504;
@@ -2915,8 +2922,10 @@ int process_response(struct session *t)
if (msg->err_pos >= 0)
http_capture_bad_message(&t->be->invalid_rep, t, rep, msg, t->fe);
buffer_shutw_now(req);
- if (t->srv)
+ if (t->srv) {
t->srv->counters.failed_resp++;
+ health_adjust(t->srv,
HANA_STATUS_HTTP_BROKEN_PIPE);
+ }
t->be->counters.failed_resp++;
rep->analysers = 0;
txn->status = 502;
@@ -2971,6 +2980,11 @@ int process_response(struct session *t)
if (n < 1 || n > 5)
n = 0;
+ if (txn->status >= 100 && txn->status < 500)
+ health_adjust(t->srv, HANA_STATUS_HTTP_OK);
+ else
+ health_adjust(t->srv, HANA_STATUS_HTTP_STS);
+
t->srv->counters.p.http.rsp[n]++;
t->be->counters.p.http.rsp[n]++;
@@ -3027,8 +3041,10 @@ int process_response(struct session *t)
if (rule_set->rsp_exp != NULL) {
if (apply_filters_to_response(t, rep,
rule_set->rsp_exp) < 0) {
return_bad_resp:
- if (t->srv)
+ if (t->srv) {
t->srv->counters.failed_resp++;
+ health_adjust(t->srv,
HANA_STATUS_HTTP_RSP);
+ }
cur_proxy->counters.failed_resp++;
return_srv_prx_502:
buffer_shutr_now(rep);
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 7bb5d7e..6f7c6d6 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -43,6 +43,7 @@
#include <proto/acl.h>
#include <proto/backend.h>
#include <proto/buffers.h>
+#include <proto/checks.h>
#include <proto/fd.h>
#include <proto/log.h>
#include <proto/port_range.h>
diff --git a/src/session.c b/src/session.c
index 447660e..da99703 100644
--- a/src/session.c
+++ b/src/session.c
@@ -246,6 +246,8 @@ int sess_update_st_cer(struct session *s, struct
stream_interface *si)
{
/* we probably have to release last session from the server */
if (s->srv) {
+ health_adjust(s->srv, HANA_STATUS_L4_ERR);
+
if (s->flags & SN_CURR_SESS) {
s->flags &= ~SN_CURR_SESS;
s->srv->cur_sess--;
@@ -324,6 +326,9 @@ void sess_establish(struct session *s, struct
stream_interface *si)
struct buffer *req = si->ob;
struct buffer *rep = si->ib;
+ if (s->srv)
+ health_adjust(s->srv, HANA_STATUS_L4_OK);
+
if (s->be->mode == PR_MODE_TCP) { /* let's allow immediate data
connection in this case */
buffer_set_rlim(rep, rep->size); /* no rewrite needed */
