Hi, The attached patchset is a first attempt at adding the possibility to compress requests, as well as responses. It adds a new keyword for compression, "tocompress" (any better alternative name would be appreciated). The valid values are "request", if you want to compress requests, "response" if you want to compress responses or "both", if you want to compress both. The default is to compress responses only.
Any comment is more than welcome. Thanks! Olivier
>From 6c3e62baa888359521091387ce6ac8376a001259 Mon Sep 17 00:00:00 2001 From: Olivier Houchard <[email protected]> Date: Mon, 3 Apr 2023 22:22:24 +0200 Subject: [PATCH 1/2] MINOR: compression: Make compression offload a flag Turn compression offload into a flag in struct comp, instead of using an int just for it. --- include/haproxy/compression-t.h | 5 ++++- src/flt_http_comp.c | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h index 062f17f76..cdbf0d14e 100644 --- a/include/haproxy/compression-t.h +++ b/include/haproxy/compression-t.h @@ -34,10 +34,13 @@ #include <haproxy/buf-t.h> +/* Compression flags */ + +#define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */ struct comp { struct comp_algo *algos; struct comp_type *types; - unsigned int offload; + unsigned int flags; }; struct comp_ctx { diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c index ceda3fd5e..507009692 100644 --- a/src/flt_http_comp.c +++ b/src/flt_http_comp.c @@ -451,8 +451,8 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc /* remove all occurrences of the header when "compression offload" is set */ if (st->comp_algo) { - if ((s->be->comp && s->be->comp->offload) || - (strm_fe(s)->comp && strm_fe(s)->comp->offload)) { + if ((s->be->comp && (s->be->comp->flags & COMP_FL_OFFLOAD)) || + (strm_fe(s)->comp && (strm_fe(s)->comp->flags & COMP_FL_OFFLOAD))) { http_remove_header(htx, &ctx); ctx.blk = NULL; while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1)) @@ -690,7 +690,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy, args[0], args[1]); ret = 1; } - comp->offload = 1; + comp->flags |= COMP_FL_OFFLOAD; } else if (strcmp(args[1], "type") == 0) { int cur_arg = 2; -- 2.39.1
>From 77a5657592338db2ed53189c50828b3c0ed52716 Mon Sep 17 00:00:00 2001 From: Olivier Houchard <[email protected]> Date: Tue, 4 Apr 2023 00:14:36 +0200 Subject: [PATCH 2/2] MEDIUM: compression: allow to compress requests too. Make it so we can compress requests, as well as responses. A new compress keyword is added, "tocompress". It takes exactly one argument. The valid values for that arguments are "response", if we want to compress only responses, "request", if we want to compress only requests, or "both", if we want to compress both. The dafault value is "response". --- doc/configuration.txt | 12 ++- include/haproxy/compression-t.h | 3 + src/flt_http_comp.c | 165 +++++++++++++++++++++++++++----- 3 files changed, 154 insertions(+), 26 deletions(-) diff --git a/doc/configuration.txt b/doc/configuration.txt index 586689ca2..8f86b285e 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -5077,7 +5077,7 @@ compression type <mime type> ... compression algo gzip compression type text/html text/plain - See also : "compression offload" + See also : "compression offload", "compression tocompress" compression offload Makes HAProxy work as a compression offloader only. @@ -5099,7 +5099,15 @@ compression offload If this setting is used in a defaults section, a warning is emitted and the option is ignored. - See also : "compression type", "compression algo" + See also : "compression type", "compression algo", "compression tocompress" + +compression tocompress <whattocompress> + Makes haproxy able to compress both requests and responses. + Valid values are "request", to compress only requests, "response", to + compress only responses, or "both", when you want to compress both. + The default value is "response". + + See also : "compression type", "compression algo", "compression offload" cookie <name> [ rewrite | insert | prefix ] [ indirect ] [ nocache ] [ postonly ] [ preserve ] [ httponly ] [ secure ] diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h index cdbf0d14e..d15d15ee3 100644 --- a/include/haproxy/compression-t.h +++ b/include/haproxy/compression-t.h @@ -37,6 +37,9 @@ /* Compression flags */ #define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */ +#define COMP_FL_COMPRESS_REQ 0x00000002 /* Compress requests */ +#define COMP_FL_COMPRESS_RES 0x00000004 /* Compress responses */ + struct comp { struct comp_algo *algos; struct comp_type *types; diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c index 507009692..7680e241f 100644 --- a/src/flt_http_comp.c +++ b/src/flt_http_comp.c @@ -33,7 +33,9 @@ struct flt_ops comp_ops; struct comp_state { struct comp_ctx *comp_ctx; /* compression context */ + struct comp_ctx *comp_ctx_req; /* compression context for request */ struct comp_algo *comp_algo; /* compression algorithm if not NULL */ + struct comp_algo *comp_algo_req; /* Algo to be used for request */ unsigned int flags; /* COMP_STATE_* */ }; @@ -49,14 +51,15 @@ static int select_compression_request_header(struct comp_state *st, static int select_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg); -static int set_compression_response_header(struct comp_state *st, - struct stream *s, - struct http_msg *msg); +static int set_compression_header(struct comp_state *st, + struct stream *s, + struct http_msg *msg, + int for_response); static int htx_compression_buffer_init(struct htx *htx, struct buffer *out); static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len, - struct buffer *out); -static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end); + struct buffer *out, int is_repsonse); +static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end, int is_response); /***********************************************************************/ static int @@ -95,7 +98,9 @@ comp_strm_init(struct stream *s, struct filter *filter) return -1; st->comp_algo = NULL; + st->comp_algo_req = NULL; st->comp_ctx = NULL; + st->comp_ctx_req = NULL; st->flags = 0; filter->ctx = st; @@ -118,25 +123,81 @@ comp_strm_deinit(struct stream *s, struct filter *filter) /* release any possible compression context */ if (st->comp_algo) st->comp_algo->end(&st->comp_ctx); + if (st->comp_algo_req) + st->comp_algo_req->end(&st->comp_ctx_req); pool_free(pool_head_comp_state, st); filter->ctx = NULL; } +static void +comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct http_msg *msg) +{ + struct htx *htx = htxbuf(&msg->chn->buf); + struct http_txn *txn = s->txn; + struct http_hdr_ctx ctx; + + ctx.blk = NULL; + /* Already compressed, don't bother */ + if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1)) + return; + /* HTTP < 1.1 should not be compressed */ + if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11)) + return; + + if (txn->meth == HTTP_METH_HEAD) + return; + if (s->be->comp->algos != NULL) + st->comp_algo_req = s->be->comp->algos; + else if (strm_fe(s)->comp->algos != NULL) + st->comp_algo_req = strm_fe(s)->comp->algos; +/* limit compression rate */ + if (global.comp_rate_lim > 0) + if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim) + goto fail; + + /* limit cpu usage */ + if (th_ctx->idle_pct < compress_min_idle) + goto fail; + + /* initialize compression */ + if (st->comp_algo_req->init(&st->comp_ctx_req, global.tune.comp_maxlevel) < 0) + goto fail; + + return; +fail: + st->comp_algo_req = NULL; +} + static int comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg) { struct comp_state *st = filter->ctx; + int comp_flags = 0; if (!strm_fe(s)->comp && !s->be->comp) goto end; - - if (!(msg->chn->flags & CF_ISRESP)) - select_compression_request_header(st, s, msg); - else { + if (strm_fe(s)->comp) + comp_flags |= strm_fe(s)->comp->flags; + if (s->be->comp) + comp_flags |= s->be->comp->flags; + + if (!(msg->chn->flags & CF_ISRESP)) { + if (comp_flags & COMP_FL_COMPRESS_REQ) { + comp_prepare_compress_request(st, s, msg); + if (st->comp_algo_req) { + if (!set_compression_header(st, s, msg, 0)) + goto end; + register_data_filter(s, msg->chn, filter); + st->flags |= COMP_STATE_PROCESSING; + } + } + if (comp_flags & COMP_FL_COMPRESS_RES) + select_compression_request_header(st, s, msg); + } else if (comp_flags & COMP_FL_COMPRESS_RES) { /* Response headers have already been checked in * comp_http_post_analyze callback. */ if (st->comp_algo) { - if (!set_compression_response_header(st, s, msg)) + if (!set_compression_header(st, s, msg, 1)) goto end; register_data_filter(s, msg->chn, filter); st->flags |= COMP_STATE_PROCESSING; @@ -183,7 +244,9 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg, enum htx_blk_type type = htx_get_blk_type(blk); uint32_t sz = htx_get_blksz(blk); struct ist v; + int is_response; + is_response = msg->chn->flags & CF_ISRESP; next = htx_get_next_blk(htx, blk); while (next && htx_get_blk_type(next) == HTX_BLK_UNUSED) next = htx_get_next_blk(htx, next); @@ -207,8 +270,8 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg, v.len = len; } - ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash); - if (ret < 0 || htx_compression_buffer_end(st, &trash, last) < 0) + ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash, is_response); + if (ret < 0 || htx_compression_buffer_end(st, &trash, last, is_response) < 0) goto error; BUG_ON(v.len != ret); @@ -228,7 +291,7 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg, case HTX_BLK_TLR: case HTX_BLK_EOT: - if (htx_compression_buffer_end(st, &trash, 1) < 0) + if (htx_compression_buffer_end(st, &trash, 1, is_response) < 0) goto error; if (b_data(&trash)) { struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash))); @@ -298,16 +361,22 @@ comp_http_end(struct stream *s, struct filter *filter, /***********************************************************************/ static int -set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg) +set_compression_header(struct comp_state *st, struct stream *s, struct http_msg *msg, int for_response) { struct htx *htx = htxbuf(&msg->chn->buf); struct htx_sl *sl; struct http_hdr_ctx ctx; + struct comp_algo *comp_algo; sl = http_get_stline(htx); if (!sl) goto error; + if (for_response) + comp_algo = st->comp_algo; + else + comp_algo = st->comp_algo_req; + /* add "Transfer-Encoding: chunked" header */ if (!(msg->flags & HTTP_MSGF_TE_CHNK)) { if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked"))) @@ -348,8 +417,8 @@ set_compression_response_header(struct comp_state *st, struct stream *s, struct * Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding * header. */ - if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) { - struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len); + if (comp_algo->cfg_name_len != 8 || memcmp(comp_algo->cfg_name, "identity", 8) != 0) { + struct ist v = ist2(comp_algo->ua_name, comp_algo->ua_name_len); if (!http_add_header(htx, ist("Content-Encoding"), v)) goto error; @@ -358,8 +427,13 @@ set_compression_response_header(struct comp_state *st, struct stream *s, struct return 1; error: - st->comp_algo->end(&st->comp_ctx); - st->comp_algo = NULL; + if (for_response) { + st->comp_algo->end(&st->comp_ctx); + st->comp_algo = NULL; + } else { + st->comp_algo_req->end(&st->comp_ctx_req); + st->comp_algo_req = NULL; + } return 0; } @@ -603,18 +677,39 @@ htx_compression_buffer_init(struct htx *htx, struct buffer *out) static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len, - struct buffer *out) + struct buffer *out, int is_response) { - return st->comp_algo->add_data(st->comp_ctx, data, len, out); + struct comp_algo *comp_algo; + void *comp_ctx; + + if (is_response) { + comp_algo = st->comp_algo; + comp_ctx = st->comp_ctx; + } else { + comp_algo = st->comp_algo_req; + comp_ctx = st->comp_ctx_req; + } + return comp_algo->add_data(comp_ctx, data, len, out); } static int -htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end) +htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end, int is_response) { + struct comp_algo *comp_algo; + void *comp_ctx; + + if (is_response) { + comp_algo = st->comp_algo; + comp_ctx = st->comp_ctx; + } else { + comp_algo = st->comp_algo_req; + comp_ctx = st->comp_ctx_req; + } + if (end) - return st->comp_algo->finish(st->comp_ctx, out); + return comp_algo->finish(comp_ctx, out); else - return st->comp_algo->flush(st->comp_ctx, out); + return comp_algo->flush(comp_ctx, out); } @@ -644,6 +739,8 @@ parse_compression_options(char **args, int section, struct proxy *proxy, if (proxy->comp == NULL) { comp = calloc(1, sizeof(*comp)); + /* Always default to compress responses */ + comp->flags = COMP_FL_COMPRESS_RES; proxy->comp = comp; } else @@ -710,8 +807,28 @@ parse_compression_options(char **args, int section, struct proxy *proxy, continue; } } + else if (strcmp(args[1], "tocompress") == 0) { + if (!args[2]) { + memprintf(err, "'%s' expects 'request', 'response', or 'both'.", args[0]); + ret = -1; + goto end; + } + if (strcmp(args[2], "request") == 0) { + comp->flags &= ~COMP_FL_COMPRESS_RES; + comp->flags |= COMP_FL_COMPRESS_REQ; + } else if (strcmp(args[2], "response") == 0) { + comp->flags &= COMP_FL_COMPRESS_REQ; + comp->flags |= COMP_FL_COMPRESS_RES; + } else if (strcmp(args[2], "both") == 0) + comp->flags |= COMP_FL_COMPRESS_REQ | COMP_FL_COMPRESS_RES; + else { + memprintf(err, "'%s' expects 'request', 'response', or 'both'.", args[0]); + ret = -1; + goto end; + } + } else { - memprintf(err, "'%s' expects 'algo', 'type' or 'offload'", + memprintf(err, "'%s' expects 'algo', 'type' 'tocompress' or 'offload'", args[0]); ret = -1; goto end; -- 2.39.1
