Hi,
On Tue, Apr 04, 2023 at 09:45:24PM +0200, Willy Tarreau wrote:
> Hi Olivier,
>
> On Tue, Apr 04, 2023 at 12:29:15AM +0200, Olivier Houchard wrote:
> > Hi,
> >
> > The attached patchset is a first attempt at adding the possibility to
> > compress requests, as well as responses.
>
> This is pretty cool, I definitely see how this can be super useful :-)
:)
> > It adds a new keyword for compression, "tocompress" (any better
> > alternative name would be appreciated).
>
> I would call it "direction" which I find more natural and understandable.
I like that a lot more. Thank you!
> > The valid values are "request",
> > if you want to compress requests, "response" if you want to compress
> > responses or "both", if you want to compress both.
> > The default is to compress responses only.
>
> I have a few below (besides renaming "tocompress").
[...]
> > diff --git a/include/haproxy/compression-t.h
> > b/include/haproxy/compression-t.h
> > index cdbf0d14e..d15d15ee3 100644
> > --- a/include/haproxy/compression-t.h
> > +++ b/include/haproxy/compression-t.h
> > @@ -37,6 +37,9 @@
> > /* Compression flags */
> >
> > #define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */
> > +#define COMP_FL_COMPRESS_REQ 0x00000002 /* Compress requests */
> > +#define COMP_FL_COMPRESS_RES 0x00000004 /* Compress responses */
>
> So these ones would be COMP_FL_DIR_{REQ,RES}.
Done!
> > struct comp {
> > struct comp_algo *algos;
> > struct comp_type *types;
> > diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
> > index 507009692..7680e241f 100644
> > --- a/src/flt_http_comp.c
> > +++ b/src/flt_http_comp.c
> > @@ -33,7 +33,9 @@ struct flt_ops comp_ops;
> >
> > struct comp_state {
> > struct comp_ctx *comp_ctx; /* compression context */
> > + struct comp_ctx *comp_ctx_req; /* compression context for request */
> > struct comp_algo *comp_algo; /* compression algorithm if not NULL */
> > + struct comp_algo *comp_algo_req; /* Algo to be used for request */
> > unsigned int flags; /* COMP_STATE_* */
>
> I find it confusing to have comp_ctx and comp_ctx_req (and same for algo).
> I'd rather see a preliminary patch renaming comp_ctx to comp_ctx_res and
> comp_algo to comp_algo_res (just perform a mechanical rename). Or you could
> also have an array [2] for each, which will even remove some ifs in the
> code.
Ok I made that an array.
> Also I don't understand how you can have different algos for each direction
> since the config language allows you to define "compression algo" and
> "compression tocompress" so you cannot (apparently) have a situation where
> the two algos differ. I think it can make sense to use different algos for
> each direction (at least compression settings, e.g. expensive compression
> for uploads, relayed over expensive links, little compression for downloads
> that possibly comes from a cache).
The idea was that while only one algo could be used for requests, the
response algorithm would depend on what was supported by the client.
> Thus maybe an approach could be to use the algo to indicate the direction
> at the same time:
>
> compression algo foobar => sets response only, for compatibility
> compression algo-res foobar => sets response only, explicit syntax
> compression algo-req foobar => sets request only, explicit syntax
>
> Then there's probably no need to have a keyword to set it at once in
> both directions since both actions are totally independent and it's
> probably not desired to encourage users to conflate the two directions
> in a single keyword.
>
> You might need the same for "compression types" I think, so that you
> don't try to compress regular forms or login:password that will confuse
> applications, but only large data.
But I did that, so now there's even more reason to have separate
algorithms.
> > -static int set_compression_response_header(struct comp_state *st,
> > - struct stream *s,
> > - struct http_msg *msg);
> > +static int set_compression_header(struct comp_state *st,
> > + struct stream *s,
> > + struct http_msg *msg,
> > + int for_response);
> ^^^^^^^^^^^^^^^^
> For this one as well I would call it "direction" (or "dir"), as we
> already do in samples and a few other parts that work in both
> directions.
Ok I was about to do that, but somebody from the haproxy team whom I
won't name told me it was even better to just deduce the direction from
the http_msg, so I did just that.
> > +static void
> > +comp_prepare_compress_request(struct comp_state *st, struct stream *s,
> > struct http_msg *msg)
> > +{
> > + struct htx *htx = htxbuf(&msg->chn->buf);
> > + struct http_txn *txn = s->txn;
> > + struct http_hdr_ctx ctx;
> > +
> > + ctx.blk = NULL;
> > + /* Already compressed, don't bother */
> > + if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
> > + return;
> > + /* HTTP < 1.1 should not be compressed */
> > + if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags &
> > HTTP_MSGF_VER_11))
> > + return;
> > +
> > + if (txn->meth == HTTP_METH_HEAD)
> > + return;
> > + if (s->be->comp->algos != NULL)
> > + st->comp_algo_req = s->be->comp->algos;
> > + else if (strm_fe(s)->comp->algos != NULL)
> > + st->comp_algo_req = strm_fe(s)->comp->algos;
> > +/* limit compression rate */
> > + if (global.comp_rate_lim > 0)
> > + if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
> > + goto fail;
>
> I had a doubt regarding this one but I think it's indeed reasonable to
> use the same global value for all directions as the goal was precisely
> to be able to enforce a total limit on all compressed streams to limit
> the resource impacts.
I think it makes sense, unless someone wants to give more priority to
request compression vs response compression, or vice versa, but I'll
rethink it if somebody gives me a valid use-case.
> > + /* limit cpu usage */
> > + if (th_ctx->idle_pct < compress_min_idle)
> > + goto fail;
> > +
> > + /* initialize compression */
> > + if (st->comp_algo_req->init(&st->comp_ctx_req,
> > global.tune.comp_maxlevel) < 0)
> > + goto fail;
> > +
> > + return;
> > +fail:
> > + st->comp_algo_req = NULL;
> > +}
> > +
> > static int
> > comp_http_headers(struct stream *s, struct filter *filter, struct http_msg
> > *msg)
> > {
> > struct comp_state *st = filter->ctx;
> > + int comp_flags = 0;
> >
> > if (!strm_fe(s)->comp && !s->be->comp)
> > goto end;
> > -
> > - if (!(msg->chn->flags & CF_ISRESP))
> > - select_compression_request_header(st, s, msg);
> > - else {
> > + if (strm_fe(s)->comp)
> > + comp_flags |= strm_fe(s)->comp->flags;
> > + if (s->be->comp)
> > + comp_flags |= s->be->comp->flags;
> > +
> > + if (!(msg->chn->flags & CF_ISRESP)) {
> > + if (comp_flags & COMP_FL_COMPRESS_REQ) {
> > + comp_prepare_compress_request(st, s, msg);
> > + if (st->comp_algo_req) {
> > + if (!set_compression_header(st, s, msg, 0))
> ^^^
> This zero is all but explicit about the fact that it's the direction.
> Maybe you should create an enum for this, or you can decide to pass
> the compression flags (which could then be reused to pass other info
> in the future) and you'd have COMP_FL_DIR_REQ here, which is way more
> explicit.
Yes this is fixed, I don't use the flag but introduced
COMP_DIR_REQ/COMP_DIR_RES, which nicely matches the index of the
corresponding arrays.
> > + goto end;
> > + register_data_filter(s, msg->chn, filter);
> > + st->flags |= COMP_STATE_PROCESSING;
> > + }
> > + }
> > + if (comp_flags & COMP_FL_COMPRESS_RES)
> > + select_compression_request_header(st, s, msg);
> > + } else if (comp_flags & COMP_FL_COMPRESS_RES) {
> > /* Response headers have already been checked in
> > * comp_http_post_analyze callback. */
> > if (st->comp_algo) {
> > - if (!set_compression_response_header(st, s, msg))
> > + if (!set_compression_header(st, s, msg, 1))
>
> Same here.
Ditto.
> > goto end;
> > register_data_filter(s, msg->chn, filter);
> > st->flags |= COMP_STATE_PROCESSING;
> > @@ -183,7 +244,9 @@ comp_http_payload(struct stream *s, struct filter
> > *filter, struct http_msg *msg,
> > enum htx_blk_type type = htx_get_blk_type(blk);
> > uint32_t sz = htx_get_blksz(blk);
> > struct ist v;
> > + int is_response;
> >
> > + is_response = msg->chn->flags & CF_ISRESP;
>
> Nitpicking but I'd do !!(msg->chn->flags & CF_ISRESP) here so that the
> day we start to extend flags to 64 bits we don't end up with a zero if
> CF_ISRESP is above 1<<31.
That is now fixed.
> > next = htx_get_next_blk(htx, blk);
> > while (next && htx_get_blk_type(next) == HTX_BLK_UNUSED)
> > next = htx_get_next_blk(htx, next);
> > @@ -207,8 +270,8 @@ comp_http_payload(struct stream *s, struct filter
> > *filter, struct http_msg *msg,
> > v.len = len;
> > }
> >
> > - ret = htx_compression_buffer_add_data(st,
> > v.ptr, v.len, &trash);
> > - if (ret < 0 || htx_compression_buffer_end(st,
> > &trash, last) < 0)
> > + ret = htx_compression_buffer_add_data(st,
> > v.ptr, v.len, &trash, is_response);
> > + if (ret < 0 || htx_compression_buffer_end(st,
> > &trash, last, is_response) < 0)
> > goto error;
> > BUG_ON(v.len != ret);
> >
> > @@ -228,7 +291,7 @@ comp_http_payload(struct stream *s, struct filter
> > *filter, struct http_msg *msg,
> >
> > case HTX_BLK_TLR:
> > case HTX_BLK_EOT:
> > - if (htx_compression_buffer_end(st, &trash, 1) <
> > 0)
> > + if (htx_compression_buffer_end(st, &trash, 1,
> > is_response) < 0)
> > goto error;
> > if (b_data(&trash)) {
> > struct htx_blk *last =
> > htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
> > @@ -298,16 +361,22 @@ comp_http_end(struct stream *s, struct filter *filter,
> >
> > /***********************************************************************/
> > static int
> > -set_compression_response_header(struct comp_state *st, struct stream *s,
> > struct http_msg *msg)
> > +set_compression_header(struct comp_state *st, struct stream *s, struct
> > http_msg *msg, int for_response)
> > {
> > struct htx *htx = htxbuf(&msg->chn->buf);
> > struct htx_sl *sl;
> > struct http_hdr_ctx ctx;
> > + struct comp_algo *comp_algo;
> >
> > sl = http_get_stline(htx);
> > if (!sl)
> > goto error;
> >
> > + if (for_response)
> > + comp_algo = st->comp_algo;
> > + else
> > + comp_algo = st->comp_algo_req;
> > +
>
> So this is one example where having an array for st->comp_algo could be nice.
And we now have one :)
> > /* add "Transfer-Encoding: chunked" header */
> > if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
> > if (!http_add_header(htx, ist("Transfer-Encoding"),
> > ist("chunked")))
> > @@ -348,8 +417,8 @@ set_compression_response_header(struct comp_state *st,
> > struct stream *s, struct
> > * Accept-Encoding header, and SHOULD NOT be used in the
> > Content-Encoding
> > * header.
> > */
> > - if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name,
> > "identity", 8) != 0) {
> > - struct ist v = ist2(st->comp_algo->ua_name,
> > st->comp_algo->ua_name_len);
> > + if (comp_algo->cfg_name_len != 8 || memcmp(comp_algo->cfg_name,
> > "identity", 8) != 0) {
> > + struct ist v = ist2(comp_algo->ua_name, comp_algo->ua_name_len);
> >
> > if (!http_add_header(htx, ist("Content-Encoding"), v))
> > goto error;
> > @@ -358,8 +427,13 @@ set_compression_response_header(struct comp_state *st,
> > struct stream *s, struct
> > return 1;
> >
> > error:
> > - st->comp_algo->end(&st->comp_ctx);
> > - st->comp_algo = NULL;
> > + if (for_response) {
> > + st->comp_algo->end(&st->comp_ctx);
> > + st->comp_algo = NULL;
> > + } else {
> > + st->comp_algo_req->end(&st->comp_ctx_req);
> > + st->comp_algo_req = NULL;
> > + }
>
> I have not seen anything updating the stats, so I suspect that request
> and response compression stats are accounted as response stats, which
> can be a problem showing compression rates larger than 100%. If you hover
> over your frontend's "Bytes / Out" fields, you'll see a yellow box with
> numbers such as:
>
> Response bytes in: 468197882863
> Compression in: 117157038990
> Compression out: 93277104103 (79%)
> Compression bypass: 0
> Total bytes saved: 23879934887 (5%)
>
> Here I'm figuring that we'll need to add the same for the upload, so
> these are extra stats. I can possibly help locate where to place them,
> but I'll need that you figure where they are updated.
Ok so I've had mixed feelings about this, because I thought a global
counter could be useful too. But ultimately I think it's better to keep
them separated, if only so that the percentages make sense. We could
have had Request + Response bytes there, but I think it's less useful.
So I added the counters, but do not expose them for requests yet, as I'm
unsure where to do that exactly, but that can easily be addressed with a
separate commit.
Thanks!
Olivier
>From 5b3199908d23f686dd23696717415d16f5bac387 Mon Sep 17 00:00:00 2001
From: Olivier Houchard <ohouch...@backtrace.io>
Date: Mon, 3 Apr 2023 22:22:24 +0200
Subject: [PATCH 1/5] MINOR: compression: Make compression offload a flag
Turn compression offload into a flag in struct comp, instead of using
an int just for it.
---
include/haproxy/compression-t.h | 5 ++++-
src/flt_http_comp.c | 6 +++---
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h
index 062f17f76..cdbf0d14e 100644
--- a/include/haproxy/compression-t.h
+++ b/include/haproxy/compression-t.h
@@ -34,10 +34,13 @@
#include <haproxy/buf-t.h>
+/* Compression flags */
+
+#define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */
struct comp {
struct comp_algo *algos;
struct comp_type *types;
- unsigned int offload;
+ unsigned int flags;
};
struct comp_ctx {
diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index ceda3fd5e..507009692 100644
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -451,8 +451,8 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
/* remove all occurrences of the header when "compression offload" is set */
if (st->comp_algo) {
- if ((s->be->comp && s->be->comp->offload) ||
- (strm_fe(s)->comp && strm_fe(s)->comp->offload)) {
+ if ((s->be->comp && (s->be->comp->flags & COMP_FL_OFFLOAD)) ||
+ (strm_fe(s)->comp && (strm_fe(s)->comp->flags & COMP_FL_OFFLOAD))) {
http_remove_header(htx, &ctx);
ctx.blk = NULL;
while (http_find_header(htx, ist("Accept-Encoding"), &ctx, 1))
@@ -690,7 +690,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
args[0], args[1]);
ret = 1;
}
- comp->offload = 1;
+ comp->flags |= COMP_FL_OFFLOAD;
}
else if (strcmp(args[1], "type") == 0) {
int cur_arg = 2;
--
2.39.1
>From a3cd5209e4cd7462d3e075cacdb94e147a0aa1ad Mon Sep 17 00:00:00 2001
From: Olivier Houchard <ohouch...@backtrace.io>
Date: Wed, 5 Apr 2023 16:25:57 +0200
Subject: [PATCH 2/5] MINOR: compression: Prepare compression code for request
compression
Make provision for storing the compression algorithm and the compression
context twice, one for requests, and the other for responses. Only the
response ones are used for now.
---
include/haproxy/compression-t.h | 5 ++
src/flt_http_comp.c | 101 ++++++++++++++++++++------------
2 files changed, 68 insertions(+), 38 deletions(-)
diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h
index cdbf0d14e..685110331 100644
--- a/include/haproxy/compression-t.h
+++ b/include/haproxy/compression-t.h
@@ -34,6 +34,11 @@
#include <haproxy/buf-t.h>
+/* Direction index */
+
+#define COMP_DIR_REQ 0
+#define COMP_DIR_RES 1
+
/* Compression flags */
#define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */
diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index 507009692..ad2b77c0b 100644
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -32,8 +32,12 @@ const char *http_comp_flt_id = "compression filter";
struct flt_ops comp_ops;
struct comp_state {
- struct comp_ctx *comp_ctx; /* compression context */
- struct comp_algo *comp_algo; /* compression algorithm if not NULL */
+ /*
+ * For both comp_ctx and comp_algo, COMP_DIR_REQ is the index
+ * for requests, and COMP_DIR_RES for responses
+ */
+ struct comp_ctx *comp_ctx[2]; /* compression context */
+ struct comp_algo *comp_algo[2]; /* compression algorithm if not NULL */
unsigned int flags; /* COMP_STATE_* */
};
@@ -49,14 +53,14 @@ static int select_compression_request_header(struct comp_state *st,
static int select_compression_response_header(struct comp_state *st,
struct stream *s,
struct http_msg *msg);
-static int set_compression_response_header(struct comp_state *st,
- struct stream *s,
- struct http_msg *msg);
+static int set_compression_header(struct comp_state *st,
+ struct stream *s,
+ struct http_msg *msg);
static int htx_compression_buffer_init(struct htx *htx, struct buffer *out);
static int htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
- struct buffer *out);
-static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end);
+ struct buffer *out, int dir);
+static int htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end, int dir);
/***********************************************************************/
static int
@@ -94,8 +98,10 @@ comp_strm_init(struct stream *s, struct filter *filter)
if (st == NULL)
return -1;
- st->comp_algo = NULL;
- st->comp_ctx = NULL;
+ st->comp_algo[COMP_DIR_REQ] = NULL;
+ st->comp_algo[COMP_DIR_RES] = NULL;
+ st->comp_ctx[COMP_DIR_REQ] = NULL;
+ st->comp_ctx[COMP_DIR_RES] = NULL;
st->flags = 0;
filter->ctx = st;
@@ -116,8 +122,10 @@ comp_strm_deinit(struct stream *s, struct filter *filter)
return;
/* release any possible compression context */
- if (st->comp_algo)
- st->comp_algo->end(&st->comp_ctx);
+ if (st->comp_algo[COMP_DIR_REQ])
+ st->comp_algo[COMP_DIR_REQ]->end(&st->comp_ctx[COMP_DIR_REQ]);
+ if (st->comp_algo[COMP_DIR_RES])
+ st->comp_algo[COMP_DIR_RES]->end(&st->comp_ctx[COMP_DIR_RES]);
pool_free(pool_head_comp_state, st);
filter->ctx = NULL;
}
@@ -135,8 +143,8 @@ comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
else {
/* Response headers have already been checked in
* comp_http_post_analyze callback. */
- if (st->comp_algo) {
- if (!set_compression_response_header(st, s, msg))
+ if (st->comp_algo[COMP_DIR_RES]) {
+ if (!set_compression_header(st, s, msg))
goto end;
register_data_filter(s, msg->chn, filter);
st->flags |= COMP_STATE_PROCESSING;
@@ -176,6 +184,12 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
struct htx_ret htxret = htx_find_offset(htx, offset);
struct htx_blk *blk, *next;
int ret, consumed = 0, to_forward = 0, last = 0;
+ int dir;
+
+ if (msg->chn->flags & CF_ISRESP)
+ dir = COMP_DIR_RES;
+ else
+ dir = COMP_DIR_REQ;
blk = htxret.blk;
offset = htxret.ret;
@@ -207,8 +221,8 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
v.len = len;
}
- ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash);
- if (ret < 0 || htx_compression_buffer_end(st, &trash, last) < 0)
+ ret = htx_compression_buffer_add_data(st, v.ptr, v.len, &trash, dir);
+ if (ret < 0 || htx_compression_buffer_end(st, &trash, last, dir) < 0)
goto error;
BUG_ON(v.len != ret);
@@ -228,7 +242,7 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
case HTX_BLK_TLR:
case HTX_BLK_EOT:
- if (htx_compression_buffer_end(st, &trash, 1) < 0)
+ if (htx_compression_buffer_end(st, &trash, 1, dir) < 0)
goto error;
if (b_data(&trash)) {
struct htx_blk *last = htx_add_last_data(htx, ist2(b_head(&trash), b_data(&trash)));
@@ -261,7 +275,7 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
if (to_forward != consumed)
flt_update_offsets(filter, msg->chn, to_forward - consumed);
- if (st->comp_ctx && st->comp_ctx->cur_lvl > 0) {
+ if (st->comp_ctx[dir] && st->comp_ctx[dir]->cur_lvl > 0) {
update_freq_ctr(&global.comp_bps_in, consumed);
_HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
_HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
@@ -285,7 +299,7 @@ comp_http_end(struct stream *s, struct filter *filter,
{
struct comp_state *st = filter->ctx;
- if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo)
+ if (!(msg->chn->flags & CF_ISRESP) || !st || !st->comp_algo[COMP_DIR_RES])
goto end;
if (strm_fe(s)->mode == PR_MODE_HTTP)
@@ -298,16 +312,25 @@ comp_http_end(struct stream *s, struct filter *filter,
/***********************************************************************/
static int
-set_compression_response_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
+set_compression_header(struct comp_state *st, struct stream *s, struct http_msg *msg)
{
struct htx *htx = htxbuf(&msg->chn->buf);
struct htx_sl *sl;
struct http_hdr_ctx ctx;
+ struct comp_algo *comp_algo;
+ int comp_index;
+
+ if (msg->chn->flags & CF_ISRESP)
+ comp_index = COMP_DIR_RES;
+ else
+ comp_index = COMP_DIR_REQ;
sl = http_get_stline(htx);
if (!sl)
goto error;
+ comp_algo = st->comp_algo[comp_index];
+
/* add "Transfer-Encoding: chunked" header */
if (!(msg->flags & HTTP_MSGF_TE_CHNK)) {
if (!http_add_header(htx, ist("Transfer-Encoding"), ist("chunked")))
@@ -348,8 +371,8 @@ set_compression_response_header(struct comp_state *st, struct stream *s, struct
* Accept-Encoding header, and SHOULD NOT be used in the Content-Encoding
* header.
*/
- if (st->comp_algo->cfg_name_len != 8 || memcmp(st->comp_algo->cfg_name, "identity", 8) != 0) {
- struct ist v = ist2(st->comp_algo->ua_name, st->comp_algo->ua_name_len);
+ if (comp_algo->cfg_name_len != 8 || memcmp(comp_algo->cfg_name, "identity", 8) != 0) {
+ struct ist v = ist2(comp_algo->ua_name, comp_algo->ua_name_len);
if (!http_add_header(htx, ist("Content-Encoding"), v))
goto error;
@@ -358,8 +381,8 @@ set_compression_response_header(struct comp_state *st, struct stream *s, struct
return 1;
error:
- st->comp_algo->end(&st->comp_ctx);
- st->comp_algo = NULL;
+ st->comp_algo[comp_index]->end(&st->comp_ctx[comp_index]);
+ st->comp_algo[comp_index] = NULL;
return 0;
}
@@ -387,7 +410,7 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
*(ctx.value.ptr + 30) < '6' ||
(*(ctx.value.ptr + 30) == '6' &&
(ctx.value.len < 54 || memcmp(ctx.value.ptr + 51, "SV1", 3) != 0)))) {
- st->comp_algo = NULL;
+ st->comp_algo[COMP_DIR_RES] = NULL;
return 0;
}
@@ -441,7 +464,7 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
if (*(ctx.value.ptr) == '*' ||
word_match(ctx.value.ptr, toklen, comp_algo->ua_name, comp_algo->ua_name_len)) {
- st->comp_algo = comp_algo;
+ st->comp_algo[COMP_DIR_RES] = comp_algo;
best_q = q;
break;
}
@@ -450,7 +473,7 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
}
/* remove all occurrences of the header when "compression offload" is set */
- if (st->comp_algo) {
+ if (st->comp_algo[COMP_DIR_RES]) {
if ((s->be->comp && (s->be->comp->flags & COMP_FL_OFFLOAD)) ||
(strm_fe(s)->comp && (strm_fe(s)->comp->flags & COMP_FL_OFFLOAD))) {
http_remove_header(htx, &ctx);
@@ -466,13 +489,13 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
(strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
- st->comp_algo = comp_algo;
+ st->comp_algo[COMP_DIR_RES] = comp_algo;
return 1;
}
}
}
- st->comp_algo = NULL;
+ st->comp_algo[COMP_DIR_RES] = NULL;
return 0;
}
@@ -488,7 +511,7 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
struct comp_type *comp_type;
/* no common compression algorithm was found in request header */
- if (st->comp_algo == NULL)
+ if (st->comp_algo[COMP_DIR_RES] == NULL)
goto fail;
/* compression already in progress */
@@ -577,13 +600,13 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
goto fail;
/* initialize compression */
- if (st->comp_algo->init(&st->comp_ctx, global.tune.comp_maxlevel) < 0)
+ if (st->comp_algo[COMP_DIR_RES]->init(&st->comp_ctx[COMP_DIR_RES], global.tune.comp_maxlevel) < 0)
goto fail;
msg->flags |= HTTP_MSGF_COMPRESSING;
return 1;
fail:
- st->comp_algo = NULL;
+ st->comp_algo[COMP_DIR_RES] = NULL;
return 0;
}
@@ -603,18 +626,20 @@ htx_compression_buffer_init(struct htx *htx, struct buffer *out)
static int
htx_compression_buffer_add_data(struct comp_state *st, const char *data, size_t len,
- struct buffer *out)
+ struct buffer *out, int dir)
{
- return st->comp_algo->add_data(st->comp_ctx, data, len, out);
+
+ return st->comp_algo[dir]->add_data(st->comp_ctx[dir], data, len, out);
}
static int
-htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end)
+htx_compression_buffer_end(struct comp_state *st, struct buffer *out, int end, int dir)
{
+
if (end)
- return st->comp_algo->finish(st->comp_ctx, out);
+ return st->comp_algo[dir]->finish(st->comp_ctx[dir], out);
else
- return st->comp_algo->flush(st->comp_ctx, out);
+ return st->comp_algo[dir]->flush(st->comp_ctx[dir], out);
}
@@ -836,8 +861,8 @@ smp_fetch_res_comp_algo(const struct arg *args, struct sample *smp,
smp->data.type = SMP_T_STR;
smp->flags = SMP_F_CONST;
- smp->data.u.str.area = st->comp_algo->cfg_name;
- smp->data.u.str.data = st->comp_algo->cfg_name_len;
+ smp->data.u.str.area = st->comp_algo[COMP_DIR_RES]->cfg_name;
+ smp->data.u.str.data = st->comp_algo[COMP_DIR_RES]->cfg_name_len;
return 1;
}
return 0;
--
2.39.1
>From 007c32166839f0b00e79690fbe3827850fc9c560 Mon Sep 17 00:00:00 2001
From: Olivier Houchard <ohouch...@backtrace.io>
Date: Wed, 5 Apr 2023 17:32:36 +0200
Subject: [PATCH 3/5] MINOR: compression: Store algo and type for both request
and response
Make provision for being able to store both compression algorithms and
content-types to compress for both requests and responses. For now only
the responses one are used.
---
include/haproxy/compression-t.h | 6 ++++--
include/haproxy/compression.h | 4 ++--
src/compression.c | 12 ++++++------
src/flt_http_comp.c | 28 ++++++++++++++--------------
src/proxy.c | 6 ++++--
5 files changed, 30 insertions(+), 26 deletions(-)
diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h
index 685110331..af51654d9 100644
--- a/include/haproxy/compression-t.h
+++ b/include/haproxy/compression-t.h
@@ -43,8 +43,10 @@
#define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */
struct comp {
- struct comp_algo *algos;
- struct comp_type *types;
+ struct comp_algo *algos_res; /* Algos available for response */
+ struct comp_algo *algo_req; /* Algo to use for request */
+ struct comp_type *types_req; /* Types to be compressed for requests */
+ struct comp_type *types_res; /* Types to be compressed for responses */
unsigned int flags;
};
diff --git a/include/haproxy/compression.h b/include/haproxy/compression.h
index ba2bdab7c..851ea2393 100644
--- a/include/haproxy/compression.h
+++ b/include/haproxy/compression.h
@@ -27,8 +27,8 @@
extern unsigned int compress_min_idle;
-int comp_append_type(struct comp *comp, const char *type);
-int comp_append_algo(struct comp *comp, const char *algo);
+int comp_append_type(struct comp_type **types, const char *type);
+int comp_append_algo(struct comp_algo **algos, const char *algo);
#ifdef USE_ZLIB
extern long zlib_used_memory;
diff --git a/src/compression.c b/src/compression.c
index 3ce2a6005..7a8a21912 100644
--- a/src/compression.c
+++ b/src/compression.c
@@ -110,7 +110,7 @@ const struct comp_algo comp_algos[] =
* Add a content-type in the configuration
* Returns 0 in case of success, 1 in case of allocation failure.
*/
-int comp_append_type(struct comp *comp, const char *type)
+int comp_append_type(struct comp_type **types, const char *type)
{
struct comp_type *comp_type;
@@ -119,8 +119,8 @@ int comp_append_type(struct comp *comp, const char *type)
return 1;
comp_type->name_len = strlen(type);
comp_type->name = strdup(type);
- comp_type->next = comp->types;
- comp->types = comp_type;
+ comp_type->next = *types;
+ *types = comp_type;
return 0;
}
@@ -129,7 +129,7 @@ int comp_append_type(struct comp *comp, const char *type)
* Returns 0 in case of success, -1 if the <algo> is unmanaged, 1 in case of
* allocation failure.
*/
-int comp_append_algo(struct comp *comp, const char *algo)
+int comp_append_algo(struct comp_algo **algos, const char *algo)
{
struct comp_algo *comp_algo;
int i;
@@ -140,8 +140,8 @@ int comp_append_algo(struct comp *comp, const char *algo)
if (!comp_algo)
return 1;
memmove(comp_algo, &comp_algos[i], sizeof(struct comp_algo));
- comp_algo->next = comp->algos;
- comp->algos = comp_algo;
+ comp_algo->next = *algos;
+ *algos = comp_algo;
return 0;
}
}
diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index ad2b77c0b..8c56d0a46 100644
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -415,8 +415,8 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
}
/* search for the algo in the backend in priority or the frontend */
- if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
- (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
+ if ((s->be->comp && (comp_algo_back = s->be->comp->algos_res)) ||
+ (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos_res))) {
int best_q = 0;
ctx.blk = NULL;
@@ -485,8 +485,8 @@ select_compression_request_header(struct comp_state *st, struct stream *s, struc
}
/* identity is implicit does not require headers */
- if ((s->be->comp && (comp_algo_back = s->be->comp->algos)) ||
- (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos))) {
+ if ((s->be->comp && (comp_algo_back = s->be->comp->algos_res)) ||
+ (strm_fe(s)->comp && (comp_algo_back = strm_fe(s)->comp->algos_res))) {
for (comp_algo = comp_algo_back; comp_algo; comp_algo = comp_algo->next) {
if (comp_algo->cfg_name_len == 8 && memcmp(comp_algo->cfg_name, "identity", 8) == 0) {
st->comp_algo[COMP_DIR_RES] = comp_algo;
@@ -571,8 +571,8 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
goto fail;
- if ((s->be->comp && (comp_type = s->be->comp->types)) ||
- (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types))) {
+ if ((s->be->comp && (comp_type = s->be->comp->types_res)) ||
+ (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types_res))) {
for (; comp_type; comp_type = comp_type->next) {
if (ctx.value.len >= comp_type->name_len &&
strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
@@ -585,8 +585,8 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
}
}
else { /* no content-type header */
- if ((s->be->comp && s->be->comp->types) ||
- (strm_fe(s)->comp && strm_fe(s)->comp->types))
+ if ((s->be->comp && s->be->comp->types_res) ||
+ (strm_fe(s)->comp && strm_fe(s)->comp->types_res))
goto fail; /* a content-type was required */
}
@@ -674,7 +674,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
else
comp = proxy->comp;
- if (strcmp(args[1], "algo") == 0) {
+ if (strcmp(args[1], "algo") == 0 || strcmp(args[1], "algo-res") == 0) {
struct comp_ctx *ctx;
int cur_arg = 2;
@@ -685,7 +685,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
goto end;
}
while (*(args[cur_arg])) {
- int retval = comp_append_algo(comp, args[cur_arg]);
+ int retval = comp_append_algo(&comp->algos_res, args[cur_arg]);
if (retval) {
if (retval < 0)
memprintf(err, "'%s' : '%s' is not a supported algorithm.",
@@ -697,8 +697,8 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
goto end;
}
- if (proxy->comp->algos->init(&ctx, 9) == 0)
- proxy->comp->algos->end(&ctx);
+ if (proxy->comp->algos_res->init(&ctx, 9) == 0)
+ proxy->comp->algos_res->end(&ctx);
else {
memprintf(err, "'%s' : Can't init '%s' algorithm.",
args[0], args[cur_arg]);
@@ -717,7 +717,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
}
comp->flags |= COMP_FL_OFFLOAD;
}
- else if (strcmp(args[1], "type") == 0) {
+ else if (strcmp(args[1], "type") == 0 || strcmp(args[1], "type-res") == 0) {
int cur_arg = 2;
if (!*args[cur_arg]) {
@@ -726,7 +726,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
goto end;
}
while (*(args[cur_arg])) {
- if (comp_append_type(comp, args[cur_arg])) {
+ if (comp_append_type(&comp->types_res, args[cur_arg])) {
memprintf(err, "'%s': out of memory.", args[0]);
ret = -1;
goto end;
diff --git a/src/proxy.c b/src/proxy.c
index 8d4de56e3..a025e18e9 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -1901,8 +1901,10 @@ static int proxy_defproxy_cpy(struct proxy *curproxy, const struct proxy *defpro
memprintf(errmsg, "proxy '%s': out of memory for default compression options", curproxy->id);
return 1;
}
- curproxy->comp->algos = defproxy->comp->algos;
- curproxy->comp->types = defproxy->comp->types;
+ curproxy->comp->algos_res = defproxy->comp->algos_res;
+ curproxy->comp->algo_req = defproxy->comp->algo_req;
+ curproxy->comp->types_res = defproxy->comp->types_res;
+ curproxy->comp->types_req = defproxy->comp->types_req;
}
if (defproxy->check_path)
--
2.39.1
>From 7e127db475fc1fc2a285d984df6cc92e63d04f15 Mon Sep 17 00:00:00 2001
From: Olivier Houchard <ohouch...@backtrace.io>
Date: Thu, 6 Apr 2023 00:33:01 +0200
Subject: [PATCH 4/5] MINOR: compression: Count separately request and response
compression
Duplicate the compression counters, so that we have separate counters
for request and response compression.
---
include/haproxy/counters-t.h | 14 ++++++++------
src/flt_http_comp.c | 12 ++++++------
src/stats.c | 12 ++++++------
3 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/include/haproxy/counters-t.h b/include/haproxy/counters-t.h
index 849f09683..2fd531125 100644
--- a/include/haproxy/counters-t.h
+++ b/include/haproxy/counters-t.h
@@ -36,9 +36,10 @@ struct fe_counters {
long long bytes_in; /* number of bytes transferred from the client to the server */
long long bytes_out; /* number of bytes transferred from the server to the client */
- long long comp_in; /* input bytes fed to the compressor */
- long long comp_out; /* output bytes emitted by the compressor */
- long long comp_byp; /* input bytes that bypassed the compressor (cpu/ram/bw limitation) */
+ /* compression counters, index 0 for requests, 1 for responses */
+ long long comp_in[2]; /* input bytes fed to the compressor */
+ long long comp_out[2]; /* output bytes emitted by the compressor */
+ long long comp_byp[2]; /* input bytes that bypassed the compressor (cpu/ram/bw limitation) */
long long denied_req; /* blocked requests because of security concerns */
long long denied_resp; /* blocked responses because of security concerns */
@@ -80,9 +81,10 @@ struct be_counters {
long long bytes_in; /* number of bytes transferred from the client to the server */
long long bytes_out; /* number of bytes transferred from the server to the client */
- long long comp_in; /* input bytes fed to the compressor */
- long long comp_out; /* output bytes emitted by the compressor */
- long long comp_byp; /* input bytes that bypassed the compressor (cpu/ram/bw limitation) */
+ /* compression counters, index 0 for requests, 1 for responses */
+ long long comp_in[2]; /* input bytes fed to the compressor */
+ long long comp_out[2]; /* output bytes emitted by the compressor */
+ long long comp_byp[2]; /* input bytes that bypassed the compressor (cpu/ram/bw limitation) */
long long denied_req; /* blocked requests because of security concerns */
long long denied_resp; /* blocked responses because of security concerns */
diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index 8c56d0a46..31c8e2171 100644
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -277,14 +277,14 @@ comp_http_payload(struct stream *s, struct filter *filter, struct http_msg *msg,
if (st->comp_ctx[dir] && st->comp_ctx[dir]->cur_lvl > 0) {
update_freq_ctr(&global.comp_bps_in, consumed);
- _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in, consumed);
- _HA_ATOMIC_ADD(&s->be->be_counters.comp_in, consumed);
+ _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_in[dir], consumed);
+ _HA_ATOMIC_ADD(&s->be->be_counters.comp_in[dir], consumed);
update_freq_ctr(&global.comp_bps_out, to_forward);
- _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out, to_forward);
- _HA_ATOMIC_ADD(&s->be->be_counters.comp_out, to_forward);
+ _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_out[dir], to_forward);
+ _HA_ATOMIC_ADD(&s->be->be_counters.comp_out[dir], to_forward);
} else {
- _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp, consumed);
- _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp, consumed);
+ _HA_ATOMIC_ADD(&strm_fe(s)->fe_counters.comp_byp[dir], consumed);
+ _HA_ATOMIC_ADD(&s->be->be_counters.comp_byp[dir], consumed);
}
return to_forward;
diff --git a/src/stats.c b/src/stats.c
index 0fe7e20fc..a90a39c43 100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -1847,13 +1847,13 @@ int stats_fill_fe_stats(struct proxy *px, struct field *stats, int len,
break;
}
case ST_F_COMP_IN:
- metric = mkf_u64(FN_COUNTER, px->fe_counters.comp_in);
+ metric = mkf_u64(FN_COUNTER, px->fe_counters.comp_in[COMP_DIR_RES]);
break;
case ST_F_COMP_OUT:
- metric = mkf_u64(FN_COUNTER, px->fe_counters.comp_out);
+ metric = mkf_u64(FN_COUNTER, px->fe_counters.comp_out[COMP_DIR_RES]);
break;
case ST_F_COMP_BYP:
- metric = mkf_u64(FN_COUNTER, px->fe_counters.comp_byp);
+ metric = mkf_u64(FN_COUNTER, px->fe_counters.comp_byp[COMP_DIR_RES]);
break;
case ST_F_COMP_RSP:
metric = mkf_u64(FN_COUNTER, px->fe_counters.p.http.comp_rsp);
@@ -2878,13 +2878,13 @@ int stats_fill_be_stats(struct proxy *px, int flags, struct field *stats, int le
metric = mkf_u64(FN_COUNTER, px->be_counters.srv_aborts);
break;
case ST_F_COMP_IN:
- metric = mkf_u64(FN_COUNTER, px->be_counters.comp_in);
+ metric = mkf_u64(FN_COUNTER, px->be_counters.comp_in[COMP_DIR_RES]);
break;
case ST_F_COMP_OUT:
- metric = mkf_u64(FN_COUNTER, px->be_counters.comp_out);
+ metric = mkf_u64(FN_COUNTER, px->be_counters.comp_out[COMP_DIR_RES]);
break;
case ST_F_COMP_BYP:
- metric = mkf_u64(FN_COUNTER, px->be_counters.comp_byp);
+ metric = mkf_u64(FN_COUNTER, px->be_counters.comp_byp[COMP_DIR_RES]);
break;
case ST_F_COMP_RSP:
metric = mkf_u64(FN_COUNTER, px->be_counters.p.http.comp_rsp);
--
2.39.1
>From e27c885fb3d37716bdbe05e4f367a48dbf2aa6bf Mon Sep 17 00:00:00 2001
From: Olivier Houchard <ohouch...@backtrace.io>
Date: Thu, 6 Apr 2023 00:33:48 +0200
Subject: [PATCH 5/5] MEDIUM: compression: Make it so we can compress requests
as well.
Add code so that compression can be used for requests as well.
New compression keywords are introduced :
"direction" that specifies what we want to compress. Valid values are
"request", "response", or "both".
"type-req" and "type-res" define content-type to be compressed for
requests and responses, respectively. "type" is kept as an alias for
"type-res" for backward compatibilty.
"algo-req" specifies the compression algorithm to be used for requests.
Only one algorithm can be provided.
"algo-res" provides the list of algorithm that can be used to compress
responses. "algo" is kept as an alias for "algo-res" for backward
compatibility.
---
doc/configuration.txt | 28 +++++-
include/haproxy/compression-t.h | 3 +
src/flt_http_comp.c | 169 +++++++++++++++++++++++++++++++-
3 files changed, 191 insertions(+), 9 deletions(-)
diff --git a/doc/configuration.txt b/doc/configuration.txt
index 3468c78f3..63e8d3440 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -5014,13 +5014,25 @@ clitcpka-intvl <timeout>
compression algo <algorithm> ...
+compression algo-req <algorithm>
+compression algo-res <algorithm>
compression type <mime type> ...
Enable HTTP compression.
May be used in sections : defaults | frontend | listen | backend
yes | yes | yes | yes
Arguments :
- algo is followed by the list of supported compression algorithms.
- type is followed by the list of MIME types that will be compressed.
+ algo is followed by the list of supported compression algorithms for
+ responses (legacy keyword)
+ algo-req is followed by compression algorithm for request (only one is
+ provided).
+ algo-res is followed by the list of supported compression algorithms for
+ responses.
+ type is followed by the list of MIME types that will be compressed for
+ responses (legacy keyword).
+ type-req is followed by the list of MIME types that will be compressed for
+ requests.
+ type-res is followed by the list of MIME types that will be compressed for
+ responses.
The currently supported algorithms are :
identity this is mostly for debugging, and it was useful for developing
@@ -5077,7 +5089,7 @@ compression type <mime type> ...
compression algo gzip
compression type text/html text/plain
- See also : "compression offload"
+ See also : "compression offload", "compression direction"
compression offload
Makes HAProxy work as a compression offloader only.
@@ -5099,7 +5111,15 @@ compression offload
If this setting is used in a defaults section, a warning is emitted and the
option is ignored.
- See also : "compression type", "compression algo"
+ See also : "compression type", "compression algo", "compression "
+
+compression direction <direction>
+ Makes haproxy able to compress both requests and responses.
+ Valid values are "request", to compress only requests, "response", to
+ compress only responses, or "both", when you want to compress both.
+ The default value is "response".
+
+ See also : "compression type", "compression algo", "compression offload"
cookie <name> [ rewrite | insert | prefix ] [ indirect ] [ nocache ]
[ postonly ] [ preserve ] [ httponly ] [ secure ]
diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h
index af51654d9..e22f64fd9 100644
--- a/include/haproxy/compression-t.h
+++ b/include/haproxy/compression-t.h
@@ -42,6 +42,9 @@
/* Compression flags */
#define COMP_FL_OFFLOAD 0x00000001 /* Compression offload */
+#define COMP_FL_DIR_REQ 0x00000002 /* Compress requests */
+#define COMP_FL_DIR_RES 0x00000004 /* Compress responses */
+
struct comp {
struct comp_algo *algos_res; /* Algos available for response */
struct comp_algo *algo_req; /* Algo to use for request */
diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index 31c8e2171..1a345e54c 100644
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -130,17 +130,112 @@ comp_strm_deinit(struct stream *s, struct filter *filter)
filter->ctx = NULL;
}
+static void
+comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct http_msg *msg)
+{
+ struct htx *htx = htxbuf(&msg->chn->buf);
+ struct http_txn *txn = s->txn;
+ struct http_hdr_ctx ctx;
+ struct comp_type *comp_type;
+
+ ctx.blk = NULL;
+ /* Already compressed, don't bother */
+ if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
+ return;
+ /* HTTP < 1.1 should not be compressed */
+ if (!(msg->flags & HTTP_MSGF_VER_11) || !(txn->req.flags & HTTP_MSGF_VER_11))
+ return;
+ comp_type = NULL;
+
+ /*
+ * We don't want to compress content-types not listed in the "compression type" directive if any. If no content-type was found but configuration
+ * requires one, we don't compress either. Backend has the priority.
+ */
+ ctx.blk = NULL;
+ if (http_find_header(htx, ist("Content-Type"), &ctx, 1)) {
+ if (ctx.value.len >= 9 && strncasecmp("multipart", ctx.value.ptr, 9) == 0)
+ goto fail;
+
+ if ((s->be->comp && (comp_type = s->be->comp->types_req)) ||
+ (strm_fe(s)->comp && (comp_type = strm_fe(s)->comp->types_req))) {
+ for (; comp_type; comp_type = comp_type->next) {
+ if (ctx.value.len >= comp_type->name_len &&
+ strncasecmp(ctx.value.ptr, comp_type->name, comp_type->name_len) == 0)
+ /* this Content-Type should be compressed */
+ break;
+ }
+ /* this Content-Type should not be compressed */
+ if (comp_type == NULL)
+ goto fail;
+ }
+ }
+ else { /* no content-type header */
+ if ((s->be->comp && s->be->comp->types_req) ||
+ (strm_fe(s)->comp && strm_fe(s)->comp->types_req))
+ goto fail; /* a content-type was required */
+ }
+
+ /* limit compression rate */
+ if (global.comp_rate_lim > 0)
+ if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
+ goto fail;
+
+ /* limit cpu usage */
+ if (th_ctx->idle_pct < compress_min_idle)
+ goto fail;
+
+ if (txn->meth == HTTP_METH_HEAD)
+ return;
+ if (s->be->comp->algo_req != NULL)
+ st->comp_algo[COMP_DIR_REQ] = s->be->comp->algo_req;
+ else if (strm_fe(s)->comp->algo_req != NULL)
+ st->comp_algo[COMP_DIR_REQ] = strm_fe(s)->comp->algo_req;
+
+
+ /* limit compression rate */
+ if (global.comp_rate_lim > 0)
+ if (read_freq_ctr(&global.comp_bps_in) > global.comp_rate_lim)
+ goto fail;
+
+ /* limit cpu usage */
+ if (th_ctx->idle_pct < compress_min_idle)
+ goto fail;
+
+ /* initialize compression */
+ if (st->comp_algo[COMP_DIR_REQ]->init(&st->comp_ctx[COMP_DIR_REQ], global.tune.comp_maxlevel) < 0)
+ goto fail;
+
+ return;
+fail:
+ st->comp_algo[COMP_DIR_REQ] = NULL;
+}
+
static int
comp_http_headers(struct stream *s, struct filter *filter, struct http_msg *msg)
{
struct comp_state *st = filter->ctx;
+ int comp_flags = 0;
if (!strm_fe(s)->comp && !s->be->comp)
goto end;
-
- if (!(msg->chn->flags & CF_ISRESP))
- select_compression_request_header(st, s, msg);
- else {
+ if (strm_fe(s)->comp)
+ comp_flags |= strm_fe(s)->comp->flags;
+ if (s->be->comp)
+ comp_flags |= s->be->comp->flags;
+
+ if (!(msg->chn->flags & CF_ISRESP)) {
+ if (comp_flags & COMP_FL_DIR_REQ) {
+ comp_prepare_compress_request(st, s, msg);
+ if (st->comp_algo[COMP_DIR_REQ]) {
+ if (!set_compression_header(st, s, msg))
+ goto end;
+ register_data_filter(s, msg->chn, filter);
+ st->flags |= COMP_STATE_PROCESSING;
+ }
+ }
+ if (comp_flags & COMP_FL_DIR_RES)
+ select_compression_request_header(st, s, msg);
+ } else if (comp_flags & COMP_FL_DIR_RES) {
/* Response headers have already been checked in
* comp_http_post_analyze callback. */
if (st->comp_algo[COMP_DIR_RES]) {
@@ -669,6 +764,8 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
if (proxy->comp == NULL) {
comp = calloc(1, sizeof(*comp));
+ /* Always default to compress responses */
+ comp->flags = COMP_FL_DIR_RES;
proxy->comp = comp;
}
else
@@ -709,6 +806,30 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
continue;
}
}
+ else if (strcmp(args[1], "algo-req") == 0) {
+ struct comp_ctx *ctx;
+ int retval = comp_append_algo(&comp->algo_req, args[2]);
+
+ if (retval) {
+ if (retval < 0)
+ memprintf(err, "'%s' : '%s' is not a supported algorithm.",
+ args[0], args[2]);
+ else
+ memprintf(err, "'%s' : out of memory while parsing algo '%s'.",
+ args[0], args[2]);
+ ret = -1;
+ goto end;
+ }
+
+ if (proxy->comp->algo_req->init(&ctx, 9) == 0)
+ proxy->comp->algo_req->end(&ctx);
+ else {
+ memprintf(err, "'%s' : Can't init '%s' algorithm.",
+ args[0], args[2]);
+ ret = -1;
+ goto end;
+ }
+ }
else if (strcmp(args[1], "offload") == 0) {
if (proxy->cap & PR_CAP_DEF) {
memprintf(err, "'%s' : '%s' ignored in 'defaults' section.",
@@ -735,8 +856,46 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
continue;
}
}
+ else if (strcmp(args[1], "type-req") == 0) {
+ int cur_arg = 2;
+
+ if (!*args[cur_arg]) {
+ memprintf(err, "'%s' expects <type>.", args[0]);
+ ret = -1;
+ goto end;
+ }
+ while (*(args[cur_arg])) {
+ if (comp_append_type(&comp->types_req, args[cur_arg])) {
+ memprintf(err, "'%s': out of memory.", args[0]);
+ ret = -1;
+ goto end;
+ }
+ cur_arg++;
+ continue;
+ }
+ }
+ else if (strcmp(args[1], "direction") == 0) {
+ if (!args[2]) {
+ memprintf(err, "'%s' expects 'request', 'response', or 'both'.", args[0]);
+ ret = -1;
+ goto end;
+ }
+ if (strcmp(args[2], "request") == 0) {
+ comp->flags &= ~COMP_FL_DIR_RES;
+ comp->flags |= COMP_FL_DIR_REQ;
+ } else if (strcmp(args[2], "response") == 0) {
+ comp->flags &= COMP_FL_DIR_REQ;
+ comp->flags |= COMP_FL_DIR_RES;
+ } else if (strcmp(args[2], "both") == 0)
+ comp->flags |= COMP_FL_DIR_REQ | COMP_FL_DIR_RES;
+ else {
+ memprintf(err, "'%s' expects 'request', 'response', or 'both'.", args[0]);
+ ret = -1;
+ goto end;
+ }
+ }
else {
- memprintf(err, "'%s' expects 'algo', 'type' or 'offload'",
+ memprintf(err, "'%s' expects 'algo', 'type' 'direction' or 'offload'",
args[0]);
ret = -1;
goto end;
--
2.39.1
