Hi everyone, please find patch attached. This is a proposed solution for 3 req.body functions: len_req_body, hash_req_body and rematch_req_body.
The idea is to iterate over the request body, wrap it into a blob (I have to thank Nils for this because I took the idea from the vmod he wrote a couple of weeks ago) and use this blob for operation such as hashing on it or regex matching. *len_req_body: check if the req.body has been buffered, if so then we return the number of consumed bytes. *hash_req_body: first the req.body is collected in a blob, then we hash on this (I had to implement a new function for hashing on binary data) *rematch_req_body: using a priv_call the regular expression given as parameter is compiled just once, then we use VRE_exec on the req.body blob to check if there's a regex match. Comments much appreciated. -- Arianna Aondio Software Developer | Varnish Software AS Mobile: +47 980 62 619 We Make Websites Fly www.varnish-software.com
From de7634c55e92dc69e75e1fc27f5ad614f1d387d2 Mon Sep 17 00:00:00 2001 From: Arianna Aondio <[email protected]> Date: Mon, 23 Mar 2015 15:14:22 +0100 Subject: [PATCH] Implementation of req.body functions and test cases. --- bin/varnishd/cache/cache_hash.c | 10 ++++ bin/varnishd/hash/hash_slinger.h | 3 +- bin/varnishtest/tests/m00022.vtc | 50 ++++++++++++++++ bin/varnishtest/tests/m00023.vtc | 77 +++++++++++++++++++++++++ bin/varnishtest/tests/m00024.vtc | 56 ++++++++++++++++++ lib/libvmod_std/vmod.vcc | 48 ++++++++++++++++ lib/libvmod_std/vmod_std.c | 119 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 bin/varnishtest/tests/m00022.vtc create mode 100644 bin/varnishtest/tests/m00023.vtc create mode 100644 bin/varnishtest/tests/m00024.vtc diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index b3d3ac8..2af4256 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -198,6 +198,16 @@ HSH_AddString(const struct req *req, const char *str) SHA256_Update(req->sha256ctx, &str, sizeof str); } +void +HSH_AddBytes(const struct req *req, const void *buf, size_t len) +{ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(req->sha256ctx); + + if (buf != NULL) + SHA256_Update(req->sha256ctx, buf, len); +} + /*--------------------------------------------------------------------- * This is a debugging hack to enable testing of boundary conditions * in the hash algorithm. diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 189f938..59524f4 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -67,7 +67,8 @@ enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **, int wait_for_busy, int always_insert); void HSH_Ref(struct objcore *o); void HSH_Init(const struct hash_slinger *slinger); -void HSH_AddString(const struct req *, const char *str); +void HSH_AddString(const struct req *req, const char *str); +void HSH_AddBytes(const struct req *req, const void *buf, size_t len); void HSH_Insert(struct worker *, const void *hash, struct objcore *); void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace, double keep); diff --git a/bin/varnishtest/tests/m00022.vtc b/bin/varnishtest/tests/m00022.vtc new file mode 100644 index 0000000..03d96a1 --- /dev/null +++ b/bin/varnishtest/tests/m00022.vtc @@ -0,0 +1,50 @@ +varnishtest "Test std.len_req_body" + +server s1 { + rxreq + txresp + rxreq + txresp + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_recv { + std.cache_req_body(110B); + set req.http.x-len = std.len_req_body(); + } + + sub vcl_deliver { + set resp.http.x-len = req.http.x-len; + } +} -start + +client c1 { + txreq -req POST -nolen -hdr "Transfer-encoding: chunked" + chunked {BLAS} + delay .2 + chunkedlen 110 + expect_close +} -run + +client c1 { + txreq -req POST -nolen -hdr "Transfer-encoding: chunked" + chunked {BLAS} + delay .2 + chunkedlen 90 + delay .2 + chunked {FOO} + delay .2 + chunkedlen 0 + rxresp + expect resp.http.x-len == 97 +} -run + +client c2 { + txreq -req POST -body "BANANE" + rxresp + expect resp.http.x-len == 6 +} -run diff --git a/bin/varnishtest/tests/m00023.vtc b/bin/varnishtest/tests/m00023.vtc new file mode 100644 index 0000000..7c3234c --- /dev/null +++ b/bin/varnishtest/tests/m00023.vtc @@ -0,0 +1,77 @@ +varnishtest "Test std.hash_req_body" + +server s1 { + rxreq + txresp + rxreq + txresp + rxreq + txresp + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import ${vmod_std}; + + sub vcl_recv { + std.cache_req_body(110B); + return (hash); + } + + sub vcl_hash { + std.hash_req_body(); + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + return (lookup); + } + + sub vcl_hit { + set req.http.x-hit = "HIT"; + } + + sub vcl_miss { + set req.http.x-miss = "MISS"; + } + + sub vcl_deliver { + set resp.http.x-hit = req.http.x-hit; + set resp.http.x-miss = req.http.x-miss; + } + +} -start + +client c1 { + txreq -req POST -nolen -hdr "Transfer-encoding: chunked" + chunked {BLAS} + delay .2 + chunkedlen 110 + expect_close +} -run + +client c1 { + txreq -req POST -nolen -hdr "Transfer-encoding: chunked" + chunked {BLAS} + delay .2 + chunkedlen 90 + delay .2 + chunked {FOO} + delay .2 + chunkedlen 0 + rxresp +} -run + +client c2 { + txreq -req POST -url "/banane" -body "FOOBAR" + rxresp +} -run + +client c2 { + txreq -req GET -url "/banane" -body "FOOBAR" + rxresp + expect resp.http.x-hit == "HIT" +} -run diff --git a/bin/varnishtest/tests/m00024.vtc b/bin/varnishtest/tests/m00024.vtc new file mode 100644 index 0000000..8e109c5 --- /dev/null +++ b/bin/varnishtest/tests/m00024.vtc @@ -0,0 +1,56 @@ +varnishtest "test rematch matchin on binary req.body" +server s1 { + rxreq + expect req.bodylen > 12 + txresp + rxreq + txresp +} -start + +# rematch_req_body is case sensitive. +varnish v1 -vcl+backend { + import ${vmod_std}; + sub vcl_recv { + std.cache_req_body(10KB); + set req.http.x-boolean1 = std.rematch_req_body(".*"); + set req.http.x-boolean2 = std.rematch_req_body("aRNI"); + set req.http.x-boolean3 = std.rematch_req_body("a"); + set req.http.x-boolean4 = std.rematch_req_body("F"); + } + + sub vcl_deliver { + set resp.http.x-boolean1 = req.http.x-boolean1; + set resp.http.x-boolean2 = req.http.x-boolean2; + set resp.http.x-boolean3 = req.http.x-boolean3; + set resp.http.x-boolean4 = req.http.x-boolean4; + } +} -start + +client c1 { + txreq -req "POST" -gzipbody {a5e2e2e1c2e2} + rxresp + expect resp.http.x-boolean1 == 1 + expect resp.http.x-boolean2 == 0 + expect resp.http.x-boolean3 == 1 + expect resp.http.x-boolean4 == 0 +} -run + +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "param.set fetch_chunksize 4k" + +client c2 { + txreq -req POST -nolen -hdr "Transfer-encoding: chunked" + chunked {a5e2e2e1c2e2} + delay .2 + chunkedlen 4090 + delay .2 + chunked {VARNISH} + delay .2 + chunked {\0} + chunked {FOO} + delay .2 + chunkedlen 0 + rxresp + expect resp.http.x-boolean1 == 1 + expect resp.http.x-boolean2 == 0 +} -run diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 288a1ed..2a7571b 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -257,6 +257,54 @@ Example | } +$Function INT len_req_body() + +Description + Returns the request body length. + + Note that the request body must be buffered. + +Example + | if (std.cache_req_body(1KB)) { + | set req.http.x-len = std.len_req_body(); + | } + +$Function VOID hash_req_body(PRIV_TOP) + +Description + Adds available request body bytes to the lookup hash key. + Note that this function can only be used in vcl_hash and + the request body must be buffered. + + Example + | sub vcl_recv { + | std.cache_req_body(1KB); + | } + | + | sub vcl_hash{ + | std.hash_req_body(); + | } + +$Function INT rematch_req_body(PRIV_TOP, PRIV_CALL, STRING re) + +Description + Returns -1 if an error occurred. + Returns 0 if the request body doesn't contain the string *re*. + Returns 1 if the request body contains the string *re*. + + Note that the comparison is case sensitive and the + request body must be buffered. + +Example + | std.cache_req_body(1KB); + | + | if (std.regex_req_body("FOO") == 1) { + | std.log("is true"); + | } + + + + SEE ALSO ======== diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 097cd45..b025ba2 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -43,7 +43,9 @@ #include "vtim.h" #include "cache/cache.h" +#include "hash/hash_slinger.h" #include "cache/cache_director.h" +#include "vcl.h" #include "vcc_if.h" @@ -248,3 +250,120 @@ vmod_strstr(VRT_CTX, VCL_STRING s1, VCL_STRING s2) return (strstr(s1, s2)); } +static int __match_proto__(req_body_iter_f) +concat_req_body(struct req *req, void *priv, void *ptr, size_t len) +{ + struct ws *ws = priv; + (void)req; + + return (!WS_Copy(ws, ptr, len)); +} + +static void +vmod_blob_req_body(VRT_CTX, struct vmod_priv *priv_top) +{ + unsigned *p; + char *ws_f; + ssize_t l; + + if (priv_top->priv) { + return; + } + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + + if (ctx->req->req_body_status != REQ_BODY_CACHED){ + VSLb(ctx->vsl, SLT_VCL_Error, + "Uncached req.body"); + return; + } + + assert(ctx->req->req_body_status == REQ_BODY_CACHED); + + p = (void*)WS_Alloc(ctx->ws, sizeof *p); + AN(p); + priv_top->priv = p; + + ws_f = WS_Snapshot(ctx->ws); + AN(ws_f); + l = VRB_Iterate(ctx->req, concat_req_body, ctx->ws); + + if (l < 0 || WS_Copy(ctx->ws, "\0", 1) == NULL) { + VSLb(ctx->vsl, SLT_VCL_Error, + "Iteration on req.body didn't succeed."); + WS_Reset(ctx->ws, ws_f); + memset(p, 0, sizeof *p); + priv_top->len = -1; + } + priv_top->priv = ws_f; + priv_top->len = l; +} + +VCL_INT __match_proto__(td_std_len_req_body) +vmod_len_req_body(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + + if (ctx->req->req_body_status != REQ_BODY_CACHED) { + VSLb(ctx->vsl, SLT_VCL_Error, + "Uncached req.body"); + return (-1); + } + + return (ctx->req->req_bodybytes); +} + +VCL_VOID __match_proto__(td_std_hash_req_body) +vmod_hash_req_body(VRT_CTX, struct vmod_priv *priv_top) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (ctx->method != VCL_MET_HASH) { + VSLb(ctx->vsl, SLT_VCL_Error, + "Hash_Req_Body can only be used in vcl_hash{}"); + return; + } + + vmod_blob_req_body(ctx, priv_top); + HSH_AddBytes(ctx->req, priv_top->priv, priv_top->len); + +} + +VCL_INT __match_proto__(td_std_regex_req_body) +vmod_rematch_req_body(VRT_CTX, struct vmod_priv *priv_top, + struct vmod_priv *priv_call, VCL_STRING re) +{ + const char *error; + int erroroffset; + vre_t *t = NULL; + int i; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(re); + + if(priv_call->priv == NULL) { + t = VRE_compile(re, 0, &error, &erroroffset); + priv_call->priv = t; + priv_call->free = free; + + if(t == NULL) { + VSLb(ctx->vsl, SLT_VCL_Error, + "Regular expression not valid"); + return (-1); + } + } + + vmod_blob_req_body(ctx, priv_top); + + i = VRE_exec(priv_call->priv, priv_top->priv, priv_top->len, 0, 0, + NULL, 0, &cache_param->vre_limits); + + if (i >= 0) + return (1); + + if (i < VRE_ERROR_NOMATCH ) + VSLb(ctx->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); + return (0); +} -- 1.9.1
_______________________________________________ varnish-dev mailing list [email protected] https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
