Hello, I am adding a "hash_always_expire" option in varnish 4.0 branch to treat any existing objects as expired. This can be used to refresh an object by triggering a background fetch, and remove the previous object when bgfetch finishes. Also this will not perform bgfetch if varnish is currently fetching the same object.
This is like an improved option as hash_always_miss, because hash_always_miss does not remove the old object, and block the refresh script. The usage is the same: set req.hash_always_expire = true in vcl_recv. Is it ok to add it? Any comments? Thanks, Jingyi Wei
From 9b66effbd72bdcfe9598b9241090985455bf4e18 Mon Sep 17 00:00:00 2001 From: Jingyi Wei <[email protected]> Date: Tue, 10 Mar 2015 15:52:18 -0400 Subject: [PATCH] Add "hash_always_expire" option to treat any existing objects as expired. This can be used to refresh an object by triggering a background fetch, and remove the previous object when bgfetch finishes. Also this will not perform bgfetch if varnish is currently fetching the same object. --- bin/varnishd/cache/cache.h | 1 + bin/varnishd/cache/cache_hash.c | 4 +-- bin/varnishd/cache/cache_http1_fsm.c | 1 + bin/varnishd/cache/cache_req_fsm.c | 6 +++-- bin/varnishd/cache/cache_vrt_var.c | 1 + bin/varnishd/hash/hash_slinger.h | 2 +- bin/varnishtest/tests/c00069.vtc | 45 ++++++++++++++++++++++++++++++++++ lib/libvcc/generate.py | 11 +++++++++ 8 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 bin/varnishtest/tests/c00069.vtc diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 660d6b7..d2ff81b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -638,6 +638,7 @@ struct req { int disable_esi; uint8_t hash_ignore_busy; uint8_t hash_always_miss; + uint8_t hash_always_expire; struct sess *sp; struct worker *wrk; diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index b666d07..195346a 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -337,7 +337,7 @@ hsh_insert_busyobj(struct worker *wrk, struct objhead *oh) enum lookup_e HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, - int wait_for_busy, int always_insert) + int wait_for_busy, int always_insert, int always_expire) { struct worker *wrk; struct objhead *oh; @@ -430,7 +430,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, if (o->vary != NULL && !VRY_Match(req, o->vary)) continue; - if (EXP_Ttl(req, o) >= req->t_req) { + if (EXP_Ttl(req, o) >= req->t_req && !always_expire) { /* If still valid, use it */ assert(oh->refcnt > 1); assert(oc->objhead == oh); diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c index b398dc2..ec9341b 100644 --- a/bin/varnishd/cache/cache_http1_fsm.c +++ b/bin/varnishd/cache/cache_http1_fsm.c @@ -227,6 +227,7 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->req_body_status = REQ_BODY_INIT; req->hash_always_miss = 0; + req->hash_always_expire = 0; req->hash_ignore_busy = 0; if (sp->fd >= 0 && req->doclose != SC_NULL) diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 40551f7..b844583 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -363,7 +363,8 @@ cnt_lookup(struct worker *wrk, struct req *req) had_objhead = 1; lr = HSH_Lookup(req, &oc, &boc, req->esi_level == 0 ? 1 : 0, - req->hash_always_miss ? 1 : 0 + req->hash_always_miss ? 1 : 0, + req->hash_always_expire ? 1 : 0 ); if (lr == HSH_BUSY) { /* @@ -747,6 +748,7 @@ cnt_recv(struct worker *wrk, struct req *req) req->d_ttl = -1; req->disable_esi = 0; req->hash_always_miss = 0; + req->hash_always_expire = 0; req->hash_ignore_busy = 0; req->client_identity = NULL; @@ -840,7 +842,7 @@ cnt_purge(struct worker *wrk, struct req *req) VRY_Prep(req); AZ(req->objcore); - lr = HSH_Lookup(req, &oc, &boc, 1, 1); + lr = HSH_Lookup(req, &oc, &boc, 1, 1, 0); assert (lr == HSH_MISS); AZ(oc); CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 45614c8..f558d9b 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -528,6 +528,7 @@ VRT_r_req_##hash_var(VRT_CTX) \ REQ_BOOL(hash_ignore_busy) REQ_BOOL(hash_always_miss) +REQ_BOOL(hash_always_expire) /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 0afdb93..744988b 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -64,7 +64,7 @@ enum lookup_e { /* cache_hash.c */ void HSH_Cleanup(struct worker *w); enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **, - int wait_for_busy, int always_insert); + int wait_for_busy, int always_insert, int always_expire); void HSH_Ref(struct objcore *o); void HSH_Drop(struct worker *, struct object **); void HSH_Init(const struct hash_slinger *slinger); diff --git a/bin/varnishtest/tests/c00069.vtc b/bin/varnishtest/tests/c00069.vtc new file mode 100644 index 0000000..fb6cfd7 --- /dev/null +++ b/bin/varnishtest/tests/c00069.vtc @@ -0,0 +1,45 @@ +varnishtest "using req.hash_always_expire to force bgfetch" + +server s1 { + rxreq + expect req.url == "/1" + txresp -bodylen 1 + rxreq + expect req.url == "/1" + txresp -bodylen 2 + +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.http.short) { + set req.hash_always_expire = true; + } + } +} -start + +client c1 { + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + + txreq -url "/1" -hdr "short: yes" + rxresp + expect resp.status == 200 + expect resp.bodylen == 1 + + # Wait for the fetch to finish. + delay 2 + + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.bodylen == 2 + +} -run diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 884fe3d..6568c23 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -301,6 +301,17 @@ sp_variables = [ always (re)fetch from the backend. """ ), + ('req.hash_always_expire', + 'BOOL', + ( 'recv',), + ( 'recv',), """ + Force a cache expire for this request. If set to true + Varnish will treat any existing objects as expired and + (re)fetch from the backend in the background if is not + currently being fetched. The previous object will be + removed after successful fetch. + """ + ), ('bereq', 'HTTP', ( 'backend',), -- 1.7.9.5
_______________________________________________ varnish-dev mailing list [email protected] https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
