Attached is a small patch set that implement soft purge support
for Varnish trunk.
Soft purges are purges where the object TTL is reduced to zero, but grace is
kept. In essence it adds a softpurge; statement in VCL that can be used
where purge; ordinarily is.
The main use case for soft purges are big Varnish setups with many
backends and automated purging systems. When doing maintenance on one of
the backends, you end up with automatically purging something that can't be
recreated right now and Varnish starts sending 503 replies.
With softpurge the grace handling will kick in and serve a stale object instead.
Git branch is available here:
https://github.com/lkarsten/Varnish-Cache/tree/softpurge
I request that these patches are merged into trunk.
--
Lasse Karstensen
Varnish Software AS
http://www.varnish-software.com/
>From 575e48193281958b0c86a5c97777d4a41ef4b35f Mon Sep 17 00:00:00 2001
From: Lasse Karstensen <[email protected]>
Date: Fri, 4 Jan 2013 13:51:05 +0100
Subject: [PATCH 1/6] Change VRT_purge format to support soft purge.
Add an additional flag in the front to say if the purge is soft or not.
---
bin/varnishtest/tests/c00033.vtc | 4 ++--
include/vrt.h | 2 +-
lib/libvcl/vcc_action.c | 13 ++++++++++++-
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/bin/varnishtest/tests/c00033.vtc b/bin/varnishtest/tests/c00033.vtc
index d4289e3..ca40445 100644
--- a/bin/varnishtest/tests/c00033.vtc
+++ b/bin/varnishtest/tests/c00033.vtc
@@ -25,13 +25,13 @@ varnish v1 -vcl+backend {
sub vcl_hit {
if (req.request == "PURGE") {
- C{ VRT_purge(req, 0, 0); }C
+ C{ VRT_purge(req, 0, 0, 0); }C
error 456 "got it";
}
}
sub vcl_miss {
if (req.request == "PURGE") {
- C{ VRT_purge(req, 0, 0); }C
+ C{ VRT_purge(req, 0, 0, 0); }C
error 456 "got it";
}
}
diff --git a/include/vrt.h b/include/vrt.h
index c98e085..35ce682 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -179,7 +179,7 @@ const char *VRT_regsub(struct req *, int all, const char *,
void VRT_ban(const struct req *, char *, ...);
void VRT_ban_string(const struct req *, const char *);
-void VRT_purge(struct req *, double ttl, double grace);
+void VRT_purge(struct req *, int soft, double ttl, double grace);
void VRT_count(struct req *, unsigned);
int VRT_rewrite(const char *, const char *);
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 1eec1c4..da19e6e 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -267,7 +267,17 @@ parse_purge(struct vcc *tl)
{
vcc_NextToken(tl);
- Fb(tl, 1, "VRT_purge(req, 0, 0);\n");
+ Fb(tl, 1, "VRT_purge(req, 0, 0, 0);\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+parse_softpurge(struct vcc *tl)
+{
+
+ vcc_NextToken(tl);
+ Fb(tl, 1, "VRT_purge(req, 1, 0, 0);\n");
}
/*--------------------------------------------------------------------*/
@@ -313,6 +323,7 @@ static struct action_table {
{ "synthetic", parse_synthetic, VCL_MET_ERROR },
{ "unset", parse_unset },
{ "purge", parse_purge, VCL_MET_MISS | VCL_MET_HIT },
+ { "softpurge", parse_softpurge, VCL_MET_MISS | VCL_MET_HIT },
{ NULL, NULL }
};
--
1.7.10.4
>From 1da4e55eb8616d5ccf27cf0b4f6b7c54328b0d8e Mon Sep 17 00:00:00 2001
From: Lasse Karstensen <[email protected]>
Date: Fri, 4 Jan 2013 13:52:26 +0100
Subject: [PATCH 2/6] Change HSH_Purge for support soft purge.
---
bin/varnishd/cache/cache_hash.c | 15 ++++++++++++---
bin/varnishd/cache/cache_vrt.c | 6 +++---
bin/varnishd/hash/hash_slinger.h | 2 +-
3 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index d0d987e..013b5b8 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -61,6 +61,7 @@
#include "hash/hash_slinger.h"
#include "vsha256.h"
+#include "vtim.h"
static const struct hash_slinger *hash;
@@ -490,11 +491,12 @@ hsh_rush(struct dstat *ds, struct objhead *oh)
*/
void
-HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace)
+HSH_Purge(struct req *req, struct objhead *oh, int soft, double ttl, double grace)
{
struct objcore *oc, **ocp;
unsigned spc, nobj, n;
struct object *o;
+ double now = VTIM_real();
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -538,8 +540,15 @@ HSH_Purge(struct req *req, struct objhead *oh, double ttl, double grace)
if (o == NULL)
continue;
CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
- o->exp.ttl = ttl;
- o->exp.grace = grace;
+ if (soft) {
+ // Only set TTL for objects that expire in the future.
+ if (o->exp.ttl > (now - o->exp.entered)) {
+ o->exp.ttl = now - o->exp.entered;
+ }
+ } else {
+ o->exp.ttl = ttl;
+ o->exp.grace = grace;
+ }
EXP_Rearm(o);
(void)HSH_Deref(&req->wrk->stats, NULL, &o);
}
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 914531e..9409849 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -516,14 +516,14 @@ VRT_ban_string(const struct req *req, const char *str)
*/
void
-VRT_purge(struct req *req, double ttl, double grace)
+VRT_purge(struct req *req, int soft, double ttl, double grace)
{
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
if (req->cur_method == VCL_MET_HIT)
- HSH_Purge(req, req->obj->objcore->objhead, ttl, grace);
+ HSH_Purge(req, req->obj->objcore->objhead, soft, ttl, grace);
else if (req->cur_method == VCL_MET_MISS)
- HSH_Purge(req, req->objcore->objhead, ttl, grace);
+ HSH_Purge(req, req->objcore->objhead, soft, ttl, grace);
}
/*--------------------------------------------------------------------
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index c385ea6..10f6de5 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -59,7 +59,7 @@ void HSH_Drop(struct worker *, struct object **);
void HSH_Init(const struct hash_slinger *slinger);
void HSH_AddString(struct req *, const char *str);
void HSH_Insert(struct worker *, const void *hash, struct objcore *);
-void HSH_Purge(struct req *, struct objhead *, double ttl, double grace);
+void HSH_Purge(struct req *, struct objhead *, int soft, double ttl, double grace);
void HSH_config(const char *h_arg);
struct objcore *HSH_NewObjCore(struct worker *wrk);
--
1.7.10.4
>From 5b3623c05fb5781d98d3388cf31bc2ca56f47403 Mon Sep 17 00:00:00 2001
From: Lasse Karstensen <[email protected]>
Date: Fri, 4 Jan 2013 13:53:01 +0100
Subject: [PATCH 3/6] Add test case for soft purge.
---
bin/varnishtest/tests/b00035.vtc | 113 ++++++++++++++++++++++++++++++++++++++
1 file changed, 113 insertions(+)
create mode 100644 bin/varnishtest/tests/b00035.vtc
diff --git a/bin/varnishtest/tests/b00035.vtc b/bin/varnishtest/tests/b00035.vtc
new file mode 100644
index 0000000..fa0d256
--- /dev/null
+++ b/bin/varnishtest/tests/b00035.vtc
@@ -0,0 +1,113 @@
+varnishtest "Basic soft purge tests"
+
+server s1 {
+ rxreq
+ txresp
+
+ # the purge restart target, avoiding vcl_error error which closes connection.
+ rxreq
+ txresp
+
+ rxreq
+ txresp
+
+ rxreq
+ txresp
+} -start
+
+varnish v1 -vcl+backend {
+ import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+ backend b1 { .host = "${s1_addr}"; .port = "${s1_port}"; }
+
+ # always sick
+ backend b2 { .host = "127.0.0.1"; .port = "12313";
+ # Keep this for better results in having the backend marked as sick. ;-)
+ .probe = { .initial = 0; }
+ }
+ sub vcl_recv {
+ set req.backend = b1;
+ if (req.http.x-sick-please) { set req.backend = b2; }
+ set req.grace = 10m;
+ if (req.request == "PURGE") { return(lookup); }
+ }
+
+ sub vcl_fetch {
+ set beresp.ttl = 1m;
+ set beresp.grace = 10m;
+
+ set req.http.x-ttl = beresp.ttl;
+ set req.http.x-grace = beresp.grace;
+ }
+
+ sub vcl_miss {
+ std.log("inside vcl_miss");
+ set req.http.x-via = "miss";
+ }
+
+ sub vcl_hit {
+ std.log("inside vcl_hit");
+ set req.http.x-via = "hit";
+ if (req.request == "PURGE") {
+ softpurge;
+ set req.url = "/exists";
+ set req.request = "GET";
+ unset req.http.x-via;
+ return(restart);
+ }
+
+ # There is no greater-than in the test language. (?)
+ if (obj.ttl < 0s) { set req.http.x-ttl-negative = true; }
+ else { set req.http.x-ttl-negative = false; }
+
+ set req.http.x-ttl = obj.ttl;
+ set req.http.x-grace = obj.grace;
+ }
+
+ sub vcl_deliver {
+ set resp.http.x-object-hits = obj.hits;
+ set resp.http.x-object-ttl = req.http.x-ttl;
+ set resp.http.x-object-grace = req.http.x-grace;
+
+ set resp.http.x-via = req.http.x-via;
+
+ set resp.http.x-ttl-negative = req.http.x-ttl-negative;
+
+ #std.log("YY: ttl is " + req.http.x-ttl);
+ #std.log("YY: grace is " + req.http.x-grace);
+ }
+} -start
+
+# Warm the cache
+client c1 {
+ txreq -url "/"
+ rxresp
+
+ txreq -url "/"
+ rxresp
+ expect resp.status == 200
+ expect resp.http.x-via == "hit"
+} -run
+
+# Softpurge the object.
+client c1 {
+ txreq -req "PURGE" -url "/"
+ rxresp
+} -run
+
+# Pretend backend is dead, should give a graced hit.
+client c2 {
+ txreq -url "/" -hdr "x-sick-please: yes"
+ rxresp
+ expect resp.http.x-via == "hit"
+ expect resp.http.x-ttl-negative == true
+} -run
+
+# check that it has been purged for normal requests.
+# this will be a miss and then fetch since we've purged the previous one.
+client c3 {
+ txreq -url "/"
+ rxresp
+ expect resp.http.x-via == "miss"
+ expect resp.status == 200
+} -run
+
--
1.7.10.4
>From 4ae2a14caae7f06618562b63c108e9bd063316c7 Mon Sep 17 00:00:00 2001
From: Lasse Karstensen <[email protected]>
Date: Fri, 4 Jan 2013 14:56:47 +0100
Subject: [PATCH 4/6] Add section describing soft purge.
---
doc/sphinx/users-guide/purging.rst | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 53facd3..f16b74d 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -173,3 +173,19 @@ cache, thus forcing a fetch from the backend. This can in turn add the
freshly fetched object to the cache, thus overriding the current one. The
old object will stay in the cache until ttl expires or it is evicted by
some other means.
+
+Soft purge
+-----------
+
+A soft purge is a variant of purge that expires the TTL of an object (with variants),
+while leaving the grace period untouched.
+
+After a soft purge the next request will use the ordinary grace handling when
+deciding if the softpurged object can be used. It will appear like any other
+object that just has expired.
+
+In the normal case the softpurged object is not used, but in situations where the backend
+server is down, and request grace handling is in use, the softpurged object can be used.
+
+Softpurge is used in VCL with the ``softpurge;`` statement where ``purge;`` ordinarily would be used.
+
--
1.7.10.4
>From d4af970cb048d83e194b640895185dc19f7a6bad Mon Sep 17 00:00:00 2001
From: Lasse Karstensen <[email protected]>
Date: Fri, 4 Jan 2013 15:03:01 +0100
Subject: [PATCH 5/6] Reference documentation for purge and softpurge.
---
doc/sphinx/reference/vcl.rst | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 325470c..bafaf16 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -327,6 +327,15 @@ regsuball(str, regex, sub)
ban(ban expression)
Bans all objects in cache that match the expression.
+Special statements
+~~~~~~~~~~~~~~~~~~
+
+purge
+ Invalidate the requested object with variants.
+
+softpurge
+ Expire the requested object with variants while keeping grace.
+
Subroutines
~~~~~~~~~~~
--
1.7.10.4
>From dd81e92efdb6465dd6e42dd7d93ce0c3d32cc9ca Mon Sep 17 00:00:00 2001
From: Lasse Karstensen <[email protected]>
Date: Fri, 4 Jan 2013 15:23:51 +0100
Subject: [PATCH 6/6] Only check the time if it is a soft purge.
---
bin/varnishd/cache/cache_hash.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 013b5b8..c5106a2 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -496,7 +496,7 @@ HSH_Purge(struct req *req, struct objhead *oh, int soft, double ttl, double grac
struct objcore *oc, **ocp;
unsigned spc, nobj, n;
struct object *o;
- double now = VTIM_real();
+ double now;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -533,6 +533,11 @@ HSH_Purge(struct req *req, struct objhead *oh, int soft, double ttl, double grac
ttl = -1.;
if (!(grace > 0.))
grace = -1.;
+
+ if (soft) {
+ now = VTIM_real();
+ }
+
for (n = 0; n < nobj; n++) {
oc = ocp[n];
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
--
1.7.10.4
_______________________________________________
varnish-dev mailing list
[email protected]
https://www.varnish-cache.org/lists/mailman/listinfo/varnish-dev
