Gilles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/243810
Change subject: Upgrade Varnish role to Varnish 4
......................................................................
Upgrade Varnish role to Varnish 4
Bug: T114702
Change-Id: I48f4994ac74753fdfdece4c8a2245f8aef188f0b
---
M puppet/modules/thumbor/files/varnish.vcl
M puppet/modules/thumbor/manifests/init.pp
M puppet/modules/varnish/files/default-subs.vcl
A puppet/modules/varnish/files/varnish-cache.list
A puppet/modules/varnish/files/varnish-cache.org-pubkey.txt
M puppet/modules/varnish/manifests/init.pp
M puppet/modules/varnish/templates/backend.vcl.erb
7 files changed, 141 insertions(+), 69 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/10/243810/1
diff --git a/puppet/modules/thumbor/files/varnish.vcl
b/puppet/modules/thumbor/files/varnish.vcl
index f499c9b..26b2ce1 100644
--- a/puppet/modules/thumbor/files/varnish.vcl
+++ b/puppet/modules/thumbor/files/varnish.vcl
@@ -1,3 +1,5 @@
+vcl 4.0;
+
# vcl_recv is called whenever a request is received
sub vcl_recv {
# Copy the thumbnail URLs so that they create variants of the same object
@@ -10,33 +12,30 @@
if (req.http.If-None-Match && !req.url ~
"^/images/thumb/.*\.(jpeg|jpg|png)") {
return(pass);
}
-}
-# Called if the cache does not have a copy of the page.
-sub vcl_miss {
# qlow jpg thumbs
if (req.url ~ "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)") {
- set bereq.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(40):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/"; +
regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\1");
+ set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(40):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/"; +
regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\1");
# regular jpg thumbs
} else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)") {
- set bereq.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(87):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/"; +
regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\1");
+ set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(87):sharpen(0.6,0.01,false)/http://127.0.0.1:8080/images/"; +
regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\1");
# png thumbs
} else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.png") {
- set bereq.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.png", "\2") +
"x/http://127.0.0.1:8080/images/"; + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.png", "\1");
+ set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.png", "\2") +
"x/http://127.0.0.1:8080/images/"; + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.png", "\1");
}
}
sub vcl_deliver {
# Thumbor doesn't do fine-grained config for the headers it returns
- if (req.url ~ "^/images/thumb/.*\.(jpeg|jpg|png)") {
+ if (req.http.X-Url ~ "^/images/thumb/.*\.(jpeg|jpg|png)") {
unset resp.http.Cache-Control;
unset resp.http.Expires;
}
}
-sub vcl_fetch {
- if (req.http.X-Url) {
- set beresp.http.X-Url = req.http.X-Url;
+sub vcl_backend_response {
+ if (bereq.http.X-Url) {
+ set beresp.http.X-Url = bereq.http.X-Url;
if (!beresp.http.Vary) {
set beresp.http.Vary = "X-Url";
} elsif (beresp.http.Vary !~ "(?i)X-Url") {
@@ -47,10 +46,12 @@
sub vcl_hash {
# For thumbnails and originals we hash on the filename, to store them all
under the same object. This will make purging any of them purge all of them.
- if (req.url ~ "^/images/thumb/") {
- hash_data("Image-" + regsub(req.url,
"^/images/thumb/[^/]+/[^/]+/([^/]+)/[^/]+$", "\1"));
- } elsif (req.url ~ "^/images/") {
- hash_data("Image-" + regsub(req.url, "^/images/[^/]+/[^/]+/(.*)",
"\1"));
+ if (req.http.X-Url ~ "^/images/thumb/") {
+ hash_data("Image-" + regsub(req.http.X-Url,
"^/images/thumb/[^/]+/[^/]+/([^/]+)/[^/]+$", "\1"));
+ } elsif (req.http.X-Url ~ "^/images/") {
+ hash_data("Image-" + regsub(req.http.X-Url,
"^/images/[^/]+/[^/]+/(.*)", "\1"));
+ } elseif (req.http.X-Url) {
+ hash_data(req.http.X-Url);
} else {
hash_data(req.url);
}
@@ -60,5 +61,5 @@
} else {
hash_data(server.ip);
}
- return (hash);
+ return (lookup);
}
diff --git a/puppet/modules/thumbor/manifests/init.pp
b/puppet/modules/thumbor/manifests/init.pp
index cbdee86..4ea5547 100644
--- a/puppet/modules/thumbor/manifests/init.pp
+++ b/puppet/modules/thumbor/manifests/init.pp
@@ -83,5 +83,6 @@
varnish::config { 'thumbor':
source => 'puppet:///modules/thumbor/varnish.vcl',
+ order => 49, # Needs to be before default for vcl_recv override
}
}
diff --git a/puppet/modules/varnish/files/default-subs.vcl
b/puppet/modules/varnish/files/default-subs.vcl
index 73579ae..80cf90c 100644
--- a/puppet/modules/varnish/files/default-subs.vcl
+++ b/puppet/modules/varnish/files/default-subs.vcl
@@ -1,8 +1,7 @@
+vcl 4.0;
+
# vcl_recv is called whenever a request is received
sub vcl_recv {
- # Serve objects up to 2 minutes past their expiry if the backend
- # is slow to respond.
- set req.grace = 120s;
set req.http.X-Forwarded-For = client.ip;
# Since we expose varnish on the default port (6081) we need to rewrite
@@ -14,28 +13,28 @@
# This uses the ACL action called "purge". Basically if a request to
# PURGE the cache comes from anywhere other than localhost, ignore it.
- if (req.request == "PURGE") {
+ if (req.method == "PURGE") {
if (!client.ip ~ purge) {
- error 405 "Not allowed.";
+ return(synth(405, "This IP is not allowed to send PURGE
requests."));
}
- return(lookup);
+ return(hash);
}
# Pass any requests that Varnish does not understand straight to the
backend.
- if (req.request != "GET" && req.request != "HEAD" &&
- req.request != "PUT" && req.request != "POST" &&
- req.request != "TRACE" && req.request != "OPTIONS" &&
- req.request != "DELETE") {
+ if (req.method != "GET" && req.method != "HEAD" &&
+ req.method != "PUT" && req.method != "POST" &&
+ req.method != "TRACE" && req.method != "OPTIONS" &&
+ req.method != "DELETE") {
return(pipe); /* Non-RFC2616 or CONNECT which is weird. */
}
# Pass anything other than GET and HEAD directly.
- if (req.request != "GET" && req.request != "HEAD") {
+ if (req.method != "GET" && req.method != "HEAD") {
return(pass);
}
# Pretend that image requests don't have cookie/auth, so that they get
cached
- if (req.url ~ "^/images/") {
+ if (req.url ~ "^/images/" || req.http.X-Url ~ "^/images/") {
unset req.http.Authorization;
unset req.http.Cookie;
}
@@ -52,7 +51,7 @@
# Force lookup if the request is a no-cache request from the client.
if (req.http.Cache-Control ~ "no-cache") {
- ban_url(req.url);
+ ban("req.url == " + req.url);
}
# Pass requests to potential non-plain reads on articles (eg. action=edit)
@@ -73,7 +72,7 @@
}
}
- return(lookup);
+ return(hash);
}
sub vcl_pipe {
@@ -89,9 +88,9 @@
# Called if the cache has a copy of the page.
sub vcl_hit {
- if (req.request == "PURGE") {
- ban_url(req.url);
- error 200 "Purged";
+ if (req.method == "PURGE") {
+ ban("req.url == " + req.url);
+ return(synth(200, "Purged"));
}
if (!obj.ttl > 0s) {
@@ -101,8 +100,8 @@
# Called if the cache does not have a copy of the page.
sub vcl_miss {
- if (req.request == "PURGE") {
- error 200 "Not in cache";
+ if (req.method == "PURGE") {
+ return(synth(200, "Not in cache"));
}
}
@@ -115,28 +114,15 @@
}
# Called after a document has been successfully retrieved from the backend.
-sub vcl_fetch {
-
- # set minimum timeouts to auto-discard stored objects
-# set beresp.prefetch = -30s;
+sub vcl_backend_response {
set beresp.grace = 120s;
if (beresp.ttl < 48h) {
set beresp.ttl = 48h;
}
- if (!beresp.ttl > 0s) {
- return(hit_for_pass);
- }
-
- if (beresp.http.Set-Cookie) {
- return(hit_for_pass);
- }
-
-# if (beresp.http.Cache-Control ~ "(private|no-cache|no-store)")
-# {return(hit_for_pass);}
-
- if (req.http.Authorization && !beresp.http.Cache-Control ~ "public") {
- return(hit_for_pass);
+ if (!beresp.ttl > 0s || beresp.http.Set-Cookie ||
(bereq.http.Authorization && !beresp.http.Cache-Control ~ "public") ||
beresp.status == 504) {
+ set beresp.uncacheable = true;
+ set beresp.ttl = 120s;
}
}
diff --git a/puppet/modules/varnish/files/varnish-cache.list
b/puppet/modules/varnish/files/varnish-cache.list
new file mode 100644
index 0000000..0517c46
--- /dev/null
+++ b/puppet/modules/varnish/files/varnish-cache.list
@@ -0,0 +1 @@
+deb https://repo.varnish-cache.org/ubuntu/ trusty varnish-4.1
\ No newline at end of file
diff --git a/puppet/modules/varnish/files/varnish-cache.org-pubkey.txt
b/puppet/modules/varnish/files/varnish-cache.org-pubkey.txt
new file mode 100644
index 0000000..414abee
--- /dev/null
+++ b/puppet/modules/varnish/files/varnish-cache.org-pubkey.txt
@@ -0,0 +1,41 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.10 (GNU/Linux)
+
+mQINBEyHLzoBEACwWIEvQ1PysbHP1zxMaRzcKC1+ZzIOB7yt59obMxgr+leU/wxq
+goKN4BmqZMwCBU7XsqjSAG2AOT/7+uAX28Y/OjC/j+L5vVpPCGwUbD7did6LsFo+
+7Dx9Fi28G0YnfLBrmnHDsZb6xSbQiAg70knLVtHRy34a4JKluMKFJKSFdwyVGAZ0
+NBMXD0zaMYgMMgKiYJ+/N7oEyeFx8/rmdczTXMO19SNtxmXkLxzr9ch4GbIRL8XZ
+OA7fehj2RmBoS2S3x+/mHwsN+nOIq6cRZMZFpHYb6FI/ORZJuuXaJs9J97Q2SnZU
+BtSihUjAtT+oUG3AyHIO8YUxirw/iyFDWvJjlAkzKlsU4TJ5FkgH4O2IDdVUTYI7
+nW8Emu1hH60GvOq5K9tzs2sONTegVdZ1J5s1+h+xU4fvRzdbmxNCGdtFX2Gy+uri
+Eo1iWtqpbWq3OAeZMDd/4HNm05Oc9N5veiSMzvNiXNWUDTLDQwrwUrfPJhWGbcAa
+1RtdfkxQFIjjoLvnnnpdV3YNCkuGT2srVefLsIIwGekW9I0VLAiiosf3eWZlNjHw
+t5GZsHvDzGa12YXpjRuNV3+6mJ2w8GGCYTOdCs7kRzzSSOesP3gQ+DdbotTS9JR/
+rl19yMoOb2UkeM6Fdo72TH/csCphHO4Wn2//MDFmVR9YazbpGRO013qPPwARAQAB
+tEB2YXJuaXNoLWNhY2hlLm9yZyByZXBvc2l0b3J5IGtleSA8c3lzYWRtaW5AdmFy
+bmlzaC1zb2Z0d2FyZS5jb20+iQJBBBMBCAArBQJMhy86AhsDBQkSzAMABwsHCQgK
+BAIHFQoJCAsDAgUWAQIDAAIeAQIXgAAKCRBg58CWxN7/60tMD/0WQOEZsbKAEZhE
+m4OJ4eAZTzxyx+l4jZRABW/cBTFkxNlll/Vm20I9KXXawNqRWxlGPyuF3a1JYCXi
+cMK2In2y5qpm/J9CVjfVGpdSJAqO52Rz0VWCSFKRuDIg2LK2s+M4uL+bZCCEj/Ih
+VOrzdjB6KIReBtzFuf7hoKoi8Bqt+uRGGlycIzkgsUXggvoICK+aVU+oVE0Xmotz
+KWjilz9h+b5xtMEkcp3O2x2i/ZlDbQy/6R+Lz2zl6L+nkVUMw9vpkL/R4SH7AikY
++GhZWABMFMA84E6Ss0VqRg8ICbCVRUmAITYfkd7h5maHnTqMeKJCmwFV/3bkHBk7
+yANRhyd/gII6fSHJA+yCrpGjUkGKy1C0izpXRJR9egGOVZ8WlFQs+6/hHHd9Zs50
+wfd5+U+5BuCWVv9AXEwbm+uP+rOOhJHduiN6UgMV7QGxWIgnBiBPGywpDBeu0i0F
+PeIqvxA1OqVgZUhexTWRNd+9TQtnPvxNawn1YQY0OE13Q2ClTC/lZskLSWpD3La3
+/tXOQt8hV7evOzkAzAaIiNQxQYEK+K7QlV6hlErwmQ5AeuzlDEs29dQoKPHVzFxR
+vPD4N7vt5GtXFkZOACH3TVbO9nmMTBHkOaeKEDqkmt0JJBkiP1cSzc00AQNnZON1
+kveIfO1K2Ay4Ya1ZDwL5ee5L6tsDzokCHAQQAQgABgUCTIi8wwAKCRC2WkhxyhnX
+F1dRD/96qfKkW2Ee+3K8DZbqUA2Pbjz+M2E9VWJYMX+XH7Q9Z2hwY8jkNUIUWLgH
+m15FDHe1cvcRIrzxa5bFCI6kqH7jKVLIhhNvNjCUQjqkDI87SxHsK1SVfniG9Uhl
+0J8gimC0PnGsxIA0VXCmEqbpwDyMY8W9VyigNgr0P0zzrOb21p30MvUHV6pEUN36
+/Kucn2h9f6yySiQ1DoyPUoovw52zxhUv9350rwoYeDW1izSNdY1/nkVUjHZ5+AUb
+1L4s2kV3bFCVPkgwtUCEGFdBrpiXML60oysM6BDCNYfHW/+NmOIE3fAjM10xeN30
+W/u1JnwfZ+Y7Rn7VBfsVF+VRJC/9GyoN2ci890TznsEivcrK0C/oIc9es8QdIKVp
+0ehFiudnQjjpF8C9nX7V4PACKPC0EcbgobCZnT6RBvaPcONNCkd1afvKjiFW1C0R
+0Y6SziGWALyo1Csg4imWnGZIO6nAyxCUKhNc13PshJ8qUP02ok0PtPLIlCP5FwK9
+tSJVtkLTHi67DEjpzHqc0dZpO9+SAieD/aXO+VJrRwg2aVLdd56oiDX4R3P1TDTg
+Hx8I2vHl+GB+HbnwpmpzA/UDFY83ygY9f7nOp2AtSnKrxsWuv43bulxTROql7uE/
+GzsBBk6TxCIUfvvB5k1gW0SU1vNIh0x+/Z+ny/ohf0J2sfUDYA==
+=EodT
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/puppet/modules/varnish/manifests/init.pp
b/puppet/modules/varnish/manifests/init.pp
index 6db63cc..f5c725e 100644
--- a/puppet/modules/varnish/manifests/init.pp
+++ b/puppet/modules/varnish/manifests/init.pp
@@ -11,8 +11,30 @@
# See
https://www.varnish-cache.org/docs/3.0/reference/vcl.html#multiple-subroutines
#
class varnish {
- package { 'varnish':
- ensure => 'present'
+ require_package('apt-transport-https')
+
+ # set up the repo pubkey
+ file { '/usr/local/share/varnish-cache.org-pubkey.txt':
+ source => 'puppet:///modules/varnish/varnish-cache.org-pubkey.txt',
+ owner => 'root',
+ group => 'root',
+ before => File['/etc/apt/sources.list.d/varnish-cache.list'],
+ notify => Exec['add_varnish_apt_key'],
+ }
+
+ # add the key
+ exec { 'add_varnish_apt_key':
+ command => '/usr/bin/apt-key add
/usr/local/share/varnish-cache.org-pubkey.txt',
+ before => File['/etc/apt/sources.list.d/varnish-cache.list'],
+ refreshonly => true,
+ }
+
+ # add the varnish repo list file
+ file { '/etc/apt/sources.list.d/varnish-cache.list':
+ source => 'puppet:///modules/varnish/varnish-cache.list',
+ owner => 'root',
+ group => 'root',
+ notify => Exec['update_package_index'],
}
$conf = '/etc/varnish/conf-d.vcl'
@@ -21,11 +43,10 @@
# This level of include indirection is annoying but necessary to escape
# endless Puppet file/file_line conflicts.
file { '/etc/varnish/default.vcl':
- content => "include \"${conf}\";\n",
+ content => "vcl 4.0;\ninclude \"${conf}\";\n",
mode => '0644',
owner => 'root',
group => 'root',
- require => Package['varnish'],
}
file { $conf:
@@ -33,7 +54,6 @@
owner => 'root',
group => 'root',
mode => '0644',
- require => Package['varnish'],
}
file { $confd:
@@ -41,14 +61,6 @@
owner => 'root',
group => 'root',
mode => '0755',
- require => Package['varnish'],
- }
-
- service { 'varnish':
- ensure => running,
- provider => init,
- require => Package['varnish'],
- subscribe => File[$conf],
}
# Ensure included config order is respected by sorting default.vcl
@@ -60,19 +72,47 @@
}
varnish::backend { 'default':
- host => '127.0.0.1',
- port => '8080',
- order => 20,
+ host => '127.0.0.1',
+ port => '8080',
+ order => 20,
+ notify => Service['varnish'],
+ before => Package['varnish'],
}
# acl for "purge": open to only localhost
varnish::config { 'acl-purge':
- content => 'acl purge { "127.0.0.1"; }',
+ content => "vcl 4.0;\nacl purge { \"127.0.0.1\"; }",
order => 10,
+ notify => Service['varnish'],
+ before => Package['varnish'],
}
varnish::config { 'default-subs':
source => 'puppet:///modules/varnish/default-subs.vcl',
order => 50,
+ notify => Service['varnish'],
+ before => Package['varnish'],
+ }
+
+ package { 'varnish':
+ ensure => 'latest',
+ require => [
+ Package['apt-transport-https'],
+ File['/etc/varnish/default.vcl'],
+ File[$conf],
+ File[$confd],
+ ]
+ }
+
+ service { 'varnish':
+ ensure => running,
+ provider => init,
+ require => Package['varnish'],
+ subscribe => File[$conf],
+ }
+
+ git::clone { 'https://github.com/varnish/libvmod-xkey':
+ directory => "/tmp/libvmod-xkey",
+ remote => 'https://github.com/varnish/libvmod-xkey',
}
}
diff --git a/puppet/modules/varnish/templates/backend.vcl.erb
b/puppet/modules/varnish/templates/backend.vcl.erb
index c52c8a8..1255677 100644
--- a/puppet/modules/varnish/templates/backend.vcl.erb
+++ b/puppet/modules/varnish/templates/backend.vcl.erb
@@ -1,3 +1,5 @@
+vcl 4.0;
+
backend <%= @title %> {
.host = "<%= @host %>";
.port = "<%= @port %>";
@@ -5,6 +7,6 @@
sub vcl_recv {
if (<%= @onlyif %>) {
- set req.backend = <%= @title %>;
+ set req.backend_hint = <%= @title %>;
}
}
--
To view, visit https://gerrit.wikimedia.org/r/243810
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I48f4994ac74753fdfdece4c8a2245f8aef188f0b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
