Ema has uploaded a new change for review.
https://gerrit.wikimedia.org/r/273490
Change subject: Maps VCL forward-port to Varnish 4
......................................................................
Maps VCL forward-port to Varnish 4
This patch uses a boolean hiera attribute, varnish_version4, to
distinguish between Varnish 3 and Varnish 4 VCL syntax.
With these changes applied, Varnish 4 starts properly on machines
deployed with role cache::maps.
The included test cases pass with Varnish 4.
Bug: T124279
Change-Id: I622f58e32878b6fa7a8578aae3e62bd448e4baac
---
M modules/role/manifests/cache/maps.pp
M modules/varnish/manifests/common/vcl.pp
M modules/varnish/manifests/instance.pp
M modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
M templates/varnish/analytics.inc.vcl.erb
M templates/varnish/errorpage.inc.vcl.erb
M templates/varnish/maps-backend.inc.vcl.erb
M templates/varnish/maps-frontend.inc.vcl.erb
M templates/varnish/misc-backend.inc.vcl.erb
M templates/varnish/misc-common.inc.vcl.erb
M templates/varnish/misc-frontend.inc.vcl.erb
M templates/varnish/text-backend.inc.vcl.erb
M templates/varnish/text-common.inc.vcl.erb
M templates/varnish/text-frontend.inc.vcl.erb
M templates/varnish/upload-backend.inc.vcl.erb
M templates/varnish/upload-frontend.inc.vcl.erb
18 files changed, 401 insertions(+), 64 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/90/273490/1
diff --git a/modules/role/manifests/cache/maps.pp
b/modules/role/manifests/cache/maps.pp
index 84bbdad..fec0f65 100644
--- a/modules/role/manifests/cache/maps.pp
+++ b/modules/role/manifests/cache/maps.pp
@@ -20,6 +20,13 @@
include role::cache::ssl::unified
+ if hiera('varnish_version4', false) {
+ $maps_director_type = 'vslp'
+ }
+ else {
+ $maps_director_type = 'chash'
+ }
+
$varnish_be_directors = {
'one' => {
'backend' => {
@@ -34,7 +41,7 @@
'two' => {
'backend' => {
'dynamic' => 'yes',
- 'type' => 'chash',
+ 'type' => $maps_director_type,
'backends' => $cluster_nodes['eqiad'],
},
'backend_random' => {
@@ -129,7 +136,7 @@
directors => {
'backend' => {
'dynamic' => 'yes',
- 'type' => 'chash',
+ 'type' => $maps_director_type,
'backends' => $site_cluster_nodes,
},
'backend_random' => {
diff --git a/modules/varnish/manifests/common/vcl.pp
b/modules/varnish/manifests/common/vcl.pp
index cff2aff..737002d 100644
--- a/modules/varnish/manifests/common/vcl.pp
+++ b/modules/varnish/manifests/common/vcl.pp
@@ -1,6 +1,8 @@
class varnish::common::vcl {
require varnish::common
+ $varnish_version4 = hiera('varnish_version4', false)
+
file { '/etc/varnish/geoip.inc.vcl':
owner => 'root',
group => 'root',
@@ -29,4 +31,9 @@
mode => '0555',
source => 'puppet:///modules/varnish/varnish-test-geoip',
}
+
+ file { '/usr/share/varnish/tests/':
+ source => 'puppet:///modules/varnish/tests',
+ recurse => true,
+ }
}
diff --git a/modules/varnish/manifests/instance.pp
b/modules/varnish/manifests/instance.pp
index 6fba0d5..809015e 100644
--- a/modules/varnish/manifests/instance.pp
+++ b/modules/varnish/manifests/instance.pp
@@ -1,3 +1,16 @@
+define varnish::wikimedia_vcl($varnish_testing, $template_path) {
+ if $varnish_testing {
+ $varnish_include_path = '/usr/share/varnish/tests/'
+ }
+
+ file { $title:
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ content => template($template_path),
+ }
+}
+
define varnish::instance(
$vcl_config,
$backend_options,
@@ -24,6 +37,56 @@
else {
$instancesuffix = "-${name}"
$extraopts = "-n ${name}"
+ }
+
+ # $varnish_version4 is used to distinguish between v4 and v3 versions of
+ # VCL code, as well as to pass the right parameters to varnishd. See
+ # varnish.systemd.erb
+ $varnish_version4 = hiera('varnish_version4', false)
+
+ if $varnish_version4 {
+ #
https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#req-request-is-now-req-method
+ $req_method = 'req.method'
+
+ $bereq_method = 'bereq.method'
+
+ # http://www.gossamer-threads.com/lists/varnish/misc/32514
+ $bereq_retries = 'bereq.retries'
+
+ # Use the following X_Y variables anywhere the upgrade from v3 to v4
+ # requires replacing the string X with Y.
+
+ #
https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#req-not-available-in-vcl-backend-response
+ $bereq_req = 'bereq'
+
+ #
https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#obj-in-vcl-error-replaced-by-beresp-in-vcl-backend-error
+ $beresp_obj = 'beresp'
+
+ #
https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#obj-is-now-read-only
+ $resp_obj = 'resp'
+
+ #
https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#some-return-values-have-been-replaced
+ # vcl_recv must now return hash instead of lookup
+ $hash_lookup = 'hash'
+
+ # vcl_pass must now return fetch instead of pass
+ $fetch_pass = 'fetch'
+
+ # The 'ip' function has been added to the Varnish Standard Module in
+ # v4. No need to use ipcast.
+ $std_ipcast = 'std'
+ }
+ else {
+ $req_method = 'req.request'
+ $bereq_method = 'req.request'
+ $bereq_retries = 'req.restarts'
+
+ $bereq_req = 'req'
+ $beresp_obj = 'obj'
+ $resp_obj = 'obj'
+ $hash_lookup = 'lookup'
+ $fetch_pass = 'pass'
+ $std_ipcast = 'ipcast'
}
# Initialize variables for templates
@@ -85,6 +148,23 @@
content => template("${module_name}/vcl/wikimedia-${layer}.vcl.erb"),
}
+ # These versions of wikimedia-common_${vcl}.vcl and wikimedia_${vcl}.vcl
+ # are exactly the same as those under /etc/varnish but without any
+ # backends defined. The goal is to make it possible to run the VTC test
+ # files under /usr/share/varnish/tests without having to modify any VCL
+ # file by hand.
+ varnish::wikimedia_vcl {
"/usr/share/varnish/tests/wikimedia-common_${vcl}.inc.vcl":
+ require => File['/usr/share/varnish/tests'],
+ varnish_testing => true,
+ template_path => "${module_name}/vcl/wikimedia-common.inc.vcl.erb",
+ }
+
+ varnish::wikimedia_vcl { "/usr/share/varnish/tests/wikimedia_${vcl}.vcl":
+ require => File['/usr/share/varnish/tests'],
+ varnish_testing => true,
+ template_path => "${module_name}/vcl/wikimedia-${layer}.vcl.erb",
+ }
+
file { "/etc/varnish/${vcl}.inc.vcl":
owner => 'root',
group => 'root',
diff --git a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
index 6329f5e..700d5ad 100644
--- a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
@@ -1,13 +1,21 @@
+<% if @varnish_version4 -%>
+vcl 4.0;
+<% end -%>
+
// common backend code for all clusters
-include "wikimedia-common_<%= @vcl %>.inc.vcl";
+include "<%= @varnish_include_path %>wikimedia-common_<%= @vcl %>.inc.vcl";
/* Include the VCL file for this role */
include "<%= @vcl %>.inc.vcl";
+sub vcl_init {
+ call wm_common_directors_init;
+}
+
sub vcl_recv {
if (client.ip !~ wikimedia_nets) {
// Do not allow direct access to non-frontend layers
- error 403 "Access denied";
+ <%= error_synth(403, "Access denied") %>
}
call wm_common_recv;
@@ -21,6 +29,13 @@
call cluster_be_hash;
// default vcl_hash invokes here!
}
+
+<% if @varnish_version4 -%>
+// http://book.varnish-software.com/4.0/chapters/Cache_Invalidation.html
+sub vcl_purge {
+ return (synth(204, "Purged"));
+}
+<% end -%>
sub vcl_hit {
call wm_common_hit;
@@ -42,18 +57,26 @@
// pass-traffic should not use consistent hashing, to avoid unecessary
// traffic focus on one node and keep things performant, *if* we're
// fairly sure that all layers/tiers make equivalent pass decisions...
+ <% if @varnish_version4 -%>
+ set req.backend_hint = backend_random.backend();
+ <% else -%>
set req.backend = backend_random;
+ <% end -%>
<% end -%>
<% end -%>
call cluster_be_pass;
- return (pass); // no default VCL (which is just "return (pass)" anyways)
+ return (<%= @fetch_pass %>); // no default VCL (which is just "return
(<%= @fetch_pass %>)" anyways)
}
+<% if @varnish_version4 -%>
+sub vcl_backend_response {
+<% else -%>
sub vcl_fetch {
- call wm_common_fetch;
- call cluster_be_fetch;
- // default vcl_fetch invokes here, unless cluster VCL unconditionally
returns!
+<% end -%>
+ call wm_common_backend_response;
+ call cluster_be_backend_response;
+ // default vcl_(fetch|backend_response) invokes here, unless cluster
VCL unconditionally returns!
}
sub vcl_deliver {
@@ -62,10 +85,33 @@
return (deliver); // no default VCL (which is just "return (deliver)"
anyways)
}
+<% if @varnish_version4 -%>
+
+// Varnish4 vcl_synth+vcl_backend_error
+
+sub vcl_synth {
+ if (resp.status > 400 && resp.status != 413) {
+ call synth_errorpage;
+ }
+ return (deliver);
+}
+
+sub vcl_backend_error {
+ if (beresp.status > 400 && beresp.status != 413) {
+ call backend_error_errorpage;
+ }
+ return (deliver);
+}
+
+<% else -%>
+
+// Varnish3 vcl_error
sub vcl_error {
- call wm_common_error;
+ call wm_common_v3_purge_error;
if (obj.status > 400 && obj.status != 413) {
call synth_errorpage;
}
return (deliver);
}
+
+<% end -%>
diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
index 5108af2..d5eb525 100644
--- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
@@ -8,6 +8,24 @@
# being set from different code.
import header;
+<% if @varnish_version4 -%>
+import directors;
+
+# This is actually only needed if a vslp backend is defined. We could make
+# this import conditional on existing vslp backend definitions.
+import vslp;
+<% end %>
+
+<%
+def error_synth(status_code, message)
+ if @varnish_version4
+ return "return (synth(#{status_code}, \"#{message}\"));\n"
+ else
+ return "error #{status_code} \"#{message}\";\n"
+ end
+end
+-%>
+
include "errorpage.inc.vcl";
<%
@@ -100,6 +118,10 @@
# Backends
+<% if @varnish_testing
+@varnish_backends = []
+end -%>
+
# List of Puppet generated backends
<%
@varnish_backends.each do |backend|
@@ -131,44 +153,89 @@
# }
if @use_dynamic_directors and @dynamic_directors -%>
include "directors.<%= @inst %>.vcl";
-
<% end -%>
+
+sub wm_common_directors_init {
+<% if not @varnish_version4 -%>
+}
+<% end -%>
+
<% @varnish_directors.keys.sort.each do |director_name|
director = @varnish_directors[director_name]
if (!@dynamic_directors or director['dynamic'] != 'yes')
backends = director['backends']
if (!backends.empty?)
-%>
+
+<% if @varnish_version4 -%>
+<% if director['type'] == 'vslp' -%>
+// Yes, the vslp director gets instantiated with vslp.vslp() instead of
+// directors.vslp(). Nobody said this would have been pretty.
+new <%= director_name %> = vslp.<%= director['type'] %>();
+<% else -%>
+new <%= director_name %> = directors.<%= director['type'] %>();
+<% end %>
+<% else -%>
director <%= director_name %> <%= director['type'] %> {
+<% end -%>
+
<% if director['type'] == 'chash' -%>
.retries = <%= chash_def_retries(99, backends.size) %>;
<% end -%>
+
<%
backends.each do |backend|
name = /^[0-9\.]+$/.match(backend) ? "ipv4_" +
backend.gsub(".", "_") : "be_" + backend.split(".")[0].gsub("-", "_")
-%>
+ <% if @varnish_version4 -%>
+ <% if director['type'] == 'vslp' -%>
+ # VSLP has no per-backend weighting. We might want to add it as a
+ # functionality to the vmod.
+ # See https://phabricator.wikimedia.org/T126206
+ <%= director_name %>.add_backend(<%= name %>);
+ <% else -%>
+ <%= director_name %>.add_backend(<%= name %>, <%=
backend_option(backend, "weight", 10) %>);
+ <% end -%>
+ <% else -%>
{
.backend = <%= name %>;
.weight = <%= backend_option(backend, "weight", 10) %>;
}
+ <% end -%>
<% end -%>
-}
+
+ <% if director['type'] == 'vslp' -%>
+ # We should think about this value, probably setting it in a
+ # backend-count-sensitive manner.
+ <%= director_name %>.init_hashcircle(150);
+ <% end -%>
+
+<% if not @varnish_version4 -%>
+} # end of director block
+<% end -%>
<% end #if !empty -%>
<% end #if !dynamic -%>
<% end #director loop -%>
+<% if @varnish_version4 -%>
+} # end wm_common_directors_init
+<% end -%>
# Functions
sub wm_common_recv_purge {
/* Support HTTP PURGE */
- if (req.request == "PURGE") {
+ if (<%= @req_method %> == "PURGE") {
if (client.ip !~ local_host) {
- error 405 "Denied.";
+ <%= error_synth(405, "Denied.") %>
} elsif (req.http.Host ~ "<%=
@vcl_config.fetch('purge_host_regex') %>") {
set req.hash_ignore_busy = true;
- return (lookup);
+ <% if @varnish_version4 -%>
+ return (purge);
+ <% else -%>
+ return (lookup);
+ <% end -%>
} else {
- error 204 "Domain not cached here.";
+ <%= error_synth(204, "Domain not cached here.") %>
}
}
}
@@ -187,13 +254,13 @@
}
}
- if (req.request !~ "<%= @vcl_config.fetch("allowed_methods",
"^(GET|HEAD|POST|PURGE)$") %>"
- && !(req.request == "OPTIONS" && req.http.Origin)) {
- error 403 "HTTP method not allowed.";
+ if (<%= @req_method %> !~ "<%= @vcl_config.fetch("allowed_methods",
"^(GET|HEAD|POST|PURGE)$") %>"
+ && !(<%= @req_method %> == "OPTIONS" && req.http.Origin)) {
+ <%= error_synth(403, "HTTP method not allowed.") %>
}
if ( req.http.host ~ "^varnishcheck" ) {
- error 200 "OK";
+ <%= error_synth(200, "OK") %>
}
<% if @vcl_config.fetch("has_def_backend", "yes") == "yes" -%>
@@ -201,6 +268,16 @@
* If an instance has no default 'backend', it must declare
has_def_backend==no,
* and its own VCL must handle all possible req.backend cases.
*/
+ <% if @varnish_version4 -%>
+ set req.backend_hint = backend.backend();
+ if (std.healthy(req.backend_hint)) {
+ # TODO: This is now handled in vcl_hit.
+ # set req.grace = 5m;
+ } else {
+ # TODO: This is now handled in vcl_hit.
+ # set req.grace = 60m;
+ }
+ <% else -%>
set req.backend = backend;
if (req.backend.healthy) {
@@ -208,23 +285,28 @@
} else {
set req.grace = 60m;
}
- <% end %>
+ <% end -%>
+ <% end -%>
}
sub wm_common_hit {
set req.http.X-CDIS = "hit";
+ <% if not @varnish_version4 -%>
if (req.request == "PURGE") {
purge;
error 204 "Purged";
}
+ <% end -%>
}
sub wm_common_miss {
set req.http.X-CDIS = "miss";
+ <% if not @varnish_version4 -%>
if (req.request == "PURGE") {
purge;
error 204 "Cache miss";
}
+ <% end -%>
}
sub wm_common_pass {
@@ -236,7 +318,7 @@
}
}
-sub wm_common_fetch {
+sub wm_common_backend_response {
<% if @vcl_config.fetch("ttl_fixed", false) -%>
// Fixed TTL (rare/questionable, only used on upload backend right now)
// Note the ttl_cap comes after this and takes precedence!
@@ -281,8 +363,12 @@
}
}
-sub wm_common_error {
+// This is not needed with Varnish 4, the Connection header is set to
+// keep-alive by default
+<% if not @varnish_version4 -%>
+sub wm_common_v3_purge_error {
if (obj.status == 204 && req.request == "PURGE") {
set obj.http.Connection = "keep-alive";
}
}
+<% end -%>
diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
index c41fe0c..9c04c3a 100644
--- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
@@ -1,10 +1,15 @@
// common frontend code for all clusters
+<% if @varnish_version4 -%>
+vcl 4.0;
+<% else -%>
+import ipcast;
+<% end -%>
+
// only used in recv_fe_ip_processing on frontends
import netmapper;
-import ipcast;
-include "wikimedia-common_<%= @vcl %>.inc.vcl";
+include "<%= @varnish_include_path %>wikimedia-common_<%= @vcl %>.inc.vcl";
include "analytics.inc.vcl";
/* Include the VCL file for this role */
@@ -18,23 +23,23 @@
// the PURGE traffic here, as purge is called after this.
sub https_recv_redirect {
if (req.http.X-Forwarded-Proto != "https") {
- if (req.request == "GET" || req.request == "HEAD") {
+ if (<%= @req_method %> == "GET" || <%= @req_method %> ==
"HEAD") {
// This is all of our unified cert wildcard domains
which are TLS-clean (cert matches all extant hostnames within)
// The lone exception now is wikimedia.org, in the next
block
if (req.http.Host ~
"(?i)((^|\.)(wikipedia|wikibooks|wikinews|wikiquote|wikisource|wikiversity|wikivoyage|wikidata|wikimediafoundation|wiktionary|mediawiki)\.org|^w\.wiki)$")
{
set req.http.Location = "https://"; +
req.http.Host + req.url;
- error 751 "TLS Redirect";
+ <%= error_synth(751, "TLS Redirect") %>
}
// wikimedia.org has multi-level subdomains used for
HTTP for which we have no certs, so they must be avoided here:
// Ref: T102826 + T102827
else if(req.http.Host ~
"(?i)^([^.]+\.)?(m\.)?wikimedia\.org$") {
set req.http.Location = "https://"; +
req.http.Host + req.url;
- error 751 "TLS Redirect";
+ <%= error_synth(751, "TLS Redirect") %>
}
}
<% if @vcl_config.fetch("secure_post", true) -%>
- if (req.request == "POST" && !(req.http.Host ~
"(?i)\.beta\.wmflabs\.org$")) {
- error 403 "Insecure POST Forbidden - use HTTPS";
+ if (<%= @req_method %> == "POST" && !(req.http.Host ~
"(?i)\.beta\.wmflabs\.org$")) {
+ <%= error_synth(403, "Insecure POST Forbidden - use
HTTPS") %>
}
<% end %>
}
@@ -42,10 +47,10 @@
// *** HTTPS error code - implements 301 response for recv code
sub https_error_redirect {
- if (obj.status == 751) {
- set obj.http.Location = req.http.Location;
- set obj.status = 301;
- set obj.http.Content-Length = "0"; // T64245
+ if (<%= @resp_obj %>.status == 751) {
+ set <%= @resp_obj %>.http.Location = req.http.Location;
+ set <%= @resp_obj %>.status = 301;
+ set <%= @resp_obj %>.http.Content-Length = "0"; // T64245
return(deliver);
}
}
@@ -149,7 +154,7 @@
if (req.http.X-Trusted-Proxy && req.http.X-Forwarded-For) {
// get last from trusted-proxy-supplied XFF
set req.http.maybe-xcip =
regsub(req.http.X-Forwarded-For, "^([^,]+, )+", "");
- if(ipcast.ip(req.http.maybe-xcip, "127.0.0.1") !~
wikimedia_nets) {
+ if(<%= @std_ipcast %>.ip(req.http.maybe-xcip,
"127.0.0.1") !~ wikimedia_nets) {
set req.http.X-Client-IP = req.http.maybe-xcip;
}
unset req.http.maybe-xcip;
@@ -171,7 +176,7 @@
// to match and XCIP gets reset to its original value.
set req.http.maybe-xcip =
regsub(req.http.X-Forwarded-For, ", [^,]+$", "");
set req.http.maybe-xcip = regsub(req.http.maybe-xcip,
"^([^,]+, )+", "");
- if(ipcast.ip(req.http.maybe-xcip, "127.0.0.1") !~
wikimedia_nets) {
+ if(<%= @std_ipcast %>.ip(req.http.maybe-xcip,
"127.0.0.1") !~ wikimedia_nets) {
set req.http.X-Client-IP = req.http.maybe-xcip;
}
unset req.http.maybe-xcip;
@@ -219,6 +224,7 @@
}
sub vcl_init {
+ call wm_common_directors_init;
// again, netmapper only used in frontends, for recv_fe_ip_processing
// args here are map-name (for .map()), data file, and seconds between
mtime checks for reload
netmapper.init("proxies", "/var/netmapper/proxies.json", 89);
@@ -246,7 +252,7 @@
// They are handled by log tailers (varnishkafka and
varnishncsa) that filter the
// Varnish shm log for reqs to these endpoints and forward them
to log processors
// for storage and analysis.
- error 204;
+ <%= error_synth(204, "") %>
}
if(req.restarts == 0) {
@@ -264,6 +270,13 @@
call cluster_fe_hash;
// default vcl_hash invokes here!
}
+
+<% if @varnish_version4 -%>
+// http://book.varnish-software.com/4.0/chapters/Cache_Invalidation.html
+sub vcl_purge {
+ return (synth(204, "Purged"));
+}
+<% end -%>
sub vcl_hit {
call wm_common_hit;
@@ -284,17 +297,25 @@
// pass-traffic should not use consistent hashing, to avoid unecessary
// traffic focus on one node and keep things performant, *if* we're
// fairly sure that all layers/tiers make equivalent pass decisions...
+ <% if @varnish_version4 -%>
+ set req.backend_hint = backend_random.backend();
+ <% else -%>
set req.backend = backend_random;
+ <% end -%>
<% end -%>
call cluster_fe_pass;
- return (pass); // no default VCL (which is just "return (pass)" anyways)
+ return (<%= @fetch_pass %>); // no default VCL (which is just "return
(<%= @fetch_pass %>)" anyways)
}
+<% if @varnish_version4 -%>
+sub vcl_backend_response {
+<% else -%>
sub vcl_fetch {
- call wm_common_fetch;
- call cluster_fe_fetch;
- // default vcl_fetch invokes here, unless cluster VCL unconditionally
returns!
+<% end -%>
+ call wm_common_backend_response;
+ call cluster_fe_backend_response;
+ // default vcl_(fetch|backend_response) invokes here, unless cluster
VCL unconditionally returns!
}
sub vcl_deliver {
@@ -336,8 +357,37 @@
return (deliver); // no default VCL (which is just "return (deliver)"
anyways)
}
+<% if @varnish_version4 -%>
+
+// Varnish4 vcl_synth+vcl_backend_error
+
+sub vcl_synth {
+<% if @vcl_config.fetch("https_redirects", false) -%>
+ call https_error_redirect;
+<% end -%>
+ call cluster_fe_err_synth;
+ if (resp.status > 400 && resp.status != 413) {
+ call synth_errorpage;
+ }
+ return (deliver);
+}
+
+sub vcl_backend_error {
+ // retry 503 once in frontend instances, to paper over transient issues
+ if (beresp.status == 503 && bereq.retries == 0) {
+ return(retry);
+ }
+ if (beresp.status > 400 && beresp.status != 413) {
+ call backend_error_errorpage;
+ }
+ return (deliver);
+}
+
+<% else -%>
+
+// Varnish3 vcl_error
sub vcl_error {
- call wm_common_error;
+ call wm_common_v3_purge_error;
<% if @vcl_config.fetch("https_redirects", false) -%>
call https_error_redirect;
@@ -356,3 +406,5 @@
return (deliver);
}
+
+<% end -%>
diff --git a/templates/varnish/analytics.inc.vcl.erb
b/templates/varnish/analytics.inc.vcl.erb
index 0d92ea8..4ab4dbb 100644
--- a/templates/varnish/analytics.inc.vcl.erb
+++ b/templates/varnish/analytics.inc.vcl.erb
@@ -36,6 +36,18 @@
/*****************************************************************************
* !!! private to analytics_last_access_deliver !!!!
+ * Use a 12h-resolution expiry so people cannot infer an exact request time.
+<% if @varnish_version4 -%>
+ ****************************************************************************/
+sub set_last_access_cookie__ {
+ header.append(resp.http.Set-Cookie,
+ "WMF-Last-Access="
+ + req.http.X-NowDay
+ + ";Path=/;HttpOnly;Expires="
+ + std.time(std.time2integer(now + 32d, 0) / 43200 * 43200, now)
+ );
+}
+<% else -%>
* This should be:
* header.append(resp.http.Set-Cookie,
* "WMF-Last-Access="
@@ -60,6 +72,7 @@
vrt_magic_string_end
);
}C }
+<% end -%>
// Call from vcl_deliver near other X-Analytics code
sub analytics_last_access_deliver_ {
diff --git a/templates/varnish/errorpage.inc.vcl.erb
b/templates/varnish/errorpage.inc.vcl.erb
index 92264b6..9f06987 100644
--- a/templates/varnish/errorpage.inc.vcl.erb
+++ b/templates/varnish/errorpage.inc.vcl.erb
@@ -1,6 +1,10 @@
-sub synth_errorpage {
- set obj.http.Content-Type = "text/html; charset=utf-8";
+sub backend_error_errorpage {
+ set <%= @beresp_obj %>.http.Content-Type = "text/html; charset=utf-8";
+ <% if @varnish_version4 -%>
+ synthetic ({"<!DOCTYPE html>
+ <% else -%>
synthetic {"<!DOCTYPE html>
+ <% end -%>
<html lang=en>
<meta charset=utf-8>
<title>Wikimedia Error</title>
@@ -26,10 +30,52 @@
<div class="footer">
<p>If you report this error to the Wikimedia System Administrators, please
include the details below.</p>
<p class="text-muted"><code>
-Request from "} + client.ip + " via " + server.hostname + " " +
server.identity + " ([" + server.ip + "]:" + server.port + "), Varnish XID " +
req.xid + "<br>" +
-regsub(req.http.X-Forwarded-For, ".+", "Forwarded for: \0<br>") +
regsub(obj.http.X-Cache, ".+", "Upstream caches: \0<br>") +
-"Error: " + obj.status + ", " + obj.response + " at " + now +
-{"
+Request from "} + <%= @bereq_req %>.http.X-Client-IP + " via " +
server.hostname + " " + server.identity + ", Varnish XID " + <%= @bereq_req
%>.xid + "<br>" + regsub(<%= @bereq_req %>.http.X-Forwarded-For, ".+",
"Forwarded for: \0<br>") + regsub(<%= @beresp_obj %>.http.X-Cache, ".+",
"Upstream caches: \0<br>") + "Error: " + <%= @beresp_obj %>.status + ", " + <%
if @varnish_version4 -%> <%= @beresp_obj %>.reason <% else -%> <%= @beresp_obj
%>.response <% end -%> + " at " + now + {"
</code></p></div></html>
+<% if @varnish_version4 -%>
+"});
+<% else -%>
"};
+<% end -%>
+}
+
+sub synth_errorpage {
+ set <%= @resp_obj %>.http.Content-Type = "text/html; charset=utf-8";
+ <% if @varnish_version4 -%>
+ synthetic ({"<!DOCTYPE html>
+ <% else -%>
+ synthetic {"<!DOCTYPE html>
+ <% end -%>
+<html lang=en>
+<meta charset=utf-8>
+<title>Wikimedia Error</title>
+<style>
+* { margin: 0; padding: 0; }
+body { background: #fff; font: 15px/1.6 sans-serif; color: #333; }
+.content { margin: 7% auto 0; padding: 2em 1em 1em; max-width: 560px; }
+.footer { clear: both; margin-top: 14%; border-top: 1px solid #e5e5e5;
background: #f9f9f9; padding: 2em 0; font-size: 0.8em; text-align: center; }
+img { float: left; margin: 0 2em 2em 0; }
+a img { border: 0; }
+h1 { margin-top: 1em; font-size: 1.2em; }
+p { margin: 0.7em 0 1em 0; }
+a { color: #0645AD; text-decoration: none; }
+a:hover { text-decoration: underline; }
+code { font-family: sans-serif; }
+.text-muted { color: #777; }
+</style>
+<div class="content" role="main">
+<a href="//www.wikimedia.org"><img
src="//www.wikimedia.org/static/images/wmf.png"
srcset="//www.wikimedia.org/static/images/wmf-2x.png 2x" alt=Wikimedia
width=135 height=135></a>
+<h1>Error</h1>
+<p>Our servers are currently experiencing a technical problem. This is
probably temporary and should be fixed soon.<br>Please <a href=""
title="Reload this page" onclick="window.location.reload(false); return
false">try again</a> in a few minutes.</p>
+</div>
+<div class="footer">
+<p>If you report this error to the Wikimedia System Administrators, please
include the details below.</p>
+<p class="text-muted"><code>
+Request from "} + req.http.X-Client-IP + " via " + server.hostname + " " +
server.identity + ", Varnish XID " + req.xid + "<br>" +
regsub(req.http.X-Forwarded-For, ".+", "Forwarded for: \0<br>") + regsub(<%=
@resp_obj %>.http.X-Cache, ".+", "Upstream caches: \0<br>") + "Error: " + <%=
@resp_obj %>.status + ", " + <% if @varnish_version4 -%> <%= @resp_obj
%>.reason <% else -%> <%= @resp_obj %>.response <% end -%> + " at " + now + {"
+</code></p></div></html>
+<% if @varnish_version4 -%>
+"});
+<% else -%>
+"};
+<% end -%>
}
diff --git a/templates/varnish/maps-backend.inc.vcl.erb
b/templates/varnish/maps-backend.inc.vcl.erb
index b9b4df6..fb20afb 100644
--- a/templates/varnish/maps-backend.inc.vcl.erb
+++ b/templates/varnish/maps-backend.inc.vcl.erb
@@ -4,5 +4,5 @@
sub cluster_be_hit { }
sub cluster_be_miss { }
sub cluster_be_pass { }
-sub cluster_be_fetch { }
+sub cluster_be_backend_response { }
sub cluster_be_deliver { }
diff --git a/templates/varnish/maps-frontend.inc.vcl.erb
b/templates/varnish/maps-frontend.inc.vcl.erb
index d1103cd..31745b7 100644
--- a/templates/varnish/maps-frontend.inc.vcl.erb
+++ b/templates/varnish/maps-frontend.inc.vcl.erb
@@ -9,7 +9,7 @@
&& req.http.referer !~
"(?i)^https?://(maps|phabricator|wikitech|incubator)\.wikimedia\.org/"
&& req.http.referer !~
"(?i)^https?://(localhost|127\.0\.0\.1)(:\d+)?/"
) {
- error 403 "Access Denied";
+ <%= error_synth(403, "Access Denied") %>
}
}
@@ -17,6 +17,6 @@
sub cluster_fe_hit { }
sub cluster_fe_miss { }
sub cluster_fe_pass { }
-sub cluster_fe_fetch { }
+sub cluster_fe_backend_response { }
sub cluster_fe_deliver { }
sub cluster_fe_err_synth { }
diff --git a/templates/varnish/misc-backend.inc.vcl.erb
b/templates/varnish/misc-backend.inc.vcl.erb
index b03f003..bd123f7 100644
--- a/templates/varnish/misc-backend.inc.vcl.erb
+++ b/templates/varnish/misc-backend.inc.vcl.erb
@@ -73,8 +73,8 @@
sub cluster_be_miss { }
sub cluster_be_pass { }
-sub cluster_be_fetch {
- call misc_fetch_large_objects;
+sub cluster_be_backend_response {
+ call misc_backend_response_large_objects;
}
sub cluster_be_deliver { }
diff --git a/templates/varnish/misc-common.inc.vcl.erb
b/templates/varnish/misc-common.inc.vcl.erb
index 0164add..d1d9816 100644
--- a/templates/varnish/misc-common.inc.vcl.erb
+++ b/templates/varnish/misc-common.inc.vcl.erb
@@ -18,7 +18,7 @@
}
}
-sub misc_fetch_large_objects {
+sub misc_backend_response_large_objects {
// Stream objects >= 1MB in size
if (std.integer(beresp.http.Content-Length, 1048576) >= 1048576 ||
beresp.http.Content-Length ~ "^[0-9]{8}") {
set beresp.do_stream = true;
diff --git a/templates/varnish/misc-frontend.inc.vcl.erb
b/templates/varnish/misc-frontend.inc.vcl.erb
index 5755d92..22d8a04 100644
--- a/templates/varnish/misc-frontend.inc.vcl.erb
+++ b/templates/varnish/misc-frontend.inc.vcl.erb
@@ -37,8 +37,8 @@
sub cluster_fe_miss { }
sub cluster_fe_pass { }
-sub cluster_fe_fetch {
- call misc_fetch_large_objects;
+sub cluster_fe_backend_response {
+ call misc_backend_response_large_objects;
}
sub cluster_fe_deliver { }
diff --git a/templates/varnish/text-backend.inc.vcl.erb
b/templates/varnish/text-backend.inc.vcl.erb
index fccd248..15d7640 100644
--- a/templates/varnish/text-backend.inc.vcl.erb
+++ b/templates/varnish/text-backend.inc.vcl.erb
@@ -78,7 +78,7 @@
call text_common_misspass_restore_cookie;
}
-sub cluster_be_fetch {
+sub cluster_be_backend_response {
// Make sure Set-Cookie responses are not cacheable, and log violations
// FIXME: exceptions are ugly; maybe get rid of the whole thing?
if (beresp.ttl > 0s && beresp.http.Set-Cookie &&
@@ -100,7 +100,7 @@
}
}
- call text_common_fetch;
+ call text_common_backend_response;
return (deliver);
}
diff --git a/templates/varnish/text-common.inc.vcl.erb
b/templates/varnish/text-common.inc.vcl.erb
index 53e3dcf..ea7f357 100644
--- a/templates/varnish/text-common.inc.vcl.erb
+++ b/templates/varnish/text-common.inc.vcl.erb
@@ -15,7 +15,7 @@
// otherwise remove the Cookie completely. In either case, in
// vcl_(pass|miss) on the way to any backend fetch, it will be
// restored to the original value.
- // Later in text_common_fetch, we'll create hit-for-pass objects for any
+ // Later in text_common_backend_response, we'll create hit-for-pass
objects for any
// response with Vary:Cookie and Cookie:Token=1. This will allow
// logged-in users to share the anonymous users' cache for
// non-Vary:Cookie responses, and then share a single hit-for-pass
@@ -98,13 +98,13 @@
}
// fe+be common fetch code
-sub text_common_fetch {
+sub text_common_backend_response {
if (req.http.X-Subdomain && (req.url ~ "mobileaction=" || req.url ~
"useformat=")) {
set beresp.ttl = 60 s;
}
// There are a couple different conditions here under which we set a
- // 601s hit-for-pass object based on response conditions in vcl_fetch:
+ // 601s hit-for-pass object based on response conditions in
vcl_backend_response:
// 1) Calculated TTL <= 0 + Status < 500 + No underlying cache hit:
// These are generally uncacheable responses. The 5xx exception
// avoids us accidentally replacing a good stale/grace object with
diff --git a/templates/varnish/text-frontend.inc.vcl.erb
b/templates/varnish/text-frontend.inc.vcl.erb
index 5ae41f0..203fb01 100644
--- a/templates/varnish/text-frontend.inc.vcl.erb
+++ b/templates/varnish/text-frontend.inc.vcl.erb
@@ -243,8 +243,8 @@
call text_common_misspass_restore_cookie;
}
-sub cluster_fe_fetch {
- call text_common_fetch;
+sub cluster_fe_backend_response {
+ call text_common_backend_response;
return (deliver);
}
diff --git a/templates/varnish/upload-backend.inc.vcl.erb
b/templates/varnish/upload-backend.inc.vcl.erb
index 10ec594..ffa25d2 100644
--- a/templates/varnish/upload-backend.inc.vcl.erb
+++ b/templates/varnish/upload-backend.inc.vcl.erb
@@ -51,7 +51,7 @@
sub cluster_be_pass { }
-sub cluster_be_fetch {
+sub cluster_be_backend_response {
// Stream large objects, >= 1 MB
if (std.integer(beresp.http.Content-Length, 0) >= 1048576 ||
beresp.http.Content-Length ~ "^[0-9]{9}") {
set beresp.do_stream = true;
diff --git a/templates/varnish/upload-frontend.inc.vcl.erb
b/templates/varnish/upload-frontend.inc.vcl.erb
index c6c8e51..6d49da2 100644
--- a/templates/varnish/upload-frontend.inc.vcl.erb
+++ b/templates/varnish/upload-frontend.inc.vcl.erb
@@ -49,7 +49,7 @@
sub cluster_fe_pass { }
-sub cluster_fe_fetch {
+sub cluster_fe_backend_response {
if (beresp.http.Content-Range) {
// Varnish itself doesn't ask for ranges, so this must have been
// a passed range request
--
To view, visit https://gerrit.wikimedia.org/r/273490
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I622f58e32878b6fa7a8578aae3e62bd448e4baac
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ema <e...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
