Ema has uploaded a new change for review. https://gerrit.wikimedia.org/r/269466
Change subject: WIP: Maps VCL forward-porting to Varnish 4 ...................................................................... WIP: Maps VCL forward-porting to Varnish 4 There is still quite a lot to do, see comments marked with TODO. However, this is a good point to upload everything to gerrit, mostly to get some initial feedback (and as a backup of my work so far). :-) Bug: T124279 Change-Id: Iee05d5f712093c0a1d939e74a340627982979404 --- M modules/varnish/manifests/common/vcl.pp M modules/varnish/manifests/instance.pp A modules/varnish/templates/vcl/wikimedia_v4.vcl.erb A templates/varnish/analytics_v4.inc.vcl.erb A templates/varnish/errorpage_v4.inc.vcl.erb A templates/varnish/maps-backend_v4.inc.vcl.erb A templates/varnish/maps-frontend_v4.inc.vcl.erb 7 files changed, 1,011 insertions(+), 6 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/66/269466/1 diff --git a/modules/varnish/manifests/common/vcl.pp b/modules/varnish/manifests/common/vcl.pp index cff2aff..d79b164 100644 --- a/modules/varnish/manifests/common/vcl.pp +++ b/modules/varnish/manifests/common/vcl.pp @@ -1,6 +1,13 @@ class varnish::common::vcl { require varnish::common + if hiera('varnish_version4', false) { + $vcl_version_suffix = '_v4' + } + else { + $vcl_version_suffix = '' + } + file { '/etc/varnish/geoip.inc.vcl': owner => 'root', group => 'root', @@ -8,18 +15,18 @@ content => template('varnish/geoip.inc.vcl.erb'), } - file { '/etc/varnish/errorpage.inc.vcl': + file { "/etc/varnish/errorpage${vcl_version_suffix}.inc.vcl": owner => 'root', group => 'root', mode => '0444', - content => template('varnish/errorpage.inc.vcl.erb'), + content => template("varnish/errorpage${vcl_version_suffix}.inc.vcl.erb"), } - file { '/etc/varnish/analytics.inc.vcl': + file { "/etc/varnish/analytics${vcl_version_suffix}.inc.vcl": owner => 'root', group => 'root', mode => '0444', - content => template('varnish/analytics.inc.vcl.erb'), + content => template("varnish/analytics${vcl_version_suffix}.inc.vcl.erb"), } # VCL unit tests diff --git a/modules/varnish/manifests/instance.pp b/modules/varnish/manifests/instance.pp index dfeefd0..f11857b 100644 --- a/modules/varnish/manifests/instance.pp +++ b/modules/varnish/manifests/instance.pp @@ -24,6 +24,13 @@ $extraopts = "-n ${name}" } + if hiera('varnish_version4', false) { + $vcl_version_suffix = '_v4' + } + else { + $vcl_version_suffix = '' + } + # Initialize variables for templates $backends_str = inline_template("<%= @directors.map{|k,v| v['backends'] }.flatten.join('|') %>") $varnish_backends = sort(unique(split($backends_str, '\|'))) @@ -74,14 +81,14 @@ group => 'root', mode => '0444', require => File["/etc/varnish/${vcl}.inc.vcl"], - content => template("${module_name}/vcl/wikimedia.vcl.erb"), + content => template("${module_name}/vcl/wikimedia${vcl_version_suffix}.vcl.erb"), } file { "/etc/varnish/${vcl}.inc.vcl": owner => 'root', group => 'root', mode => '0444', - content => template("varnish/${vcl}.inc.vcl.erb"), + content => template("varnish/${vcl}${vcl_version_suffix}.inc.vcl.erb"), notify => Exec["load-new-vcl-file${instancesuffix}"], } diff --git a/modules/varnish/templates/vcl/wikimedia_v4.vcl.erb b/modules/varnish/templates/vcl/wikimedia_v4.vcl.erb new file mode 100644 index 0000000..f573d48 --- /dev/null +++ b/modules/varnish/templates/vcl/wikimedia_v4.vcl.erb @@ -0,0 +1,680 @@ +# This file is managed by Puppet! + +vcl 4.0; +import std; +import directors; + +# this is needed by geoip.inc.vcl and zero.inc.vcl, and in general is the only +# way to sanely do Set-Cookie in the face of multiple independent cookies +# being set from different code. +import header; + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> +# only used in recv_fe_ip_processing on frontends +import netmapper; +// TODO: possible replacement for chash in varnish 4 +import vslp; +<% end %> + +<% +def backend_option(backend, option, default) + if @varnish_backend_options.kind_of?(Array) + # List of hashes of options, 'backend_match' key is a regexp against the FQDN + @varnish_backend_options.each do |be_options| + if Regexp.new(be_options.fetch("backend_match", "^.*$")).match(backend) + if be_options.has_key?(option) + return be_options[option] + end + end + end + return default + else + return @varnish_backend_options.fetch(option, default) + end +end + +# Calculates number of director-level retries necessary for chash to hit all +# "n" backends with probability percentage "p", given they're randomly-mixed +# into an array considerably larger in size than "n". This is an +# overestimation in that it assumes an infinite array, but the values still +# come out reasonably small compared to doing anything based on our actual +# weight*num_backends. +# Blame _joe_ for the math! :) +def chash_def_retries(p, n) + x = n - 1 + if (x <= 0) + return n + end + return ((Math.log10(100 - p) - 2) / (Math.log10(x) - Math.log10(n))).ceil +end +-%> + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> +// defines analytics_(recv|deliver) subs +include "analytics_v4.inc.vcl"; +<% end -%> + +# ACLs + +acl local_host { + "127.0.0.1"; + "<%= @ipaddress %>"; // note this matches nginx proxy_pass for TLS +} + +acl wikimedia_nets { +<% scope.lookupvar('::network::constants::all_networks_lo').each do |entry| + subnet, mask = entry.split("/", 2) +-%> + "<%= subnet %>"/<%= mask %>; +<% end -%> +} + +# Backend probes + +# frontends in front of other varnish instances should send +# probes that don't depend on the app backend + +# TODO: unused probe according to varnish 4. Commeting it out for now. +#probe varnish { +# .request = +# "GET /check HTTP/1.1" +# "Host: varnishcheck" +# "User-agent: Varnish backend check" +# "Connection: close"; +# .timeout = 500ms; +# .interval = 100ms; +# .window = 3; +# .threshold = 2; +#} + +# TODO: unused probe according to varnish 4. Commeting it out for now. +#probe logstash { +# .url = "/status"; +# .interval = 5s; +# .timeout = 1s; +# .window = 5; +# .threshold = 3; +#} + +probe maps { + .url = "/_info"; + .interval = 5s; + .timeout = 1s; + .window = 5; + .threshold = 3; +} + +# TODO: unused probe according to varnish 4. Commeting it out for now. +#probe wdqs { +# .url = "/"; +# .interval = 5s; +# .timeout = 1s; +# .window = 5; +# .threshold = 3; +#} + +# Backends + +# List of Puppet generated backends +<% +@varnish_backends.each do |backend| + name = /^[0-9\.]+$/.match(backend) ? "ipv4_" + backend.gsub(".", "_") : "be_" + backend.split(".")[0].gsub("-", "_") + probe = backend_option(backend, "probe", nil) +-%> +backend <%= name %> { + .host = "<%= backend %>"; + .port = "<%= backend_option(backend, "port", "80") %>"; + .connect_timeout = <%= backend_option(backend, "connect_timeout", "2s") %>; + .first_byte_timeout = <%= backend_option(backend, "first_byte_timeout", "35s") %>; + .between_bytes_timeout = <%= backend_option(backend, "between_bytes_timeout", "2s") %>; + .max_connections = <%= backend_option(backend, "max_connections", "100") %>; +<% if probe -%> + .probe = <%= probe %>; +<% end -%> +} + +<% end -%> + +<% +# Expected directors data format: (all keys required!) +# @varnish_directors = { +# 'director name' => { +# 'dynamic' => 'yes', # or 'no' +# 'type' => 'chash', +# 'backends' => [ "backend1", "backend2" ], +# } +# } +if @use_dynamic_directors and @dynamic_directors -%> +include "directors.<%= @inst %>.vcl"; + +<% end -%> + +sub vcl_init { +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + // again, netmapper only used in frontends, for recv_fe_ip_processing + // args here are map-name (for .map()), data file, and seconds between mtime checks for reload + netmapper.init("proxies", "/var/netmapper/proxies.json", 89); + netmapper.init("carriers", "/var/netmapper/carriers.json", 89); +<% end %> + +<% @varnish_directors.keys.sort.each do |director_name| +director = @varnish_directors[director_name] +if (!@dynamic_directors or director['dynamic'] != 'yes') + backends = director['backends'] + if (!backends.empty?) +-%> + # See https://www.varnish-cache.org/docs/trunk/whats-new/upgrade-4.0.html#directors-have-been-moved-to-the-vmod-directors + new <%= director_name %> = directors.<%= director['type'] %>(); + + // director <%= director_name %> <%= director['type'] %> { +<% if director['type'] == 'chash' -%> + // .retries = <%= chash_def_retries(99, backends.size) %>; + <% + # TODO: vslp might replace chash. Add init_hashcircle(). + director['type'] = 'vslp' + -%> +<% end -%> +<% + backends.each do |backend| + name = /^[0-9\.]+$/.match(backend) ? "ipv4_" + backend.gsub(".", "_") : "be_" + backend.split(".")[0].gsub("-", "_") +-%> + <%= director_name %>.add_backend(<%= name %>, <%= backend_option(backend, "weight", 10) %>); + + //{ + // .backend = <%= name %>; + // .weight = <%= backend_option(backend, "weight", 10) %>; + //} +<% end -%> +//} +<% end #if !empty -%> +<% end #if !dynamic -%> +<% end #director loop -%> +} # end vcl_init + +# Functions + +// start frontend-only block for HTTPS +<% if @vcl_config.fetch("layer", "") == "frontend" && @vcl_config.fetch("https_redirects", false) -%> + +// *** HTTPS recv code - domain-based 301/302->HTTPS decisions happen here +// if GET/HEAD filter is modified/removed later, keep in mind we need to not affect +// the PURGE traffic here, as purge is called after this. +sub https_recv_redirect { + if (req.http.X-Forwarded-Proto != "https") { + if (req.method == "GET" || req.method == "HEAD") { + // This is all of our unified cert wildcard domains which are TLS-clean (cert matches all extant hostnames within) + // The lone exception now is wikimedia.org, in the next block + if (req.http.Host ~ "(?i)((^|\.)(wikipedia|wikibooks|wikinews|wikiquote|wikisource|wikiversity|wikivoyage|wikidata|wikimediafoundation|wiktionary|mediawiki)\.org|^w\.wiki)$") { + set req.http.Location = "https://" + req.http.Host + req.url; + return (synth(751, "TLS Redirect")); + } + // wikimedia.org has multi-level subdomains used for HTTP for which we have no certs, so they must be avoided here: + // Ref: T102826 + T102827 + else if(req.http.Host ~ "(?i)^([^.]+\.)?(m\.)?wikimedia\.org$") { + set req.http.Location = "https://" + req.http.Host + req.url; + return (synth(751, "TLS Redirect")); + } + } +<% if @vcl_config.fetch("secure_post", true) -%> + if (req.method == "POST" && !(req.http.Host ~ "(?i)\.beta\.wmflabs\.org$")) { + return (synth(403, "Insecure POST Forbidden - use HTTPS")); + } +<% end %> + } +} + +// *** HTTPS error code - implements 301 response for recv code +sub https_error_redirect { + if (obj.status == 751) { + set obj.http.Location = req.http.Location; + set obj.status = 301; + set obj.http.Content-Length = "0"; // T64245 + return(deliver); + } +} + +// *** HTTPS deliver code - domain-based HSTS headers +sub https_deliver_hsts { + // The reason we don't need the stricter domain restrictions here, + // like we do on the recv side for redirects, is that in order for + // HSTS to reach a client, the client implicitly has to have already + // successfully reached us over HTTPS for the given domainname. + if (req.http.X-Forwarded-Proto == "https") { + // This is the same regex as the first one in https_recv_redirect (all unified except wikimedia.org) + if (req.http.Host ~ "(?i)((^|\.)(wikipedia|wikibooks|wikinews|wikiquote|wikisource|wikiversity|wikivoyage|wikidata|wikimediafoundation|wiktionary|mediawiki)\.org|^w\.wiki)$") { + set resp.http.Strict-Transport-Security = "max-age=31536000; includeSubDomains; preload"; + } + else { + set resp.http.Strict-Transport-Security = "max-age=31536000"; + } + } +} + +<% end -%> +// ^ end frontend + https_redirects block + +// We shouldn't even legally be receiving proxy-style requests, as we're not a +// proxy from any client's point of view. Just in case, we support it anyways +// according to RFC7230 rules: we ignore any Host header sent along with it +// and set a new Host header based on the host part we strip from the abs URI. +// ref: http://tools.ietf.org/html/rfc7230#section-5.4 + +// TODO: varnish4 says this is unused. Commenting out for now. + +//sub rewrite_proxy_urls { +// if(req.url ~ "(?i)^https?://[^/]") { +// set req.http.Host = regsub(req.url, "(?i)^https?://([^/]+).*$", "\1"); +// set req.url = regsub(req.url, "(?i)^https?://[^/]+", ""); +// } +//} + +sub recv_purge { + /* Support HTTP PURGE */ + if (req.method == "PURGE") { + if (client.ip !~ local_host) { + return (synth(405, "Denied.")); + } elsif (req.http.Host ~ "<%= @vcl_config.fetch('purge_host_regex') %>") { + set req.hash_ignore_busy = true; + return (hash); + } else { + return (synth(204, "Domain not cached here.")); + } + } +} + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> +// Must be done at the top of vcl_recv, in our varnish-frontend layer only, +// and should be guarded against running on request restarts. +sub recv_fe_ip_processing { + // this subroutine "owns" these 3 headers - nothing else in our VCL or + // anywhere in our network should be setting them. + unset req.http.X-Trusted-Proxy; + unset req.http.X-Carrier; + unset req.http.X-Carrier-Meta; + + // unset this one just because it's well-known and some default + // software configs may look at it, and an external client may spoof + // it. We don't set or use this header internally (we use X-Client-IP) + unset req.http.X-Real-IP; + + if (client.ip !~ wikimedia_nets) { + // Ensure we only accept XFP from our own networks. Ideally + // it should only be set by our nginx TLS terminator + // specifically, but there are known cases where internal apps + // set XFP to fake HTTPS when making a request to our public + // endpoints from the inside. + unset req.http.X-Forwarded-Proto; + } + + if (client.ip !~ local_host) { + // only the local nginx TLS terminator should set this one at + // all - there are no other internal exceptions to that rule + unset req.http.X-Client-IP; + } + + if (req.http.X-Forwarded-For) { + // To make further parsing/sanitizing simpler, convert all whitespace + // in XFF to single spaces, and make sure all commas have a space + // suffix but no space prefix. + set req.http.X-Forwarded-For = regsuball(req.http.X-Forwarded-For, "[ \t]+", " "); + set req.http.X-Forwarded-For = regsuball(req.http.X-Forwarded-For, " ?, ?", ", "); + + // Now fully-sanitize it to only the strict form "X(, X)*", where X is + // a string of legal characters in IPv[46] addresses. Note + // that injections can still leave well-formed junk on the + // left, but it's up to the trusted proxy code to ignore that, + // e.g.: + // "junk2, 123.123.123.123" -> "2, 123.123.123.123" + set req.http.X-Forwarded-For = regsub(req.http.X-Forwarded-For, + "^.*?([0-9A-Fa-f:.]+(, [0-9A-Fa-f:.]+)*)? ?$", "\1"); + + // Clear header if empty after all the above, to avoid messing + // up our normal XFF-append code later + if (req.http.X-Forwarded-For == "") { + unset req.http.X-Forwarded-For; + } + } + + // There are two possible cases here: either nginx acted as our TLS + // proxy and already set X-Client-IP (as well as appended the same value + // to XFF), or the traffic was direct to varnish-fe, in which case + // XCIP is not yet set and XFF is directly from external. + if (!req.http.X-Client-IP) { + // direct-to-port-80 case, set XCIP ourselves + set req.http.X-Client-IP = client.ip; + set req.http.X-Trusted-Proxy = netmapper.map("proxies", req.http.X-Client-IP); + // normalize to boolean post-netmapper (varnish-3.0.4...) + if (req.http.X-Trusted-Proxy == "") { + unset req.http.X-Trusted-Proxy; + } + if (req.http.X-Trusted-Proxy && req.http.X-Forwarded-For) { + // get last from trusted-proxy-supplied XFF + set req.http.maybe-xcip = regsub(req.http.X-Forwarded-For, "^([^,]+, )+", ""); + if(std.ip(req.http.maybe-xcip, "127.0.0.1") !~ wikimedia_nets) { + set req.http.X-Client-IP = req.http.maybe-xcip; + } + unset req.http.maybe-xcip; + } + } else { + // XCIP from nginx, XFF set/appended by nginx and contains at + // least XCIP at the end, possibly prepended by other addrs + // set externally by some proxy. + set req.http.X-Trusted-Proxy = netmapper.map("proxies", req.http.X-Client-IP); + // normalize to boolean post-netmapper (varnish-3.0.4...) + if (req.http.X-Trusted-Proxy == "") { + unset req.http.X-Trusted-Proxy; + } + if (req.http.X-Trusted-Proxy) { + // We want the second-to-last XFF entry here, assuming + // there's two or more IPs. Note that with the + // regsub's below if there was only one (which would + // alias XCIP by definition), there would be no commas + // to match and XCIP gets reset to its original value. + set req.http.maybe-xcip = regsub(req.http.X-Forwarded-For, ", [^,]+$", ""); + set req.http.maybe-xcip = regsub(req.http.maybe-xcip, "^([^,]+, )+", ""); + if(std.ip(req.http.maybe-xcip, "127.0.0.1") !~ wikimedia_nets) { + set req.http.X-Client-IP = req.http.maybe-xcip; + } + unset req.http.maybe-xcip; + } + } + + // Now check carrier database for setting X-Carrier based on XCIP + set req.http.X-Carrier = netmapper.map("carriers", req.http.X-Client-IP); + // normalize to boolean post-netmapper (varnish-3.0.4...) + if (req.http.X-Carrier == "") { + unset req.http.X-Carrier; + } + else { + // Split X-Carrier data from raw form with optional trailing metadata, + // such as "123-45|wap|mobile", so that X-Carrier contains only + // MCC-MNC and X-Carrier-Meta contains the trailing attributes + set req.http.X-Carrier-Meta = regsub(req.http.X-Carrier, "^[^|]*\|?", ""); + if (req.http.X-Carrier-Meta != "") { + set req.http.X-Carrier = regsub(req.http.X-Carrier, "\|.*$", ""); + } + else { + unset req.http.X-Carrier-Meta; + } + } + + // From this (very early) point forward, regardless of cache tier/layer: + // req.http.X-Client-IP -> + // This is our standard notion of the Client/UA's real IP, after + // decoding XFF for our internal infrastructure addresses as well + // as any trusted proxies. + // req.http.X-Trusted-Proxy -> + // If the traffic pass through a trusted proxy in our "proxies" + // database (such as OperaMini), this will be the official name of + // the trusted proxy. Otherwise it will be unset (boolean false). + // req.http.X-Carrier -> + // If X-Client-IP matches a network in our "carriers" database, + // this will contain the MCC-MNC code for that carrier. Otherwise + // it will be undefined. + // req.http.X-Carrier-Meta -> + // If X-Carrier is defined: for some carriers, the database + // contains extra metadata in the form of one or more labels like + // "wap" or "residential". They'll be separated by "|" if more + // than one, and this header is undefined if there was no such + // metadata. +} + +<% end %> + +sub vcl_recv { + unset req.http.X-CDIS; // clear internal cache-disposition header + // IP processing is req->req mangling that shouldn't be re-done on + // restart, and XFF-appending is non-idempotent for restart purposes.. + if (req.restarts == 0) { +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + call recv_fe_ip_processing; +<% end %> + // All layers need to update XFF with client.ip hop-by-hop so that it + // looks right to layers beneath, including the app layer + if (req.http.X-Forwarded-For) { + set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip; + } else { + set req.http.X-Forwarded-For = client.ip; + } + } + +<% if @vcl_config.fetch("layer", "") != "frontend" -%> + if (client.ip !~ wikimedia_nets) { + // Do not allow direct access to non-frontend layers + return (synth(403, "Access denied")); + } +<% end -%> + + if (req.method !~ "<%= @vcl_config.fetch("allowed_methods", "^(GET|HEAD|POST|PURGE)$") %>" + && !(req.method == "OPTIONS" && req.http.Origin)) { + return (synth(403, "HTTP method not allowed.")); + } + + <% if @vcl_config.fetch("has_def_backend", "yes") == "yes" -%> + /* Select the default backend/director, which is always the one named 'backend'. + * If an instance has no default 'backend', it must declare has_def_backend==no, + * and its own VCL must handle all possible req.backend_hint cases. + */ + set req.backend_hint = backend.backend(); + + if (std.healthy(req.backend_hint)) { + # + # This is now handled in vcl_hit. + # + # set req.grace = 5m; + } else { + # + # This is now handled in vcl_hit. + # + # set req.grace = 60m; + } + <% end -%> + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + call rewrite_proxy_urls; +<% end -%> + +<% if @vcl_config.fetch("layer", "") == "frontend" && @vcl_config.fetch("https_redirects", false) -%> + call https_recv_redirect; +<% end -%> + + if ( req.http.host ~ "^varnishcheck" ) { + return (synth(200, "OK")); + } + + if (req.url ~ "^/beacon\/[^/?]+") { + // Logging beacon endpoints + // + // They are handled by log tailers (varnishkafka and varnishncsa) that filter the + // Varnish shm log for reqs to these endpoints and forward them to log processors + // for storage and analysis. + return (synth(204)); + } + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + if(req.restarts == 0) { + call analytics_recv; + } +<% end -%> + /* Function vcl_recv in <%= @vcl %>.inc.vcl will be appended here */ +} + +sub vcl_backend_response { + // default hard cap of max 30d life on all cache objects everywhere + if (beresp.ttl > 30d) { + set beresp.ttl = 30d; + } + + /* Don't cache private, no-cache, no-store objects */ + if (beresp.http.Cache-Control ~ "(private|no-cache|no-store)") { + set beresp.ttl = 0s; + /* This should be translated into hit_for_pass later */ + } + elsif (beresp.status >= 400 && beresp.status <= 499 && beresp.ttl > <%= @vcl_config.fetch("cache4xx", "5m") %>) { + set beresp.ttl = <%= @vcl_config.fetch("cache4xx", "5m") %>; + } + + set beresp.grace = 60m; + +<% if @vcl_config.fetch("do_gzip", false) -%> + // Compress compressible things if the backend didn't already + if (beresp.http.content-type ~ "json|text|html|script|xml|icon|ms-fontobject|ms-opentype|x-font") { + set beresp.do_gzip = true; + } +<% end -%> + + /* Function vcl_backend_response in <%= @vcl %>.inc.vcl will be appended here */ +} + +sub vcl_hit { + set req.http.X-CDIS = "hit"; + if (req.method == "PURGE") { + # TODO + return (synth(204, "Purged")); + } + + /* Function vcl_hit in <%= @vcl %>.inc.vcl will be appended here */ +} + +sub vcl_miss { + set req.http.X-CDIS = "miss"; + if (req.method == "PURGE") { + # TODO + return (synth(204, "Cache miss")); + } + + /* Function vcl_miss in <%= @vcl %>.inc.vcl will be appended here */ +} + +sub vcl_pass { + if (req.http.X-CDIS) { + // _pass can theoretically be called after moving through _hit or _miss + set req.http.X-CDIS = req.http.X-CDIS + "+pass"; + } else { + set req.http.X-CDIS = "pass"; + } + +// All cache clusters are dual-tier/layer, and all tier-two backends and all +// frontends have exactly two backends: "backend" and "backend_random". The +// regular backend is a chash on the URL to help with cacheable objects, +// whereas the randomized one avoids focusing pass/hit-for-pass traffic onto a +// single node in the varnish-backend layers. All pass and hit-for-pass +// traffic comes through vcl_pass, so we set backend_random here for all such +// requests. +// Note parsoid is specially excluded, because it's legacy / non-standard. +<% if scope.function_hiera(["cluster"]) != "cache_parsoid" %> +<% if @vcl_config.fetch("layer", "") == "frontend" || @site_tier == "two" -%> + set req.backend_hint = backend_random; +<% end -%> +<% end -%> +} + +sub vcl_deliver { +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + std.collect(resp.http.Via); + std.collect(resp.http.X-Varnish); + + // Set CP ('Connection Properties') cookie + if (req.http.X-Connection-Properties ~ "SPDY=3") { + if (req.http.X-Orig-Cookie !~ "(^|;\s*)CP=H2" && req.http.Cookie !~ "(^|;\s*)CP=H2") { + header.append(resp.http.Set-Cookie, "CP=H2; Path=/"); + } + } else { + // Explicitly unset the cookie if it exists. Support for SPDY in a browser session can + // flip if a device moves networks and thus behind a proxy. + if (req.http.X-Orig-Cookie ~ "(^|;\s*)CP=H2" || req.http.Cookie ~ "(^|;\s*)CP=H2") { + header.append(resp.http.Set-Cookie, "CP=H1; Expires=Thu, 01-Jan-1970 00:00:01 GMT; Path=/"); + } + } + +<% end -%> + + if (!req.http.X-CDIS) { + set req.http.X-CDIS = "int"; // internally-generated response (not a cache object hit, and not a miss|pass to a deeper layer either) + } + if (resp.http.X-Cache) { + set resp.http.X-Cache = resp.http.X-Cache + ", <%= @hostname + (@name.empty? ? "" : " " + @name) %> " + req.http.X-CDIS + "(" + obj.hits + ")"; + } else { + set resp.http.X-Cache = "<%= @hostname + (@name.empty? ? "" : " " + @name) %> " + req.http.X-CDIS + "(" + obj.hits + ")"; + } + +<% if @vcl_config.fetch("layer", "") == "frontend" && @vcl_config.fetch("https_redirects", false) -%> + call https_deliver_hsts; +<% end -%> + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + call analytics_deliver; +<% end -%> + +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + // echo metadata about the client back to the client (analytics looks at this as well) + set resp.http.X-Client-IP = req.http.X-Client-IP; + // note mobile apps look at X-C + X-C-M below + if (req.http.X-Carrier) { + set resp.http.X-Carrier = req.http.X-Carrier; + if (req.http.X-Carrier-Meta) { + set resp.http.X-Carrier-Meta = req.http.X-Carrier-Meta; + } + } +<% end -%> + + /* Function vcl_deliver in <%= @vcl %>.inc.vcl will be appended here */ +} + +sub vcl_backend_error { +<% if @vcl_config.fetch("layer", "") == "frontend" && @vcl_config.fetch("https_redirects", false) -%> + call https_error_redirect; +<% end -%> + + if (beresp.status == 400 || beresp.status == 413) { + return(deliver); + } + +<% if scope.function_hiera(["cluster"]) != "cache_parsoid" -%> +<% if @vcl_config.fetch("layer", "") == "frontend" -%> + // retry 503 once in frontend instances, to paper over transient issues + if (beresp.status == 503 && bereq.retries == 0) { + return (retry); + } +<% end -%> +<% end -%> + + if (beresp.status == 204 && bereq.method == "PURGE") { + set beresp.http.Connection = "keep-alive"; + } + + /* Function vcl_error in <%= @vcl %>.inc.vcl will be appended here */ +} + +/* Include the VCL file for this role */ +include "<%= @vcl %>.inc.vcl"; + +#Generated by varnish3to4 +#TODO: do we really need to duplicate this here? +#sub vcl_synth { +#<% if @vcl_config.fetch("layer", "") == "frontend" && @vcl_config.fetch("https_redirects", false) -%> +# call https_error_redirect; +#<% end -%> +# +# if (resp.status == 400 || resp.status == 413) { +# return(deliver); +# } +# +#<% if scope.function_hiera(["cluster"]) != "cache_parsoid" -%> +#<% if @vcl_config.fetch("layer", "") == "frontend" -%> +# // retry 503 once in frontend instances, to paper over transient issues +# if (resp.status == 503 && req.restarts == 0) { +# return(restart); +# } +#<% end -%> +#<% end -%> +# +# if (resp.status == 204 && req.method == "PURGE") { +# set resp.http.Connection = "keep-alive"; +# } +# +# /* Function vcl_error in <%= @vcl %>.inc.vcl will be appended here */ +#} +# +#/* Include the VCL file for this role */ +#include "<%= @vcl %>.inc.vcl"; diff --git a/templates/varnish/analytics_v4.inc.vcl.erb b/templates/varnish/analytics_v4.inc.vcl.erb new file mode 100644 index 0000000..edd912b --- /dev/null +++ b/templates/varnish/analytics_v4.inc.vcl.erb @@ -0,0 +1,200 @@ +/***************************************************************************** + * Varnish VCL for WMF-Last-Access Cookie + * Please see what this cookie is trying to acomplish: + * https://wikitech.wikimedia.org/wiki/Analytics/Unique_clients/Last_visit_solution + * + * General notes on timestamp format strings used here: + * "now" stringifies as "Wed, 01 Jan 2000 01:01:01 GMT", which is the same + * format used by Set-Cookie "Expires" data. The format for the last access + * value, and thus X-NowDay and X-WMF-LastStamp as well, is "01-Jan-2000" + * (because the other info is redundant or too-specific, and cookie values + * shouldn't have whitespace or commas). + ****************************************************************************/ + +/***************************************************************************** + * This must be called *before* any vcl_recv cookie munging. It more-properly + * belongs in _deliver, but putting it here avoids all of the issues + * surrounding consistent access to Cookie vs X-Orig-Cookie in vcl_deliver + * It does so at the cost of sending a pointless and unintended + * "X-WMF-LastStamp: 01-Jan-2000" header to the application layer as well on + * cache miss/bypass. + * Note we don't validate that the cookie's 3-letter month abbreviation is + * legal, or that the numeric values for the date/year are legal, just that + * they have the right count of the right kinds of characters. + ****************************************************************************/ +sub analytics_last_access_recv_ { + unset req.http.X-WMF-LastStamp; // clear any sent by the user + if (req.http.Cookie ~ "(^|;\s*)WMF-Last-Access=[0-9]{2}-[A-Za-z]{3}-[0-9]{4}(;|$)") { + // Save the value for use later in _deliver + set req.http.X-WMF-LastStamp = regsub( + req.http.Cookie, + "^(?:.*;\s*)?WMF-Last-Access=([^;]+).*$", + "\1" + ); + } +} + +/***************************************************************************** + * !!! private to analytics_last_access_deliver !!!! + * This should be: + * header.append(resp.http.Set-Cookie, + * "WMF-Last-Access=" + * + req.http.X-NowDay + * + ";Path=/;HttpOnly;Expires=" + * + (now + 32d) + * ); + * However, varnish3 is buggy wrt str + (time + duration), so we're forced to + * drop to inline C a bit here and do what the VCL compiler should have done + * for us above. On top of all that, the C code now floors the expiry to the + * next-lower 12 hour mark, which would've been a bit trickier in VCL... + ****************************************************************************/ + +sub set_last_access_cookie__ {} + +/* + +TODO: Inline C code not allowed in Varnish 4 + +C{#include <time.h>}C +sub set_last_access_cookie__ { C{ + Vmod_Func_header.append(sp, HDR_RESP, "\013Set-Cookie:", + "WMF-Last-Access=", + VRT_GetHdr(sp, HDR_REQ, "\011X-NowDay:"), + ";Path=/;HttpOnly;Expires=", + VRT_time_string(sp, (double)( + ((time_t)VRT_r_now(sp) + 2764800) / 43200 * 43200 + )), + vrt_magic_string_end + ); +}C } +*/ + +// Call from vcl_deliver near other X-Analytics code +sub analytics_last_access_deliver_ { + // Create X-NowDay in "01-Jan-2000" form, from "now" + set req.http.X-NowDay = regsub( + now, "^..., (..) (...) (....) .*$", "\1-\2-\3" + ); + + if(req.http.X-WMF-LastStamp) { + set resp.http.X-Analytics = resp.http.X-Analytics + + ";WMF-Last-Access=" + + req.http.X-WMF-LastStamp; + + // re-set the cookie if it's not from today + if (req.http.X-NowDay != req.http.X-WMF-LastStamp) { + call set_last_access_cookie__; + } + + } + else { + // sets the initial cookie if no valid one existed + call set_last_access_cookie__; + } + + // we could clean up req.http.X-WMF-LastStamp + req.http.X-NowDay + // here, but they're not being sent anywhere (else) at this point + // anyways, so why bother? +} + +/***************************************************************************** + * Analytics for "wprov" Provenance data + * See https://www.mediawiki.org/wiki/Provenance for reserved values. + ****************************************************************************/ + +sub analytics_provenance_recv_ { + // Avoid cache fragmentation for well-formed provenance parameters + // Refer to discussion starting from + // https://lists.wikimedia.org/pipermail/analytics/2015-February/003426.html + // Look for wprov parameter with a value + if (req.url ~ "(?i)[?&]wprov=[^&]+") { + // Ready a variable for later X-Analytics tagging in vcl_deliver. + + // Grab just the value of the wprov parameter, excluding the rest of the URL + set req.http.X-WMF-WPROV = regsub(req.url, "(?i).+[?&]wprov=([^&]+).*", "\1"); + + // Remove the wprov=X parameter from req.url to avoid cache + // fragmentation using two regexes to cover distinct cases: + + // (1) Simple strip if final query arg: + set req.url = regsub(req.url, "(?i)[?&]wprov=[^&]+$", ""); + + // (2) When not the final arg, we need to capture the leading + // [?&] to reuse with the parameter that follows: + set req.url = regsub(req.url, "(?i)([?&])wprov=[^&]+&", "\1"); + } +} + +sub analytics_provenance_deliver_ { + // In case there was a provenance parameter with a value, add it to X-Analytics + if (req.http.X-WMF-WPROV) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";wprov=" + req.http.X-WMF-WPROV; + } +} + +/***************************************************************************** + * Combined analytics recv and deliver hooks, to be included directly in + * vcl_recv and vcl_deliver in common wikimedia.vcl - these are the only + * "public" interfaces in this file! + ****************************************************************************/ + +sub analytics_recv { + // If this request had no cookies whatsoever mark it as such + // to later report this fact to X-Analytics + if (!req.http.Cookie) { + set req.http.X-WMF-NOCOOKIES = 1; + } + + call analytics_last_access_recv_; + call analytics_provenance_recv_; +} + +sub analytics_deliver { + // Create empty header if none, to avoid tons of if/else clauses; will + // clean up at the end. Note that if we defined one of the k=v pairs as + // required (having a real value for the false/negative case), we could + // set that one first and this would get a bit cleaner... + if (!resp.http.X-Analytics) { + set resp.http.X-Analytics = ""; + } + + call analytics_last_access_deliver_; + call analytics_provenance_deliver_; + + if (req.http.X-Carrier) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";zero=" + req.http.X-Carrier; + if (req.http.X-Carrier-Meta) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";zeronet=" + req.http.X-Carrier-Meta; + } + } + + if (req.http.X-Trusted-Proxy) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";proxy=" + req.http.X-Trusted-Proxy; + } + + if (req.http.X-Forwarded-Proto) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";https=1"; + } + + if (req.http.X-WMF-UUID) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";wmfuuid=" + req.http.X-WMF-UUID; + } + + // Add proxy=IORG X-Analytics tag if appropriate. + // Although Via: Internet.org usually comes via proxying, it isn't guaranteed to come that way. + // Nonetheless, as it is tagged with Via and the equipment is under Internet.org, we proxy tag. + if (req.http.Via ~ "(?i)Internet\.org") { + set resp.http.X-Analytics = resp.http.X-Analytics + ";proxy=IORG"; + } + + if (req.http.X-WMF-NOCOOKIES) { + set resp.http.X-Analytics = resp.http.X-Analytics + ";nocookies=1"; + } + + // Clean up header from setting to empty at the start... + if (resp.http.X-Analytics == "") { + unset resp.http.X-Analytics; + } else { + set resp.http.X-Analytics = regsub(resp.http.X-Analytics, "^;", ""); + } +} diff --git a/templates/varnish/errorpage_v4.inc.vcl.erb b/templates/varnish/errorpage_v4.inc.vcl.erb new file mode 100644 index 0000000..845440b --- /dev/null +++ b/templates/varnish/errorpage_v4.inc.vcl.erb @@ -0,0 +1,47 @@ +<% + # Source: https://www.wikimedia.org/static/images/wmf.png + wmf_png = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIcAAACHCAYAAAA850oKAAAABmJLR0QA/wD/AP+gvaeTAAATD0lEQVR42u1dC3gVxRVeFBQReYkoilSR+lEQhARBFBUrbwjUSnxhMEDuvQkYFR9AC5agVKwoapTk3ptAEAotVFQqfiKtgAUfpTwUUKk85VXeiggGCEnP2TvLNxl29u7M3XuT3Tvn+853787Ozs458++ZM2dmZzVNkSJFihQpUqRIkSJFVUf5mlYvpGmdgdPDmuYLatpDwH2LNa2l0k4SEgChMQBhBPAS+H8SuILDO4Bfg3x3KK15nKZpWl2wCuOhwY9ZAMKUASDL0MIoLXqQABRdoHF3i4KC4XLgN4BrKY16hOCpz4AG/TlGYNC89A1Nu1Rp1v0WY4iDoKD5C+A6SsPu7kpOxgkcyG9VaFoNpWl3DlH/F0dgGI7qY0rb7rMaE+MNDMKHSjStgdK4S6hI0y6HRvspQeBA6/GC0rp7glxPJgoYhA/M17TzlebdAY6PEwyOCujGbleajzcNm36lllXUTvZyjD9AY5VVATimSMucl3ee5g/1VI1vRf5QU+BNwIc0X7i9pNW4M9HAIOD4KAZgTAcuB5nVyMeURky7AhT0DXAFYSmAQCMNrQpwAG+NARiGzOVaIJirwEDT0BmXMcCQBgg0Uk4VgeNAjMCgAZKtQIGUPv8CAMBKEyVJAaQKwbHbAWAYXAYA6a3AEQiGLZQkDBDoVgZVETg2OAQMg3/Qhhe3TGZgZNtQkhBAwprWvirAAff9m4PAMPgr4PrJB4zswk4g+EkBRdkCCAajoLEOVgFARjoMDIPnJ5+f4Q9tlFCULYBAQ81MMDDKp2na1XEARoQDwUHJ1J1MlFaUP3QCwNErit+RlmBwrLKUd9TUi0DmxTHIvB/8j0bJAIwbQdhTkko6DtffFbVr17TzoMHWJTAAlhZV7tz8CwHU78cAkNnJAI7PJJVTqmUV3SYwpO2ZIHB8LNidLpIGSHZhN+8CwxfuL6kYDC0PloiWzoszMI4XaZrYPBCOPvyh9ZJ6+NSjyKioAcKtlVTKZKuSOwQDHW8K+9qbWI86wGvi5YRiTMXM+ewYDAxJn5/On8LPKWgBMh2QdE77etFq3CMJjKU6sPjAuD015D8KfMgMIPBkN4OG3OI0MMKa9pQZMFJC/ulQl4rUsG9ut7y8mhYWpKduEcX1sdZKH+4kf+hLCUUcAz/jWl6RAIY7oCGO640RYVOAFGtaI3ydwKmuBIBxryUwCHcM+RdYWhB/KCRpPdK8A4xAsIukErgTUCnFw1tCAxygG8MKIPgiEvCzsSwfhG7kM4zA2gHGWQ77+EsIh02/BOTcLqGbRV6yGjIBoP/wzGfn/Nx6oPjNpo1hARACkqbAQUGQfAmguJsX4OICw7AgwcBwi+72bgndlMHI5Sr3AyPydByTsBo9eEWCuZ5h1RjRAIJUomm1ocH7E6CsAt6PXQbwEeDNwIvIWtTrrCKf0YBB+Ce0dBaO+ucSABnngS4l5JcQfDkXGMHAQBuNYQsgMZF9YOgMeT/h+h8Y2BPX0Tb3O6YyQR9OFLRbSWZtUPQOAXDEByCCwLDVvchYD4w2u5YwZOwP/SQo9Le8JyI1GHhCtEEcB4gkMAjv6jJ11EUc6zFUwnqMdi84MNzrkMBtpo2oC8o9KNkozgAkNmAY/CTHctTRF/iI6eofbh7CPi8o7Bl9PamZ1Qj5s2JslNgA4gwwkLfpU/jOxD1K9RnfpJhk84X/zSsKlLrGgYaRA4hzwIj4HmFfH46+Bjo5qqvucyk/Cgr7rOkIpTD7BqcaRhggDgND52BgnsWwX3Q5w+NuHKU0l/A3unKsxliHwWEPIPEARoQPc4e1OIwX01nIjRNtvYSjfpz+E5S5NA4NZA2Q+AHD4K4ccLwoqLcVbrQcowSF3Gwa28jLqwmKLI1bIwUD73G6slvjCAzksZyHKlN4Ta0LwfG6oJALOVajVZwbaRHnvl3jed+OIf9szvC/k3B3jL6Ky8AxU1DIlzjh8kFeBAfwWlO95RQ0FAbHiGlXuw0cCwSHsb9zOCpa3cFx2EJ35YK6a+M2h3SJ4BMwkjMDm+dRcJRZgON7Id1lFd3s7QBYIDjErBgYMUz1KDgq2s3KuJgDju8EHyyXbfwSCK0WFPBBTrcyzavgAK7PAccWQd31c5vPIRbMwXUf5o002avg4C5AxjfcHAgeemcdRyD4BMdyjPcoOH620N0JQYe0vdvAMUcQ/RM4Q9khHgXHfzl6qyUx7XCd28ARErQcf66iSGXVgCPse5/jyLeWeE2yidvAMU5QSNM31dsW5DQEZZZ7MEL6IkdvvxF+udx1a0lxTwkxIY9arOXY6DlwhH0DOHobI6i3L9w4t9JW2DxmFf3KtKHCvkKPgeNMl+LhjTjgeFtQb391HzgyS2rry/4ciJKmhH29vAQO6FI+5i0TAB0ccWKBlBusx7eCgr7FaahaMS4url6WI+wbwdFXisRIJd2t4AgK+x1occznWF7xCDhOADfm6GuC8L4lnAXZbph8u19iwex9nMZqDnza9eAA/8lcWfqa282C+lqvuZZw/C0+/fy+xahljsvBcQr4Os6DdIvEg/Sq5moS3+LoNG9PjvaF2ddg2NnF4HjNQk+zhMGB22i5HBzjJIQu5FqPYOB5l4LjENfXiGwDdVpQT0f0101dTcOLfyGxvVGp/lEeE2qZn3shKHmDC8GR7thUg2tfSXBi+j7CRbziOhZmd3Cwe4k7OFJC/lkWumkltS9rduGt3gCH3Bvk5VbL31KCgYfcAY7AavitYwGOpRK62eqdTePwZSV/aJ+EEtZp6fO5G67BEzmpmoPju85FWc0sHpoHpPZK89znvvyhsZJbTf7RqtjUsO/lagqOnTcV5LSw0Ae+LnpYQh/73PtmPY9y8+sJr6o2tmWIsoAWRjATY5jWjwc4vulUlHWtBTBwQc8nkjssjtE8SfJfS9gfbbVTStg3mISmqxYcYd8SXIMSJXKcL6mHI+57u80uRbYY2C2pmM3R5hE6BAOtoYHWVRE4SjuG/KO5G7PE3r0iP6J5mnDuRFY5uLlLlM9atZmffgE00jhorGMJA0fYtyylKKudDdmHSG5pHdnW2sI59w6Jvw1Hv8KwGoa4l0e7BTRaU+ACG/GQWMCxDobUv7X5UORKrG+h/a7OWlKQL3y9vvZR3rxu1kPONqhdYXYT4rBucQgc2H0swAVI9oTVZ1snxSCr5XSCVwHii0lhuC+F4Jte0LCd8d1bXIlFWRQ74NgGACvpGPZltC/JbGD7hpklDaCO78Qo51f6ToNJR+Lvtph9pOdl/QtIgoTbLuH0OYbiOeBojGDivrIYjSL7bGyLUb7j+qsKSUmR0cumGBWI/DXwndVCJrQWkaHqaQfkelhLaoqsUj/kgCKxb55bZW9/RQJbWcB7HZHFHyrQFOmK7Ryjg1p5sRB2V4na0CSyhXeO5HdT+G//RYuXJJmDOsAhU0z7Iyv1t/dxKyXn4zVd9FGE3PyIFTA+kvGhvE+RANkpR5VtLB7yhZfB7zP6OgjOCvco1q05lJEBdSyB/zvjUMfItpFJ+e16+xakj8SXFmSCStv1r0Tj0+8LvwDHvwd+VN+YH18U8oem6j5MZBOaH+Ncn8huip6bbY2fD3IoAQ1SPdgXLk6O0LizUdQNHgdGWeRzXF77FGhiLEgd0sd7ERi7RT6/rogPkoelPiJYfflD977GWB0p8orDQpeD4kAk6qm6kXj5Iv0dmLNINJ/RR0UYWlcUZ8Ihny/8lIOh6nhxuT4r67rd/rxAkQ1iRgLvqIajkDladuENqpGq3mGtBU/nPcDvxinCapdx5+E8uwuRFCXecW2kL8SNvIJZmgBA7CBR1luUo+m2OIkv3B1+JwN/KvHdVrPuYqseVseVbMpCeA4wTcnHkAP62k7cAAXD1/7QfDLX8q4+ZY5bVuGcC75EhF0W+g+u3/JAkSJFihQpUqRIkSKvUJ8+fVr37t27HLiC8Dmfu+zWrVtNSN9p5OnVq1c/k3IGU2Ug55P0HDodj41rsBzmmslMmT0hrYw6vwWu0fcT79u3b1um3DnGdd27d2/OlLsqLS3N9KUhONcO+DiVdwVzvsIGT6byr+DkOQG8B3ghyJAFOj1nGaOVTCb67mByj+ccBwgUuoy+CVT+Rvo8HN/LVGKxSRnLqfPlcE3LWMCB18PxEeocNmA7O4o0AQfyXDhVg1HwZcDfMfniBQ6WN4OMbWTBAednm5R5MD093dlliFDJexhwvMJU5FOmEmeMxidCXc9Yn8VUAwiDY8CAAZdAvg3MdffZVSQHHMhnv2sLSrwAjlea5LECx1Go8wsm3JsHDjhXAL8vQv3egt/dTHm7+vfvf5UoOIh8p81khGt8joKDdBu7qJvsT01NrWVhvioBCPL8iTnfKwZw1IDft5n0l0VMsAU4yiBfGrn3dE4eLjjQytiwwpXAgXWhAQn3DTP3mykKDjg3hcqHIFlDHX/FWkgnupbxTOMPJEp8kwOO7wFUdckTuJ82l3nUyzqi4ACewKR9BPc43yFwIP8A/JrFeSvLsadfv34tWKb9BytwIOFDB+l7qTwne/bs2cQuOFDnqHsq3wLIN4j3cDrVtTTFilKN/w72yUza10zl/WyXBGm5TL8uAo41dPeETyrWway+guDYRMvB8HoHfI6udsFB8sxk6p5mFxyoXyZPT3xA4XeflU/ohPX4C41o9gkjXcxa6hj9gg+oPMdAGfVjAAfbgAcgf7NYwYHn4F4jTRp1DxmtJRocU5j6DbEDDrTIaJmpPFsNK02s7tkBAcrl9LD2VkZo2slcRvIMsVBSvkmZIuCYZuKIrjYbhoqCg9yP7u9PQhmdEcx2wYHmHMoaY8LNEmE5sKtnh67QrTVEhq4plW4vlNVx6wGFruN4wWmUl7/XJE85VLBVjOCYTIawh6MNQ2XAQY9Q4D5DSd76iXJIic9Qm8Q7zubp0aPHL+2Agw05ROET8FA1dtp6+ExuVMkDZp1Xq35OZigLv3cxwS9szNGxggMJnT9Ie5rKmyhw1IDrb0YHm7nfSjsy8UaNUXi8o+AA5V3MeMPnjJ0RkSTidzYPXNffKXCQc4+xw1DgvrGCwyRmUF/A5ziJ3RzLUNdJPHCQINt24FKTxjuOctiRiR01wnEGdiU0w/UdGcd0L1pLpx3TqVQF95lF3Zj+u9Lw1QlwkGtnsMNQDLhVITgqOF3uHIkI6UHgbnasITuSJJac13aTzZxdJ/0OI3R9BB0uizkZPQ/kf8SimxrEPGmDKEG6Mk9gpXKIj/A2c/0cjHtgHXlPLyj5Ct45s7gBnRfuV8zUf3U0ZixHMS8fnFsKPA/4UWOOiNW7WbmQP5MpJ5MnD8ZdrORRpEiRIkWKFClSpCj5yO/318nJyWlIM6SxG6PVGDFixBVwrsWwYcPO+aYIdV0taj6hppGO1+OQmTquVLaRjtdgQm5ubj26Plb1HzVq1EX0fYhMtViZaM7IyLgY82VmZjbAYzamgKMqkPNKlBfqciEzT1LTTEe0vHle2YoShCwArmD4bETR5/PdC8ebqHOnIe1d+G1OlWGc60eldaXS62N+6ngcla8+ld6VpK1n6nMqEAj8C/gutv6Q9pmRD+rVi1zfz0QmmueQfD+Q4wcNoML/UcB7qLzHodxiAwyUXOWQPpgjb3OvgeMbUPQYZPifQ86lUwKjIr8FLiPH2/EJlwRHBXUPK3B8APwM8N/JcRnUr4dxj6ysrJvpMuHcYuPe8H81MvzfTc6foNImmYEDfidQ5e0H3kodryDgoeU6BQAZkAzg+JDtVuB3Izn3T+x+qAYpJU+qLwZw6E9eFHCMpcqbSdI+p6zGPJL2OvCPWCaktWbkG0vyrDeR/Sw4UD7gYwRkYWOhEqmjXj+Q/TZGLt2yQPqvvQ6OSt0K9rWUub6fuWYFUeI0GhxwPJDqju6wAMfn5Pck5MuwAw7SvelPKzYcKe808Bn0DeB8odGwMuCABm5HyXEjk+8QSc9lQLCKsqqjkq1bOUHOPU07awgeoqwpJF8pyTeaKjeLUlYdGhzQEKlw7XsGQGxajhwDHGje8d7k+GsARnc4fsLoPoAbi4IjOzv7GgocaUae4cOHNzK6UrjPMNZCkG6qgupuPQmORSbnZpFzR4GfJD7IQqNbAMV0IfmWk7S9kPYQybeNpG0keWjL0ZYAZjljsc4BB3r+0CjtDacY/i+BkVNd+P89z+GEOowXBQfppoyG3oVdJspCObylOGJju8uhQ4dexjjsyQEOeJqaQPoGkwZAYEykTH57w/QyfIxq8ErgoJzRNTZGKwbvAqvTjJh3PN49cuTISylf6REDpMbwVAQcIEcbyoGlWe/+zHwp4oc1g/87vQgOdMZCqFheHAEUkwnnZxAH8CV0Sk3KaYxdEuSdC/9nYxeDTxVjnvE+IQDd2Xc+yJOHAA1Bnpakkf5g5EWG43z4fdSIeZCuLwS/D9B1IBYlSK5pQ/L2Jsd/MBkGv0rq04mp5+OQ/02UBfg5OG5FnW9p1At1Y5aOZagImiJFihQpUqRIkSJFipKT/g/ofre4o5fwfwAAAABJRU5ErkJggg==" +-%> + +sub errorpage { + if (resp.status >= 400) { + call synth_errorpage; + return (deliver); + } +} + +sub synth_errorpage { + set resp.http.Content-Type = "text/html; charset=utf-8"; + synthetic ({"<!DOCTYPE html> +<html lang=en> +<meta charset=utf-8> +<title>Wikimedia Error</title> +<style> +* { margin: 0; padding: 0; } +body { background: #fff; font: 15px/1.6 sans-serif; color: #333; } +.content { margin: 7% auto 0; padding: 2em 1em 1em; max-width: 560px; } +.footer { clear: both; margin-top: 14%; border-top: 1px solid #e5e5e5; background: #f9f9f9; padding: 2em 0; font-size: 0.8em; text-align: center; } +img { float: left; margin: 0 2em 2em 0; } +a img { border: 0; } +h1 { margin-top: 1em; font-size: 1.2em; } +p { margin: 0.7em 0 1em 0; } +a { color: #0645AD; text-decoration: none; } +a:hover { text-decoration: underline; } +code { font-family: sans-serif; } +.text-muted { color: #777; } +</style> +<div class="content" role="main"> +<a href="//www.wikimedia.org"><img src="<%= wmf_png %>" srcset="//www.wikimedia.org/static/images/wmf-2x.png 2x" alt=Wikimedia width=135 height=135></a> +<h1>Error</h1> +<p>Our servers are currently experiencing a technical problem. This is probably temporary and should be fixed soon.<br>Please <a href="" title="Reload this page" onclick="window.location.reload(false); return false">try again</a> in a few minutes.</p> +</div> +<div class="footer"> +<p>If you report this error to the Wikimedia System Administrators, please include the details below.</p> +<p class="text-muted"><code> +Request from "} + client.ip + " via " + server.hostname + " " + server.identity + " ([" + server.ip + "]:" + std.port(server.ip) + "), Varnish XID " + req.xid + "<br>" + +regsub(req.http.X-Forwarded-For, ".+", "Forwarded for: \0<br>") + regsub(resp.http.X-Cache, ".+", "Upstream caches: \0<br>") + +"Error: " + resp.status + ", " + resp.reason + " at " + now + +{" +</code></p></div></html> +"}); +} diff --git a/templates/varnish/maps-backend_v4.inc.vcl.erb b/templates/varnish/maps-backend_v4.inc.vcl.erb new file mode 100644 index 0000000..dbef527 --- /dev/null +++ b/templates/varnish/maps-backend_v4.inc.vcl.erb @@ -0,0 +1,29 @@ +// Varnish VCL include file for maps backends + +include "errorpage_v4.inc.vcl"; + +sub vcl_recv { + call recv_purge; + return (hash); +} + +sub vcl_backend_response { + // Cap TTL to 1 day for now (purging still hasn't been sorted out...) + if (beresp.ttl > 1d) { + set beresp.ttl = 1d; + } + return (deliver); +} + +// TODO: we cannot call errorpage from vcl_backend_error as resp. is not +// available. Commenting it out for now + +//sub vcl_backend_error { +// call errorpage; +// return (deliver); +//} + +sub vcl_synth { + call errorpage; + return (deliver); +} diff --git a/templates/varnish/maps-frontend_v4.inc.vcl.erb b/templates/varnish/maps-frontend_v4.inc.vcl.erb new file mode 100644 index 0000000..2ed4f2a --- /dev/null +++ b/templates/varnish/maps-frontend_v4.inc.vcl.erb @@ -0,0 +1,35 @@ +// Varnish VCL include file for upload frontends + +include "errorpage_v4.inc.vcl"; + +sub vcl_recv { + call recv_purge; + if (req.http.referer + && req.url != "/" + && req.http.referer !~ "(?i)^https?://([-a-zA-Z0-9.]+\.)?(mediawiki|wikivoyage|wikivoyage-ev|wmflabs)\.org/" + && req.http.referer !~ "(?i)^https?://(maps|phabricator|wikitech|incubator)\.wikimedia\.org/" + && req.http.referer !~ "(?i)^https?://(localhost|127\.0\.0\.1)(:\d+)?/" + ) { + return (synth(403, "Access Denied")); + } + + return (hash); +} + +sub vcl_backend_response { + // Cap TTL to 1 day for now (purging still hasn't been sorted out...) + if (beresp.ttl > 1d) { + set beresp.ttl = 1d; + } + return (deliver); +} + +sub vcl_backend_error { + call errorpage; + return (deliver); +} + +sub vcl_synth { + call errorpage; + return (deliver); +} -- To view, visit https://gerrit.wikimedia.org/r/269466 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iee05d5f712093c0a1d939e74a340627982979404 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ema <e...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits