BBlack has submitted this change and it was merged.
Change subject: varnish: use same VCL files for text+mobile
......................................................................
varnish: use same VCL files for text+mobile
Note this adds some $cluster conditionals for the few remaining
true diffs for now, while merging up the majority common code.
Bug: T109286
Change-Id: I4d0727654dbe85ce2e8e1d4f904fb68625986d1a
---
M modules/role/manifests/cache/mobile.pp
D templates/varnish/mobile-backend.inc.vcl.erb
D templates/varnish/mobile-frontend.inc.vcl.erb
M templates/varnish/text-backend.inc.vcl.erb
M templates/varnish/text-frontend.inc.vcl.erb
5 files changed, 108 insertions(+), 314 deletions(-)
Approvals:
BBlack: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/role/manifests/cache/mobile.pp
b/modules/role/manifests/cache/mobile.pp
index 25f6d62..3b32c14 100644
--- a/modules/role/manifests/cache/mobile.pp
+++ b/modules/role/manifests/cache/mobile.pp
@@ -98,7 +98,7 @@
varnish::instance { 'mobile-backend':
name => '',
- vcl => 'mobile-backend',
+ vcl => 'text-backend',
extra_vcl => ['text-common'],
ports => [ 3128 ],
admin_port => 6083,
@@ -133,7 +133,7 @@
varnish::instance { 'mobile-frontend':
name => 'frontend',
- vcl => 'mobile-frontend',
+ vcl => 'text-frontend',
extra_vcl => ['text-common', 'zero'],
ports => [ 80 ],
admin_port => 6082,
diff --git a/templates/varnish/mobile-backend.inc.vcl.erb
b/templates/varnish/mobile-backend.inc.vcl.erb
deleted file mode 100644
index e073f40..0000000
--- a/templates/varnish/mobile-backend.inc.vcl.erb
+++ /dev/null
@@ -1,144 +0,0 @@
-// Varnish VCL include file for mobile backends
-
-include "errorpage.inc.vcl";
-include "text-common.inc.vcl";
-
-sub vcl_recv {
- call vcl_recv_purge;
-
- if (req.http.host ~ "^test\.") {
- set req.http.X-Wikimedia-Debug = "1";
- }
-
-<% if @site_tier == "one" -%>
-
- if (req.url ~ "^/w/api\.php") {
- set req.backend = api;
- } else if (req.url ~ "^/w/thumb(_handler)?\.php") {
- set req.backend = rendering;
- } else if (req.url ~ "^/api/rest_v1/") {
- set req.url = "/" + req.http.host + regsub(req.url,
"^/api/rest_v1/", "/v1/");
- set req.backend = restbase_backend;
- }
-
- if (req.http.X-Wikimedia-Debug == "1") {
- set req.backend = test_wikipedia;
- }
-
-<% if @varnish_directors.include?('security_audit') &&
!@varnish_directors['security_audit']['backends'].empty? %>
- if (req.http.X-Wikimedia-Security-Audit == "1") {
- set req.backend = security_audit;
- }
-<% end %>
-
-<% end %> // end site_tier == one
-
- if (req.url ~ "^/wiki/Special:CentralAutoLogin/") {
-<% if @site_tier != "one" -%>
- set req.backend = backend_random;
-<% end -%>
- set req.hash_ignore_busy = true;
- }
-
- if (req.request != "GET" && req.request != "HEAD") {
- return (pass);
- }
-
- call evaluate_cookie;
- call pass_authorization;
- return (lookup);
-}
-
-<% if @site_tier == "one" -%>
-// BITS: legacy bits.wm.o domain support
-sub bitscompat_1be_mangle {
- // Transform backend url: /<sitename>/load.php -> /w/load.php
- // Set host header for backend to <sitename>
- if (req.url ~
"^/([a-zA-Z0-9-]+\.)?([a-zA-Z0-9-]+\.)?([a-zA-Z0-9-]+)\.<%=
Regexp.escape(@vcl_config.fetch('top_domain')) %>/load\.php") {
- set bereq.http.host = regsub(req.url, "^/([^/]+)/(.*)$", "\1");
- set bereq.url = regsub(req.url, "^/([^/]+)/load\.php(.*)?",
"/w/load.php\2");
- }
-}
-<% end -%>
-
-sub vcl_pass {
-<% if @site_tier == "one" -%>
- // BITS: legacy bits.wm.o domain support
- if (req.http.host == "<%= @vcl_config.fetch('bits_domain') %>") {
- call bitscompat_1be_mangle;
- return (pass);
- }
-<% end -%>
- call restore_cookie;
-}
-
-sub vcl_miss {
-<% if @site_tier == "one" -%>
- // BITS: legacy bits.wm.o domain support
- if (req.http.host == "<%= @vcl_config.fetch('bits_domain') %>") {
- call bitscompat_1be_mangle;
- return (fetch);
- }
-<% end -%>
- call restore_cookie;
-}
-
-sub vcl_fetch {
- // Make sure Set-Cookie responses are not cacheable, and log violations
- // FIXME: exceptions are ugly; maybe get rid of the whole thing?
- if (beresp.ttl > 0s && beresp.http.Set-Cookie &&
- req.url !~ "^/wiki/Special:HideBanners") {
- std.log("Cacheable object with Set-Cookie found. req.url: " +
req.url + " Cache-Control: " + beresp.http.Cache-Control + " Set-Cookie: " +
beresp.http.Set-Cookie);
- set beresp.http.Cache-Control = "private, max-age=0,
s-maxage=0";
- set beresp.ttl = 0s;
- return (hit_for_pass);
- }
-
- // FIXME: Fix up missing Vary headers on Apache redirects
- if ((beresp.status == 301 || beresp.status == 302)
- && beresp.http.Location ~ "^http"
- && beresp.http.Vary !~ "X-Forwarded-Proto") {
- if (beresp.http.Vary) {
- set beresp.http.Vary = beresp.http.Vary +
",X-Forwarded-Proto";
- } else {
- set beresp.http.Vary = "X-Forwarded-Proto";
- }
- }
-
- if (req.url ~ "mobileaction=" || req.url ~ "useformat=") {
- set beresp.ttl = 60 s;
- }
-
- if (beresp.ttl <= 0s || req.http.X-Wikimedia-Debug == "1" ||
req.http.X-Wikimedia-Security-Audit == "1") {
- set beresp.ttl = 120s;
- return (hit_for_pass);
- }
-
- return (deliver);
-}
-
-sub vcl_error {
- call errorpage;
- return (deliver);
-}
-
-sub vcl_hash {
- // The cookies below represent prefrences that can be set for anonymous
users.
-
- // Split the cache for the images-disabled variant of the mobile site.
- if (req.http.X-Orig-Cookie ~ "(^|;\s*)disableImages=1" ||
req.http.Cookie ~ "(^|;\s*)disableImages=1") {
- hash_data("disableImages=1");
- }
-
- // Split the cache if the NetSpeed cookie is set and if its value is
'B'. The only other
- // permissable value is 'A', which is equivalent to not having the
cookie at all, so we
- // don't need to update the hash in that case.
- if (req.http.X-Orig-Cookie ~ "(^|;\s*)NetSpeed=B" || req.http.Cookie ~
"(^|;\s*)NetSpeed=B") {
- hash_data("NetSpeed=B");
- }
-
- // Split the cache for the beta variant of the mobile site.
- if (req.http.X-Orig-Cookie ~ "(^|;\s*)optin=beta" || req.http.Cookie ~
"(^|;\s*)optin=beta") {
- hash_data("optin=beta");
- }
-}
diff --git a/templates/varnish/mobile-frontend.inc.vcl.erb
b/templates/varnish/mobile-frontend.inc.vcl.erb
deleted file mode 100644
index b9fb452..0000000
--- a/templates/varnish/mobile-frontend.inc.vcl.erb
+++ /dev/null
@@ -1,168 +0,0 @@
-// Varnish VCL include file for mobile frontends
-
-include "errorpage.inc.vcl";
-include "text-common.inc.vcl";
-include "zero.inc.vcl";
-
-// Note that analytics.inc.vcl will set an X-Analytics value of proxy=IORG
-// without inspecting whether there's an existing proxy=<proxy> key-
-// value pair inside X-Analytics. We do this because if the traffic
-// had come from a known proxy (e.g., Opera or Nokia), that would
-// imply that Internet.org was not the rightmost endpoint. In time
-// we will need to add the notion of proxy chaining to record whether
-// something came through both a known proxy and had Via: Internet.org
-// with a corresponding unknown rightmost endpoint (the rightmost
-// Internet.org endpoint with an unpredictable Internet-facing IP
-// address) in X-Forwarded-For, even if it's the only value, as in
-// the example of traffic sourced directly by satellite or something.
-
-sub vcl_recv {
- call filter_headers;
- call filter_noise;
-
- // FIXME: we're seeing an issue with Range requests and gzip/gunzip.
- // Disable Range requests for now.
- unset req.http.Range;
-
- // Only do tag_carrier logic on first start, and only for (m|zero).wp
- if (req.restarts == 0) {
- // Always set X-Subdomain
- if (req.http.host ~ "^([a-zA-Z0-9-]+\.)?zero\.") {
- set req.http.X-Subdomain = "ZERO";
- } else {
- set req.http.X-Subdomain = "M";
- }
-
- if (req.http.host ~ "^([a-zA-Z0-9-]+\.)?(m|zero)\.wikipedia\.")
{
- call tag_carrier;
- }
- }
-
- // Rewrite m.mediawiki.org to mediawiki.org etc
- if (req.http.host == "m.mediawiki.org") {
- set req.http.host = "www.mediawiki.org";
- } else if (req.http.host == "m.wikimediafoundation.org") {
- set req.http.host = "wikimediafoundation.org";
- } else if (req.http.host == "m.wikisource.org") {
- set req.http.host = "wikisource.org";
- } else if (req.http.host == "m.wikidata.org") {
- set req.http.host = "www.wikidata.org";
- } else {
- // Replace <language>.(m|zero).<project>.org by
<language>.<project>.org
- set req.http.host = regsub(req.http.host,
"^([a-zA-Z0-9-]+)\.(m|zero)\.", "\1.");
- }
-
- // Allow purging
- call vcl_recv_purge;
-
- // BITS: legacy bits.wm.o domain support
- if (req.http.host == "<%= @vcl_config.fetch('bits_domain') %>") {
- if (req.url ~ "^/event\.gif") {
- error 204;
- }
- return (lookup);
- }
-
- // Don't decode percent-encoded slashes in paths for REST APIs
- if (req.url !~ "^/api/rest_v1/") {
- call normalize_path;
- }
-
- # normalize all /static to the same hostname for caching
- if (req.url ~ "^/static/") { set req.http.host = "<%=
@vcl_config.fetch("static_host") %>"; }
-
- if (req.http.host ~ "^test\.") {
- set req.http.X-Wikimedia-Debug = "1";
- }
-
- // Users that just logged out, should not get a 304 for their
- // (locally cached) logged in pages.
- if (req.http.If-Modified-Since && req.http.Cookie ~ "LoggedOut") {
- unset req.http.If-Modified-Since;
- }
-
- if (req.url ~ "^/wiki/Special:CentralAutoLogin/") {
- set req.backend = backend_random;
- set req.hash_ignore_busy = true;
- }
-
- if (req.request != "GET" && req.request != "HEAD") {
- return (pass);
- }
-
- call evaluate_cookie;
- call pass_authorization;
- return (lookup);
-}
-
-sub vcl_pass {
- call restore_cookie;
-}
-
-sub vcl_miss {
- call restore_cookie;
-}
-
-sub vcl_fetch {
- if (req.url ~ "mobileaction=" || req.url ~ "useformat=") {
- set beresp.ttl = 60 s;
- }
-
- if (beresp.ttl <= 0s || req.http.X-Wikimedia-Debug == "1" ||
req.http.X-Wikimedia-Security-Audit == "1") {
- set beresp.ttl = 120s;
- return (hit_for_pass);
- }
-
- return (deliver);
-}
-
-sub vcl_error {
- // BITS: legacy bits.wm.o domain support
- if (req.http.host == "<%= @vcl_config.fetch('bits_domain') %>") {
- if (obj.status == 204) {
- set obj.http.Connection = "keep-alive";
- return (deliver);
- }
- }
-
- call errorpage;
- return (deliver);
-}
-
-sub vcl_deliver {
- // Strip s-maxage Cache-Control of wiki pages. The s-maxage still
applies to Varnish (sent
- // by MediaWiki $wgUseSquid, sends purges internally). But pages musn't
be cached elsewhere.
- // NOTE: Only apply to pages. Don't steal cachability of api.php,
load.php, etc. (T102898, T113007)
- if ((req.url ~ "^/wiki/" || req.url ~ "^/w/index\.php") &&
resp.http.Cache-Control ~ "s-maxage=[1-9]") {
- set resp.http.Cache-Control = "private, s-maxage=0, max-age=0,
must-revalidate";
- }
-
-<% if @vcl_config.fetch("enable_geoiplookup", false) -%>
- // Perform GeoIP look-up and send the result as a session cookie
- if (req.http.X-Orig-Cookie !~ "(^|;\s*)GeoIP=[^;]"
- && req.http.Cookie !~ "(^|;\s*)GeoIP=[^;]") {
- call geoip_cookie;
- }
-<% end -%>
-}
-
-sub vcl_hash {
- // The cookies below represent prefrences that can be set for anonymous
users.
-
- // Split the cache for the images-disabled variant of the mobile site.
- if (req.http.X-Orig-Cookie ~ "(^|;\s*)disableImages=1" ||
req.http.Cookie ~ "(^|;\s*)disableImages=1") {
- hash_data("disableImages=1");
- }
-
- // Split the cache if the NetSpeed cookie is set and if its value is
'B'. The only other
- // permissable value is 'A', which is equivalent to not having the
cookie at all, so we
- // don't need to update the hash in that case.
- if (req.http.X-Orig-Cookie ~ "(^|;\s*)NetSpeed=B" || req.http.Cookie ~
"(^|;\s*)NetSpeed=B") {
- hash_data("NetSpeed=B");
- }
-
- // Split the cache for the beta variant of the mobile site.
- if (req.http.X-Orig-Cookie ~ "(^|;\s*)optin=beta" || req.http.Cookie ~
"(^|;\s*)optin=beta") {
- hash_data("optin=beta");
- }
-}
diff --git a/templates/varnish/text-backend.inc.vcl.erb
b/templates/varnish/text-backend.inc.vcl.erb
index ebebcd9..034bede 100644
--- a/templates/varnish/text-backend.inc.vcl.erb
+++ b/templates/varnish/text-backend.inc.vcl.erb
@@ -106,6 +106,12 @@
}
}
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+ if (req.url ~ "mobileaction=" || req.url ~ "useformat=") {
+ set beresp.ttl = 60 s;
+ }
+<% end -%>
+
if (beresp.ttl <= 0s || req.http.X-Wikimedia-Debug == "1" ||
req.http.X-Wikimedia-Security-Audit == "1") {
set beresp.ttl = 120s;
return (hit_for_pass);
@@ -118,3 +124,26 @@
call errorpage;
return (deliver);
}
+
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+sub vcl_hash {
+ // The cookies below represent prefrences that can be set for anonymous
users.
+
+ // Split the cache for the images-disabled variant of the mobile site.
+ if (req.http.X-Orig-Cookie ~ "(^|;\s*)disableImages=1" ||
req.http.Cookie ~ "(^|;\s*)disableImages=1") {
+ hash_data("disableImages=1");
+ }
+
+ // Split the cache if the NetSpeed cookie is set and if its value is
'B'. The only other
+ // permissable value is 'A', which is equivalent to not having the
cookie at all, so we
+ // don't need to update the hash in that case.
+ if (req.http.X-Orig-Cookie ~ "(^|;\s*)NetSpeed=B" || req.http.Cookie ~
"(^|;\s*)NetSpeed=B") {
+ hash_data("NetSpeed=B");
+ }
+
+ // Split the cache for the beta variant of the mobile site.
+ if (req.http.X-Orig-Cookie ~ "(^|;\s*)optin=beta" || req.http.Cookie ~
"(^|;\s*)optin=beta") {
+ hash_data("optin=beta");
+ }
+}
+<% end -%>
diff --git a/templates/varnish/text-frontend.inc.vcl.erb
b/templates/varnish/text-frontend.inc.vcl.erb
index 7d837b5..c9ee56e 100644
--- a/templates/varnish/text-frontend.inc.vcl.erb
+++ b/templates/varnish/text-frontend.inc.vcl.erb
@@ -2,6 +2,9 @@
include "errorpage.inc.vcl";
include "text-common.inc.vcl";
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+include "zero.inc.vcl";
+<% end -%>
// Note that analytics.inc.vcl will set an X-Analytics value of proxy=IORG
// without inspecting whether there's an existing proxy=<proxy> key-
@@ -20,6 +23,7 @@
// mobile-frontend.inc.vcl.erb for (m|zero).wikipedia.org and its
// subdomains.
+<% if scope.function_hiera(["cluster"]) == "cache_text" -%>
sub mobile_redirect {
if ((req.request == "GET" || req.request == "HEAD")
&& (req.http.User-Agent ~
"(?i)(mobi|240x240|240x320|320x320|alcatel|android|audiovox|bada|benq|blackberry|cdm-|compal-|docomo|ericsson|hiptop|htc[-_]|huawei|ipod|kddi-|kindle|meego|midp|mitsu|mmp\/|mot-|motor|ngm_|nintendo|opera.m|palm|panasonic|philips|phone|playstation|portalmmm|sagem-|samsung|sanyo|sec-|semc-browser|sendo|sharp|silk|softbank|symbian|teleca|up.browser|vodafone|webos)"
@@ -49,6 +53,7 @@
unset req.http.MobileHost;
}
}
+<% end -%>
sub vcl_recv {
call filter_headers;
@@ -57,6 +62,36 @@
// FIXME: we're seeing an issue with Range requests and gzip/gunzip.
// Disable Range requests for now.
unset req.http.Range;
+
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+ // Only do tag_carrier logic on first start, and only for (m|zero).wp
+ if (req.restarts == 0) {
+ // Always set X-Subdomain
+ if (req.http.host ~ "^([a-zA-Z0-9-]+\.)?zero\.") {
+ set req.http.X-Subdomain = "ZERO";
+ } else {
+ set req.http.X-Subdomain = "M";
+ }
+
+ if (req.http.host ~ "^([a-zA-Z0-9-]+\.)?(m|zero)\.wikipedia\.")
{
+ call tag_carrier;
+ }
+ }
+
+ // Rewrite m.mediawiki.org to mediawiki.org etc
+ if (req.http.host == "m.mediawiki.org") {
+ set req.http.host = "www.mediawiki.org";
+ } else if (req.http.host == "m.wikimediafoundation.org") {
+ set req.http.host = "wikimediafoundation.org";
+ } else if (req.http.host == "m.wikisource.org") {
+ set req.http.host = "wikisource.org";
+ } else if (req.http.host == "m.wikidata.org") {
+ set req.http.host = "www.wikidata.org";
+ } else {
+ // Replace <language>.(m|zero).<project>.org by
<language>.<project>.org
+ set req.http.host = regsub(req.http.host,
"^([a-zA-Z0-9-]+)\.(m|zero)\.", "\1.");
+ }
+<% end -%>
// Allow purging
call vcl_recv_purge;
@@ -74,7 +109,9 @@
call normalize_path;
}
+<% if scope.function_hiera(["cluster"]) == "cache_text" -%>
call mobile_redirect;
+<% end -%>
# normalize all /static to the same hostname for caching
if (req.url ~ "^/static/") { set req.http.host = "<%=
@vcl_config.fetch("static_host") %>"; }
@@ -112,6 +149,11 @@
}
sub vcl_fetch {
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+ if (req.url ~ "mobileaction=" || req.url ~ "useformat=") {
+ set beresp.ttl = 60 s;
+ }
+<% end -%>
if (beresp.ttl <= 0s || req.http.X-Wikimedia-Debug == "1" ||
req.http.X-Wikimedia-Security-Audit == "1") {
set beresp.ttl = 120s;
return (hit_for_pass);
@@ -129,6 +171,7 @@
}
}
+<% if scope.function_hiera(["cluster"]) == "cache_text" -%>
// Support mobile redirects
if (obj.status == 666) {
set obj.http.Location = req.http.Location;
@@ -137,6 +180,7 @@
set obj.http.Content-Length = "0"; // BZ #62245
return (deliver);
}
+<% end -%>
call errorpage;
return (deliver);
@@ -149,12 +193,22 @@
// Instead of writing regexps for every edge-case, we should impose
some order and coherence
// on our URL routing schemes.
// NOTE: Only apply to pages. Don't steal cachability of api.php,
load.php, etc. (T102898, T113007)
+
+<% if scope.function_hiera(["cluster"]) == "cache_text" -%>
if (req.url ~ "^/wiki/" || req.url ~ "^/w/index\.php" || req.url ~
"^/\?title=") {
// ...but exempt CentralNotice banner special pages
if (req.url !~ "^/wiki/Special\:Banner") {
set resp.http.Cache-Control = "private, s-maxage=0,
max-age=0, must-revalidate";
}
}
+<% end -%>
+
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+ if ((req.url ~ "^/wiki/" || req.url ~ "^/w/index\.php") &&
resp.http.Cache-Control ~ "s-maxage=[1-9]") {
+ set resp.http.Cache-Control = "private, s-maxage=0, max-age=0,
must-revalidate";
+ }
+<% end -%>
+
<% if @vcl_config.fetch("enable_geoiplookup", false) -%>
// Perform GeoIP look-up and send the result as a session cookie
if (req.http.X-Orig-Cookie !~ "(^|;\s*)GeoIP=[^;]"
@@ -163,3 +217,26 @@
}
<% end -%>
}
+
+<% if scope.function_hiera(["cluster"]) == "cache_mobile" -%>
+sub vcl_hash {
+ // The cookies below represent prefrences that can be set for anonymous
users.
+
+ // Split the cache for the images-disabled variant of the mobile site.
+ if (req.http.X-Orig-Cookie ~ "(^|;\s*)disableImages=1" ||
req.http.Cookie ~ "(^|;\s*)disableImages=1") {
+ hash_data("disableImages=1");
+ }
+
+ // Split the cache if the NetSpeed cookie is set and if its value is
'B'. The only other
+ // permissable value is 'A', which is equivalent to not having the
cookie at all, so we
+ // don't need to update the hash in that case.
+ if (req.http.X-Orig-Cookie ~ "(^|;\s*)NetSpeed=B" || req.http.Cookie ~
"(^|;\s*)NetSpeed=B") {
+ hash_data("NetSpeed=B");
+ }
+
+ // Split the cache for the beta variant of the mobile site.
+ if (req.http.X-Orig-Cookie ~ "(^|;\s*)optin=beta" || req.http.Cookie ~
"(^|;\s*)optin=beta") {
+ hash_data("optin=beta");
+ }
+}
+<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/257699
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4d0727654dbe85ce2e8e1d4f904fb68625986d1a
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <[email protected]>
Gerrit-Reviewer: BBlack <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
