BBlack has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/351663 )
Change subject: maps->upload functional cluster-level changes
......................................................................
maps->upload functional cluster-level changes
Note that many bits of upload-specific VCL are left in play for
both cases, as they're ultimately not going to do much. We'd
rather avoid excess VCL conditional complexity than try to cut all
of these out of the maps request path.
varnishmedia: filters for /thumbs/ which continues to do its job
webrequest: needs discussion with analytics, as maps reqs will
move to the webrequest_upload data source (but differentiable on
request hostname)
Bug: T164608
Change-Id: I2963112222a64b5349415813f347abea20547655
---
M hieradata/labs.yaml
M hieradata/role/common/cache/upload.yaml
M modules/role/manifests/cache/base.pp
M modules/role/manifests/cache/upload.pp
M modules/varnish/templates/upload-common.inc.vcl.erb
M modules/varnish/templates/upload-frontend.inc.vcl.erb
6 files changed, 32 insertions(+), 17 deletions(-)
Approvals:
Ema: Looks good to me, but someone else must approve
BBlack: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index 0c946f5..f0264ee 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -51,9 +51,10 @@
eqiad:
- 'deployment-cache-upload04.deployment-prep.eqiad.wmflabs'
role::cache::base::zero_site: 'https://zero.wikimedia.beta.wmflabs.org'
-role::cache::base::purge_host_only_upload_re: '^upload\.beta\.wmflabs\.org$'
-role::cache::base::purge_host_not_upload_re: '^(?!upload\.beta\.wmflabs\.org)'
+role::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.beta\.wmflabs\.org$'
+role::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.beta\.wmflabs\.org)'
role::cache::upload::upload_domain: 'upload.beta.wmflabs.org'
+role::cache::upload::maps_domain: 'maps.beta.wmflabs.org'
role::cache::text::static_host: 'deployment.wikimedia.beta.wmflabs.org'
role::cache::text::shortener_domain: 'w-beta.wmflabs.org'
role::cache::text::top_domain: 'beta.wmflabs.org'
diff --git a/hieradata/role/common/cache/upload.yaml
b/hieradata/role/common/cache/upload.yaml
index 869a257..7871407 100644
--- a/hieradata/role/common/cache/upload.yaml
+++ b/hieradata/role/common/cache/upload.yaml
@@ -30,9 +30,15 @@
backends:
eqiad: 'ms-fe-thumbs.svc.eqiad.wmnet'
# codfw: 'ms-fe-thumbs.svc.codfw.wmnet'
+ kartotherian:
+ backends:
+ eqiad: 'kartotherian.svc.eqiad.wmnet'
+ # codfw: 'kartotherian.svc.codfw.wmnet'
cache::req_handling:
- default:
+ upload.wikimedia.org:
director: 'swift'
subpaths:
'^/+[^/]+/[^/]+/thumb/':
director: 'swift_thumbs'
+ maps.wikimedia.org:
+ director: 'kartotherian'
diff --git a/modules/role/manifests/cache/base.pp
b/modules/role/manifests/cache/base.pp
index 701d6a4..88abc95 100644
--- a/modules/role/manifests/cache/base.pp
+++ b/modules/role/manifests/cache/base.pp
@@ -1,7 +1,7 @@
class role::cache::base(
$zero_site = 'https://zero.wikimedia.org',
- $purge_host_only_upload_re = '^upload\.wikimedia\.org$',
- $purge_host_not_upload_re = '^(?!upload\.wikimedia\.org)',
+ $purge_host_only_upload_re = '^(upload|maps)\.wikimedia\.org$',
+ $purge_host_not_upload_re = '^(?!(upload|maps)\.wikimedia\.org)',
$storage_parts = ['sda3', 'sdb3'],
) {
include ::standard
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index 4fefa16..1540224 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -1,6 +1,7 @@
# filtertags: labs-project-deployment-prep
class role::cache::upload(
$upload_domain = 'upload.wikimedia.org',
+ $maps_domain = 'maps.wikimedia.org',
) {
include role::cache::base
include role::cache::ssl::unified
@@ -9,8 +10,8 @@
class { 'prometheus::node_vhtcpd': }
class { 'varnish::htcppurger':
- host_regex => 'upload',
- mc_addrs => [ '239.128.0.112', '239.128.0.113' ],
+ host_regex => '[um][pa][lp][os]', # like 'uplo|maps', but avoiding
shell metachar...
+ mc_addrs => [ '239.128.0.112', '239.128.0.113', '239.128.0.114' ],
}
class { '::lvs::realserver':
@@ -36,6 +37,7 @@
$common_vcl_config = {
'purge_host_regex' => $::role::cache::base::purge_host_only_upload_re,
'upload_domain' => $upload_domain,
+ 'maps_domain' => $maps_domain,
'allowed_methods' => '^(GET|HEAD|OPTIONS|PURGE)$',
'req_handling' => hiera('cache::req_handling'),
}
diff --git a/modules/varnish/templates/upload-common.inc.vcl.erb
b/modules/varnish/templates/upload-common.inc.vcl.erb
index 2043693..402fe4a 100644
--- a/modules/varnish/templates/upload-common.inc.vcl.erb
+++ b/modules/varnish/templates/upload-common.inc.vcl.erb
@@ -2,7 +2,9 @@
sub upload_common_recv {
unset req.http.X-Range;
- if (req.http.host != "<%= @vcl_config.fetch('upload_domain') %>") {
+
+ if (req.http.host != "<%= @vcl_config.fetch('upload_domain') %>"
+ && req.http.host != "<%= @vcl_config.fetch('maps_domain') %>") {
return (synth(404, "Requested domainname does not exist on this
server"));
}
diff --git a/modules/varnish/templates/upload-frontend.inc.vcl.erb
b/modules/varnish/templates/upload-frontend.inc.vcl.erb
index bc2e439..98efb4f 100644
--- a/modules/varnish/templates/upload-frontend.inc.vcl.erb
+++ b/modules/varnish/templates/upload-frontend.inc.vcl.erb
@@ -9,9 +9,11 @@
return (synth(667, "OK"));
}
- // Homepage redirect to commons
- if (req.url == "/") {
- return (synth(666, "Moved Permanently"));
+ if (req.http.host == "<%= @vcl_config.fetch('upload_domain') %>") {
+ // Homepage redirect to commons
+ if (req.url == "/") {
+ return (synth(666, "Moved Permanently"));
+ }
}
call upload_common_recv;
@@ -127,12 +129,14 @@
// (e9cbfd539edd3). See <https://phabricator.wikimedia.org/T76020>.
set resp.http.Timing-Allow-Origin = "*";
- // Restrict uploads from loading external resources. (T117618)
- // Currently experimental and testing on large but not huge wikis.
- if (req.url ~ "^/wikipedia/(el|fr|ru|it|de|uk|ja|id|he|fi|zh|test)") {
- set resp.http.Content-Security-Policy-Report-Only =
"default-src 'none'; style-src 'unsafe-inline' data:; font-src data:; img-src
data: https://upload.wikimedia.org/favicon.ico; media-src data:; sandbox;
report-uri
https://commons.wikimedia.org/w/api.php?reportonly=1&source=image&action=cspreport&format=json&";;
- set resp.http.X-Content-Security-Policy-Report-Only =
resp.http.Content-Security-Policy-Report-Only;
- set resp.http.X-Webkit-CSP-Report-Only =
resp.http.Content-Security-Policy-Report-Only;
+ if (req.http.host == "<%= @vcl_config.fetch('upload_domain') %>") {
+ // Restrict uploads from loading external resources. (T117618)
+ // Currently experimental and testing on large but not huge
wikis.
+ if (req.url ~
"^/wikipedia/(el|fr|ru|it|de|uk|ja|id|he|fi|zh|test)") {
+ set resp.http.Content-Security-Policy-Report-Only =
"default-src 'none'; style-src 'unsafe-inline' data:; font-src data:; img-src
data: https://upload.wikimedia.org/favicon.ico; media-src data:; sandbox;
report-uri
https://commons.wikimedia.org/w/api.php?reportonly=1&source=image&action=cspreport&format=json&";;
+ set resp.http.X-Content-Security-Policy-Report-Only =
resp.http.Content-Security-Policy-Report-Only;
+ set resp.http.X-Webkit-CSP-Report-Only =
resp.http.Content-Security-Policy-Report-Only;
+ }
}
}
--
To view, visit https://gerrit.wikimedia.org/r/351663
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2963112222a64b5349415813f347abea20547655
Gerrit-PatchSet: 7
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <[email protected]>
Gerrit-Reviewer: BBlack <[email protected]>
Gerrit-Reviewer: Ema <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
