BBlack has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/339668 )
Change subject: varnish: switch all clusters to req_handling [WIP, 2/4]
......................................................................
varnish: switch all clusters to req_handling [WIP, 2/4]
This also adds "debug_director" to support the text cluster's
unique debug backends logic.
Bug: T134404
Change-Id: I757b420c4791398d947c6e725b5508a7f70db4f3
---
M modules/role/manifests/cache/maps.pp
M modules/role/manifests/cache/text.pp
M modules/role/manifests/cache/upload.pp
M modules/varnish/templates/maps-backend.inc.vcl.erb
M modules/varnish/templates/misc-backend.inc.vcl.erb
M modules/varnish/templates/misc-common.inc.vcl.erb
M modules/varnish/templates/text-backend.inc.vcl.erb
M modules/varnish/templates/text-common.inc.vcl.erb
M modules/varnish/templates/upload-backend.inc.vcl.erb
M modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
12 files changed, 171 insertions(+), 165 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/68/339668/1
diff --git a/modules/role/manifests/cache/maps.pp
b/modules/role/manifests/cache/maps.pp
index 247fd1b..f49b4bb 100644
--- a/modules/role/manifests/cache/maps.pp
+++ b/modules/role/manifests/cache/maps.pp
@@ -36,7 +36,6 @@
'max_connections' => 1000,
}
- $apps = hiera('cache::maps::apps')
$app_directors = {
'kartotherian' => {
'backend' => 'kartotherian.svc.eqiad.wmnet',
@@ -47,6 +46,7 @@
'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re,
'ttl_cap' => '1d',
'pass_random' => true,
+ 'req_handling' => { director => 'kartotherian' },
}
$common_runtime_params = ['default_ttl=86400']
diff --git a/modules/role/manifests/cache/text.pp
b/modules/role/manifests/cache/text.pp
index 4303d79..8b71c85 100644
--- a/modules/role/manifests/cache/text.pp
+++ b/modules/role/manifests/cache/text.pp
@@ -76,12 +76,41 @@
},
}
+ $req_handling = {
+ 'cxserver.wikimedia.org' => {
+ 'director' => 'cxserver_backend',
+ 'caching' => 'pass',
+ },
+ 'citoid.wikimedia.org' => {
+ 'director' => 'citoid_backend',
+ 'caching' => 'pass',
+ },
+ 'default' => {
+ 'director' => 'appservers',
+ 'debug_director' => 'appservers_debug',
+ 'subpaths' => {
+ '^/api/rest_v1/' => {
+ 'director' => 'restbase_backend'
+ },
+ '^/w/api\.php' => {
+ 'director' => 'api',
+ 'debug_director' => 'appservers_debug',
+ },
+ '^/w/thumb(_handler)?\.php' => {
+ 'director' => 'rendering',
+ 'debug_director' => 'appservers_debug',
+ }
+ }
+ },
+ }
+
$common_vcl_config = {
'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re,
'static_host' => $static_host,
'top_domain' => $top_domain,
'shortener_domain' => $shortener_domain,
'pass_random' => true,
+ 'req_handling' => $req_handling,
}
$be_vcl_config = $common_vcl_config
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index 482f5b4..95ed0c9 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -40,7 +40,6 @@
'max_connections' => 10000,
}
- $apps = hiera('cache::upload::apps')
$app_directors = {
'swift' => {
'backend' => 'ms-fe.svc.eqiad.wmnet',
@@ -50,10 +49,20 @@
},
}
+ $req_handling = {
+ 'director' => 'swift',
+ 'subpaths' => {
+ '^/+[^/]+/[^/]+/thumb/' => {
+ director => 'swift_thumbs',
+ },
+ },
+ }
+
$common_vcl_config = {
'purge_host_regex' => $::role::cache::base::purge_host_only_upload_re,
'upload_domain' => $upload_domain,
'allowed_methods' => '^(GET|HEAD|OPTIONS|PURGE)$',
+ 'req_handling' => $req_handling,
}
# Note pass_random true in BE, false in FE below.
diff --git a/modules/varnish/templates/maps-backend.inc.vcl.erb
b/modules/varnish/templates/maps-backend.inc.vcl.erb
index 3c3b110..cc0330c 100644
--- a/modules/varnish/templates/maps-backend.inc.vcl.erb
+++ b/modules/varnish/templates/maps-backend.inc.vcl.erb
@@ -1,11 +1,6 @@
// Varnish VCL include file for maps backends
sub cluster_be_recv_pre_purge { }
-
-sub cluster_be_recv_applayer_backend {
- set req.backend_hint = kartotherian.backend();
-}
-
sub cluster_be_recv { }
sub cluster_be_hash { }
sub cluster_be_hit { }
diff --git a/modules/varnish/templates/misc-backend.inc.vcl.erb
b/modules/varnish/templates/misc-backend.inc.vcl.erb
index 77a4cfe..feebcab 100644
--- a/modules/varnish/templates/misc-backend.inc.vcl.erb
+++ b/modules/varnish/templates/misc-backend.inc.vcl.erb
@@ -2,45 +2,6 @@
sub cluster_be_recv_pre_purge { }
-sub cluster_be_recv_applayer_backend {
-<%
- def set_director(dirname)
- return "set req.backend_hint = #{dirname}.backend();"
- end
-
- def_action = 'return (synth(404, "Domain not served here"));'
- if_stmts = []
- @vcl_config['req_handling'].keys.sort.each do |reqhost|
- options = @vcl_config['req_handling'][reqhost]
- host_action = set_director(options['director'])
- if options.has_key?('subpaths')
- path_ifs = []
- options['subpaths'].keys.sort.each do |subpath|
- path_action =
set_director(options['subpaths'][subpath]['director'])
- path_ifs.push(%Q[if (req.url ~ "#{subpath}") {\n
#{path_action}\n }])
- end
- path_ifs.push(%Q[e {\n #{host_action}\n }])
- host_action = path_ifs.join(' els')
- end
-
- if reqhost == 'default'
- def_action = host_action
- else
- if reqhost =~ /^[-.A-Za-z0-9]+$/
- hostop = '=='
- else
- hostop = '~'
- end
- hostcmp = %Q[req.http.Host #{hostop} "#{reqhost}"]
- if_stmts.push("if (#{hostcmp}) {\n #{host_action}\n }")
- end
- end
- if_stmts.push("e {\n #{def_action}\n }")
- set_backend = if_stmts.join(' els')
-%>
- <%= set_backend %>
-}
-
sub cluster_be_recv {
call misc_recv_pass;
}
diff --git a/modules/varnish/templates/misc-common.inc.vcl.erb
b/modules/varnish/templates/misc-common.inc.vcl.erb
index 1ca2111..db2e8fd 100644
--- a/modules/varnish/templates/misc-common.inc.vcl.erb
+++ b/modules/varnish/templates/misc-common.inc.vcl.erb
@@ -1,75 +1,4 @@
sub misc_recv_pass {
-<%
- def uc_action(uc)
- if uc == "pass"
- return "return (pass);"
- elsif uc == "pipe"
- return "return (pipe);"
- elsif uc == "websockets"
- return %Q[if (req.http.upgrade ~ "(?i)websocket") { return (pipe);
} else { return (pass); }]
- elsif uc == "normal"
- return ""
- else
- #throw error XXX
- end
- end
-
- caching = []
- caching_default = false
- @vcl_config['req_handling'].keys.sort.each do |reqhost|
- if reqhost == 'default'
- host_cmp = %Q[e] # "else" when join(' els') applied later
- elsif reqhost =~ /^[-.A-Za-z0-9]+$/
- host_cmp = %Q[if (req.http.host == "#{reqhost}")]
- else
- host_cmp = %Q[if (req.http.host ~ "#{reqhost}")]
- end
-
- need_host = false
- host_action = false
- options = @vcl_config['req_handling'][reqhost]
- if options.has_key?('caching')
- host_action = uc_action(options['caching'])
- need_host = true
- end
-
- if options.has_key?('subpaths')
- path_ifs = []
- options['subpaths'].keys.sort.each do |subpath|
- path_options = options['subpaths'][subpath]
- path_cmp = %Q[if (req.url ~ "#{subpath}")]
- if path_options.has_key?('caching')
- need_host = true
- path_action = uc_action(path_options['caching'])
- path_ifs.push(%Q[#{path_cmp} {\n
#{path_action}\n }])
- else
- path_ifs.push(%Q[#{path_cmp} { }])
- end
- end
- if host_action
- path_ifs.push(%Q[e { #{host_action} }])
- end
- if need_host
- host_action = path_ifs.join(' els')
- end
- end
-
- if need_host
- if reqhost == 'default'
- caching_default = %Q[#{host_cmp} { #{host_action} }]
- else
- caching.push(%Q[#{host_cmp} { #{host_action} }])
- end
- end
- end
-
- if caching_default
- caching.push(caching_default)
- end
- caching_vcl = caching.join("\n ");
-%>
- <%= caching_vcl %>
-
if (req.method != "GET" && req.method != "HEAD") {
// We only deal with GET and HEAD
return (pass);
diff --git a/modules/varnish/templates/text-backend.inc.vcl.erb
b/modules/varnish/templates/text-backend.inc.vcl.erb
index 8a3cfc6..62a54ef 100644
--- a/modules/varnish/templates/text-backend.inc.vcl.erb
+++ b/modules/varnish/templates/text-backend.inc.vcl.erb
@@ -8,40 +8,14 @@
}
}
-sub cluster_be_recv_applayer_backend {
- if (req.http.Host == "cxserver.wikimedia.org" ) { # LEGACY: to be
removed eventually
- set req.backend_hint = cxserver_backend.backend();
- } else if (req.http.Host == "citoid.wikimedia.org" ) { # LEGACY: to be
removed eventually
- set req.backend_hint = citoid_backend.backend();
- } else { // default for all other hostnames
- if (req.url ~ "^/api/rest_v1/") {
- set req.backend_hint = restbase_backend.backend();
- } else if (req.url ~ "^/w/api\.php") {
- set req.backend_hint = api.backend();
- set req.http.X-Backend-is-Mediawiki = 1;
- } else if (req.url ~ "^/w/thumb(_handler)?\.php") {
- set req.backend_hint = rendering.backend();
- set req.http.X-Backend-is-Mediawiki = 1;
- } else {
- set req.backend_hint = appservers.backend();
- set req.http.X-Backend-is-Mediawiki = 1;
- }
- }
-
- if (req.http.X-Wikimedia-Debug && req.http.X-Backend-is-Mediawiki) {
- set req.backend_hint = appservers_debug.backend();
- unset req.http.X-Backend-is-Mediawiki;
- }
-
+sub cluster_be_recv {
+ call text_common_recv;
+// This only applies for beta, which only has a fake "eqiad" and always goes
"direct"...
<% if @realm != "production" %>
if (req.http.X-Wikimedia-Security-Audit == "1") {
set req.backend_hint = security_audit.backend();
}
<% end %>
-}
-
-sub cluster_be_recv {
- call text_common_recv;
}
sub cluster_be_hash {
diff --git a/modules/varnish/templates/text-common.inc.vcl.erb
b/modules/varnish/templates/text-common.inc.vcl.erb
index 4848fb6..d21afda 100644
--- a/modules/varnish/templates/text-common.inc.vcl.erb
+++ b/modules/varnish/templates/text-common.inc.vcl.erb
@@ -88,11 +88,6 @@
return (pass);
}
- # LEGACY: to be removed eventually
- if (req.http.Host ~ "^(cxserver|citoid)\.wikimedia\.org$" ) {
- return (pass);
- }
-
call evaluate_cookie;
call pass_authorization;
}
diff --git a/modules/varnish/templates/upload-backend.inc.vcl.erb
b/modules/varnish/templates/upload-backend.inc.vcl.erb
index f8c278a..7635586 100644
--- a/modules/varnish/templates/upload-backend.inc.vcl.erb
+++ b/modules/varnish/templates/upload-backend.inc.vcl.erb
@@ -29,14 +29,6 @@
sub cluster_be_recv_pre_purge { }
-sub cluster_be_recv_applayer_backend {
- if (req.url ~ "^/+[^/]+/[^/]+/thumb/") {
- set req.backend_hint = swift_thumbs.backend();
- } else {
- set req.backend_hint = swift.backend();
- }
-}
-
sub cluster_be_recv {
call upload_common_recv;
}
diff --git a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
index bd894e6..71b16cd 100644
--- a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
@@ -10,6 +10,58 @@
call wm_common_directors_init;
}
+sub set_backend_app__ {
+<%
+ def set_director(dirname, debug_dir)
+ if debug_dir.nil?
+ return "set req.backend_hint = #{dirname}.backend();"
+ else
+ return "if (req.http.X-Wikimedia-Debug) { set req.backend_hint =
#{debug_dir}.backend(); } else { set req.backend_hint = #{dirname}.backend(); }"
+ end
+ end
+
+ def_action = 'return (synth(404, "Domain not served here"));'
+ if_stmts = []
+ @vcl_config['req_handling'].keys.sort.each do |reqhost|
+ options = @vcl_config['req_handling'][reqhost]
+ host_action = set_director(options['director'],
options['debug_director'])
+ if options.has_key?('subpaths')
+ path_ifs = []
+ options['subpaths'].keys.sort.each do |subpath|
+ path_action =
set_director(options['subpaths'][subpath]['director'],
options['subpaths'][subpath]['debug_director'])
+ path_ifs.push(%Q[if (req.url ~ "#{subpath}") {\n
#{path_action}\n }])
+ end
+ path_ifs.push(%Q[e {\n #{host_action}\n }])
+ host_action = path_ifs.join(' els')
+ end
+
+ if reqhost == 'default'
+ def_action = host_action
+ else
+ if reqhost =~ /^[-.A-Za-z0-9]+$/
+ hostop = '=='
+ else
+ hostop = '~'
+ end
+ hostcmp = %Q[req.http.Host #{hostop} "#{reqhost}"]
+ if_stmts.push("if (#{hostcmp}) {\n #{host_action}\n }")
+ end
+ end
+ if_stmts.push("e {\n #{def_action}\n }")
+ set_backend = if_stmts.join(' els')
+%>
+ <%= set_backend %>
+}
+
+sub set_backend__ {
+<% if @cache_route == 'direct' -%>
+ // tier-one caches must select an applayer backend
+ call set_backend_app__;
+<% else -%>
+ set req.backend_hint = cache_<%= @cache_route %>.backend();
+<% end -%>
+}
+
sub vcl_recv {
if (client.ip !~ wikimedia_trust) {
// Do not allow direct access to non-frontend layers
@@ -34,17 +86,13 @@
}
}
-<% if @cache_route == 'direct' -%>
- // tier-one caches must select an applayer backend
- call cluster_be_recv_applayer_backend;
-<% else -%>
- set req.backend_hint = cache_<%= @cache_route %>.backend();
-<% end -%>
+ call set_backend__;
call wm_common_recv_grace;
call cluster_be_recv_pre_purge;
call wm_common_recv_purge;
call cluster_be_recv;
+ call wm_common_recv_pass;
return (hash); // no default VCL
}
diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
index d95dec1..50ea0d2 100644
--- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
@@ -216,6 +216,79 @@
# Functions
+sub wm_common_recv_pass {
+<%
+ def uc_action(uc)
+ if uc == "pass"
+ return "return (pass);"
+ elsif uc == "pipe"
+ return "return (pipe);"
+ elsif uc == "websockets"
+ return %Q[if (req.http.upgrade ~ "(?i)websocket") { return (pipe);
} else { return (pass); }]
+ elsif uc == "normal"
+ return ""
+ else
+ #throw error XXX
+ end
+ end
+
+ caching = []
+ caching_default = false
+ @vcl_config['req_handling'].keys.sort.each do |reqhost|
+ if reqhost == 'default'
+ host_cmp = %Q[e] # "else" when join(' els') applied later
+ elsif reqhost =~ /^[-.A-Za-z0-9]+$/
+ host_cmp = %Q[if (req.http.host == "#{reqhost}")]
+ else
+ host_cmp = %Q[if (req.http.host ~ "#{reqhost}")]
+ end
+
+ need_host = false
+ host_action = false
+ options = @vcl_config['req_handling'][reqhost]
+ if options.has_key?('caching')
+ host_action = uc_action(options['caching'])
+ need_host = true
+ end
+
+ if options.has_key?('subpaths')
+ path_ifs = []
+ options['subpaths'].keys.sort.each do |subpath|
+ path_options = options['subpaths'][subpath]
+ path_cmp = %Q[if (req.url ~ "#{subpath}")]
+ if path_options.has_key?('caching')
+ need_host = true
+ path_action = uc_action(path_options['caching'])
+ path_ifs.push(%Q[#{path_cmp} {\n
#{path_action}\n }])
+ else
+ path_ifs.push(%Q[#{path_cmp} { }])
+ end
+ end
+ if host_action
+ path_ifs.push(%Q[e { #{host_action} }])
+ end
+ if need_host
+ host_action = path_ifs.join(' els')
+ end
+ end
+
+ if need_host
+ if reqhost == 'default'
+ caching_default = %Q[#{host_cmp} { #{host_action} }]
+ else
+ caching.push(%Q[#{host_cmp} { #{host_action} }])
+ end
+ end
+ end
+
+ if caching_default
+ caching.push(caching_default)
+ end
+ caching_vcl = caching.join("\n ");
+%>
+ <%= caching_vcl %>
+}
+
sub wm_common_recv_purge {
/* Support HTTP PURGE */
if (req.method == "PURGE") {
diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
index 59af6d7..515b799 100644
--- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
@@ -268,6 +268,7 @@
call cluster_fe_recv_pre_purge;
call wm_common_recv_purge;
call cluster_fe_recv;
+ call wm_common_recv_pass;
return (hash); // no default VCL
}
--
To view, visit https://gerrit.wikimedia.org/r/339668
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I757b420c4791398d947c6e725b5508a7f70db4f3
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <bbl...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
