BBlack has uploaded a new change for review.
https://gerrit.wikimedia.org/r/282151
Change subject: cache_misc: declarative req.http.host=>backend map
......................................................................
cache_misc: declarative req.http.host=>backend map
Bug: T131501
Change-Id: I5f7ed0cfaf71553818d9e162f7d9ff9cd58d89a6
---
M modules/role/manifests/cache/misc.pp
M templates/varnish/misc-backend.inc.vcl.erb
2 files changed, 81 insertions(+), 59 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/51/282151/1
diff --git a/modules/role/manifests/cache/misc.pp
b/modules/role/manifests/cache/misc.pp
index 3a2dc85..aa4a286 100644
--- a/modules/role/manifests/cache/misc.pp
+++ b/modules/role/manifests/cache/misc.pp
@@ -33,138 +33,199 @@
'max_connections' => 100,
}
+ # misc-cluster specific (for now!):
+ # every director must have exactly one of...
+ # 'req_host' => request hostname (or array of them)
+ # 'req_host_re' => request hostname regex
+ # ...and for sanity's sake, there should be no overlap among them
$app_directors = {
'analytics1027' => { # Hue (Hadoop GUI)
'dynamic' => 'no',
'type' => 'random',
'backends' => ['analytics1027.eqiad.wmnet'],
'be_opts' => merge($app_def_be_opts, { 'port' => 8888 }),
+ 'req_host' => 'hue.wikimedia.org',
},
'antimony' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['antimony.wikimedia.org'],
'be_opts' => merge($app_def_be_opts, { 'port' => 8080 }),
+ 'req_host' => 'git.wikimedia.org',
},
'bromine' => { # ganeti VM for misc. static HTML sites
'dynamic' => 'no',
'type' => 'random',
'backends' => ['bromine.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'static-bugzilla.wikimedia.org',
+ 'annual.wikimedia.org',
+ 'transparency.wikimedia.org',
+ '15.wikipedia.org',
+ 'releases.wikimedia.org'
+ ],
},
'bohrium' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['bohrium.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'piwik.wikimedia.org',
},
'californium' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['californium.wikimedia.org'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'horizon.wikimedia.org',
},
'labtestweb2001' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['labtestweb2001.wikimedia.org'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'labtesthorizon.wikimedia.org',
},
'dataset1001' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['dataset1001.wikimedia.org'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'download.wikimedia.org',
},
'etherpad1001' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['etherpad1001.eqiad.wmnet'],
'be_opts' => merge($app_def_be_opts, { 'port' => 9001 }),
+ 'req_host' => 'etherpad.wikimedia.org',
},
'gallium' => { # CI server
'dynamic' => 'no',
'type' => 'random',
'backends' => ['gallium.wikimedia.org' ],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'doc.wikimedia.org',
+ 'integration.wikimedia.org'
+ ],
},
'graphite1001' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['graphite1001.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'performance.wikimedia.org',
+ 'graphite.wikimedia.org'
+ ],
},
'iridium' => { # main phab
'dynamic' => 'no',
'type' => 'random',
'backends' => ['iridium.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'phabricator.wikimedia.org',
+ 'phab.wmfusercontent.org',
+ 'bugzilla.wikimedia.org',
+ 'bugs.wikimedia.org'
+ ],
},
'krypton' => { # ganeti VM for misc. PHP apps
'dynamic' => 'no',
'type' => 'random',
'backends' => ['krypton.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'scholarships.wikimedia.org',
+ 'iegreview.wikimedia.org',
+ 'racktables.wikimedia.org',
+ 'grafana.wikimedia.org',
+ 'grafana-admin.wikimedia.org'
+ ],
},
'magnesium' => { # RT and racktables
'dynamic' => 'no',
'type' => 'random',
'backends' => ['magnesium.wikimedia.org'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'rt.wikimedia.org',
},
'netmon1001' => { # servermon
'dynamic' => 'no',
'type' => 'random',
'backends' => ['netmon1001.wikimedia.org'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'servermon.wikimedia.org',
+ 'smokeping.wikimedia.org',
+ 'torrus.wikimedia.org'
+ ],
},
'noc' => { # noc.wikimedia.org and dbtree.wikimedia.org
'dynamic' => 'no',
'type' => 'random',
'backends' => ['mw1152.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'noc.wikimedia.org',
+ 'dbtree.wikimedia.org'
+ ],
},
'palladium' => {
'dynamic' => 'no',
'type' => 'random',
'backends' => ['palladium.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'config-master.wikimedia.org',
},
'planet1001' => {
- 'dynamic' => 'no',
- 'type' => 'random',
- 'backends' => ['planet1001.eqiad.wmnet'],
- 'be_opts' => $app_def_be_opts,
+ 'dynamic' => 'no',
+ 'type' => 'random',
+ 'backends' => ['planet1001.eqiad.wmnet'],
+ 'be_opts' => $app_def_be_opts,
+ 'req_host_re' => '(?i)^([^.]+\.)?planet\.wikimedia\.org$'
},
'ruthenium' => { # parsoid rt test server
'dynamic' => 'no',
'type' => 'random',
'backends' => ['ruthenium.eqiad.wmnet'],
'be_opts' => merge($app_def_be_opts, { 'port' => 8001 }),
+ 'req_host' => 'parsoid-tests.wikimedia.org',
},
'rutherfordium' => { # people.wikimedia.org
'dynamic' => 'no',
'type' => 'random',
'backends' => ['rutherfordium.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'people.wikimedia.org',
},
'stat1001' => { # metrics and metrics-api
'dynamic' => 'no',
'type' => 'random',
'backends' => ['stat1001.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => [
+ 'metrics.wikimedia.org',
+ 'stats.wikimedia.org',
+ 'datasets.wikimedia.org'
+ ],
},
'mendelevium' => { # OTRS
'dynamic' => 'no',
'type' => 'random',
'backends' => ['mendelevium.eqiad.wmnet'],
'be_opts' => $app_def_be_opts,
+ 'req_host' => 'ticket.wikimedia.org',
},
'ytterbium' => { # Gerrit
'dynamic' => 'no',
'type' => 'random',
'backends' => ['ytterbium.wikimedia.org'],
'be_opts' => merge($app_def_be_opts, { 'port' => 8080 }),
+ 'req_host' => 'gerrit.wikimedia.org',
},
'logstash_director' => {
'dynamic' => 'no',
@@ -175,6 +236,7 @@
'logstash1003.eqiad.wmnet',
],
'be_opts' => merge($app_def_be_opts, { 'probe' => 'logstash' }),
+ 'req_host' => 'logstash.wikimedia.org',
},
'wdqs_director' => {
'dynamic' => 'no',
@@ -184,6 +246,7 @@
'wdqs1002.eqiad.wmnet'
],
'be_opts' => merge($app_def_be_opts, { 'probe' => 'wdqs' }),
+ 'req_host' => 'query.wikidata.org',
},
}
diff --git a/templates/varnish/misc-backend.inc.vcl.erb
b/templates/varnish/misc-backend.inc.vcl.erb
index a91f1bf..9be4313 100644
--- a/templates/varnish/misc-backend.inc.vcl.erb
+++ b/templates/varnish/misc-backend.inc.vcl.erb
@@ -3,61 +3,20 @@
sub cluster_be_recv_pre_purge { }
sub cluster_be_recv_applayer_backend {
- if (req.http.Host == "git.wikimedia.org") {
- set req.backend = antimony;
- } elsif (req.http.Host == "doc.wikimedia.org" || req.http.Host ==
"integration.wikimedia.org") {
- set req.backend = gallium;
- } elsif (req.http.Host == "download.wikimedia.org") {
- set req.backend = dataset1001;
- } elsif (req.http.Host == "gerrit.wikimedia.org") {
- set req.backend = ytterbium;
- } elsif (req.http.Host == "performance.wikimedia.org" || req.http.Host ==
"graphite.wikimedia.org") {
- set req.backend = graphite1001;
- } elsif (req.http.Host == "logstash.wikimedia.org"){
- set req.backend = logstash_director;
- } elsif (req.http.Host == "query.wikidata.org"){
- set req.backend = wdqs_director;
- } elsif (req.http.Host == "releases.wikimedia.org") {
- set req.backend = bromine;
- } elsif (req.http.Host == "piwik.wikimedia.org") {
- set req.backend = bohrium;
- } elsif (req.http.Host == "grafana.wikimedia.org" || req.http.host ==
"grafana-admin.wikimedia.org") {
- set req.backend = krypton;
- } elsif (req.http.Host == "parsoid-tests.wikimedia.org") {
- set req.backend = ruthenium;
- } elsif (req.http.Host == "horizon.wikimedia.org") {
- set req.backend = californium;
- } elsif (req.http.Host == "labtesthorizon.wikimedia.org") {
- set req.backend = labtestweb2001;
- } elsif (req.http.Host == "phabricator.wikimedia.org" || req.http.Host ==
"phab.wmfusercontent.org" || req.http.Host == "bugzilla.wikimedia.org" ||
req.http.Host == "bugs.wikimedia.org") {
- set req.backend = iridium;
- } elsif (req.http.Host == "static-bugzilla.wikimedia.org" || req.http.Host
== "annual.wikimedia.org" || req.http.Host == "transparency.wikimedia.org" ||
req.http.Host == "15.wikipedia.org") {
- set req.backend = bromine;
- } elsif (req.http.Host == "servermon.wikimedia.org" || req.http.Host ==
"smokeping.wikimedia.org" || req.http.Host == "torrus.wikimedia.org") {
- set req.backend = netmon1001;
- } elsif (req.http.Host == "ticket.wikimedia.org") {
- set req.backend = mendelevium;
- } elsif (req.http.Host == "people.wikimedia.org") {
- set req.backend = rutherfordium;
- } elsif (req.http.Host == "rt.wikimedia.org") {
- set req.backend = magnesium;
- } elsif (req.http.Host == "metrics.wikimedia.org" || req.http.Host ==
"stats.wikimedia.org") {
- set req.backend = stat1001;
- } elsif (req.http.Host == "datasets.wikimedia.org") {
- set req.backend = stat1001;
- } elsif (req.http.Host == "config-master.wikimedia.org") {
- set req.backend = palladium;
- } elsif (req.http.Host == "noc.wikimedia.org" || req.http.Host ==
"dbtree.wikimedia.org") {
- set req.backend = noc;
- } elsif (req.http.Host == "scholarships.wikimedia.org" || req.http.Host ==
"iegreview.wikimedia.org" || req.http.Host == "racktables.wikimedia.org") {
- set req.backend = krypton;
- } elsif (req.http.Host == "hue.wikimedia.org") {
- set req.backend = analytics1027;
- } elsif (req.http.Host ~ ".*\.planet\.wikimedia\.org" || req.http.Host ==
"planet.wikimedia.org") {
- set req.backend = planet1001;
- } elsif (req.http.Host == "etherpad.wikimedia.org") {
- set req.backend = etherpad1001;
- } else {
+<%
+ if_stmts = []
+ @app_directors.keys.sort.each do |dirname|
+ dir = @app_directors[dirname]
+ if dir.key?('req_hosts')
+ hostcmp = [*dir['req_host']].map { |h| %Q[req.http.Host == "#{h}"]
}.join(' || ')
+ else
+ hostcmp = %Q[req.http.Host ~ "#{dir['req_host_re']}"]
+ end
+ if_stmts.push("if (%{hostcmp}) {\n set req.backend =
#{dirname};\n }")
+ end
+ all_ifs = if_stmts.join(' els')
+%>
+ <%= all_ifs -%> else {
error 404 "Domain not served here";
}
}
--
To view, visit https://gerrit.wikimedia.org/r/282151
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5f7ed0cfaf71553818d9e162f7d9ff9cd58d89a6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
