Ema has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/382464 )
Change subject: varnish: add support for version 5
......................................................................
varnish: add support for version 5
Introduce a new hiera setting, profile::cache::base::varnish_version,
defaulting to 4.
Use the shard built-in director on Varnish 5, VSLP on Varnish 4.
Bug: T168529
Change-Id: I5c4e53a6a3242a62a951dba202b4ff5feb9c1382
---
M hieradata/role/common/cache/canary.yaml
M modules/profile/manifests/cache/base.pp
M modules/varnish/manifests/common.pp
M modules/varnish/manifests/common/directors.pp
M modules/varnish/manifests/packages.pp
M modules/varnish/manifests/wikimedia_vcl.pp
M modules/varnish/templates/vcl/directors.vcl.tpl.erb
M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
8 files changed, 40 insertions(+), 6 deletions(-)
Approvals:
Ema: Verified; Looks good to me, approved
diff --git a/hieradata/role/common/cache/canary.yaml
b/hieradata/role/common/cache/canary.yaml
index 433d1ec..a7f8984 100644
--- a/hieradata/role/common/cache/canary.yaml
+++ b/hieradata/role/common/cache/canary.yaml
@@ -1,5 +1,6 @@
bbr_congestion_control: true
varnish::dynamic_backend_caches: false
+apt::use_experimental: true
cache::lua_support: true
cluster: cache_canary
cache::cluster: canary
@@ -78,6 +79,7 @@
profile::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.wikimedia\.org$'
profile::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.wikimedia\.org)'
profile::cache::base::storage_parts: ['sda3', 'sdb3']
+profile::cache::base::varnish_version: 5
# Profile::cache::ssl::unified
profile::cache::ssl::unified::monitoring: true
profile::cache::ssl::unified::letsencrypt: false
diff --git a/modules/profile/manifests/cache/base.pp
b/modules/profile/manifests/cache/base.pp
index 428832e..447e69a 100644
--- a/modules/profile/manifests/cache/base.pp
+++ b/modules/profile/manifests/cache/base.pp
@@ -13,7 +13,8 @@
$purge_host_only_upload_re =
hiera('profile::cache::base::purge_host_only_upload_re'),
$purge_host_not_upload_re =
hiera('profile::cache::base::purge_host_not_upload_re'),
$storage_parts = hiera('profile::cache::base::purge_host_not_upload_re'),
- $packages_version = hiera('profile::cache::base::packages_version',
'installed')
+ $packages_version = hiera('profile::cache::base::packages_version',
'installed'),
+ $varnish_version = hiera('profile::cache::base::varnish_version', 4),
) {
# There is no better way to do this, so it can't be a class parameter. In
fact,
# I consider our requirement to make hiera calls parameters
@@ -52,11 +53,15 @@
}
# Basic varnish classes
class { '::varnish::packages':
- version => $packages_version,
+ version => $packages_version,
+ varnish_version => $varnish_version,
+ }
+
+ class { '::varnish::common':
+ varnish_version => $varnish_version,
}
class { [
- '::varnish::common',
'::varnish::common::errorpage',
'::varnish::common::browsersec',
]:
diff --git a/modules/varnish/manifests/common.pp
b/modules/varnish/manifests/common.pp
index 620484b..4da4bc1 100644
--- a/modules/varnish/manifests/common.pp
+++ b/modules/varnish/manifests/common.pp
@@ -1,4 +1,4 @@
-class varnish::common {
+class varnish::common($varnish_version=4) {
require ::varnish::packages
# Mount /var/lib/varnish as tmpfs to avoid Linux flushing mlocked
diff --git a/modules/varnish/manifests/common/directors.pp
b/modules/varnish/manifests/common/directors.pp
index c8f8502..d1f4894 100644
--- a/modules/varnish/manifests/common/directors.pp
+++ b/modules/varnish/manifests/common/directors.pp
@@ -20,6 +20,8 @@
# usual old trick
$group = hiera('cluster', $::cluster)
+ $varnish_version = $::varnish::common::varnish_version
+
#
https://bibwild.wordpress.com/2012/04/12/ruby-hash-select-1-8-7-and-1-9-3-simultaneously-compatible/
$keyspaces_str = inline_template("<%= @directors.values.map{ |v|
\"#{@conftool_namespace}/#{v['dc']}/#{@group}/#{v['service']}\" }.join('|') %>")
$keyspaces = sort(unique(split($keyspaces_str, '\|')))
diff --git a/modules/varnish/manifests/packages.pp
b/modules/varnish/manifests/packages.pp
index f46df41..f3235ea 100644
--- a/modules/varnish/manifests/packages.pp
+++ b/modules/varnish/manifests/packages.pp
@@ -1,4 +1,4 @@
-class varnish::packages($version='installed') {
+class varnish::packages($version='installed', $varnish_version=4) {
package { [
'varnish',
'varnish-dbg',
@@ -12,8 +12,13 @@
'varnish-modules',
'libvmod-netmapper',
'libvmod-tbf',
- 'libvmod-vslp',
]:
ensure => 'installed',
}
+
+ if ($varnish_version == 4) {
+ package { 'libvmod-vslp':
+ ensure => 'installed',
+ }
+ }
}
diff --git a/modules/varnish/manifests/wikimedia_vcl.pp
b/modules/varnish/manifests/wikimedia_vcl.pp
index 94774f0..b9f5774 100644
--- a/modules/varnish/manifests/wikimedia_vcl.pp
+++ b/modules/varnish/manifests/wikimedia_vcl.pp
@@ -25,6 +25,8 @@
# TODO: fix this horror through proper parameter-passing or scoping
$cache_route = $::cacheproxy::instance_pair::cache_route
+ $varnish_version = $::varnish::common::varnish_version
+
if $generate_extra_vcl {
$extra_vcl_name = regsubst($title, '^([^ ]+) .*$', '\1')
$extra_vcl_filename = "/etc/varnish/${extra_vcl_name}.inc.vcl"
diff --git a/modules/varnish/templates/vcl/directors.vcl.tpl.erb
b/modules/varnish/templates/vcl/directors.vcl.tpl.erb
index 0a8fc4c..c662f5b 100644
--- a/modules/varnish/templates/vcl/directors.vcl.tpl.erb
+++ b/modules/varnish/templates/vcl/directors.vcl.tpl.erb
@@ -1,12 +1,20 @@
<% @directors.keys.sort.each do |director_name|
director = @directors[director_name] -%>
+<% if @varnish_version == 5 -%>
+new <%= director_name %> = directors.shard();
+<% else -%>
new <%= director_name %> = vslp.vslp();
+<% end -%>
new <%= director_name %>_random = directors.random();
<%- keyspace =
"#{@conftool_namespace}/#{director['dc']}/#{@group}/#{director['service']}" -%>
{{range $node := ls "<%= keyspace %>/"}}{{ $key := printf "<%= keyspace
%>/%s" $node }}{{ $data := json (getv $key) }}{{ if eq $data.pooled "yes"}}
<%= director_name %>.add_backend(be_{{ $parts := split $node "." }}{{ join
$parts "_" }});
<%= director_name %>_random.add_backend(be_{{ $parts := split $node "."
}}{{ join $parts "_" }}, {{ $data.weight }});
{{end}}{{end}}
+<% if @varnish_version == 5 -%>
+<%= director_name %>.reconfigure();
+<% else -%>
<%= director_name %>.init_hashcircle(150);
+<% end -%>
<% end # directors loop %>
diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
index eb5c727..0549da2 100644
--- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
@@ -10,9 +10,11 @@
import directors;
+<% if @varnish_version == 4 -%>
# This is actually only needed if a vslp backend is defined. We could make
# this import conditional on existing vslp backend definitions.
import vslp;
+<% end -%>
include "errorpage.inc.vcl";
@@ -185,7 +187,11 @@
<% @backend_caches.keys.sort.each do |director_name|
director = @backend_caches[director_name]
backends = director['backends'] -%>
+<% if @varnish_version == 5 -%>
+ new <%= director_name %> = directors.shard();
+<% else -%>
new <%= director_name %> = vslp.vslp();
+<% end -%>
new <%= director_name %>_random = directors.random();
<%
backends.each do |backend|
@@ -200,7 +206,11 @@
<%= director_name %>.add_backend(<%= name %>);
<%= director_name %>_random.add_backend(<%= name %>, 100);
<% end #backends loop -%>
+<% if @varnish_version == 5 -%>
+ <%= director_name %>.reconfigure();
+<% else -%>
<%= director_name %>.init_hashcircle(150);
+<% end -%>
<% end #director loop -%>
<% end #dynamic_backend_caches -%>
--
To view, visit https://gerrit.wikimedia.org/r/382464
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5c4e53a6a3242a62a951dba202b4ff5feb9c1382
Gerrit-PatchSet: 14
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ema <[email protected]>
Gerrit-Reviewer: BBlack <[email protected]>
Gerrit-Reviewer: Ema <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
