Giuseppe Lavagetto has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/382683 )
Change subject: role::cache::base: convert to profile [1]
......................................................................
role::cache::base: convert to profile [1]
I am not fully sure this is the best way to go, I might end up chopping
the whole thing into pieces later.
Change-Id: I28093780badaa38713847028cd5b5569dc81d6ca
---
M hieradata/labs.yaml
M hieradata/role/common/cache/canary.yaml
M hieradata/role/common/cache/misc.yaml
M hieradata/role/common/cache/text.yaml
M hieradata/role/common/cache/upload.yaml
R modules/profile/manifests/cache/base.pp
M modules/role/manifests/cache/misc.pp
M modules/role/manifests/cache/text.pp
M modules/role/manifests/cache/upload.pp
M modules/varnish/manifests/zero_update.pp
10 files changed, 69 insertions(+), 30 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/83/382683/1
diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index fed4f4a..423c9c2 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -57,9 +57,9 @@
cache::upload::nodes:
eqiad:
- 'deployment-cache-upload04.deployment-prep.eqiad.wmflabs'
-role::cache::base::zero_site: 'https://zero.wikimedia.beta.wmflabs.org'
-role::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.beta\.wmflabs\.org$'
-role::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.beta\.wmflabs\.org)'
+profile::cache::base::zero_site: 'https://zero.wikimedia.beta.wmflabs.org'
+profile::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.beta\.wmflabs\.org$'
+profile::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.beta\.wmflabs\.org)'
role::cache::upload::upload_domain: 'upload.beta.wmflabs.org'
role::cache::upload::maps_domain: 'maps.beta.wmflabs.org'
role::cache::text::static_host: 'deployment.wikimedia.beta.wmflabs.org'
diff --git a/hieradata/role/common/cache/canary.yaml
b/hieradata/role/common/cache/canary.yaml
index 264688c..ab8be26 100644
--- a/hieradata/role/common/cache/canary.yaml
+++ b/hieradata/role/common/cache/canary.yaml
@@ -73,3 +73,8 @@
debug_director: 'appservers_debug'
cache::fe_transient_gb: 5
cache::be_transient_gb: 2
+# Profile::cache::base
+profile::cache::base::zero_site: 'https://zero.wikimedia.org'
+profile::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.wikimedia\.org$',
+profile::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.wikimedia\.org)',
+profile::cache::base::storage_parts: ['sda3', 'sdb3']
diff --git a/hieradata/role/common/cache/misc.yaml
b/hieradata/role/common/cache/misc.yaml
index 00bf122..eff990c 100644
--- a/hieradata/role/common/cache/misc.yaml
+++ b/hieradata/role/common/cache/misc.yaml
@@ -272,3 +272,8 @@
director: 'bromine'
yarn.wikimedia.org:
director: 'thorium'
+# Profile::cache::base
+profile::cache::base::zero_site: 'https://zero.wikimedia.org'
+profile::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.wikimedia\.org$',
+profile::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.wikimedia\.org)',
+profile::cache::base::storage_parts: ['sda3', 'sdb3']
diff --git a/hieradata/role/common/cache/text.yaml
b/hieradata/role/common/cache/text.yaml
index 6187ec5..fcc2695 100644
--- a/hieradata/role/common/cache/text.yaml
+++ b/hieradata/role/common/cache/text.yaml
@@ -78,3 +78,8 @@
debug_director: 'appservers_debug'
cache::fe_transient_gb: 5
cache::be_transient_gb: 2
+# Profile::cache::base
+profile::cache::base::zero_site: 'https://zero.wikimedia.org'
+profile::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.wikimedia\.org$',
+profile::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.wikimedia\.org)',
+profile::cache::base::storage_parts: ['sda3', 'sdb3']
diff --git a/hieradata/role/common/cache/upload.yaml
b/hieradata/role/common/cache/upload.yaml
index 37f6314..826a914 100644
--- a/hieradata/role/common/cache/upload.yaml
+++ b/hieradata/role/common/cache/upload.yaml
@@ -48,3 +48,9 @@
director: 'swift_thumbs'
maps.wikimedia.org:
director: 'kartotherian'
+# Profile parameters
+# Profile::cache::base
+profile::cache::base::zero_site: 'https://zero.wikimedia.org'
+profile::cache::base::purge_host_only_upload_re:
'^(upload|maps)\.wikimedia\.org$',
+profile::cache::base::purge_host_not_upload_re:
'^(?!(upload|maps)\.wikimedia\.org)',
+profile::cache::base::storage_parts: ['sda3', 'sdb3']
diff --git a/modules/role/manifests/cache/base.pp
b/modules/profile/manifests/cache/base.pp
similarity index 86%
rename from modules/role/manifests/cache/base.pp
rename to modules/profile/manifests/cache/base.pp
index 2851017..36d0ee4 100644
--- a/modules/role/manifests/cache/base.pp
+++ b/modules/profile/manifests/cache/base.pp
@@ -1,24 +1,30 @@
-class role::cache::base(
- $zero_site = 'https://zero.wikimedia.org',
- $purge_host_only_upload_re = '^(upload|maps)\.wikimedia\.org$',
- $purge_host_not_upload_re = '^(?!(upload|maps)\.wikimedia\.org)',
- $storage_parts = ['sda3', 'sdb3'],
+# === Class profile::cache::base
+#
+# Sets up some common things for cache instances:
+# - LVS/conftool
+# - monitoring
+# - logging/analytics
+# - storage
+#
+class profile::cache::base(
+ $cache_cluster = hiera('cache::cluster'),
+ $statsd_host = hiera('statsd'),
+ $zero_site = hiera('profile::cache::base::zero_site'),
+ $purge_host_only_upload_re =
hiera('profile::cache::base::purge_host_only_upload_re'),
+ $purge_host_not_upload_re =
hiera('profile::cache::base::purge_host_not_upload_re'),
+ $storage_parts = hiera('profile::cache::base::purge_host_not_upload_re'),
) {
- include ::standard
+ # Needed profiles
require ::profile::conftool::client
+ require ::profile::cache::kafka::webrequest
+ include ::standard
+
+ # Other includes - to fix
include ::nrpe
include lvs::configuration
include network::constants
include conftool::scripts
include ::role::prometheus::varnish_exporter
-
-
- $cache_cluster = hiera('cache::cluster')
- $statsd_host = hiera('statsd')
-
- system::role { "cache::${cache_cluster}":
- description => "${cache_cluster} Varnish cache server",
- }
# Only production needs system perf tweaks
if $::realm == 'production' {
@@ -53,7 +59,6 @@
###########################################################################
# Install a varnishkafka producer to send
# varnish webrequest logs to Kafka.
- require ::profile::cache::kafka::webrequest
class { '::varnish::logging':
cache_cluster => $cache_cluster,
diff --git a/modules/role/manifests/cache/misc.pp
b/modules/role/manifests/cache/misc.pp
index 05d3ac1..de7eca1 100644
--- a/modules/role/manifests/cache/misc.pp
+++ b/modules/role/manifests/cache/misc.pp
@@ -1,5 +1,9 @@
class role::cache::misc {
- include role::cache::base
+ system::role { "cache::misc":
+ description => "misc Varnish cache server",
+ }
+
+ include profile::cache::base
include role::cache::ssl::unified
class { 'tlsproxy::prometheus': }
@@ -31,7 +35,7 @@
$common_vcl_config = {
'allowed_methods' =>
'^(GET|DELETE|HEAD|PATCH|POST|PURGE|PUT|OPTIONS)$',
- 'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re,
+ 'purge_host_regex' =>
$::profile::cache::base::purge_host_not_upload_re,
'pass_random' => true,
'req_handling' => hiera('cache::req_handling'),
}
@@ -53,7 +57,7 @@
be_vcl_config => $common_vcl_config,
fe_extra_vcl => ['misc-common', 'zero'],
be_extra_vcl => ['misc-common'],
- be_storage => $::role::cache::base::file_storage_args,
+ be_storage => $::profile::cache::base::file_storage_args,
fe_cache_be_opts => $fe_cache_be_opts,
be_cache_be_opts => $be_cache_be_opts,
cluster_nodes => hiera('cache::misc::nodes'),
diff --git a/modules/role/manifests/cache/text.pp
b/modules/role/manifests/cache/text.pp
index a531c04..28830ff 100644
--- a/modules/role/manifests/cache/text.pp
+++ b/modules/role/manifests/cache/text.pp
@@ -5,9 +5,14 @@
$top_domain = 'org',
$shortener_domain = 'w.wiki',
) {
+
+ system::role { "cache::text":
+ description => "text Varnish cache server",
+ }
+
require geoip
require geoip::dev # for VCL compilation using libGeoIP
- include role::cache::base
+ include profile::cache::base
include role::cache::ssl::unified
include ::standard
@@ -39,7 +44,7 @@
}
$common_vcl_config = {
- 'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re,
+ 'purge_host_regex' =>
$::profile::cache::base::purge_host_not_upload_re,
'static_host' => $static_host,
'top_domain' => $top_domain,
'shortener_domain' => $shortener_domain,
@@ -55,7 +60,7 @@
$common_runtime_params = ['default_ttl=86400']
- $text_storage_args = $::role::cache::base::file_storage_args
+ $text_storage_args = $::profile::cache::base::file_storage_args
class { 'role::cache::instances':
cache_type => 'text',
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index 3b4db65..ed804ff 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -3,7 +3,11 @@
$upload_domain = 'upload.wikimedia.org',
$maps_domain = 'maps.wikimedia.org',
) {
- include role::cache::base
+ system::role { "cache::upload":
+ description => "upload Varnish cache server",
+ }
+
+ include profile::cache::base
include role::cache::ssl::unified
include ::standard
@@ -35,7 +39,7 @@
}
$common_vcl_config = {
- 'purge_host_regex' => $::role::cache::base::purge_host_only_upload_re,
+ 'purge_host_regex' =>
$::profile::cache::base::purge_host_only_upload_re,
'upload_domain' => $upload_domain,
'maps_domain' => $maps_domain,
'allowed_methods' => '^(GET|HEAD|OPTIONS|PURGE)$',
@@ -58,9 +62,9 @@
})
# See T145661 for storage binning rationale
- $sda = $::role::cache::base::storage_parts[0]
- $sdb = $::role::cache::base::storage_parts[1]
- $ssm = $::role::cache::base::storage_size * 2 * 1024
+ $sda = $::profile::cache::base::storage_parts[0]
+ $sdb = $::profile::cache::base::storage_parts[1]
+ $ssm = $::profile::cache::base::storage_size * 2 * 1024
$bin0_size = floor($ssm * 0.04)
$bin1_size = floor($ssm * 0.23)
$bin2_size = floor($ssm * 0.40)
diff --git a/modules/varnish/manifests/zero_update.pp
b/modules/varnish/manifests/zero_update.pp
index e174434..1b010ef 100644
--- a/modules/varnish/manifests/zero_update.pp
+++ b/modules/varnish/manifests/zero_update.pp
@@ -1,6 +1,6 @@
# Zero-specific update stuff
class varnish::zero_update($site) {
- require '::varnish::netmapper_update_common'
+ class { '::varnish::netmapper_update_common': }
package { 'python-requests':
ensure => installed;
--
To view, visit https://gerrit.wikimedia.org/r/382683
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I28093780badaa38713847028cd5b5569dc81d6ca
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
