Giuseppe Lavagetto has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/344479 )
Change subject: profile::redis::multidc: generic profile for redis cross-dc
replica
......................................................................
profile::redis::multidc: generic profile for redis cross-dc replica
This allows to manage the cross-dc replica depending on etcd discovery,
and should be general enough to accomodate both clusters we have
currently, and any other we might want to set up in the future.
Also, add a global data structure for redis shards of all kinds, much
like lvs::configuration is, in order to be used by other parts of the
code too.
Change-Id: Ibe461edbd43bf5a182c1125c68eb08f5e98edf59
---
A hieradata/common/redis.yaml
M hieradata/role/common/jobqueue_redis.yaml
M hieradata/role/common/jobqueue_redis/master.yaml
D modules/profile/manifests/jobqueue_redis/master.pp
A modules/profile/manifests/redis/multidc.pp
R modules/profile/manifests/redis/multidc_instance.pp
M modules/role/manifests/jobqueue_redis/master.pp
7 files changed, 291 insertions(+), 270 deletions(-)
Approvals:
Giuseppe Lavagetto: Verified; Looks good to me, approved
diff --git a/hieradata/common/redis.yaml b/hieradata/common/redis.yaml
new file mode 100644
index 0000000..470ddcf
--- /dev/null
+++ b/hieradata/common/redis.yaml
@@ -0,0 +1,211 @@
+redis::shards:
+ jobqueue:
+ eqiad:
+ rdb1-aggr: #rdb1001
+ host: 10.64.32.76
+ port: 6378
+ rdb1-6379:
+ host: 10.64.32.76
+ port: 6379
+ rdb1-6380:
+ host: 10.64.32.76
+ port: 6380
+ rdb1-6381:
+ host: 10.64.32.76
+ port: 6381
+ rdb2-aggr: # rdb1003
+ host: 10.64.0.201
+ port: 6378
+ rdb2-6379:
+ host: 10.64.0.201
+ port: 6379
+ rdb2-6380:
+ host: 10.64.0.201
+ port: 6380
+ rdb2-6381:
+ host: 10.64.0.201
+ port: 6381
+ rdb3-aggr: # rdb1005
+ host: 10.64.0.24
+ port: 6378
+ rdb3-6379:
+ host: 10.64.0.24
+ port: 6379
+ rdb3-6380:
+ host: 10.64.0.24
+ port: 6380
+ rdb3-6381:
+ host: 10.64.0.24
+ port: 6381
+ rdb4-aggr: # rdb1007
+ host: 10.64.32.18
+ port: 6378
+ rdb4-6379:
+ host: 10.64.32.18
+ port: 6379
+ rdb4-6380:
+ host: 10.64.32.18
+ port: 6380
+ rdb4-6381:
+ host: 10.64.32.18
+ port: 6381
+ codfw:
+ rdb1-aggr: #rdb2001
+ host: 10.192.0.119
+ port: 6378
+ rdb1-6379:
+ host: 10.192.0.119
+ port: 6379
+ rdb1-6380:
+ host: 10.192.0.119
+ port: 6380
+ rdb1-6381:
+ host: 10.192.0.119
+ port: 6381
+ rdb2-aggr: # rdb2003
+ host: 10.192.16.122
+ port: 6378
+ rdb2-6379:
+ host: 10.192.16.122
+ port: 6379
+ rdb2-6380:
+ host: 10.192.16.122
+ port: 6380
+ rdb2-6381:
+ host: 10.192.16.122
+ port: 6381
+ rdb3-aggr: # rdb2005
+ host: 10.192.32.133
+ port: 6378
+ rdb3-6379:
+ host: 10.192.32.133
+ port: 6379
+ rdb3-6380:
+ host: 10.192.32.133
+ port: 6380
+ rdb3-6381:
+ host: 10.192.32.133
+ port: 6381
+ rdb4-aggr: # rdb2005 - higher ports
+ host: 10.192.32.133
+ port: 6478
+ rdb4-6379:
+ host: 10.192.32.133
+ port: 6479
+ rdb4-6380:
+ host: 10.192.32.133
+ port: 6480
+ rdb4-6381:
+ host: 10.192.32.133
+ port: 6481
+ sessions:
+ eqiad:
+ shard01:
+ host: 10.64.0.180
+ port: 6379
+ shard02:
+ host: 10.64.0.181
+ port: 6379
+ shard03:
+ host: 10.64.0.182
+ port: 6379
+ shard04:
+ host: 10.64.0.183
+ port: 6379
+ shard05:
+ host: 10.64.0.184
+ port: 6379
+ shard06:
+ host: 10.64.0.185
+ port: 6379
+ shard07:
+ host: 10.64.32.161
+ port: 6379
+ shard08:
+ host: 10.64.32.162
+ port: 6379
+ shard09:
+ host: 10.64.32.163
+ port: 6379
+ shard10:
+ host: 10.64.32.164
+ port: 6379
+ shard11:
+ host: 10.64.32.165
+ port: 6379
+ shard12:
+ host: 10.64.32.166
+ port: 6379
+ shard13:
+ host: 10.64.48.101
+ port: 6379
+ shard14:
+ host: 10.64.48.102
+ port: 6379
+ shard15:
+ host: 10.64.48.103
+ port: 6379
+ shard16:
+ host: 10.64.48.104
+ port: 6379
+ shard17:
+ host: 10.64.48.95
+ port: 6379
+ shard18:
+ host: 10.64.48.96
+ port: 6379
+ codfw:
+ shard01:
+ host: 10.192.0.83
+ port: 6379
+ shard02:
+ host: 10.192.0.84
+ port: 6379
+ shard03:
+ host: 10.192.0.85
+ port: 6379
+ shard04:
+ host: 10.192.0.86
+ port: 6379
+ shard05:
+ host: 10.192.16.60
+ port: 6379
+ shard06:
+ host: 10.192.16.61
+ port: 6379
+ shard07:
+ host: 10.192.16.62
+ port: 6379
+ shard08:
+ host: 10.192.16.194
+ port: 6379
+ shard09:
+ host: 10.192.32.159
+ port: 6379
+ shard10:
+ host: 10.192.32.160
+ port: 6379
+ shard11:
+ host: 10.192.32.161
+ port: 6379
+ shard12:
+ host: 10.192.32.162
+ port: 6379
+ shard13:
+ host: 10.192.32.163
+ port: 6379
+ shard14:
+ host: 10.192.48.76
+ port: 6379
+ shard15:
+ host: 10.192.48.77
+ port: 6379
+ shard16:
+ host: 10.192.48.78
+ port: 6379
+ shard17:
+ host: 10.192.48.79
+ port: 6379
+ shard18:
+ host: 10.192.48.80
+ port: 6379
diff --git a/hieradata/role/common/jobqueue_redis.yaml
b/hieradata/role/common/jobqueue_redis.yaml
index 27f7e30..1136271 100644
--- a/hieradata/role/common/jobqueue_redis.yaml
+++ b/hieradata/role/common/jobqueue_redis.yaml
@@ -1,101 +1 @@
cluster: redis
-redis::shards:
- eqiad:
- rdb1-aggr: #rdb1001
- host: 10.64.32.76
- port: 6378
- rdb1-6379:
- host: 10.64.32.76
- port: 6379
- rdb1-6380:
- host: 10.64.32.76
- port: 6380
- rdb1-6381:
- host: 10.64.32.76
- port: 6381
- rdb2-aggr: # rdb1003
- host: 10.64.0.201
- port: 6378
- rdb2-6379:
- host: 10.64.0.201
- port: 6379
- rdb2-6380:
- host: 10.64.0.201
- port: 6380
- rdb2-6381:
- host: 10.64.0.201
- port: 6381
- rdb3-aggr: # rdb1005
- host: 10.64.0.24
- port: 6378
- rdb3-6379:
- host: 10.64.0.24
- port: 6379
- rdb3-6380:
- host: 10.64.0.24
- port: 6380
- rdb3-6381:
- host: 10.64.0.24
- port: 6381
- rdb4-aggr: # rdb1007
- host: 10.64.32.18
- port: 6378
- rdb4-6379:
- host: 10.64.32.18
- port: 6379
- rdb4-6380:
- host: 10.64.32.18
- port: 6380
- rdb4-6381:
- host: 10.64.32.18
- port: 6381
- codfw:
- rdb1-aggr: #rdb2001
- host: 10.192.0.119
- port: 6378
- rdb1-6379:
- host: 10.192.0.119
- port: 6379
- rdb1-6380:
- host: 10.192.0.119
- port: 6380
- rdb1-6381:
- host: 10.192.0.119
- port: 6381
- rdb2-aggr: # rdb2003
- host: 10.192.16.122
- port: 6378
- rdb2-6379:
- host: 10.192.16.122
- port: 6379
- rdb2-6380:
- host: 10.192.16.122
- port: 6380
- rdb2-6381:
- host: 10.192.16.122
- port: 6381
- rdb3-aggr: # rdb2005
- host: 10.192.32.133
- port: 6378
- rdb3-6379:
- host: 10.192.32.133
- port: 6379
- rdb3-6380:
- host: 10.192.32.133
- port: 6380
- rdb3-6381:
- host: 10.192.32.133
- port: 6381
- rdb4-aggr: # rdb2005 - higher ports
- host: 10.192.32.133
- port: 6478
- rdb4-6379:
- host: 10.192.32.133
- port: 6479
- rdb4-6380:
- host: 10.192.32.133
- port: 6480
- rdb4-6381:
- host: 10.192.32.133
- port: 6481
-
diff --git a/hieradata/role/common/jobqueue_redis/master.yaml
b/hieradata/role/common/jobqueue_redis/master.yaml
index 58d39d5..f4d845f 100644
--- a/hieradata/role/common/jobqueue_redis/master.yaml
+++ b/hieradata/role/common/jobqueue_redis/master.yaml
@@ -1,100 +1,15 @@
cluster: redis
-redis_shards:
- eqiad:
- rdb1-aggr: #rdb1001
- host: 10.64.32.76
- port: 6378
- rdb1-6379:
- host: 10.64.32.76
- port: 6379
- rdb1-6380:
- host: 10.64.32.76
- port: 6380
- rdb1-6381:
- host: 10.64.32.76
- port: 6381
- rdb2-aggr: # rdb1003
- host: 10.64.0.201
- port: 6378
- rdb2-6379:
- host: 10.64.0.201
- port: 6379
- rdb2-6380:
- host: 10.64.0.201
- port: 6380
- rdb2-6381:
- host: 10.64.0.201
- port: 6381
- rdb3-aggr: # rdb1005
- host: 10.64.0.24
- port: 6378
- rdb3-6379:
- host: 10.64.0.24
- port: 6379
- rdb3-6380:
- host: 10.64.0.24
- port: 6380
- rdb3-6381:
- host: 10.64.0.24
- port: 6381
- rdb4-aggr: # rdb1007
- host: 10.64.32.18
- port: 6378
- rdb4-6379:
- host: 10.64.32.18
- port: 6379
- rdb4-6380:
- host: 10.64.32.18
- port: 6380
- rdb4-6381:
- host: 10.64.32.18
- port: 6381
- codfw:
- rdb1-aggr: #rdb2001
- host: 10.192.0.119
- port: 6378
- rdb1-6379:
- host: 10.192.0.119
- port: 6379
- rdb1-6380:
- host: 10.192.0.119
- port: 6380
- rdb1-6381:
- host: 10.192.0.119
- port: 6381
- rdb2-aggr: # rdb2003
- host: 10.192.16.122
- port: 6378
- rdb2-6379:
- host: 10.192.16.122
- port: 6379
- rdb2-6380:
- host: 10.192.16.122
- port: 6380
- rdb2-6381:
- host: 10.192.16.122
- port: 6381
- rdb3-aggr: # rdb2005
- host: 10.192.32.133
- port: 6378
- rdb3-6379:
- host: 10.192.32.133
- port: 6379
- rdb3-6380:
- host: 10.192.32.133
- port: 6380
- rdb3-6381:
- host: 10.192.32.133
- port: 6381
- rdb4-aggr: # rdb2005 - higher ports
- host: 10.192.32.133
- port: 6478
- rdb4-6379:
- host: 10.192.32.133
- port: 6479
- rdb4-6380:
- host: 10.192.32.133
- port: 6480
- rdb4-6381:
- host: 10.192.32.133
- port: 6481
+profile::redis::multidc::category: jobqueue
+profile::redis::multidc::discovery: "appservers-rw"
+profile::redis::multidc::settings:
+ bind: '0.0.0.0'
+ appendonly: true
+ auto_aof_rewrite_min_size: '512mb'
+ client_output_buffer_limit: 'slave 2048mb 512mb 60'
+ dir: '/srv/redis'
+ maxmemory: '8Gb'
+ no_appendfsync_on_rewrite: true
+ save: '""'
+ stop_writes_on_bgsave_error: false
+ slave_read_only: false
+profile::redis::multidc::aof: true
diff --git a/modules/profile/manifests/jobqueue_redis/master.pp
b/modules/profile/manifests/jobqueue_redis/master.pp
deleted file mode 100644
index 76cb5c9..0000000
--- a/modules/profile/manifests/jobqueue_redis/master.pp
+++ /dev/null
@@ -1,68 +0,0 @@
-class profile::jobqueue_redis::master(
- $shards = hiera('redis_shards'),
- $conftool_prefix = hiera('conftool_prefix'),
-) {
- include ::passwords::redis
- $ip = $facts['ipaddress_primary']
- $instances = redis_get_instances($ip, $shards)
- $password = $passwords::redis::main_password
- $uris = apply_format("localhost:%s/${password}", $instances)
- $redis_ports = join($instances, ' ')
-
- system::role { 'role::jobqueue_redis::master': }
-
- # Set up ipsec
- class { 'redis::multidc::ipsec':
- shards => $shards
- }
-
- class { '::ferm::ipsec_allow': }
-
- file { '/etc/redis/replica/':
- ensure => directory,
- owner => 'root',
- group => 'root',
- mode => '0444',
- }
-
- # Now the redis instances. We watch etcd every 5 minutes to fix config
- # based on the mediawiki master datacenter
- class { 'confd':
- interval => 300,
- prefix => $conftool_prefix,
- }
-
- profile::multidc_redis::instances{ $instances:
- ip => $ip,
- shards => $shards,
- discovery => 'appservers-rw',
- aof => true,
- settings => {
- bind => '0.0.0.0',
- appendonly => true,
- auto_aof_rewrite_min_size => '512mb',
- client_output_buffer_limit => 'slave 2048mb 512mb 60',
- dir => '/srv/redis',
- masterauth => $passwords::redis::main_password,
- maxmemory => '8Gb',
- no_appendfsync_on_rewrite => true,
- requirepass => $passwords::redis::main_password,
- save => '""',
- stop_writes_on_bgsave_error => false,
- slave_read_only => false,
- },
- }
-
- # Add monitoring, using nrpe and not remote checks anymore
- redis::monitoring::nrpe_instance { $instances: }
-
- diamond::collector { 'Redis':
- settings => { instances => join($uris, ', ') }
- }
-
- ferm::service { 'redis_jobqueue_role':
- proto => 'tcp',
- notrack => true,
- port => inline_template('(<%= @redis_ports %>)'),
- }
-}
diff --git a/modules/profile/manifests/redis/multidc.pp
b/modules/profile/manifests/redis/multidc.pp
new file mode 100644
index 0000000..d9e2343
--- /dev/null
+++ b/modules/profile/manifests/redis/multidc.pp
@@ -0,0 +1,63 @@
+class profile::redis::multidc(
+ $category = hiera('profile::redis::multidc::category'),
+ $all_shards = hiera('redis::shards'),
+ $conftool_prefix = hiera('conftool_prefix'),
+ $settings = hiera('profile::redis::multidc::settings'),
+ $discovery = hiera('profile::redis::multidc::discovery'),
+ $aof = hiera('profile::redis::multidc::aof', false)
+) {
+ require ::passwords::redis
+ $shards = $all_shards[$category]
+ $ip = $facts['ipaddress_primary']
+ $instances = redis_get_instances($ip, $shards)
+ $password = $passwords::redis::main_password
+ $uris = apply_format("localhost:%s/${password}", $instances)
+ $redis_ports = join($instances, ' ')
+ $auth_settings = {
+ 'masterauth' => $password,
+ 'requirepass' => $password,
+ }
+
+ system::role { "profile::redis::${category}": }
+ # Set up ipsec
+ class { 'redis::multidc::ipsec':
+ shards => $shards
+ }
+
+ class { '::ferm::ipsec_allow': }
+
+ file { '/etc/redis/replica/':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+
+ # Now the redis instances. We watch etcd every 5 minutes to fix config
+ # based on the active datacenter for the chosen discovery label
+ class { 'confd':
+ interval => 300,
+ prefix => $conftool_prefix,
+ }
+
+ profile::redis::multidc_instance{ $instances:
+ ip => $ip,
+ shards => $shards,
+ discovery => $discovery,
+ aof => $aof,
+ settings => merge($settings, $auth_settings),
+ }
+
+ # Add monitoring, using nrpe and not remote checks anymore
+ redis::monitoring::nrpe_instance { $instances: }
+
+ diamond::collector { 'Redis':
+ settings => { instances => join($uris, ', ') }
+ }
+
+ ferm::service { "redis_${category}_role":
+ proto => 'tcp',
+ notrack => true,
+ port => inline_template('(<%= @redis_ports %>)'),
+ }
+}
diff --git a/modules/profile/manifests/multidc_redis/instances.pp
b/modules/profile/manifests/redis/multidc_instance.pp
similarity index 95%
rename from modules/profile/manifests/multidc_redis/instances.pp
rename to modules/profile/manifests/redis/multidc_instance.pp
index 9020ead..f9def03 100644
--- a/modules/profile/manifests/multidc_redis/instances.pp
+++ b/modules/profile/manifests/redis/multidc_instance.pp
@@ -1,7 +1,7 @@
-define profile::multidc_redis::instances(
+define profile::redis::multidc_instance(
$ip,
+ $shards,
$settings = {},
- $shards = {},
$discovery = undef,
$aof = false,
) {
diff --git a/modules/role/manifests/jobqueue_redis/master.pp
b/modules/role/manifests/jobqueue_redis/master.pp
index cb483c9..aa0dfd8 100644
--- a/modules/role/manifests/jobqueue_redis/master.pp
+++ b/modules/role/manifests/jobqueue_redis/master.pp
@@ -1,5 +1,5 @@
class role::jobqueue_redis::master {
include ::standard
include ::base::firewall
- include ::profile::jobqueue_redis::master
+ include ::profile::redis::multidc
}
--
To view, visit https://gerrit.wikimedia.org/r/344479
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibe461edbd43bf5a182c1125c68eb08f5e98edf59
Gerrit-PatchSet: 5
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
