Giuseppe Lavagetto has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/344479 )
Change subject: profile::redis::multidc: generic profile for redis cross-dc
replica
......................................................................
profile::redis::multidc: generic profile for redis cross-dc replica
This allows to manage the cross-dc replica depending on etcd discovery,
and should be general enough to accomodate both clusters we have
currently, and any other we might want to set up in the future.
Also, add a global data structure for redis shards of all kinds, much
like lvs::configuration is, in order to be used by other parts of the
code too.
Change-Id: Ibe461edbd43bf5a182c1125c68eb08f5e98edf59
---
A hieradata/common/redis.yaml
M hieradata/role/common/jobqueue_redis.yaml
M hieradata/role/common/jobqueue_redis/master.yaml
D modules/profile/manifests/jobqueue_redis/master.pp
A modules/profile/manifests/redis/multidc.pp
R modules/profile/manifests/redis/multidc_instance.pp
M modules/role/manifests/jobqueue_redis/master.pp
7 files changed, 288 insertions(+), 270 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/79/344479/1
diff --git a/hieradata/common/redis.yaml b/hieradata/common/redis.yaml
new file mode 100644
index 0000000..470ddcf
--- /dev/null
+++ b/hieradata/common/redis.yaml
@@ -0,0 +1,211 @@
+redis::shards:
+ jobqueue:
+ eqiad:
+ rdb1-aggr: #rdb1001
+ host: 10.64.32.76
+ port: 6378
+ rdb1-6379:
+ host: 10.64.32.76
+ port: 6379
+ rdb1-6380:
+ host: 10.64.32.76
+ port: 6380
+ rdb1-6381:
+ host: 10.64.32.76
+ port: 6381
+ rdb2-aggr: # rdb1003
+ host: 10.64.0.201
+ port: 6378
+ rdb2-6379:
+ host: 10.64.0.201
+ port: 6379
+ rdb2-6380:
+ host: 10.64.0.201
+ port: 6380
+ rdb2-6381:
+ host: 10.64.0.201
+ port: 6381
+ rdb3-aggr: # rdb1005
+ host: 10.64.0.24
+ port: 6378
+ rdb3-6379:
+ host: 10.64.0.24
+ port: 6379
+ rdb3-6380:
+ host: 10.64.0.24
+ port: 6380
+ rdb3-6381:
+ host: 10.64.0.24
+ port: 6381
+ rdb4-aggr: # rdb1007
+ host: 10.64.32.18
+ port: 6378
+ rdb4-6379:
+ host: 10.64.32.18
+ port: 6379
+ rdb4-6380:
+ host: 10.64.32.18
+ port: 6380
+ rdb4-6381:
+ host: 10.64.32.18
+ port: 6381
+ codfw:
+ rdb1-aggr: #rdb2001
+ host: 10.192.0.119
+ port: 6378
+ rdb1-6379:
+ host: 10.192.0.119
+ port: 6379
+ rdb1-6380:
+ host: 10.192.0.119
+ port: 6380
+ rdb1-6381:
+ host: 10.192.0.119
+ port: 6381
+ rdb2-aggr: # rdb2003
+ host: 10.192.16.122
+ port: 6378
+ rdb2-6379:
+ host: 10.192.16.122
+ port: 6379
+ rdb2-6380:
+ host: 10.192.16.122
+ port: 6380
+ rdb2-6381:
+ host: 10.192.16.122
+ port: 6381
+ rdb3-aggr: # rdb2005
+ host: 10.192.32.133
+ port: 6378
+ rdb3-6379:
+ host: 10.192.32.133
+ port: 6379
+ rdb3-6380:
+ host: 10.192.32.133
+ port: 6380
+ rdb3-6381:
+ host: 10.192.32.133
+ port: 6381
+ rdb4-aggr: # rdb2005 - higher ports
+ host: 10.192.32.133
+ port: 6478
+ rdb4-6379:
+ host: 10.192.32.133
+ port: 6479
+ rdb4-6380:
+ host: 10.192.32.133
+ port: 6480
+ rdb4-6381:
+ host: 10.192.32.133
+ port: 6481
+ sessions:
+ eqiad:
+ shard01:
+ host: 10.64.0.180
+ port: 6379
+ shard02:
+ host: 10.64.0.181
+ port: 6379
+ shard03:
+ host: 10.64.0.182
+ port: 6379
+ shard04:
+ host: 10.64.0.183
+ port: 6379
+ shard05:
+ host: 10.64.0.184
+ port: 6379
+ shard06:
+ host: 10.64.0.185
+ port: 6379
+ shard07:
+ host: 10.64.32.161
+ port: 6379
+ shard08:
+ host: 10.64.32.162
+ port: 6379
+ shard09:
+ host: 10.64.32.163
+ port: 6379
+ shard10:
+ host: 10.64.32.164
+ port: 6379
+ shard11:
+ host: 10.64.32.165
+ port: 6379
+ shard12:
+ host: 10.64.32.166
+ port: 6379
+ shard13:
+ host: 10.64.48.101
+ port: 6379
+ shard14:
+ host: 10.64.48.102
+ port: 6379
+ shard15:
+ host: 10.64.48.103
+ port: 6379
+ shard16:
+ host: 10.64.48.104
+ port: 6379
+ shard17:
+ host: 10.64.48.95
+ port: 6379
+ shard18:
+ host: 10.64.48.96
+ port: 6379
+ codfw:
+ shard01:
+ host: 10.192.0.83
+ port: 6379
+ shard02:
+ host: 10.192.0.84
+ port: 6379
+ shard03:
+ host: 10.192.0.85
+ port: 6379
+ shard04:
+ host: 10.192.0.86
+ port: 6379
+ shard05:
+ host: 10.192.16.60
+ port: 6379
+ shard06:
+ host: 10.192.16.61
+ port: 6379
+ shard07:
+ host: 10.192.16.62
+ port: 6379
+ shard08:
+ host: 10.192.16.194
+ port: 6379
+ shard09:
+ host: 10.192.32.159
+ port: 6379
+ shard10:
+ host: 10.192.32.160
+ port: 6379
+ shard11:
+ host: 10.192.32.161
+ port: 6379
+ shard12:
+ host: 10.192.32.162
+ port: 6379
+ shard13:
+ host: 10.192.32.163
+ port: 6379
+ shard14:
+ host: 10.192.48.76
+ port: 6379
+ shard15:
+ host: 10.192.48.77
+ port: 6379
+ shard16:
+ host: 10.192.48.78
+ port: 6379
+ shard17:
+ host: 10.192.48.79
+ port: 6379
+ shard18:
+ host: 10.192.48.80
+ port: 6379
diff --git a/hieradata/role/common/jobqueue_redis.yaml
b/hieradata/role/common/jobqueue_redis.yaml
index 27f7e30..1136271 100644
--- a/hieradata/role/common/jobqueue_redis.yaml
+++ b/hieradata/role/common/jobqueue_redis.yaml
@@ -1,101 +1 @@
cluster: redis
-redis::shards:
- eqiad:
- rdb1-aggr: #rdb1001
- host: 10.64.32.76
- port: 6378
- rdb1-6379:
- host: 10.64.32.76
- port: 6379
- rdb1-6380:
- host: 10.64.32.76
- port: 6380
- rdb1-6381:
- host: 10.64.32.76
- port: 6381
- rdb2-aggr: # rdb1003
- host: 10.64.0.201
- port: 6378
- rdb2-6379:
- host: 10.64.0.201
- port: 6379
- rdb2-6380:
- host: 10.64.0.201
- port: 6380
- rdb2-6381:
- host: 10.64.0.201
- port: 6381
- rdb3-aggr: # rdb1005
- host: 10.64.0.24
- port: 6378
- rdb3-6379:
- host: 10.64.0.24
- port: 6379
- rdb3-6380:
- host: 10.64.0.24
- port: 6380
- rdb3-6381:
- host: 10.64.0.24
- port: 6381
- rdb4-aggr: # rdb1007
- host: 10.64.32.18
- port: 6378
- rdb4-6379:
- host: 10.64.32.18
- port: 6379
- rdb4-6380:
- host: 10.64.32.18
- port: 6380
- rdb4-6381:
- host: 10.64.32.18
- port: 6381
- codfw:
- rdb1-aggr: #rdb2001
- host: 10.192.0.119
- port: 6378
- rdb1-6379:
- host: 10.192.0.119
- port: 6379
- rdb1-6380:
- host: 10.192.0.119
- port: 6380
- rdb1-6381:
- host: 10.192.0.119
- port: 6381
- rdb2-aggr: # rdb2003
- host: 10.192.16.122
- port: 6378
- rdb2-6379:
- host: 10.192.16.122
- port: 6379
- rdb2-6380:
- host: 10.192.16.122
- port: 6380
- rdb2-6381:
- host: 10.192.16.122
- port: 6381
- rdb3-aggr: # rdb2005
- host: 10.192.32.133
- port: 6378
- rdb3-6379:
- host: 10.192.32.133
- port: 6379
- rdb3-6380:
- host: 10.192.32.133
- port: 6380
- rdb3-6381:
- host: 10.192.32.133
- port: 6381
- rdb4-aggr: # rdb2005 - higher ports
- host: 10.192.32.133
- port: 6478
- rdb4-6379:
- host: 10.192.32.133
- port: 6479
- rdb4-6380:
- host: 10.192.32.133
- port: 6480
- rdb4-6381:
- host: 10.192.32.133
- port: 6481
-
diff --git a/hieradata/role/common/jobqueue_redis/master.yaml
b/hieradata/role/common/jobqueue_redis/master.yaml
index 58d39d5..ebff9a5 100644
--- a/hieradata/role/common/jobqueue_redis/master.yaml
+++ b/hieradata/role/common/jobqueue_redis/master.yaml
@@ -1,100 +1,17 @@
cluster: redis
-redis_shards:
- eqiad:
- rdb1-aggr: #rdb1001
- host: 10.64.32.76
- port: 6378
- rdb1-6379:
- host: 10.64.32.76
- port: 6379
- rdb1-6380:
- host: 10.64.32.76
- port: 6380
- rdb1-6381:
- host: 10.64.32.76
- port: 6381
- rdb2-aggr: # rdb1003
- host: 10.64.0.201
- port: 6378
- rdb2-6379:
- host: 10.64.0.201
- port: 6379
- rdb2-6380:
- host: 10.64.0.201
- port: 6380
- rdb2-6381:
- host: 10.64.0.201
- port: 6381
- rdb3-aggr: # rdb1005
- host: 10.64.0.24
- port: 6378
- rdb3-6379:
- host: 10.64.0.24
- port: 6379
- rdb3-6380:
- host: 10.64.0.24
- port: 6380
- rdb3-6381:
- host: 10.64.0.24
- port: 6381
- rdb4-aggr: # rdb1007
- host: 10.64.32.18
- port: 6378
- rdb4-6379:
- host: 10.64.32.18
- port: 6379
- rdb4-6380:
- host: 10.64.32.18
- port: 6380
- rdb4-6381:
- host: 10.64.32.18
- port: 6381
- codfw:
- rdb1-aggr: #rdb2001
- host: 10.192.0.119
- port: 6378
- rdb1-6379:
- host: 10.192.0.119
- port: 6379
- rdb1-6380:
- host: 10.192.0.119
- port: 6380
- rdb1-6381:
- host: 10.192.0.119
- port: 6381
- rdb2-aggr: # rdb2003
- host: 10.192.16.122
- port: 6378
- rdb2-6379:
- host: 10.192.16.122
- port: 6379
- rdb2-6380:
- host: 10.192.16.122
- port: 6380
- rdb2-6381:
- host: 10.192.16.122
- port: 6381
- rdb3-aggr: # rdb2005
- host: 10.192.32.133
- port: 6378
- rdb3-6379:
- host: 10.192.32.133
- port: 6379
- rdb3-6380:
- host: 10.192.32.133
- port: 6380
- rdb3-6381:
- host: 10.192.32.133
- port: 6381
- rdb4-aggr: # rdb2005 - higher ports
- host: 10.192.32.133
- port: 6478
- rdb4-6379:
- host: 10.192.32.133
- port: 6479
- rdb4-6380:
- host: 10.192.32.133
- port: 6480
- rdb4-6381:
- host: 10.192.32.133
- port: 6481
+profile::redis::multidc::category: jobqueue
+profile::redis::multidc::discovery: "appservers-rw"
+profile::redis::multidc::settings:
+ bind: '0.0.0.0'
+ appendonly: true
+ auto_aof_rewrite_min_size: '512mb'
+ client_output_buffer_limit: 'slave 2048mb 512mb 60'
+ dir: '/srv/redis'
+ masterauth: "%{::passwords::redis::main_password}"
+ maxmemory: '8Gb'
+ no_appendfsync_on_rewrite: true
+ requirepass: "%{::passwords::redis::main_password}"
+ save: '""'
+ stop_writes_on_bgsave_error: false
+ slave_read_only: false
+profile::redis::multidc::aof: false
diff --git a/modules/profile/manifests/jobqueue_redis/master.pp
b/modules/profile/manifests/jobqueue_redis/master.pp
deleted file mode 100644
index 76cb5c9..0000000
--- a/modules/profile/manifests/jobqueue_redis/master.pp
+++ /dev/null
@@ -1,68 +0,0 @@
-class profile::jobqueue_redis::master(
- $shards = hiera('redis_shards'),
- $conftool_prefix = hiera('conftool_prefix'),
-) {
- include ::passwords::redis
- $ip = $facts['ipaddress_primary']
- $instances = redis_get_instances($ip, $shards)
- $password = $passwords::redis::main_password
- $uris = apply_format("localhost:%s/${password}", $instances)
- $redis_ports = join($instances, ' ')
-
- system::role { 'role::jobqueue_redis::master': }
-
- # Set up ipsec
- class { 'redis::multidc::ipsec':
- shards => $shards
- }
-
- class { '::ferm::ipsec_allow': }
-
- file { '/etc/redis/replica/':
- ensure => directory,
- owner => 'root',
- group => 'root',
- mode => '0444',
- }
-
- # Now the redis instances. We watch etcd every 5 minutes to fix config
- # based on the mediawiki master datacenter
- class { 'confd':
- interval => 300,
- prefix => $conftool_prefix,
- }
-
- profile::multidc_redis::instances{ $instances:
- ip => $ip,
- shards => $shards,
- discovery => 'appservers-rw',
- aof => true,
- settings => {
- bind => '0.0.0.0',
- appendonly => true,
- auto_aof_rewrite_min_size => '512mb',
- client_output_buffer_limit => 'slave 2048mb 512mb 60',
- dir => '/srv/redis',
- masterauth => $passwords::redis::main_password,
- maxmemory => '8Gb',
- no_appendfsync_on_rewrite => true,
- requirepass => $passwords::redis::main_password,
- save => '""',
- stop_writes_on_bgsave_error => false,
- slave_read_only => false,
- },
- }
-
- # Add monitoring, using nrpe and not remote checks anymore
- redis::monitoring::nrpe_instance { $instances: }
-
- diamond::collector { 'Redis':
- settings => { instances => join($uris, ', ') }
- }
-
- ferm::service { 'redis_jobqueue_role':
- proto => 'tcp',
- notrack => true,
- port => inline_template('(<%= @redis_ports %>)'),
- }
-}
diff --git a/modules/profile/manifests/redis/multidc.pp
b/modules/profile/manifests/redis/multidc.pp
new file mode 100644
index 0000000..b40c1d5
--- /dev/null
+++ b/modules/profile/manifests/redis/multidc.pp
@@ -0,0 +1,58 @@
+class profile::redis::multidc(
+ $category = hiera('profile::redis::multidc::category'),
+ $all_shards = hiera('redis::shards'),
+ $conftool_prefix = hiera('conftool_prefix'),
+ $settings = hiera('profile::redis::multidc::settings'),
+ $discovery = hiera('profile::redis::multidc::discovery'),
+ $aof = hiera('profile::redis::multidc::aof', false)
+) {
+ include ::passwords::redis
+ $shards = $all_shards[$category]
+ $ip = $facts['ipaddress_primary']
+ $instances = redis_get_instances($ip, $shards)
+ $password = $passwords::redis::main_password
+ $uris = apply_format("localhost:%s/${password}", $instances)
+ $redis_ports = join($instances, ' ')
+
+ # Set up ipsec
+ class { 'redis::multidc::ipsec':
+ shards => $shards
+ }
+
+ class { '::ferm::ipsec_allow': }
+
+ file { '/etc/redis/replica/':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+
+ # Now the redis instances. We watch etcd every 5 minutes to fix config
+ # based on the active datacenter for the chosen discovery label
+ class { 'confd':
+ interval => 300,
+ prefix => $conftool_prefix,
+ }
+
+ profile::redis::multidc_instance{ $instances:
+ ip => $ip,
+ shards => $shards,
+ discovery => $discovery,
+ aof => $aof,
+ settings => $settings
+ }
+
+ # Add monitoring, using nrpe and not remote checks anymore
+ redis::monitoring::nrpe_instance { $instances: }
+
+ diamond::collector { 'Redis':
+ settings => { instances => join($uris, ', ') }
+ }
+
+ ferm::service { 'redis_role':
+ proto => 'tcp',
+ notrack => true,
+ port => inline_template('(<%= @redis_ports %>)'),
+ }
+}
diff --git a/modules/profile/manifests/multidc_redis/instances.pp
b/modules/profile/manifests/redis/multidc_instance.pp
similarity index 95%
rename from modules/profile/manifests/multidc_redis/instances.pp
rename to modules/profile/manifests/redis/multidc_instance.pp
index 9020ead..f9def03 100644
--- a/modules/profile/manifests/multidc_redis/instances.pp
+++ b/modules/profile/manifests/redis/multidc_instance.pp
@@ -1,7 +1,7 @@
-define profile::multidc_redis::instances(
+define profile::redis::multidc_instance(
$ip,
+ $shards,
$settings = {},
- $shards = {},
$discovery = undef,
$aof = false,
) {
diff --git a/modules/role/manifests/jobqueue_redis/master.pp
b/modules/role/manifests/jobqueue_redis/master.pp
index cb483c9..aa0dfd8 100644
--- a/modules/role/manifests/jobqueue_redis/master.pp
+++ b/modules/role/manifests/jobqueue_redis/master.pp
@@ -1,5 +1,5 @@
class role::jobqueue_redis::master {
include ::standard
include ::base::firewall
- include ::profile::jobqueue_redis::master
+ include ::profile::redis::multidc
}
--
To view, visit https://gerrit.wikimedia.org/r/344479
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibe461edbd43bf5a182c1125c68eb08f5e98edf59
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
