Giuseppe Lavagetto has uploaded a new change for review.
https://gerrit.wikimedia.org/r/276980
Change subject: jobqueue_redis: set up encryption and cross-dc replication
......................................................................
jobqueue_redis: set up encryption and cross-dc replication
Since the topology is slightly different in eqiad and codfw, the logic
of cross-dc replication is moved into a data structure. The dc-local
redis masters are then set up for replication and cross-dc encryption
based on it. The local slaves have no IPSec encryption and only need to
know the instances present on their master, of which they are an exact
mirror. In case of need, a slave can be promoted to be master by
removing the slaveof definition and promoting it into the master-master
data structure.
Bug: T124672
Change-Id: I2bd075501661bb9e9527bb8e1858a1322a4a1535
---
M hieradata/role/common/jobqueue_redis.yaml
M manifests/role/jobqueue_redis.pp
M modules/mediawiki/manifests/jobqueue_redis.pp
3 files changed, 132 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/80/276980/1
diff --git a/hieradata/role/common/jobqueue_redis.yaml
b/hieradata/role/common/jobqueue_redis.yaml
index 1136271..27f7e30 100644
--- a/hieradata/role/common/jobqueue_redis.yaml
+++ b/hieradata/role/common/jobqueue_redis.yaml
@@ -1 +1,101 @@
cluster: redis
+redis::shards:
+ eqiad:
+ rdb1-aggr: #rdb1001
+ host: 10.64.32.76
+ port: 6378
+ rdb1-6379:
+ host: 10.64.32.76
+ port: 6379
+ rdb1-6380:
+ host: 10.64.32.76
+ port: 6380
+ rdb1-6381:
+ host: 10.64.32.76
+ port: 6381
+ rdb2-aggr: # rdb1003
+ host: 10.64.0.201
+ port: 6378
+ rdb2-6379:
+ host: 10.64.0.201
+ port: 6379
+ rdb2-6380:
+ host: 10.64.0.201
+ port: 6380
+ rdb2-6381:
+ host: 10.64.0.201
+ port: 6381
+ rdb3-aggr: # rdb1005
+ host: 10.64.0.24
+ port: 6378
+ rdb3-6379:
+ host: 10.64.0.24
+ port: 6379
+ rdb3-6380:
+ host: 10.64.0.24
+ port: 6380
+ rdb3-6381:
+ host: 10.64.0.24
+ port: 6381
+ rdb4-aggr: # rdb1007
+ host: 10.64.32.18
+ port: 6378
+ rdb4-6379:
+ host: 10.64.32.18
+ port: 6379
+ rdb4-6380:
+ host: 10.64.32.18
+ port: 6380
+ rdb4-6381:
+ host: 10.64.32.18
+ port: 6381
+ codfw:
+ rdb1-aggr: #rdb2001
+ host: 10.192.0.119
+ port: 6378
+ rdb1-6379:
+ host: 10.192.0.119
+ port: 6379
+ rdb1-6380:
+ host: 10.192.0.119
+ port: 6380
+ rdb1-6381:
+ host: 10.192.0.119
+ port: 6381
+ rdb2-aggr: # rdb2003
+ host: 10.192.16.122
+ port: 6378
+ rdb2-6379:
+ host: 10.192.16.122
+ port: 6379
+ rdb2-6380:
+ host: 10.192.16.122
+ port: 6380
+ rdb2-6381:
+ host: 10.192.16.122
+ port: 6381
+ rdb3-aggr: # rdb2005
+ host: 10.192.32.133
+ port: 6378
+ rdb3-6379:
+ host: 10.192.32.133
+ port: 6379
+ rdb3-6380:
+ host: 10.192.32.133
+ port: 6380
+ rdb3-6381:
+ host: 10.192.32.133
+ port: 6381
+ rdb4-aggr: # rdb2005 - higher ports
+ host: 10.192.32.133
+ port: 6478
+ rdb4-6379:
+ host: 10.192.32.133
+ port: 6479
+ rdb4-6380:
+ host: 10.192.32.133
+ port: 6480
+ rdb4-6381:
+ host: 10.192.32.133
+ port: 6481
+
diff --git a/manifests/role/jobqueue_redis.pp b/manifests/role/jobqueue_redis.pp
index b35efe3..13f12c9 100644
--- a/manifests/role/jobqueue_redis.pp
+++ b/manifests/role/jobqueue_redis.pp
@@ -6,17 +6,29 @@
$password = $passwords::redis::main_password
$slaveof = hiera('jobqueue_redis_slaveof', undef)
- $instances = apply_format("localhost:%s/${password}", range(6378, 6382))
+ $shards = hiera('redis::shards')
- # Aggregator backend
- mediawiki::jobqueue_redis { 6378: slaveof => $slaveof }
+ if ($slaveof == undef) { # Local master
+ $ip = $::main_ipaddress
+ $instances = redis_get_instances($ip, $shards)
+ # find out the replication topology
+ $replica_map = redis_add_replica({}, $ip, $shards, $::mw_primary)
- # Queues
- mediawiki::jobqueue_redis { 6379: slaveof => $slaveof }
- mediawiki::jobqueue_redis { 6380: slaveof => $slaveof }
- mediawiki::jobqueue_redis { 6381: slaveof => $slaveof }
+ # Encrypt the replication
+ if os_version('Debian >= jessie') {
+ class { 'redis::multidc::ipsec':
+ shards => $shards
+ }
+ }
+ mediawiki::jobqueue_redis {$instances: slaveof => $replica_map}
+ } else {
+ # Slave: the slave has the same instances as its master
+ $instances = redis_get_instances($slaveof, $shards)
+ mediawiki::jobqueue_redis { $instances: slaveof => $slaveof}
+ }
+ $uris = apply_format("localhost:%s/${password}", $instances)
diamond::collector { 'Redis':
- settings => { instances => join($instances, ', ') }
+ settings => { instances => join($uris, ', ') }
}
}
diff --git a/modules/mediawiki/manifests/jobqueue_redis.pp
b/modules/mediawiki/manifests/jobqueue_redis.pp
index 8706be8..84e2097 100644
--- a/modules/mediawiki/manifests/jobqueue_redis.pp
+++ b/modules/mediawiki/manifests/jobqueue_redis.pp
@@ -21,7 +21,7 @@
define mediawiki::jobqueue_redis(
$port = $title,
$slaveof = undef
-) {
+ ) {
include ::passwords::redis
ferm::service { "redis-server-${port}":
@@ -30,10 +30,16 @@
srange => '$ALL_NETWORKS',
}
- $slaveof_actual = $slaveof ? {
- /^\S+ \d+$/ => $slaveof,
- /^\S+$/ => "${slaveof} ${port}",
- default => undef,
+ if is_hash($slaveof) {
+ $map = $slaveof
+ $slaveof_actual = undef
+ } else {
+ $map = {}
+ $slaveof_actual = $slaveof ? {
+ /^\S+ \d+$/ => $slaveof,
+ /^\S+$/ => "${slaveof} ${port}",
+ default => undef,
+ }
}
redis::instance { $port:
@@ -54,5 +60,6 @@
dbfilename => "${::hostname}-${port}.rdb",
slaveof => $slaveof_actual,
},
+ map => $map,
}
}
--
To view, visit https://gerrit.wikimedia.org/r/276980
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2bd075501661bb9e9527bb8e1858a1322a4a1535
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
