Jcrespo has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/374560 )
Change subject: mariadb: Implement regular logical backups using mydumper
......................................................................
mariadb: Implement regular logical backups using mydumper
Bug: T169516
Change-Id: I671fb54b35e92fadcc65e622381900443dca8729
---
A modules/role/files/mariadb/dump_shards.sh
A modules/role/manifests/mariadb/backup_mydumper.pp
2 files changed, 77 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/60/374560/1
diff --git a/modules/role/files/mariadb/dump_shards.sh
b/modules/role/files/mariadb/dump_shards.sh
new file mode 100644
index 0000000..5c28071
--- /dev/null
+++ b/modules/role/files/mariadb/dump_shards.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+numthreads=16
+backupdir="backups"
+shards="s2 s3 s5 s6 s7 x1"
+host="localhost"
+rowsmax=20000000
+
+mkdir -p "/srv/$backupdir"
+chmod go-rwx "/srv/$backupdir"
+
+find "/srv/$backupdir" -mtime +14 -type f -delete
+find "/srv/$backupdir" -mtime +14 -type d -exec rmdir '{}' ';'
+
+for backup_shard in $shards; do
+ /usr/bin/mydumper --compress --host="$host" --threads="$numthreads"
--user="`whoami`" --socket="/run/mysqld/mysqld.$backup_shard.sock" --triggers
--routines --events --rows="$rowsmax"
--logfile="/srv/$backupdir/dump.$backup_shard.log"
--outputdir="/srv/$backupdir/$backup_shard.`date +%Y%m%d%H%M%S`";
+done
+
diff --git a/modules/role/manifests/mariadb/backup_mydumper.pp
b/modules/role/manifests/mariadb/backup_mydumper.pp
new file mode 100644
index 0000000..14aca34
--- /dev/null
+++ b/modules/role/manifests/mariadb/backup_mydumper.pp
@@ -0,0 +1,59 @@
+# Create mydumper logical backups using the dump_shards.sh hosts
+# of all production core (s*, x*) and misc (m*) hosts.
+# Do that using a cron job on all backup hosts, all datacenters.
+# If we are on the active datacenter, also send the backups to the
+# long-term storage.
+class role::mariadb::backup_mydumper {
+ include ::profile::backup::host
+ include passwords::mysql::dump
+
+ user { 'dumps':
+ ensure => present,
+ gid => 'root',
+ shell => '/bin/false',
+ home => '/srv/backups',
+ system => true,
+ managehome => false,
+ }
+
+ file { '/srv/backups':
+ ensure => directory,
+ owner => 'dumps',
+ group => 'root',
+ mode => '0600', # implicitly 0700 for dirs
+ }
+
+ file { '/usr/local/bin/dump_shards.sh':
+ ensure => present,
+ owner => 'dumps',
+ group => 'root',
+ mode => '0755',
+ source => 'puppet:///modules/role/mariadb/dump_shards.sh',
+ }
+
+ file { '/etc/mysql/dump_shards.cnf':
+ ensure => present,
+ owner => 'dumps',
+ group => 'root',
+ mode => '0400',
+ content =>
"[client]\nuser=${passwords::mysql::dump::user}\npassword=${passwords::mysql::dump::pass}\n",
+ }
+
+ cron { 'dumps-shards':
+ minute => 0,
+ hour => 22,
+ weekday => 2,
+ user => 'dumps',
+ command => '/usr/local/bin/dump_shards.sh >/dev/null 2>&1',
+ require => [File['/usr/local/bin/dump_shards.sh'],
+ File['/srv/backups'],
+ ],
+ }
+
+ # produce logical backups on all backups hosts, but only send them
+ # to the long term storage on one datacenter at a time
+ $backups_active = hiera('role::mariadb::backups_mydumer::active', false)
+ if $backups_active {
+ backup::set { 'mysql-srv-backups': }
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/374560
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I671fb54b35e92fadcc65e622381900443dca8729
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
