Nikerabbit has uploaded a new change for review.
https://gerrit.wikimedia.org/r/92176
Change subject: Add backup module
......................................................................
Add backup module
Change-Id: I59fab3f631b2bf70ca67296aa719fe85a3f647c7
---
A puppet/modules/backup/files/backup.sh
A puppet/modules/backup/files/duplicity.conf
R puppet/modules/backup/files/logrotate
A puppet/modules/backup/manifests/init.pp
A puppet/modules/backup/templates/backup.erb
A puppet/modules/backup/templates/dump-databases.sh.erb
M puppet/modules/base/manifests/init.pp
M puppet/modules/logrotate/manifests/init.pp
M puppet/modules/wiki/manifests/init.pp
D puppet/modules/wiki/templates/wikibackup.erb
M puppet/site.pp
11 files changed, 113 insertions(+), 15 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/translatewiki
refs/changes/76/92176/1
diff --git a/puppet/modules/backup/files/backup.sh
b/puppet/modules/backup/files/backup.sh
new file mode 100644
index 0000000..bc88cd3
--- /dev/null
+++ b/puppet/modules/backup/files/backup.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# file managed by puppet
+
+# uncomment for debug
+#set -x
+
+source /root/.duplicity.conf
+
+# duplicity command
+SSHOPTS="--ssh-options \"-oIdentityFile=/root/.ssh/id_dsa_duplicity_backup\""
+
+DUPEXEC="--encrypt-key $ENCRKEY --sign-key $SIGNKEY $SSHOPTS $DUPOPTS $*"
+# loop on directories
+echo -n "---- Incremental backup of $HOSTNAME ---- "; date
+for i in $BACKDIRS
+do
+ echo "Starting backup of directory /$i"
+ # create dirs and then backup
+ $MKDIR $LPATH/$i && duplicity $DUPEXEC /$i $RPATH/$i
+ # clean up
+ duplicity remove-older-than 2M --force $DUPEXEC $RPATH/$i
+ duplicity clean --force $DUPEXEC $RPATH/$i
+ echo
+done
+# if local, fix permissions
+if [ -z $HOST ]; then chown -R $NAME.$NAME $LPATH; fi
+echo -n "---- Finished backup on $HOSTNAME ---- "; date
+echo
+echo
diff --git a/puppet/modules/backup/files/duplicity.conf
b/puppet/modules/backup/files/duplicity.conf
new file mode 100644
index 0000000..8a11e93
--- /dev/null
+++ b/puppet/modules/backup/files/duplicity.conf
@@ -0,0 +1,31 @@
+# file managed by puppet
+
+# path to backup to
+LPATH=/work/users/nike/backups/twn
+
+# remote settings
+HOST=lakka.kapsi.fi
+NAME=nike
+RPATH=scp://$NAME@$HOST/$LPATH
+SSHID="/root/.ssh/id_dsa_duplicity_backup"
+
+# complete with root gpg signature and encryption key
+SIGNKEY=D4D02B43
+ENCRKEY=$SIGNKEY
+export PASSPHRASE=$(</root/secrets/backup-passphrase)
+
+# local list of directories to backup
+BACKDIRS='etc var/log var/lib/awstats home www root'
+
+# duplicity options (backup.sh also accepts command line arguments)
+DUPOPTS="--ssh-options \"-oIdentityFile=$SSHID\""
+DUPOPTS="$DUPOPTS --full-if-older-than 1M"
+
+export GNUPGHOME=/root/.gnupg
+export HOME=/root
+
+if [ -z $HOST ]; then
+ MKDIR="mkdir -p"
+else
+ MKDIR="ssh -i $SSHID $NAME@$HOST mkdir -p"
+fi
diff --git a/puppet/modules/logrotate/files/twn-database-backup
b/puppet/modules/backup/files/logrotate
similarity index 77%
rename from puppet/modules/logrotate/files/twn-database-backup
rename to puppet/modules/backup/files/logrotate
index ba3d905..680e2f0 100644
--- a/puppet/modules/logrotate/files/twn-database-backup
+++ b/puppet/modules/backup/files/logrotate
@@ -1,5 +1,5 @@
# file managed by puppet
-/home/betawiki/twn-db-backup.sql {
+/root/db-backup-*.sql {
rotate 1
daily
compress
diff --git a/puppet/modules/backup/manifests/init.pp
b/puppet/modules/backup/manifests/init.pp
new file mode 100644
index 0000000..a8a8374
--- /dev/null
+++ b/puppet/modules/backup/manifests/init.pp
@@ -0,0 +1,36 @@
+# = Class: backup
+#
+# Handles translatewiki.net offsite backups. We backup certain directories
+# and a database dump daily to offsite with duplicity. Backups are encrypted.
+#
+# == Parameters:
+#
+# $databases:: What databases to dump and backup.
+#
+class backup ($databases) {
+ package { 'duplicity':
+ ensure => present,
+ }
+
+ file { "/etc/cron.d/backup":
+ # Enable when new server is primary
+ ensure => absent,
+ content => template("backup/backup.erb"),
+ }
+
+ file { "/root/backup.sh":
+ source => 'puppet:///modules/backup/backup.sh',
+ }
+
+ file { "/root/.duplicity.conf":
+ source => 'puppet:///modules/backup/duplicity.conf',
+ }
+
+ file { "/root/dump-databases.sh":
+ content => template("backup/dump-databases.sh.erb"),
+ }
+
+ file { '/etc/logrotate.d/twn-database-backup':
+ source => 'puppet:///modules/backup/lograte'
+ }
+}
diff --git a/puppet/modules/backup/templates/backup.erb
b/puppet/modules/backup/templates/backup.erb
new file mode 100644
index 0000000..840fa96
--- /dev/null
+++ b/puppet/modules/backup/templates/backup.erb
@@ -0,0 +1,3 @@
+# file managed by puppet
+00 02 * * * root /root/dump-databases.sh
+00 03 * * * root /root/backup.sh
diff --git a/puppet/modules/backup/templates/dump-databases.sh.erb
b/puppet/modules/backup/templates/dump-databases.sh.erb
new file mode 100644
index 0000000..b77b4af
--- /dev/null
+++ b/puppet/modules/backup/templates/dump-databases.sh.erb
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+<% @databases.each do |db| -%>
+nice mysqldump --opt --single-transaction \
+ --user=root --password="" <%= db %> \
+ > /root/db-backup-<%= db =%>.sql
+<% end -%>
diff --git a/puppet/modules/base/manifests/init.pp
b/puppet/modules/base/manifests/init.pp
index 8539219..152a782 100644
--- a/puppet/modules/base/manifests/init.pp
+++ b/puppet/modules/base/manifests/init.pp
@@ -4,7 +4,6 @@
'ack-grep',
'bash-completion',
'doxygen',
- 'duplicity',
'fontconfig',
'htop',
'iotop', # IO view
diff --git a/puppet/modules/logrotate/manifests/init.pp
b/puppet/modules/logrotate/manifests/init.pp
index f2afee6..2cd019b 100644
--- a/puppet/modules/logrotate/manifests/init.pp
+++ b/puppet/modules/logrotate/manifests/init.pp
@@ -2,9 +2,4 @@
file { '/etc/logrotate.d/twn':
source => 'puppet:///modules/logrotate/twn'
}
-
- # @todo Should eventually end up in a backup module
- file { '/etc/logrotate.d/twn-database-backup':
- source => 'puppet:///modules/logrotate/twn-database-backup'
- }
}
diff --git a/puppet/modules/wiki/manifests/init.pp
b/puppet/modules/wiki/manifests/init.pp
index d6f3abc..fae5295 100644
--- a/puppet/modules/wiki/manifests/init.pp
+++ b/puppet/modules/wiki/manifests/init.pp
@@ -17,10 +17,6 @@
# }
#
class wiki ($config, $user) {
- file { "/etc/cron.d/wikibackup":
- content => template("wiki/wikibackup.erb"),
- }
-
file { "/etc/cron.d/wikimaintenance":
content => template("wiki/wikimaintenance.erb"),
}
diff --git a/puppet/modules/wiki/templates/wikibackup.erb
b/puppet/modules/wiki/templates/wikibackup.erb
deleted file mode 100644
index 4852ee5..0000000
--- a/puppet/modules/wiki/templates/wikibackup.erb
+++ /dev/null
@@ -1,2 +0,0 @@
-# file managed by puppet
-@daily <%= @user %> /home/betawiki/backup.sh
diff --git a/puppet/site.pp b/puppet/site.pp
index cb9c2d8..0b16b5f 100644
--- a/puppet/site.pp
+++ b/puppet/site.pp
@@ -1,6 +1,6 @@
File {
- owner => 'root',
- group => 'root',
+ owner => 'root',
+ group => 'root',
}
node default {
@@ -15,6 +15,10 @@
include exim-conf
include logrotate
+ class { 'backup':
+ databases => ['mediawiki'],
+ }
+
class { 'wiki':
config => '/home/betawiki/config',
user => 'betawiki',
--
To view, visit https://gerrit.wikimedia.org/r/92176
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I59fab3f631b2bf70ca67296aa719fe85a3f647c7
Gerrit-PatchSet: 1
Gerrit-Project: translatewiki
Gerrit-Branch: master
Gerrit-Owner: Nikerabbit <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
