Ottomata has submitted this change and it was merged.
Change subject: Add mysql_wmf::mylvmbackup define, use this for backups of
analytics-meta mysql instance
......................................................................
Add mysql_wmf::mylvmbackup define, use this for backups of analytics-meta mysql
instance
mylvmbackup's prebackup hook allows us to define our own backup method. This
hook runs after
the LVM snapshot has been taken and mounted at $mountdir. We rsync $mountdir
to $dest at this
time.
Bug: T127991
Change-Id: I6b0c6e0b819c837b8dbffc97be0c08ee4f66a452
---
A hieradata/role/eqiad/analytics_cluster/database/meta/backup.yaml
M manifests/site.pp
A modules/mysql_wmf/files/mylvmbackup.logrotate
A modules/mysql_wmf/manifests/mylvmbackup.pp
A modules/mysql_wmf/templates/mylvmbackup-prebackup.erb
A modules/role/manifests/analytics_cluster/database/meta/backup.pp
A modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
7 files changed, 195 insertions(+), 1 deletion(-)
Approvals:
Ottomata: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/eqiad/analytics_cluster/database/meta/backup.yaml
b/hieradata/role/eqiad/analytics_cluster/database/meta/backup.yaml
new file mode 100644
index 0000000..ca00d5f
--- /dev/null
+++ b/hieradata/role/eqiad/analytics_cluster/database/meta/backup.yaml
@@ -0,0 +1,6 @@
+# Configure regular backups of the analytics-meta MySQL instance
+# in the Analytics Cluster to back up via rsync to
+# analytics1002. This works because the analytics1002 has the
+# role::analyitcs_cluster::database::meta::backup_dest class
+# applied to it.
+dest: analytics1002.eqiad.wmnet::backup/mysql/analytics-meta
\ No newline at end of file
diff --git a/manifests/site.pp b/manifests/site.pp
index b489ec3..54df25e 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -50,7 +50,14 @@
# analytics1002 is the Hadoop standby NameNode and ResourceManager.
node 'analytics1002.eqiad.wmnet' {
role analytics_cluster::hadoop::standby,
- analytics_cluster::users
+ analytics_cluster::users,
+ # analytics1002 is usually inactive, and it has a
+ # decent amount of disk space. We use it to
+ # store backups of the analytics_cluster::database::meta
+ # (MySQL analytics-meta) instance. If you move this,
+ # make sure /srv/backup/mysql/analytics-meta has
+ # enough space to store backups.
+ analytics_cluster::database::meta::backup_dest
include standard
include base::firewall
@@ -64,6 +71,10 @@
node 'analytics1015.eqiad.wmnet' {
role analytics_cluster::client,
analytics_cluster::database::meta,
+ # Back up analytics-meta MySQL instance
+ # to analytics1002. $dest is configured in
+ # hieradata/role/eqiad/analytics_cluster/database/meta/backup.yaml
+ analytics_cluster::database::meta::backup,
analytics_cluster::hive::metastore::database,
analytics_cluster::oozie::server::database,
analytics_cluster::hive::metastore,
diff --git a/modules/mysql_wmf/files/mylvmbackup.logrotate
b/modules/mysql_wmf/files/mylvmbackup.logrotate
new file mode 100644
index 0000000..4703f04
--- /dev/null
+++ b/modules/mysql_wmf/files/mylvmbackup.logrotate
@@ -0,0 +1,9 @@
+/var/log/mylvmbackup/*.log
+{
+ rotate 4
+ weekly
+ missingok
+ notifempty
+ compress
+ nocreate
+}
diff --git a/modules/mysql_wmf/manifests/mylvmbackup.pp
b/modules/mysql_wmf/manifests/mylvmbackup.pp
new file mode 100644
index 0000000..11550c4
--- /dev/null
+++ b/modules/mysql_wmf/manifests/mylvmbackup.pp
@@ -0,0 +1,108 @@
+# == Define mysql_wmf::mylvmbackup
+# Uses mylvmbackup to periodically rsync an LVM snapshot of the
+# MySQL data directory to a destination.
+#
+# == Parameters
+#
+# [*dest*]
+# Rsync destination of backup.
+#
+# [*vgname*]
+# Specifies the volume group of the logical volume where the MySQL data
+# directory is located. Default: $::hostname-vg
+#
+# [*lvname*]
+# Specifies the name of the logical volume where the MySQL data directory is
+# located. Default: mysql
+#
+# [*mountdir*]
+# Path for mounting the snapshot volume to.
+# See: --mountdir option for mylvmbackup.
+# Default: /var/cache/mylvmbackup/mnt/${title}
+#
+# ...standard cron resource parameters...
+#
+# == Usage
+#
+# # Back up the mysql data directory to host.example.org every hour.
+# mysql_wmf::mylvmbackup { 'myinstance':
+# dest => 'host.example.org::rsync_module/path/to/backups/'
+# minute => 0,
+# }
+#
+define mysql_wmf::mylvmbackup(
+ $dest,
+ $vgname = "${::hostname}-vg",
+ $lvname = 'mysql',
+ $mountdir = "/var/cache/mylvmbackup/mnt/${title}",
+ $hour = undef,
+ $minute = undef,
+ $month = undef,
+ $monthday = undef,
+ $weekday = undef,
+ $ensure = 'present'
+)
+{
+ require_package('mylvmbackup')
+
+ if !defined(File['/var/log/mylvmbackup']) {
+ file { '/var/log/mylvmbackup':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+ }
+ if !defined(Logrotate::Conf['mylvmbackup']) {
+ logrotate::conf { 'mylvmbackup':
+ ensure => $ensure,
+ source => 'puppet:///modules/mysql_wmf/mylvmbackup.logrotate',
+ }
+ }
+
+ # The prebackup rsync hook for this mylvmbackup
+ # job will be stored in this directory.
+ $hooksdir = "/usr/share/mylvmbackup/${title}"
+ file { $hooksdir:
+ ensure => ensure_directory($ensure),
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ }
+ # Always use the preflush.pm that ships with mylvmbackup
+ file { "${hooksdir}/preflush.pm":
+ ensure => ensure_link($ensure),
+ target => '/usr/share/mylvmbackup/preflush.pm',
+ require => Package['mylvmbackup'],
+ }
+
+ # prebackup rsyncs the LVM snapshot to $dest.
+ file { "${hooksdir}/prebackup":
+ ensure => $ensure,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ content => template('mysql_wmf/mylvmbackup-prebackup.erb'),
+ require => Package['mylvmbackup'],
+ }
+
+ # --backuptype none will mean no copy of the lvm snapshot will be made
+ # by mylvmbackup. Instead, this is handled by the prebackup hook,
+ # which just rsyncs the lvm snapshot to a destination.
+ $command = "/usr/bin/mylvmbackup --hooksdir ${hooksdir} --vgname ${vgname}
--lvname ${lvname} --mountdir ${mountdir} --backuptype none 2>&1 >>
/var/log/mylvmbackup/${title}.log"
+ cron { "mylvmbackup-${title}":
+ ensure => $ensure,
+ command => $command,
+ user => 'root',
+ hour => $hour,
+ minute => $minute,
+ month => $month,
+ monthday => $monthday,
+ weekday => $weekday,
+ require => [
+ Package['mylvmbackup'],
+ File["${hooksdir}/preflush.pm"],
+ File["${hooksdir}/prebackup"],
+ ],
+ }
+}
diff --git a/modules/mysql_wmf/templates/mylvmbackup-prebackup.erb
b/modules/mysql_wmf/templates/mylvmbackup-prebackup.erb
new file mode 100644
index 0000000..38f6b8a
--- /dev/null
+++ b/modules/mysql_wmf/templates/mylvmbackup-prebackup.erb
@@ -0,0 +1,7 @@
+# NOTE: This file is managed by Puppet.
+
+# This command will be run instead of using one of mylvmbackup's
+# default backup types. This allows us to do incremental backups
+# directly from mysql data directories.
+
+/usr/bin/rsync --progress --delete -av <%= @mountdir %>/ <%= @dest %>/
diff --git a/modules/role/manifests/analytics_cluster/database/meta/backup.pp
b/modules/role/manifests/analytics_cluster/database/meta/backup.pp
new file mode 100644
index 0000000..881a613
--- /dev/null
+++ b/modules/role/manifests/analytics_cluster/database/meta/backup.pp
@@ -0,0 +1,23 @@
+# == Class role::analytics_cluster::database::meta::backup
+# Uses mysql_wmf::mylvmbackup to take backups of the
+# analytics-meta mysql instance.
+#
+# == Parameters
+#
+# [*dest*]
+# Rsync path for backup. Default: /srv/backups/mysql/analytics-meta/
+# You could include role::analytics_cluster::database::meta::backup_dest
+# on some other node, and then set $dest here to
+# other.node.org::backup/mysql/analytics-meta/ and backups would
+# be rsynced elsewhere.
+#
+class role::analytics_cluster::database::meta::backup(
+ $dest = '/srv/backup/mysql/analytics-meta'
+) {
+ # Take hourly backups of the analytics-meta instance
+ # and rsync those backups to $dest.
+ mysql_wmf::mylvmbackup { 'analytics-meta':
+ dest => $dest,
+ minute => 0,
+ }
+}
diff --git
a/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
b/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
new file mode 100644
index 0000000..aa2a867
--- /dev/null
+++ b/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
@@ -0,0 +1,30 @@
+# == Class role::analytics_cluster::database::meta::backup_dest
+#
+class role::analytics_cluster::database::meta::backup_dest {
+ file { [
+ '/srv/backup',
+ '/srv/backup/mysql',
+ '/srv/backup/mysql/analytics-meta',
+ ]:
+ ensure => 'directory',
+ owner => 'root',
+ group => 'analytics-admins',
+ mode => '0750',
+ }
+
+ # This will allow $hosts_allow to host public data files
+ # generated by the analytics cluster.
+ # Note that this requires that cdh::hadoop::mount
+ # be present and mounted at /mnt/hdfs
+ rsync::server::module { 'backup':
+ path => '/srv/backup',
+ read_only => 'no',
+ list => 'no',
+ # These are probably the same host, but in case they
+ # aren't, allow either use of this rsync server module.
+ hosts_allow => [
+ hiera('cdh::hive::metastore_host'),
+ hiera('cdh::oozie::oozie_host'),
+ ],
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/277640
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6b0c6e0b819c837b8dbffc97be0c08ee4f66a452
Gerrit-PatchSet: 12
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Jcrespo <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
