Ottomata has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/377352 )
Change subject: Fetch Hadoop NameNode fsimage backups daily and also save them in bacula ...................................................................... Fetch Hadoop NameNode fsimage backups daily and also save them in bacula Change-Id: I5d8b772c8501e914858cac16aaa6d3c9f3a3c028 Needs: https://gerrit.wikimedia.org/r/#/c/377350/ --- M modules/cdh M modules/profile/manifests/backup/director.pp A modules/role/manifests/analytics_cluster/backup.pp M modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp M modules/role/manifests/analytics_cluster/hadoop/standby.pp 5 files changed, 45 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/52/377352/1 diff --git a/modules/cdh b/modules/cdh index 1d5b40f..d96a904 160000 --- a/modules/cdh +++ b/modules/cdh @@ -1 +1 @@ -Subproject commit 1d5b40f8adf3e9e9698494d32c610bae2952734b +Subproject commit d96a9040669178f034c9489d7e48d3060441bfdb diff --git a/modules/profile/manifests/backup/director.pp b/modules/profile/manifests/backup/director.pp index d55e62f..c034fe1 100644 --- a/modules/profile/manifests/backup/director.pp +++ b/modules/profile/manifests/backup/director.pp @@ -233,6 +233,10 @@ includes => [ '/var/lib/rancid' ] } + bacula::director::fileset { 'hadoop-namenode-backup': + includes => [ '/srv/backup/hadoop/namenode' ] + } + # The console should be on the director class { 'bacula::console': director => $::fqdn, diff --git a/modules/role/manifests/analytics_cluster/backup.pp b/modules/role/manifests/analytics_cluster/backup.pp new file mode 100644 index 0000000..b3bc9a1 --- /dev/null +++ b/modules/role/manifests/analytics_cluster/backup.pp @@ -0,0 +1,10 @@ +# == Class role::analytics_cluster::backup +# Simple wrapper class to create and manage /srv/backup +class role::analytics_cluster::backup { + file { '/srv/backup': + ensure => 'directory', + owner => 'root', + group => 'analytics-admins', + mode => '0750', + } +} diff --git a/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp b/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp index c85f0a1..fd691d6 100644 --- a/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp +++ b/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp @@ -1,8 +1,10 @@ # == Class role::analytics_cluster::database::meta::backup_dest # class role::analytics_cluster::database::meta::backup_dest { + # Ensure /srv/backup exists + include ::role::analytics_cluster::backup + file { [ - '/srv/backup', '/srv/backup/mysql', '/srv/backup/mysql/analytics-meta', ]: diff --git a/modules/role/manifests/analytics_cluster/hadoop/standby.pp b/modules/role/manifests/analytics_cluster/hadoop/standby.pp index 61a63f8..70152d3 100644 --- a/modules/role/manifests/analytics_cluster/hadoop/standby.pp +++ b/modules/role/manifests/analytics_cluster/hadoop/standby.pp @@ -53,6 +53,33 @@ } } + if $::realm == 'production' { + include ::role::analytics_cluster::backup + + file { [ + '/srv/backup/hadoop', + '/srv/backup/hadoop/namenode', + ]: + ensure => 'directory', + owner => 'hdfs', + group => 'analytics-admins', + mode => '0750', + } + + # Use this standby name node to also take daily fsimage namenode + # metadata backups, backed up with bacula. + class { '::cdh::hadoop::namenode::backup': + destination => '/srv/backup/hadoop/namenode', + # Delete fsimages older than $retention_days + retention_days => 30, + hour => 0 + } + + class { 'backup::host': + sets => ['hadoop-namenode-backup', ] + } + } + # Firewall include ::role::analytics_cluster::hadoop::ferm::namenode -- To view, visit https://gerrit.wikimedia.org/r/377352 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5d8b772c8501e914858cac16aaa6d3c9f3a3c028 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <ao...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
