Elukey has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/393257 )
Change subject: role::analytics_cluster::database::meta::backup*: move to
profiles
......................................................................
role::analytics_cluster::database::meta::backup*: move to profiles
Bug: T167790
Change-Id: Ib0c195bde8471589d3c14afa5f1244ea202dbba9
---
M hieradata/role/common/analytics_cluster/coordinator.yaml
M manifests/site.pp
M modules/role/manifests/analytics_cluster/coordinator.pp
D modules/role/manifests/analytics_cluster/database/meta/backup.pp
D modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
M modules/role/manifests/analytics_cluster/hadoop/standby.pp
6 files changed, 18 insertions(+), 88 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/57/393257/1
diff --git a/hieradata/role/common/analytics_cluster/coordinator.yaml
b/hieradata/role/common/analytics_cluster/coordinator.yaml
index c0d91dc..08a27d7 100644
--- a/hieradata/role/common/analytics_cluster/coordinator.yaml
+++ b/hieradata/role/common/analytics_cluster/coordinator.yaml
@@ -5,4 +5,8 @@
profile::hive::client::zookeeper_cluster_name: main-eqiad
profile::hive::client::server_host: analytics1003.eqiad.wmnet
-profile::hive::client::server_port: 10000
\ No newline at end of file
+profile::hive::client::server_port: 10000
+
+
+# Periodical backups of the analytics-meta mysql database.
+profile::analytics::database::meta::backup::rsync_dest:
'/srv/backups/mysql/analytics-meta'
diff --git a/manifests/site.pp b/manifests/site.pp
index 39081b8..7c81ad6 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -46,15 +46,7 @@
# analytics1002 is the Hadoop standby NameNode and ResourceManager.
node 'analytics1002.eqiad.wmnet' {
- role(analytics_cluster::hadoop::standby,
- # analytics1002 is usually inactive, and it has a
- # decent amount of disk space. We use it to
- # store some backups, including fsimage snapshots
- # of Hadoop NameNode metadata, and of the
- # analytics_cluster::database::meta (MySQL analytics-meta) instance.
- # If you move these, make sure /srv/backup has
- # enough space to store backups.
- analytics_cluster::database::meta::backup_dest)
+ role(analytics_cluster::hadoop::standby)
}
node 'analytics1003.eqiad.wmnet' {
diff --git a/modules/role/manifests/analytics_cluster/coordinator.pp
b/modules/role/manifests/analytics_cluster/coordinator.pp
index 78270e7..cfd8a00 100644
--- a/modules/role/manifests/analytics_cluster/coordinator.pp
+++ b/modules/role/manifests/analytics_cluster/coordinator.pp
@@ -21,9 +21,8 @@
include ::role::analytics_cluster::database::meta
# Back up analytics-meta MySQL instance
- # to analytics1002. $dest is configured in
- # hieradata/role/eqiad/analytics_cluster/database/meta/backup.yaml
- include ::role::analytics_cluster::database::meta::backup
+ # to analytics1002.
+ include ::profile::analytics::database::meta::backup
# SQL-like queries to data stored in HDFS
include ::role::analytics_cluster::hive::metastore
diff --git a/modules/role/manifests/analytics_cluster/database/meta/backup.pp
b/modules/role/manifests/analytics_cluster/database/meta/backup.pp
deleted file mode 100644
index 3f6dd10..0000000
--- a/modules/role/manifests/analytics_cluster/database/meta/backup.pp
+++ /dev/null
@@ -1,26 +0,0 @@
-# == Class role::analytics_cluster::database::meta::backup
-# Uses mariadb::mylvmbackup to take backups of the
-# analytics-meta mysql instance.
-#
-# == Hiera Variables
-#
-# [*analytics_cluster_meta_database_backup_rsync_dest*]
-# Rsync path for backup. Default: /srv/backups/mysql/analytics-meta
-# You could include role::analytics_cluster::database::meta::backup_dest
-# on some other node, and then set $dest here to
-# other.node.org::backup/mysql/analytics-meta/ and backups would
-# be rsynced elsewhere.
-#
-class role::analytics_cluster::database::meta::backup {
- $dest = hiera(
- 'analytics_cluster_meta_database_backup_rsync_dest',
- '/srv/backups/mysql/analytics-meta'
- )
-
- # Take hourly backups of the analytics-meta instance
- # and rsync those backups to $dest.
- mariadb::mylvmbackup { 'analytics-meta':
- dest => $dest,
- minute => 0,
- }
-}
diff --git
a/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
b/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
deleted file mode 100644
index 335000f..0000000
--- a/modules/role/manifests/analytics_cluster/database/meta/backup_dest.pp
+++ /dev/null
@@ -1,49 +0,0 @@
-# == Class role::analytics_cluster::database::meta::backup_dest
-#
-class role::analytics_cluster::database::meta::backup_dest {
-
- if !defined(File['/srv/backup']) {
- file { '/srv/backup':
- ensure => 'directory',
- owner => 'root',
- group => 'analytics-admins',
- mode => '0755',
- }
- }
-
- file { [
- '/srv/backup/mysql',
- '/srv/backup/mysql/analytics-meta',
- ]:
- ensure => 'directory',
- owner => 'root',
- group => 'analytics-admins',
- mode => '0750',
- }
-
- include ::rsync::server
-
- # These are probably the same host, but in case they
- # aren't, allow either use of this rsync server module.
- $hosts_allow = [
- hiera('cdh::hive::metastore_host'),
- hiera('cdh::oozie::oozie_host'),
- ]
- # This will allow $hosts_allow to host public data files
- # generated by the analytics cluster.
- # Note that this requires that cdh::hadoop::mount
- # be present and mounted at /mnt/hdfs
- rsync::server::module { 'backup':
- path => '/srv/backup',
- read_only => 'no',
- list => 'no',
- hosts_allow => $hosts_allow,
- }
-
- $rsync_clients_ferm = join($hosts_allow, ' ')
- ferm::service { 'analytics_cluster-backup-rsync':
- proto => 'tcp',
- port => '873',
- srange => "@resolve((${rsync_clients_ferm}))",
- }
-}
diff --git a/modules/role/manifests/analytics_cluster/hadoop/standby.pp
b/modules/role/manifests/analytics_cluster/hadoop/standby.pp
index a2edfdf..9c64903 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/standby.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/standby.pp
@@ -12,6 +12,16 @@
include ::profile::hadoop::firewall::master
include ::profile::hadoop::users
include ::profile::hadoop::backup::namenode
+
+ # analytics1002 is usually inactive, and it has a
+ # decent amount of disk space. We use it to
+ # store some backups, including fsimage snapshots
+ # of Hadoop NameNode metadata, and of the
+ # analytics_cluster::database::meta (MySQL analytics-meta) instance.
+ # If you move these, make sure /srv/backup has
+ # enough space to store backups.
+ include ::profile::analytics::database::meta::backup_dest
+
include ::profile::base::firewall
class { 'standard': }
--
To view, visit https://gerrit.wikimedia.org/r/393257
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib0c195bde8471589d3c14afa5f1244ea202dbba9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
