Alexandros Kosiaris has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169680
Change subject: Modularize backups.pp
......................................................................
Modularize backups.pp
Move all the DRY defines in a WMF specific module
Add (incomplete) RSpec tests
Add README.md
Delete backup::mysql which was unused
Change-Id: I7e2d1f933d57dc27f5933d826fadec2f64f74e47
---
D manifests/backups.pp
M manifests/site.pp
A modules/backup/.rspec
A modules/backup/README.md
A modules/backup/Rakefile
A modules/backup/manifests/mysqlset.pp
A modules/backup/manifests/schedule.pp
A modules/backup/manifests/set.pp
A modules/backup/manifests/weeklyschedule.pp
A modules/backup/spec/defines/schedule_spec.rb
A modules/backup/spec/defines/set_spec.rb
A modules/backup/spec/defines/weeklyschedule_spec.rb
A modules/backup/spec/fixtures/manifests/site.pp
A modules/backup/spec/spec_helper.rb
14 files changed, 254 insertions(+), 133 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/80/169680/1
diff --git a/manifests/backups.pp b/manifests/backups.pp
deleted file mode 100644
index a316ba9..0000000
--- a/manifests/backups.pp
+++ /dev/null
@@ -1,132 +0,0 @@
-# backup::set is probably what you want
-define backup::set($jobdefaults=$role::backup::host::jobdefaults) {
- if $jobdefaults != undef {
- @bacula::client::job { "${name}-${jobdefaults}":
- fileset => $name,
- jobdefaults => $jobdefaults,
- }
-
- $motd_content = "#!/bin/sh\necho \"Backed up on this host: ${name}\""
- @file { "/etc/update-motd.d/06-backups-${name}":
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0555',
- content => $motd_content,
- tag => 'backup-motd',
- }
- }
-}
-
-define backup::mysqlset($method='bpipe',
- $xtrabackup=true,
- $per_db=false,
- $innodb_only=false,
- $local_dump_dir=undef,
- $password_file=undef,
- $mysql_binary=undef,
- $mysqldump_binary=undef,
- $jobdefaults=$role::backup::host::jobdefaults,
-) {
-
- $allowed_methods = [ 'bpipe', 'predump' ]
- if !($method in $allowed_methods) {
- fail("${method} is not allowed")
- }
-
- if !defined(Package['pigz']) {
- package { 'pigz':
- ensure => present,
- }
- }
- if $method == 'predump' {
- $extras = {
- 'ClientRunBeforeJob' => '/etc/bacula/scripts/predump',
- }
- $basefileset = regsubst(regsubst($local_dump_dir,'/',''),'/','-','G')
- $fileset = "mysql-${basefileset}"
-
- file { '/etc/bacula/scripts/predump':
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0500',
- content => template('backups/mysql-predump.erb'),
- }
-
- } elsif $method == 'bpipe' {
- bacula::client::mysql-bpipe {
"mysql-bpipe-x${xtrabackup}-p${per_db}-i${innodb_only}":
- per_database => $per_db,
- xtrabackup => $xtrabackup,
- mysqldump_innodb_only => $innodb_only,
- local_dump_dir => $local_dump_dir,
- password_file => $password_file,
- mysql_binary => $mysql_binary,
- mysqldump_binary => $mysqldump_binary,
- }
- $extras = undef
- $fileset = "mysql-${method}-x${xtrabackup}-p${per_db}-i${innodb_only}"
- }
-
- if $jobdefaults != undef {
- @bacula::client::job { "mysql-${method}-${name}-${jobdefaults}":
- fileset => $fileset,
- jobdefaults => $jobdefaults,
- extras => $extras,
- }
- }
-}
-
-# Utility definition used internally to deduplicate code
-define backup::schedule($pool) {
- bacula::director::schedule { "Monthly-1st-${name}":
- runs => [
- { 'level' => 'Full',
- 'at' => "1st ${name} at 02:05",
- },
- { 'level' => 'Differential',
- 'at' => "3rd ${name} at 03:05",
- },
- { 'level' => 'Incremental',
- 'at' => 'at 04:05',
- },
- ],
- }
-
- bacula::director::jobdefaults { "Monthly-1st-${name}-${pool}":
- when => "Monthly-1st-${name}",
- pool => $pool,
- }
-
-}
-# Same for weekly backups
-define backup::weeklyschedule($pool) {
- bacula::director::schedule { "Weekly-${name}":
- runs => [
- { 'level' => 'Full',
- 'at' => "${name} at 02:05",
- },
- ],
- }
-
- bacula::director::jobdefaults { "Weekly-${name}-${pool}":
- when => "Weekly-${name}",
- pool => $pool,
- }
-}
-
-class backup::mysql {
- file { '/usr/local/sbin/snaprotate.pl':
- owner => 'root',
- group => 'root',
- mode => '0755',
- source => 'puppet:///files/backup/snaprotate.pl',
- }
-
- cron { 'snaprotate':
- command => '/usr/local/sbin/snaprotate.pl -a swap -V tank -s data -L
20G -c 1',
- user => 'root',
- hour => 1,
- minute => 0,
- }
-}
diff --git a/manifests/site.pp b/manifests/site.pp
index 6123c30..d140c69 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2,7 +2,6 @@
# site.pp
import 'realm.pp' # These ones first
-import 'backups.pp'
import 'certs.pp'
import 'dns.pp'
import 'facilities.pp'
diff --git a/modules/backup/.rspec b/modules/backup/.rspec
new file mode 100644
index 0000000..f449dae
--- /dev/null
+++ b/modules/backup/.rspec
@@ -0,0 +1,2 @@
+--format doc
+--color
diff --git a/modules/backup/README.md b/modules/backup/README.md
new file mode 100644
index 0000000..ccc42c4
--- /dev/null
+++ b/modules/backup/README.md
@@ -0,0 +1,21 @@
+# Backups Puppet Module #
+
+A Puppet module for deduplicating configuration puppet stanzas for WMF
+The bacula puppet module is generic enough to allow being used by third
+parties. At the same time however some simple defines must be used to
+deduplicate stuff and allow easy of use. This module contains that,
+mostly WMF specific code
+
+## Requirements ##
+- The bacula puppet module and whatever that requires
+
+## Notes ##
+
+The idea behind this module is to make it extra easy to get a host up
+and running in WMF backups.
+
+
+## Usage ##
+
+include role::backup::host
+backup::set { 'home': }
diff --git a/modules/backup/Rakefile b/modules/backup/Rakefile
new file mode 100644
index 0000000..d9226a8
--- /dev/null
+++ b/modules/backup/Rakefile
@@ -0,0 +1,37 @@
+require 'rake'
+require 'fileutils'
+
+require 'rspec/core/rake_task'
+
+modulename = File.basename(File.expand_path(File.dirname(__FILE__)))
+
+symlinks = { 'spec/fixtures/modules/%s/files' % modulename =>
'../../../../files',
+ 'spec/fixtures/modules/%s/manifests' % modulename => '../../../../manifests',
+ 'spec/fixtures/modules/%s/templates' % modulename => '../../../../templates',
+}
+
+
+task :setup do
+ FileUtils.mkdir_p('spec/fixtures/modules/%s' % modulename)
+ symlinks.each do |x|
+ if !File.exist?(x[0])
+ FileUtils.ln_s(x[1], x[0])
+ end
+ end
+end
+
+task :teardown do
+ symlinks.each { |x| FileUtils.rm(x[0], :force => true) }
+ FileUtils.rmdir('spec/fixtures/modules/%s' % modulename)
+ FileUtils.rmdir('spec/fixtures/modules')
+end
+
+RSpec::Core::RakeTask.new(:realspec) do |t|
+ t.fail_on_error = false
+ t.pattern = 'spec/*/*_spec.rb'
+end
+
+task :spec => [ :setup, :realspec, :teardown]
+
+task :default => :spec do
+end
diff --git a/modules/backup/manifests/mysqlset.pp
b/modules/backup/manifests/mysqlset.pp
new file mode 100644
index 0000000..8f412e8
--- /dev/null
+++ b/modules/backup/manifests/mysqlset.pp
@@ -0,0 +1,58 @@
+define backup::mysqlset($method='bpipe',
+ $xtrabackup=true,
+ $per_db=false,
+ $innodb_only=false,
+ $local_dump_dir=undef,
+ $password_file=undef,
+ $mysql_binary=undef,
+ $mysqldump_binary=undef,
+ $jobdefaults=$role::backup::host::jobdefaults,
+) {
+
+ $allowed_methods = [ 'bpipe', 'predump' ]
+ if !($method in $allowed_methods) {
+ fail("${method} is not allowed")
+ }
+
+ if !defined(Package['pigz']) {
+ package { 'pigz':
+ ensure => present,
+ }
+ }
+ if $method == 'predump' {
+ $extras = {
+ 'ClientRunBeforeJob' => '/etc/bacula/scripts/predump',
+ }
+ $basefileset = regsubst(regsubst($local_dump_dir,'/',''),'/','-','G')
+ $fileset = "mysql-${basefileset}"
+
+ file { '/etc/bacula/scripts/predump':
+ ensure => 'present',
+ owner => 'root',
+ group => 'root',
+ mode => '0500',
+ content => template('backups/mysql-predump.erb'),
+ }
+
+ } elsif $method == 'bpipe' {
+ bacula::client::mysql-bpipe {
"mysql-bpipe-x${xtrabackup}-p${per_db}-i${innodb_only}":
+ per_database => $per_db,
+ xtrabackup => $xtrabackup,
+ mysqldump_innodb_only => $innodb_only,
+ local_dump_dir => $local_dump_dir,
+ password_file => $password_file,
+ mysql_binary => $mysql_binary,
+ mysqldump_binary => $mysqldump_binary,
+ }
+ $extras = undef
+ $fileset = "mysql-${method}-x${xtrabackup}-p${per_db}-i${innodb_only}"
+ }
+
+ if $jobdefaults != undef {
+ @bacula::client::job { "mysql-${method}-${name}-${jobdefaults}":
+ fileset => $fileset,
+ jobdefaults => $jobdefaults,
+ extras => $extras,
+ }
+ }
+}
diff --git a/modules/backup/manifests/schedule.pp
b/modules/backup/manifests/schedule.pp
new file mode 100644
index 0000000..1ff68ad
--- /dev/null
+++ b/modules/backup/manifests/schedule.pp
@@ -0,0 +1,22 @@
+# Utility definition used internally to deduplicate code
+define backup::schedule($pool) {
+ bacula::director::schedule { "Monthly-1st-${name}":
+ runs => [
+ { 'level' => 'Full',
+ 'at' => "1st ${name} at 02:05",
+ },
+ { 'level' => 'Differential',
+ 'at' => "3rd ${name} at 03:05",
+ },
+ { 'level' => 'Incremental',
+ 'at' => 'at 04:05',
+ },
+ ],
+ }
+
+ bacula::director::jobdefaults { "Monthly-1st-${name}-${pool}":
+ when => "Monthly-1st-${name}",
+ pool => $pool,
+ }
+
+}
diff --git a/modules/backup/manifests/set.pp b/modules/backup/manifests/set.pp
new file mode 100644
index 0000000..238320a
--- /dev/null
+++ b/modules/backup/manifests/set.pp
@@ -0,0 +1,19 @@
+# backup::set is probably what you want
+define backup::set($jobdefaults=$role::backup::host::jobdefaults) {
+ if $jobdefaults != undef {
+ @bacula::client::job { "${name}-${jobdefaults}":
+ fileset => $name,
+ jobdefaults => $jobdefaults,
+ }
+
+ $motd_content = "#!/bin/sh\necho \"Backed up on this host: ${name}\""
+ @file { "/etc/update-motd.d/06-backups-${name}":
+ ensure => 'present',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ content => $motd_content,
+ tag => 'backup-motd',
+ }
+ }
+}
diff --git a/modules/backup/manifests/weeklyschedule.pp
b/modules/backup/manifests/weeklyschedule.pp
new file mode 100644
index 0000000..040d400
--- /dev/null
+++ b/modules/backup/manifests/weeklyschedule.pp
@@ -0,0 +1,15 @@
+# Same for weekly backups
+define backup::weeklyschedule($pool) {
+ bacula::director::schedule { "Weekly-${name}":
+ runs => [
+ { 'level' => 'Full',
+ 'at' => "${name} at 02:05",
+ },
+ ],
+ }
+
+ bacula::director::jobdefaults { "Weekly-${name}-${pool}":
+ when => "Weekly-${name}",
+ pool => $pool,
+ }
+}
diff --git a/modules/backup/spec/defines/schedule_spec.rb
b/modules/backup/spec/defines/schedule_spec.rb
new file mode 100644
index 0000000..0962a86
--- /dev/null
+++ b/modules/backup/spec/defines/schedule_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe 'backup::schedule', :type => :define do
+ let(:title) { 'something' }
+ let(:params) { {
+ :pool => 'unimportant',
+ }
+ }
+ let(:pre_condition) do
+ [
+ 'define bacula::director::jobdefaults($when, $pool) {}',
+ 'define bacula::director::schedule($runs) {}',
+ ]
+ end
+ it 'should create bacula::director::jobdefaults' do
+ should
contain_bacula__director__jobdefaults("Monthly-1st-#{title}-unimportant").with({
+ 'when' => "Monthly-1st-#{title}",
+ 'pool' => 'unimportant',
+ })
+ end
+ it 'should create bacula::director::schedule' do
+ should contain_bacula__director__schedule("Monthly-1st-#{title}")
+ end
+end
diff --git a/modules/backup/spec/defines/set_spec.rb
b/modules/backup/spec/defines/set_spec.rb
new file mode 100644
index 0000000..0ed7fc4
--- /dev/null
+++ b/modules/backup/spec/defines/set_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe 'backup::set', :type => :define do
+ let(:title) { 'something' }
+ let(:params) { {
+ :jobdefaults => 'unimportant',
+ }
+ }
+ let(:pre_condition) do
+ [
+ 'File <| |>',
+ 'define bacula::client::job($fileset, $jobdefaults) {}',
+ ]
+ end
+ it 'should create valid content for
/etc/update-motd.d/06-backups-something' do
+ should contain_file("/etc/update-motd.d/06-backups-#{title}").with({
+ 'ensure' => 'present',
+ 'owner' => 'root',
+ 'group' => 'root',
+ 'mode' => '0555',
+ }) \
+ .with_content(/Backed up on this host: something/)
+ end
+end
diff --git a/modules/backup/spec/defines/weeklyschedule_spec.rb
b/modules/backup/spec/defines/weeklyschedule_spec.rb
new file mode 100644
index 0000000..6a35dd2
--- /dev/null
+++ b/modules/backup/spec/defines/weeklyschedule_spec.rb
@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe 'backup::weeklyschedule', :type => :define do
+ let(:title) { 'something' }
+ let(:params) { {
+ :pool => 'unimportant',
+ }
+ }
+ let(:pre_condition) do
+ [
+ 'define bacula::director::jobdefaults($when, $pool) {}',
+ 'define bacula::director::schedule($runs) {}',
+ ]
+ end
+ it 'should create bacula::director::jobdefaults' do
+ should
contain_bacula__director__jobdefaults("Weekly-#{title}-unimportant").with({
+ 'when' => "Weekly-#{title}",
+ 'pool' => 'unimportant',
+ })
+ end
+ it 'should create bacula::director::schedule' do
+ should contain_bacula__director__schedule("Weekly-#{title}")
+ end
+end
diff --git a/modules/backup/spec/fixtures/manifests/site.pp
b/modules/backup/spec/fixtures/manifests/site.pp
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/modules/backup/spec/fixtures/manifests/site.pp
diff --git a/modules/backup/spec/spec_helper.rb
b/modules/backup/spec/spec_helper.rb
new file mode 100644
index 0000000..d3923f8
--- /dev/null
+++ b/modules/backup/spec/spec_helper.rb
@@ -0,0 +1,8 @@
+require 'rspec-puppet'
+
+fixture_path = File.expand_path(File.join(__FILE__, '..', 'fixtures'))
+
+RSpec.configure do |c|
+ c.module_path = File.join(fixture_path, 'modules')
+ c.manifest_dir = File.join(fixture_path, 'manifests')
+end
--
To view, visit https://gerrit.wikimedia.org/r/169680
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7e2d1f933d57dc27f5933d826fadec2f64f74e47
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
