ArielGlenn has submitted this change and it was merged.
Change subject: basic snapshot role for dumps
......................................................................
basic snapshot role for dumps
set up config files dblists etc
scap3 deploy
cron job
hiera settings for things which vary from host to host
(only one host needs to run en wiki, only one host needs to
reserve cpus because it runs cron jobs)
Change-Id: I8d48878300de584bd5385a0c46b2fc0e25e5bf41
---
A hieradata/hosts/snapshot1005.yaml
A hieradata/hosts/snapshot1006.yaml
A hieradata/hosts/snapshot1007.yaml
M modules/role/manifests/snapshot/dumper.pp
M modules/snapshot/manifests/dumps.pp
M modules/snapshot/manifests/dumps/configs.pp
M modules/snapshot/manifests/dumps/cron.pp
M modules/snapshot/manifests/dumps/dblists.pp
M modules/snapshot/templates/dumps/fulldumps.sh.erb
M modules/snapshot/templates/initscripts/dumps-monitor.systemd.erb
M modules/snapshot/templates/initscripts/dumps-monitor.upstart.erb
11 files changed, 62 insertions(+), 46 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/hosts/snapshot1005.yaml
b/hieradata/hosts/snapshot1005.yaml
new file mode 100644
index 0000000..8535061
--- /dev/null
+++ b/hieradata/hosts/snapshot1005.yaml
@@ -0,0 +1,4 @@
+snapshot::dumps:
+ runtype: hugewikis
+ maxjobs: 28
+snapshot::cron::misc: false
diff --git a/hieradata/hosts/snapshot1006.yaml
b/hieradata/hosts/snapshot1006.yaml
new file mode 100644
index 0000000..b594de8
--- /dev/null
+++ b/hieradata/hosts/snapshot1006.yaml
@@ -0,0 +1,4 @@
+snapshot::dumps:
+ runtype: regular
+ maxjobs: 28
+snapshot::cron::misc: false
diff --git a/hieradata/hosts/snapshot1007.yaml
b/hieradata/hosts/snapshot1007.yaml
new file mode 100644
index 0000000..191de55
--- /dev/null
+++ b/hieradata/hosts/snapshot1007.yaml
@@ -0,0 +1,4 @@
+snapshot::dumps:
+ runtype: regular
+ maxjobs: 20
+snapshot::cron::misc: true
diff --git a/modules/role/manifests/snapshot/dumper.pp
b/modules/role/manifests/snapshot/dumper.pp
index 798843c..af22313 100644
--- a/modules/role/manifests/snapshot/dumper.pp
+++ b/modules/role/manifests/snapshot/dumper.pp
@@ -2,8 +2,13 @@
# meaning sql/xml dumps every couple of weeks or so
class role::snapshot::dumper {
- # set up scap3 deployment of dump scripts except for
- # all the misc cron jobs (not handled in this class)
+ # config, stages files, dblists, html templates
+ include snapshot::dumps
+
+ # scap3 deployment of dump scripts
include dataset::user
include snapshot::deployment
+
+ # cron job for running the dumps
+ class { 'snapshot::dumps::cron': user => 'datasets' }
}
diff --git a/modules/snapshot/manifests/dumps.pp
b/modules/snapshot/manifests/dumps.pp
index fb099af..8a55540 100644
--- a/modules/snapshot/manifests/dumps.pp
+++ b/modules/snapshot/manifests/dumps.pp
@@ -1,27 +1,16 @@
class snapshot::dumps(
$enable = true,
- $hugewikis = false,
) {
-
- if ($enable) {
- $ensure = 'present'
- }
- else {
- $ensure = 'absent'
- }
-
class { 'snapshot::dumps::configs':
- enable => $enable,
- hugewikis_enable => $hugewikis,
+ enable => $enable,
}
class { 'snapshot::dumps::dblists':
- enable => $enable,
- hugewikis_enable => $hugewikis,
+ enable => $enable,
}
class { 'snapshot::dumps::templates':
- enable => 'enable'
+ enable => $enable,
}
class { 'snapshot::dumps::stagesconfig':
- enable => 'enable'
+ enable => $enable,
}
}
diff --git a/modules/snapshot/manifests/dumps/configs.pp
b/modules/snapshot/manifests/dumps/configs.pp
index fc5c8e0..929c6ad 100644
--- a/modules/snapshot/manifests/dumps/configs.pp
+++ b/modules/snapshot/manifests/dumps/configs.pp
@@ -1,6 +1,5 @@
class snapshot::dumps::configs(
$enable = true,
- $hugewikis_enable = true,
) {
include snapshot::dumps::dirs
@@ -132,11 +131,9 @@
configtype => 'bigwikis',
config => $config,
}
- if ($hugewikis_enable) {
- snapshot::dumps::wikiconf { 'wikidump.conf.hugewikis':
- configtype => 'hugewikis',
- config => $config,
- }
+ snapshot::dumps::wikiconf { 'wikidump.conf.hugewikis':
+ configtype => 'hugewikis',
+ config => $config,
}
snapshot::dumps::wikiconf { 'wikidump.conf.monitor':
configtype => 'monitor',
diff --git a/modules/snapshot/manifests/dumps/cron.pp
b/modules/snapshot/manifests/dumps/cron.pp
index f1a62f7..f5e2180 100644
--- a/modules/snapshot/manifests/dumps/cron.pp
+++ b/modules/snapshot/manifests/dumps/cron.pp
@@ -11,6 +11,7 @@
$ensure = 'absent'
}
+ $maxjobs = hiera('snapshot::dumps::maxjobs', 28)
file { '/usr/local/bin/fulldumps.sh':
ensure => 'present',
path => '/usr/local/bin/fulldumps.sh',
@@ -19,4 +20,17 @@
group => root,
content => template('snapshot/dumps/fulldumps.sh.erb'),
}
+
+ # fixme there is an implicit dependency on
+ # wikidump.conf.* plus some stage files, make explicit
+ $runtype = hiera('snapshot::dumps::runtype', 'regular')
+ cron { 'fulldumps_rest':
+ ensure => 'present',
+ environment => '[email protected]',
+ user => $user,
+ command => "/usr/local/bin/fulldumps.sh 01 14 ${runtype} >
/dev/null",
+ minute => '05',
+ hour => '02',
+ monthday => '05-14',
+ }
}
diff --git a/modules/snapshot/manifests/dumps/dblists.pp
b/modules/snapshot/manifests/dumps/dblists.pp
index c8f3305..1961a45 100644
--- a/modules/snapshot/manifests/dumps/dblists.pp
+++ b/modules/snapshot/manifests/dumps/dblists.pp
@@ -22,16 +22,13 @@
$dblistsdir = $snapshot::dumps::dirs::dblistsdir
- if ($hugewikis_enable) {
- # this host will run enwiki dumps
- file { "${dblistsdir}/hugewikis.dblist":
- ensure => 'present',
- path => "${dblistsdir}/hugewikis.dblist",
- mode => '0644',
- owner => 'root',
- group => 'root',
- content => "${hugewikis_dblist}\n",
- }
+ file { "${dblistsdir}/hugewikis.dblist":
+ ensure => 'present',
+ path => "${dblistsdir}/hugewikis.dblist",
+ mode => '0644',
+ owner => 'root',
+ group => 'root',
+ content => "${hugewikis_dblist}\n",
}
file { "${dblistsdir}/bigwikis.dblist":
ensure => 'present',
diff --git a/modules/snapshot/templates/dumps/fulldumps.sh.erb
b/modules/snapshot/templates/dumps/fulldumps.sh.erb
index 3f9ab98..a4dcd31 100644
--- a/modules/snapshot/templates/dumps/fulldumps.sh.erb
+++ b/modules/snapshot/templates/dumps/fulldumps.sh.erb
@@ -38,7 +38,7 @@
getcommandargs(){
slots="$1"
stagesfile="$2"
- commandargs="${scriptsdir}/dumpscheduler.py --slots ${slots} --commands
${dumpsdir}/stages/${stagesfile} --cache ${dumpsdir}/cache/running_cache.txt
--directory $dumpsdir"
+ commandargs="${repodir}/dumpscheduler.py --slots ${slots} --commands
${dumpsdir}/stages/${stagesfile} --cache ${dumpsdir}/cache/running_cache.txt
--directory $dumpsdir"
}
maybe_do_createdirs(){
@@ -117,9 +117,8 @@
python="/usr/bin/python"
dumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dumpsdir') %>"
-scriptsdir="<%= scope.lookupvar('snapshot::dumps::dirs::scriptsdir') %>"
-#soon: scriptsdir="<%=
scope.lookupvar('snapshot::deployment::common::scriptsdir') %>"
-cd $scriptsdir
+repodir="<%= scope.lookupvar('snapshot::deployment::dirs::repodir') %>"
+cd $repodir
today=`/bin/date +%d`
if [[ "$today" < "$startdate" || "$today" > "$enddate" ]]; then
@@ -133,6 +132,8 @@
case $wikitype in
'hugewikis')
maybe_do_createdirs "wikidump.conf.hugewikis"
"stages_create_hugewikis" "hugewikis"
+ maybe_do_createdirs "wikidump.conf.bigwikis" "stages_create_bigwikis"
"bigwikis"
+ maybe_do_createdirs "wikidump.conf" "stages_create_smallwikis"
"smallwikis"
;;
'regular')
maybe_do_createdirs "wikidump.conf.bigwikis" "stages_create_bigwikis"
"bigwikis"
@@ -144,27 +145,28 @@
;;
esac
+maxjobs=<%= scope.lookupvar('snapshot::dumps::cron::maxjobs') %>
case $wikitype in
'hugewikis')
- maybe_do_dumps "wikidump.conf.hugewikis"
"stages_normal_nocreate_hugewikis" "hugewikis" "27"
+ maybe_do_dumps "wikidump.conf.hugewikis"
"stages_normal_nocreate_hugewikis" "hugewikis" "$maxjobs"
# After huge wikis are done, check and do some of the rest if needed
# If not all the big wikis are done we will start a dump run covering
small and big wikis
# Only wikis without complete dumps will be updated.
- maybe_do_dumps "wikidump.conf.bigwikis" "stages_normal_nocreate"
"bigwikis" "16"
+ maybe_do_dumps "wikidump.conf.bigwikis" "stages_normal_nocreate"
"bigwikis" "$maxjobs"
if [ $? -ne 0 ]; then
# If we did not start a dump run above for small and big wikis,
and the small wikis
# are not done, start such a run now. Only wikis without complete
dumps will be updated.
- maybe_do_dumps "wikidump.conf" "stages_normal_nocreate"
"smallwikis" "16"
+ maybe_do_dumps "wikidump.conf" "stages_normal_nocreate"
"smallwikis" "$maxjobs"
fi
;;
'regular')
# If not all the big wikis are done we will start a dump run covering
small and big wikis
# Only wikis without complete dumps will be updated.
- maybe_do_dumps "wikidump.conf.bigwikis" "stages_normal_nocreate"
"bigwikis" "8"
+ maybe_do_dumps "wikidump.conf.bigwikis" "stages_normal_nocreate"
"bigwikis" "$maxjobs"
if [ $? -ne 0 ]; then
# If we did not start a dump run above for small and big wikis,
and the small wikis
# are not done, start such a run now. Only wikis without complete
dumps will be updated.
- maybe_do_dumps "wikidump.conf" "stages_normal_nocreate"
"smallwikis" "8"
+ maybe_do_dumps "wikidump.conf" "stages_normal_nocreate"
"smallwikis" "$maxjobs"
fi
;;
'*')
diff --git a/modules/snapshot/templates/initscripts/dumps-monitor.systemd.erb
b/modules/snapshot/templates/initscripts/dumps-monitor.systemd.erb
index 6a9a059..8c012e1 100644
--- a/modules/snapshot/templates/initscripts/dumps-monitor.systemd.erb
+++ b/modules/snapshot/templates/initscripts/dumps-monitor.systemd.erb
@@ -8,8 +8,8 @@
[Service]
User=datasets
-WorkingDirectory=<%= scope.lookupvar('snapshot::dirs::dumpsdir') -%>
-ExecStart=/bin/bash <%= scope.lookupvar('snapshot::dirs::dumpsdir')
-%>/monitor --configfile confs/wikidump.conf.monitor
+WorkingDirectory=<%= scope.lookupvar('snapshot::deployment::dirs::repodir') -%>
+ExecStart=/bin/bash <%= scope.lookupvar('snapshot::deployment::dirs::repodir')
-%>/monitor --configfile confs/wikidump.conf.monitor
[Install]
WantedBy=multi-user.target
diff --git a/modules/snapshot/templates/initscripts/dumps-monitor.upstart.erb
b/modules/snapshot/templates/initscripts/dumps-monitor.upstart.erb
index 1eb832b..67df23a 100644
--- a/modules/snapshot/templates/initscripts/dumps-monitor.upstart.erb
+++ b/modules/snapshot/templates/initscripts/dumps-monitor.upstart.erb
@@ -12,6 +12,6 @@
respawn
-chdir <%= scope.lookupvar('snapshot::dumps::dirs::scriptsdir') -%>
+chdir <%= scope.lookupvar('snapshot::deployment::dirs::repodir') -%>
-exec /bin/bash <%= scope.lookupvar('snapshot::dumps::dirs::scriptsdir')
-%>/monitor --configfile confs/wikidump.conf.monitor
+exec /bin/bash <%= scope.lookupvar('snapshot::deployment::dirs::repodir')
-%>/monitor --configfile confs/wikidump.conf.monitor
--
To view, visit https://gerrit.wikimedia.org/r/281530
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8d48878300de584bd5385a0c46b2fc0e25e5bf41
Gerrit-PatchSet: 5
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
