ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/395612 )
Change subject: move adds-changes (so-called incrementals) dump cron to
dumpsdata hosts
......................................................................
move adds-changes (so-called incrementals) dump cron to dumpsdata hosts
Bug: T179942
Change-Id: I8bbccbe13226bb3f12754651da6d0a4d37060d5f
---
M modules/dumps/manifests/web/cleanups/miscdumps.pp
M modules/snapshot/manifests/addschanges.pp
M modules/snapshot/manifests/cron.pp
M modules/snapshot/templates/addschanges.conf.erb
4 files changed, 16 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/12/395612/1
diff --git a/modules/dumps/manifests/web/cleanups/miscdumps.pp
b/modules/dumps/manifests/web/cleanups/miscdumps.pp
index 5a4531c..a89ce64 100644
--- a/modules/dumps/manifests/web/cleanups/miscdumps.pp
+++ b/modules/dumps/manifests/web/cleanups/miscdumps.pp
@@ -12,7 +12,6 @@
}
$keep_generator=['categoriesrdf:3', 'cirrussearch:3',
'contenttranslation:3', 'globalblocks:3', 'imageinfo:3', 'mediatitles:3',
'pagetitles:3', 'wikibase/wikidatawiki:3']
- # FIXME check for imageinfo, mediatitles, pagetitles that we really
want/need all those
$keep_replicas=['categoriesrdf:11', 'cirrussearch:11',
'contenttranslation:14', 'globalblocks:13', 'imageinfo:32', 'mediatitles:90',
'pagetitles:90', 'wikibase/wikidatawiki:20']
if ($isreplica == true) {
$content= join($keep_replicas, "\n")
@@ -29,10 +28,21 @@
content => "${content}\n"
}
+ $cleanup_miscdumps = "/bin/bash /usr/local/bin/cleanup_old_miscdumps.sh
--miscdumpsdir ${miscdumpsdir} --configfile /etc/dumps/confs/cleanup_misc.conf"
+
+ if ($isreplica == true) {
+ $addschanges_keeps = "40"
+ } else {
+ $addschanges_keeps = "7"
+ }
+
+ # adds-changes dumps cleanup; these are in incr/wikiname/YYYYMMDD for each
day, so they can't go into the above config/cron setup
+ $cleanup_addschanges = "find ${miscdumpsdir}/incr -mindepth 2 -maxdepth 2
-type d -mtime +${addschanges_keeps} -exec rm -rf {} \\;"
+
cron { 'cleanup-misc-dumps':
ensure => 'present',
environment => '[email protected]',
- command => "/bin/bash /usr/local/bin/cleanup_old_miscdumps.sh
--miscdumpsdir ${miscdumpsdir} --configfile /etc/dumps/confs/cleanup_misc.conf",
+ command => "${cleanup_miscdumps} ; ${cleanup_addschanges}",
user => root,
minute => '15',
hour => '7',
diff --git a/modules/snapshot/manifests/addschanges.pp
b/modules/snapshot/manifests/addschanges.pp
index 1a55562..62c97b1 100644
--- a/modules/snapshot/manifests/addschanges.pp
+++ b/modules/snapshot/manifests/addschanges.pp
@@ -6,6 +6,7 @@
$apachedir = $snapshot::dumps::dirs::apachedir
$dblistsdir = $snapshot::dumps::dirs::dblistsdir
$templsdir = $snapshot::dumps::dirs::templsdir
+ $cronsdir = $snapshot::dumps::dirs::cronsdir
file { "${confsdir}/addschanges.conf":
ensure => 'present',
diff --git a/modules/snapshot/manifests/cron.pp
b/modules/snapshot/manifests/cron.pp
index f05b378..7dabd1a 100644
--- a/modules/snapshot/manifests/cron.pp
+++ b/modules/snapshot/manifests/cron.pp
@@ -27,5 +27,5 @@
group => $group,
}
class { '::snapshot::cron::contentxlation': user => $depr_user }
- class { '::snapshot::addschanges': user => $depr_user }
+ class { '::snapshot::addschanges': user => $user }
}
diff --git a/modules/snapshot/templates/addschanges.conf.erb
b/modules/snapshot/templates/addschanges.conf.erb
index 540d16b..14f2a87 100644
--- a/modules/snapshot/templates/addschanges.conf.erb
+++ b/modules/snapshot/templates/addschanges.conf.erb
@@ -14,10 +14,10 @@
adminsettings=private/PrivateSettings.php
[output]
-dumpdir=<%= scope.lookupvar('snapshot::dumps::dirs::otherdir') -%>/incr
+dumpdir=<%= scope.lookupvar('snapshot::dumps::dirs::cronsdir') -%>/incr
templatedir=<%= scope.lookupvar('snapshot::dumps::dirs::templsdir') %>
indextmpl=<%= scope.lookupvar('snapshot::dumps::dirs::templsdir')
-%>/incrs-index.html
-temp=<%= scope.lookupvar('snapshot::dumps::dirs::miscdumpsdir') -%>/temp
+temp=<%= scope.lookupvar('snapshot::dumps::dirs::xmldumpsdir') -%>/temp
webroot=http://download.wikimedia.org
fileperms=0644
# revisions must be at least this much older than time of current run
--
To view, visit https://gerrit.wikimedia.org/r/395612
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8bbccbe13226bb3f12754651da6d0a4d37060d5f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
