ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/328158 )
Change subject: make (most) snapshot shell scripts files instead of templates
......................................................................
make (most) snapshot shell scripts files instead of templates
and add a little file of directory paths to be sourced by
said scripts
Change-Id: I926df4424d70ba458c1ec85fb3a62a9c33304f29
---
R modules/snapshot/files/cron/create-media-per-project-lists.sh
R modules/snapshot/files/cron/dumpcentralauth.sh
R modules/snapshot/files/cron/dumpcirrussearch.sh
R modules/snapshot/files/cron/dumpcontentxlation.sh
A modules/snapshot/files/cron/list-last-good-dumps.sh
R modules/snapshot/files/cron/wikidatadumps-shared.sh
M modules/snapshot/manifests/dumps/dirs.pp
D modules/snapshot/templates/cron/list-last-good-dumps.sh.erb
A modules/snapshot/templates/set_dump_dirs.sh.erb
9 files changed, 50 insertions(+), 26 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/58/328158/1
diff --git
a/modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb
b/modules/snapshot/files/cron/create-media-per-project-lists.sh
similarity index 78%
rename from
modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb
rename to modules/snapshot/files/cron/create-media-per-project-lists.sh
index f7e3c0c..7d65e56 100755
--- a/modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb
+++ b/modules/snapshot/files/cron/create-media-per-project-lists.sh
@@ -2,13 +2,13 @@
#############################################################
# This file is maintained by puppet!
-# modules/snapshot/templates/cron/create-media-per-project-lists.sh.erb
+# modules/snapshot/cron/create-media-per-project-lists.sh
#############################################################
+source /usr/local/etc/set_dump_dirs.sh
+
DATE=`/bin/date '+%Y%m%d'`
-outputdir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir')
-%>/public/other/imageinfo/$DATE"
-repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>"
-confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') %>"
+outputdir="${datadir}/public/other/imageinfo/$DATE"
configfile="${confsdir}/wikidump.conf.media"
errors=0
diff --git a/modules/snapshot/templates/cron/dumpcentralauth.sh.erb
b/modules/snapshot/files/cron/dumpcentralauth.sh
similarity index 95%
rename from modules/snapshot/templates/cron/dumpcentralauth.sh.erb
rename to modules/snapshot/files/cron/dumpcentralauth.sh
index 523d9f7..c68f633 100755
--- a/modules/snapshot/templates/cron/dumpcentralauth.sh.erb
+++ b/modules/snapshot/files/cron/dumpcentralauth.sh
@@ -1,8 +1,10 @@
#!/bin/bash
#############################################################
# This file is maintained by puppet!
-# modules/snapshot/templates/cron/dumpcentralauth.sh.erb
+# modules/snapshot/cron/dumpcentralauth.sh
#############################################################
+
+source /usr/local/etc/set_dump_dirs.sh
usage() {
echo "Usage: $0 --site <datacenter> [--config <pathtofile>] [--dryrun]"
@@ -14,7 +16,6 @@
exit 1
}
-confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>"
configfile="${confsdir}/wikidump.conf"
dryrun="false"
site=""
diff --git a/modules/snapshot/templates/cron/dumpcirrussearch.sh.erb
b/modules/snapshot/files/cron/dumpcirrussearch.sh
similarity index 93%
rename from modules/snapshot/templates/cron/dumpcirrussearch.sh.erb
rename to modules/snapshot/files/cron/dumpcirrussearch.sh
index 2e04961..29f30cc 100644
--- a/modules/snapshot/templates/cron/dumpcirrussearch.sh.erb
+++ b/modules/snapshot/files/cron/dumpcirrussearch.sh
@@ -1,20 +1,24 @@
#!/bin/bash
#############################################################
# This file is maintained by puppet!
-# modules/snapshot/templates/cron/dumpcirrussearch.sh.erb
+# modules/snapshot/cron/dumpcirrussearch.sh
#############################################################
#
# Generate a json dump of cirrussearch indices for all enabled
# wikis and remove old ones.
+
+source /usr/local/etc/set_dump_dirs.sh
+
usage() {
echo "Usage: $0 [--config <pathtofile>] [--dryrun]"
echo
echo " --config path to configuration file for dump generation"
- echo " (default value: <%=
scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf"
+ echo " (default value: ${confsdir}/wikidump.conf"
echo " --dryrun don't run dump, show what would have been done"
exit 1
}
-configFile="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir')
-%>/wikidump.conf"
+
+configFile="${confsdir}/wikidump.conf"
dryrun="false"
while [ $# -gt 0 ]; do
diff --git a/modules/snapshot/templates/cron/dumpcontentxlation.sh.erb
b/modules/snapshot/files/cron/dumpcontentxlation.sh
similarity index 88%
rename from modules/snapshot/templates/cron/dumpcontentxlation.sh.erb
rename to modules/snapshot/files/cron/dumpcontentxlation.sh
index baa3a40..8b9f28f 100644
--- a/modules/snapshot/templates/cron/dumpcontentxlation.sh.erb
+++ b/modules/snapshot/files/cron/dumpcontentxlation.sh
@@ -1,8 +1,10 @@
#!/bin/bash
#############################################################
# This file is maintained by puppet!
-# modules/snapshot/templates/cron/dumpcontentxlation.sh.erb
+# modules/snapshot/cron/dumpcontentxlation.sh
#############################################################
+
+source /usr/local/etc/set_dump_dirs.sh
checkval() {
setting=$1
@@ -49,10 +51,8 @@
# MAIN
#####################
-confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>"
-repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>"
configfile="${confsdir}/wikidump.conf"
-otherdir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir')
-%>/public/other"
+otherdir="${datadir}/public/other"
dryrun="false"
#####################
diff --git a/modules/snapshot/files/cron/list-last-good-dumps.sh
b/modules/snapshot/files/cron/list-last-good-dumps.sh
new file mode 100644
index 0000000..84d8ee0
--- /dev/null
+++ b/modules/snapshot/files/cron/list-last-good-dumps.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+#############################################################
+# This file is maintained by puppet!
+# /modules/snapshot/cron/list-last-good-dumps.sh
+#############################################################
+
+source /usr/local/etc/set_dump_dirs.sh
+
+# generate lists of most recent completed successful dumps for rsync (dirs,
files)
+python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--dirlisting 'rsync-dirlist-last-%s-good.txt' --configfile
"${confsdir}/wikidump.conf.monitor" --rsynclists --relpath --outputdir
"${datadir}/public/"
+python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--filelisting 'rsync-filelist-last-%s-good.txt' --configfile
"${confsdir}/wikidump.conf.monitor" --rsynclists --relpath --outputdir
"${datadir}/public/" --toplevel
+# these lists can be used for rsync excl/incl on our side, providing shares
that "just work" for the mirrors
+python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--rsynclisting 'rsync-inc-last-%s.txt' --configfile
"${confsdir}/wikidump.conf.monitor" --relpath --outputdir "${datadir}/public/"
--toplevel
diff --git a/modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb
b/modules/snapshot/files/cron/wikidatadumps-shared.sh
similarity index 92%
rename from modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb
rename to modules/snapshot/files/cron/wikidatadumps-shared.sh
index 4c42206..bcb65b7 100644
--- a/modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb
+++ b/modules/snapshot/files/cron/wikidatadumps-shared.sh
@@ -1,14 +1,14 @@
#!/bin/bash
#############################################################
# This file is maintained by puppet!
-# modules/snapshot/templates/cron/wikidatadumps-shared.sh.erb
+# modules/snapshot/cron/wikidatadumps-shared.sh
#############################################################
#
# Shared variable and function declarations for creating Wikidata dumps
#
# Marius Hoch < h...@online.de >
-confdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>"
+source /usr/local/etc/set_dump_dirs.sh
configfile="${confdir}/wikidump.conf"
today=`date +'%Y%m%d'`
diff --git a/modules/snapshot/manifests/dumps/dirs.pp
b/modules/snapshot/manifests/dumps/dirs.pp
index ceb53a8..e1c3f3e 100644
--- a/modules/snapshot/manifests/dumps/dirs.pp
+++ b/modules/snapshot/manifests/dumps/dirs.pp
@@ -57,4 +57,14 @@
}
$repodir = '/srv/deployment/dumps/dumps/xmldumps-backup'
+
+ file { '/usr/local/etc/set_dump_dirs.sh':
+ ensure => 'present',
+ path => '/usr/local/etc/set_dump_dirs.sh',
+ mode => '0755',
+ owner => 'root',
+ group => 'root',
+ content => template('snapshot/set_dump_dirs.sh.erb'),
+ }
+
}
diff --git a/modules/snapshot/templates/cron/list-last-good-dumps.sh.erb
b/modules/snapshot/templates/cron/list-last-good-dumps.sh.erb
deleted file mode 100644
index 05aa945..0000000
--- a/modules/snapshot/templates/cron/list-last-good-dumps.sh.erb
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#############################################################
-# This file is maintained by puppet!
-# /modules/snapshot/templates/cron/list-last-good-dumps.sh.erb
-#############################################################
-
-# generate lists of most recent completed successful dumps for rsync (dirs,
files)
-python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--dirlisting 'rsync-dirlist-last-%s-good.txt' --configfile <%=
scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf.monitor
--rsynclists --relpath --outputdir <%=
scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/
-python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--filelisting 'rsync-filelist-last-%s-good.txt' --configfile <%=
scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf.monitor
--rsynclists --relpath --outputdir <%=
scope.lookupvar('snapshot::dumps::dirs::datadir') -%>/public/ --toplevel
-# these lists can be used for rsync excl/incl on our side, providing shares
that "just work" for the mirrors
-python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber 1,2,3,4,5
--rsynclisting 'rsync-inc-last-%s.txt' --configfile <%=
scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>/wikidump.conf.monitor
--relpath --outputdir <%= scope.lookupvar('snapshot::dumps::dirs::datadir')
-%>/public/ --toplevel
diff --git a/modules/snapshot/templates/set_dump_dirs.sh.erb
b/modules/snapshot/templates/set_dump_dirs.sh.erb
new file mode 100644
index 0000000..26cecee
--- /dev/null
+++ b/modules/snapshot/templates/set_dump_dirs.sh.erb
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# set some directory paths many cron jobs need
+confsdir="<%= scope.lookupvar('snapshot::dumps::dirs::confsdir') -%>"
+repodir="<%= scope.lookupvar('snapshot::dumps::dirs::repodir') -%>"
+datadir="<%= scope.lookupvar('snapshot::dumps::dirs::datadir') -%>"
+dumpsdir="<%= scope.lookupvar('snapshot::dumps::dirs::dumpsdir') -%>"
--
To view, visit https://gerrit.wikimedia.org/r/328158
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I926df4424d70ba458c1ec85fb3a62a9c33304f29
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
