ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/392625 )
Change subject: rsync misc dumps (everything but xml/sql) to fallback hosts,
labstore1006
......................................................................
rsync misc dumps (everything but xml/sql) to fallback hosts, labstore1006
Not everything is generated on the source host yet, but this
will rsync jobs as they get moved over.
Bug: T179942
Change-Id: I78de4a6d913427b278a376cfe1384966582f1672
---
M modules/dumps/files/generation/rsync-to-peers.sh
M modules/dumps/manifests/generation/server/rsyncer.pp
M modules/dumps/templates/initscripts/dumps-rsync-peers.systemd.erb
M modules/dumps/templates/initscripts/dumps-rsync-peers.upstart.erb
M modules/profile/manifests/dumps/generation/server/primary.pp
5 files changed, 59 insertions(+), 29 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/25/392625/1
diff --git a/modules/dumps/files/generation/rsync-to-peers.sh
b/modules/dumps/files/generation/rsync-to-peers.sh
index 8193c50..a7b4a43 100644
--- a/modules/dumps/files/generation/rsync-to-peers.sh
+++ b/modules/dumps/files/generation/rsync-to-peers.sh
@@ -5,35 +5,49 @@
# puppet:///modules/dumps/generation/rsync-to-peers.sh
##############################
-# This script rsyncs xml/sql dumps to fallback dumps generation nfs servers,
-# i.e. to its peers.
-# Eventually it will also rsync all output from misc dump cron jobs to
-# these servers.
+# This script rsyncs xml/sql dumps and misc dumps to fallback dumps
+# generation nfs servers, i.e. to its peers.
+#
# This ensures that we never have more than one rsync connection going
# at a time on the host where dumps are written as they are generated.
usage() {
cat<<EOF
-Usage: $0 --dumpsdir <path> --remotedirs <path>,<path>,<path>...
+Usage: $0 --dumpsdir <path> --xmlremotedirs <path>,<path>,<path>...
- --dumpsdir path to root of dumps tree for rsync to peer hosts
- --remotedir comma-separated list of remote destinations to which to rsync
+ --xmldumpsdir path to root of xml/sql dumps tree for rsync to peer hosts
+ --xmlremotedirs comma-separated list of remote destinations to which to
rsync
-Example: $0 --dumpsdir /data/xmldatadumps \\
- --remotedirs
dumpsdata1002.eqiad.wmnet::data/xmldatadumps/public/,dumpsdata1003.eqiad.wmnet::data/xmldatadumps/public/
+ --miscdumpsdir path to root of misc dumps tree for rsync to peer hosts
+ --miscremotedirs comma-separated list of remote destinations to which to
rsync
+
+Example:
+
+ $0 --xmldumpsdir /data/xmldatadumps \\
+ --xmlremotedirs
dumpsdata1002.eqiad.wmnet::data/xmldatadumps/public/,dumpsdata1003.eqiad.wmnet::data/xmldatadumps/public/
\\
+ --miscdumpsdir /data/otherdumps \\
+ --miscremotedirs
dumpsdata1002.eqiad.wmnet::data/otherdumps/,dumpsdata1003.eqiad.wmnet::data/otherdumps/
EOF
exit 1
}
-dumpsdir=""
-remotedirs=""
+xmldumpsdir=""
+xmlremotedirs=""
+miscdumpsdir=""
+miscremotedirs=""
while [ $# -gt 0 ]; do
- if [ $1 == "--dumpsdir" ]; then
- dumpsdir="$2"
+ if [ $1 == "--xmldumpsdir" ]; then
+ xmldumpsdir="$2"
shift; shift
- elif [ $1 == "--remotedirs" ]; then
- remotedirs="$2"
+ elif [ $1 == "--xmlremotedirs" ]; then
+ xmlremotedirs="$2"
+ shift; shift
+ elif [ $1 == "--miscdumpdir" ]; then
+ xmlremotedirs="$2"
+ shift; shift
+ elif [ $1 == "--miscremotedirs" ]; then
+ xmlremotedirs="$2"
shift; shift
else
echo "$0: Unknown option $1" >& 2
@@ -41,27 +55,39 @@
fi
done
-if [ -z "$dumpsdir" ]; then
- echo "$0: missing argument --dumpsdir"
+if [ -z "$xmldumpsdir" ]; then
+ echo "$0: missing argument --xmldumpsdir"
usage && exit 1
-fi
-if [ -z "$remotedirs" ]; then
- echo "$0: missing argument --remotedirs"
+elif [ -z "$xmlremotedirs" ]; then
+ echo "$0: missing argument --xmlremotedirs"
+ usage && exit 1
+elif [ -z "$miscdumpsdir" ]; then
+ echo "$0: missing argument --miscdumpsdir"
+ usage && exit 1
+elif [ -z "$miscemotedirs" ]; then
+ echo "$0: missing argument --miscremotedirs"
usage && exit 1
fi
IFS_SAVE=$IFS
IFS=','
-read -a remotedirs_list <<<$remotedirs
+read -a xmlremotedirs_list <<<$xmlremotedirs
+read -a miscremotedirs_list <<<$miscremotedirs
IFS=$IFS_SAVE
while [ 1 ]; do
- for dest in $remotedirs_list; do
- /usr/bin/rsync -a --contimeout=600 --timeout=600
${dumpsdir}/public/*html "$dest" > /dev/null 2>&1
+ # rsync of xml/sql dumps for public wikis
+ for dest in $xmlremotedirs_list; do
+ /usr/bin/rsync -a --contimeout=600 --timeout=600
${xmldumpsdir}/public/*html "$dest" > /dev/null 2>&1
/usr/bin/rsync -a --contimeout=600 --timeout=600 --exclude='**bad/'
--exclude='**save/' --exclude='**not/' --exclude='**temp/' --exclude='**tmp/'
--exclude='*.inprog' ${dumpsdir}/public/*wik* "$dest" > /dev/null 2>&1
done
+ # rsync of misc dumps, not necessarily to/from the same tree as the public
wikis
+ for dest in $miscremotedirs_list; do
+ /usr/bin/rsync -a --contimeout=600 --timeout=600 ${miscdumpsdir}/*
"$dest" > /dev/null 2>&1
+ done
+
# when dumps aren't being generated, no reason to try over and over again
to push new files.
# take a break in between.
sleep 600
diff --git a/modules/dumps/manifests/generation/server/rsyncer.pp
b/modules/dumps/manifests/generation/server/rsyncer.pp
index 6f038bb..e29f45c 100644
--- a/modules/dumps/manifests/generation/server/rsyncer.pp
+++ b/modules/dumps/manifests/generation/server/rsyncer.pp
@@ -1,6 +1,8 @@
class dumps::generation::server::rsyncer(
- $dumpsdir = undef,
- $remotedirs = undef,
+ $xmldumpsdir = undef,
+ $xmlremotedirs = undef,
+ $miscdumpsdir = undef,
+ $miscremotedirs = undef,
) {
file { '/usr/local/bin/rsync-to-peers.sh':
ensure => 'present',
diff --git a/modules/dumps/templates/initscripts/dumps-rsync-peers.systemd.erb
b/modules/dumps/templates/initscripts/dumps-rsync-peers.systemd.erb
index 2582b92..8e84e28 100644
--- a/modules/dumps/templates/initscripts/dumps-rsync-peers.systemd.erb
+++ b/modules/dumps/templates/initscripts/dumps-rsync-peers.systemd.erb
@@ -8,7 +8,7 @@
[Service]
User=root
-ExecStart=/bin/bash /usr/local/bin/rsync-to-peers.sh --dumpsdir <%= @dumpsdir
%> --remotedirs <%= @remotedirs -%>
+ExecStart=/bin/bash /usr/local/bin/rsync-to-peers.sh --xmldumpsdir <%=
@xmldumpsdir %> --xmlremotedirs <%= @xmlremotedirs -%> --miscdumpsdir <%=
@miscdumpsdir %> --miscremotedirs <%= @miscremotedirs -%>
[Install]
WantedBy=multi-user.target
diff --git a/modules/dumps/templates/initscripts/dumps-rsync-peers.upstart.erb
b/modules/dumps/templates/initscripts/dumps-rsync-peers.upstart.erb
index 9e63b88..cf9f2b5 100644
--- a/modules/dumps/templates/initscripts/dumps-rsync-peers.upstart.erb
+++ b/modules/dumps/templates/initscripts/dumps-rsync-peers.upstart.erb
@@ -12,4 +12,4 @@
respawn
-exec /bin/bash /usr/local/bin/rsync-to-peers.sh --dumpsdir <%= @publicdir -%>
--remotedirs <%= @remotedirs -%>
+exec /bin/bash /usr/local/bin/rsync-to-peers.sh --xmldumpsdir <%= @xmldumpsdir
-%> --xmlremotedirs <%= @xmlremotedirs -%> --miscdumpsdir <%= @miscdumpsdir -%>
--miscremotedirs <%= @miscremotedirs -%>
diff --git a/modules/profile/manifests/dumps/generation/server/primary.pp
b/modules/profile/manifests/dumps/generation/server/primary.pp
index b5d48b7..028288a 100644
--- a/modules/profile/manifests/dumps/generation/server/primary.pp
+++ b/modules/profile/manifests/dumps/generation/server/primary.pp
@@ -7,7 +7,9 @@
}
class { '::dumps::generation::server::rsyncer':
- dumpsdir => '/data/xmldatadumps',
- remotedirs =>
'dumpsdata1002.eqiad.wmnet::data/xmldatadumps/public/,dataset1001.wikimedia.org::data/xmldatadumps/public/,labstore1006.wikimedia.org::data/xmldatadumps/public/',
+ xmldumpsdir => '/data/xmldatadumps',
+ xmlremotedirs =>
'dumpsdata1002.eqiad.wmnet::data/xmldatadumps/public/,dataset1001.wikimedia.org::data/xmldatadumps/public/,labstore1006.wikimedia.org::data/xmldatadumps/public/',
+ miscdumpsdir => '/data/otherdumps',
+ miscremotedirs =>
'dumpsdata1002.eqiad.wmnet::data/otherdumps/,dataset1001.wikimedia.org::data/xmldatadumps/public/other/,labstore1006.wikimedia.org::data/xmldatadumps/public/other/',
}
}
--
To view, visit https://gerrit.wikimedia.org/r/392625
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I78de4a6d913427b278a376cfe1384966582f1672
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
