[MediaWiki-commits] [Gerrit] option to run until all wikis have dumps more recent than cu... - change (operations/dumps)
ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/62800
Change subject: option to run until all wikis have dumps more recent than
cutoff date
..
option to run until all wikis have dumps more recent than cutoff date
--cutoff option in wrapper shell script and in main python script
wrapper shell script cleanup (break stuff into functions for
readability)
Change-Id: I405aae9f4c3894eb337d59d487b87e111f37306c
---
M xmldumps-backup/worker
M xmldumps-backup/worker.py
2 files changed, 162 insertions(+), 81 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/00/62800/1
diff --git a/xmldumps-backup/worker b/xmldumps-backup/worker
index afd9944..ced94f8 100755
--- a/xmldumps-backup/worker
+++ b/xmldumps-backup/worker
@@ -1,92 +1,150 @@
#!/bin/bash
-# default number of failures of worker.py in a row before we decide
-# something serious is broken and we refuse to run
-MAXFAILS=3
-# default: don't pass special config file
-CONFIGFILE=
-# default: no logging to file
-LOG=
+usage() {
+echo $0: Unknown option $1
+echo Usage: $0 [--configfile filename] [--log] [--maxfails num]
[--basedir dir] [--wiki wikiname]
+echo [--cutoff date]
+echo --configfile use specified file for config file (default:
wikidump.conf)
+echo --log write log of (almost) everything written to stderr
(default: no logging)
+echo --maxfails if more than this many dumps fail in a row, exit
(default: 3)
+echo --basedir scripts and files are relative to this dir (default:
location of this script)
+echo --cutoff dump wikis until all have a dump produced more recent
than the specified cutoff,
+echothen exit. format: mmdd OR 'today'
+echo --wiki name of specific wiki db to dump; otherwise all wikis
in list referenced by
+echoconfig file will be dumped
+echo
+echo If the file maintenance.txt is present, no more jobs will be run,
and
+echo this script will check the status again in 5 minutes.
+echo If the file exit.txt is present, no more new jobs will be started
and
+echo this script will terminate.
+exit 1
+}
-failures=0
-WIKIDUMP_BASE=`dirname $0`
+set_defaults() {
+# default number of failures of worker.py in a row before we decide
+# something serious is broken and we refuse to run
+MAXFAILS=3
+# default: don't pass special config file
+CONFIGFILE=
+# default: no logging to file
+LOG=
+# default: no cutoff date
+CUTOFF=
+}
-while [ $# -gt 0 ]; do
-if [ $1 == --configfile ]; then
- CONFIGFILE=$2
- shift; shift
-elif [ $1 == --maxfails ]; then
- MAXFAILS=$2
- shift; shift
-elif [ $1 == --basedir ]; then
- WIKIDUMP_BASE=$2
- shift; shift
-elif [ $1 == --log ]; then
- LOG=true
- shift;
-elif [ $1 == --wiki ]; then
- WIKI=$2
- shift; shift
-else
- echo $0: Unknown option $1
- echo Usage: $0 [--configfile filename] [--log] [--maxfails num]
[--basedir dir] [--wiki wikiname]
- echo --configfile use specified file for config file (default:
wikidump.conf)
- echo --log write log of (almost) everything written to stderr
(default: no logging)
- echo --maxfails if more than this many dumps fail in a row, exit
(default: 3)
- echo --basedir scripts and files are relative to this dir
(default: location of this script)
- echo --wiki name of specific wiki db to dump; otherwise all
wikis in list referenced by
- echoconfig file will be dumped
- echo
- echo If the file maintenance.txt is present, no more jobs will be run,
and
- echo this script will check the status again in 5 minutes.
- echo If the file exit.txt is present, no more new jobs will be started
and
- echo this script will terminate.
- exit 1
+process_opts() {
+while [ $# -gt 0 ]; do
+ if [ $1 == --configfile ]; then
+ CONFIGFILE=$2
+ shift; shift
+ elif [ $1 == --maxfails ]; then
+ MAXFAILS=$2
+ shift; shift
+ elif [ $1 == --basedir ]; then
+ WIKIDUMP_BASE=$2
+ shift; shift
+ elif [ $1 == --log ]; then
+ LOG=true
+ shift;
+ elif [ $1 == --cutoff ]; then
+ CUTOFF=$2
+ shift; shift
+ elif [ $1 == --wiki ]; then
+ WIKI=$2
+ shift; shift
+ else
+ usage
+ fi
+done
+}
+
+setup_python_args() {
+# set up the command
+pythonargs=( $WIKIDUMP_BASE/worker.py )
+if [ ! -z $CONFIGFILE ]; then
+ pythonargs=( ${pythonargs[@]} --configfile $CONFIGFILE )
fi
-done
+if [ ! -z $LOG ]; then
+ pythonargs=( ${pythonargs[@]} --log )
+fi
+if [ ! -z $CUTOFF ];
[MediaWiki-commits] [Gerrit] option to run until all wikis have dumps more recent than cu... - change (operations/dumps)
ArielGlenn has submitted this change and it was merged.
Change subject: option to run until all wikis have dumps more recent than
cutoff date
..
option to run until all wikis have dumps more recent than cutoff date
--cutoff option in wrapper shell script and in main python script
wrapper shell script cleanup (break stuff into functions for
readability)
Change-Id: I405aae9f4c3894eb337d59d487b87e111f37306c
---
M xmldumps-backup/worker
M xmldumps-backup/worker.py
2 files changed, 162 insertions(+), 81 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/xmldumps-backup/worker b/xmldumps-backup/worker
index afd9944..ced94f8 100755
--- a/xmldumps-backup/worker
+++ b/xmldumps-backup/worker
@@ -1,92 +1,150 @@
#!/bin/bash
-# default number of failures of worker.py in a row before we decide
-# something serious is broken and we refuse to run
-MAXFAILS=3
-# default: don't pass special config file
-CONFIGFILE=
-# default: no logging to file
-LOG=
+usage() {
+echo $0: Unknown option $1
+echo Usage: $0 [--configfile filename] [--log] [--maxfails num]
[--basedir dir] [--wiki wikiname]
+echo [--cutoff date]
+echo --configfile use specified file for config file (default:
wikidump.conf)
+echo --log write log of (almost) everything written to stderr
(default: no logging)
+echo --maxfails if more than this many dumps fail in a row, exit
(default: 3)
+echo --basedir scripts and files are relative to this dir (default:
location of this script)
+echo --cutoff dump wikis until all have a dump produced more recent
than the specified cutoff,
+echothen exit. format: mmdd OR 'today'
+echo --wiki name of specific wiki db to dump; otherwise all wikis
in list referenced by
+echoconfig file will be dumped
+echo
+echo If the file maintenance.txt is present, no more jobs will be run,
and
+echo this script will check the status again in 5 minutes.
+echo If the file exit.txt is present, no more new jobs will be started
and
+echo this script will terminate.
+exit 1
+}
-failures=0
-WIKIDUMP_BASE=`dirname $0`
+set_defaults() {
+# default number of failures of worker.py in a row before we decide
+# something serious is broken and we refuse to run
+MAXFAILS=3
+# default: don't pass special config file
+CONFIGFILE=
+# default: no logging to file
+LOG=
+# default: no cutoff date
+CUTOFF=
+}
-while [ $# -gt 0 ]; do
-if [ $1 == --configfile ]; then
- CONFIGFILE=$2
- shift; shift
-elif [ $1 == --maxfails ]; then
- MAXFAILS=$2
- shift; shift
-elif [ $1 == --basedir ]; then
- WIKIDUMP_BASE=$2
- shift; shift
-elif [ $1 == --log ]; then
- LOG=true
- shift;
-elif [ $1 == --wiki ]; then
- WIKI=$2
- shift; shift
-else
- echo $0: Unknown option $1
- echo Usage: $0 [--configfile filename] [--log] [--maxfails num]
[--basedir dir] [--wiki wikiname]
- echo --configfile use specified file for config file (default:
wikidump.conf)
- echo --log write log of (almost) everything written to stderr
(default: no logging)
- echo --maxfails if more than this many dumps fail in a row, exit
(default: 3)
- echo --basedir scripts and files are relative to this dir
(default: location of this script)
- echo --wiki name of specific wiki db to dump; otherwise all
wikis in list referenced by
- echoconfig file will be dumped
- echo
- echo If the file maintenance.txt is present, no more jobs will be run,
and
- echo this script will check the status again in 5 minutes.
- echo If the file exit.txt is present, no more new jobs will be started
and
- echo this script will terminate.
- exit 1
+process_opts() {
+while [ $# -gt 0 ]; do
+ if [ $1 == --configfile ]; then
+ CONFIGFILE=$2
+ shift; shift
+ elif [ $1 == --maxfails ]; then
+ MAXFAILS=$2
+ shift; shift
+ elif [ $1 == --basedir ]; then
+ WIKIDUMP_BASE=$2
+ shift; shift
+ elif [ $1 == --log ]; then
+ LOG=true
+ shift;
+ elif [ $1 == --cutoff ]; then
+ CUTOFF=$2
+ shift; shift
+ elif [ $1 == --wiki ]; then
+ WIKI=$2
+ shift; shift
+ else
+ usage
+ fi
+done
+}
+
+setup_python_args() {
+# set up the command
+pythonargs=( $WIKIDUMP_BASE/worker.py )
+if [ ! -z $CONFIGFILE ]; then
+ pythonargs=( ${pythonargs[@]} --configfile $CONFIGFILE )
fi
-done
+if [ ! -z $LOG ]; then
+ pythonargs=( ${pythonargs[@]} --log )
+fi
+if [ ! -z $CUTOFF ]; then
+ if [ $CUTOFF == today ];
