Yuvipanda has uploaded a new change for review.
https://gerrit.wikimedia.org/r/176293
Change subject: Replace webservice bash script with python script
......................................................................
Replace webservice bash script with python script
Too big to bash
Change-Id: I8eb49476544900629837768f4539215f0dc897f8
---
M misctools/webservice
1 file changed, 149 insertions(+), 101 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/labs/toollabs
refs/changes/93/176293/1
diff --git a/misctools/webservice b/misctools/webservice
index e5ef365..e5f6e10 100755
--- a/misctools/webservice
+++ b/misctools/webservice
@@ -1,114 +1,162 @@
-#! /bin/bash
+#!/usr/bin/python
+import os
+import pwd
+import re
+import subprocess
+import argparse
+import xml.etree.ElementTree as ET
-prefix=$(/bin/cat /etc/wmflabs-project)
-tool=$(/usr/bin/id -nu|sed -e "s/^$prefix.//")
-user="$prefix.$tool"
-server="lighttpd"
-public="public_html"
-memlimit=4g
-if [ -r "/data/project/.system/config/$tool.web-memlimit" ]; then
- memlimit=$(cat "/data/project/.system/config/$tool.web-memlimit")
-fi
+def read_file(path, default=None):
+ """
+ Helper function to return contents of file if it exists, or a default
value.
-case "$1" in
- -tomcat)
- server="tomcat"
- public="public_tomcat"
- shift
- ;;
- -lighttpd)
- shift
- ;;
- -*)
- echo "Unknown webservice type $1" >&2
- exit 1
- ;;
-esac
+ :param path: Path to file to read from
+ :param default: Value to return if the file does not exist
+ :return: String containing either contents of the file, or default value
+ """
+ if os.path.exists(path):
+ with open(path) as f:
+ return f.read()
+ return default
-home=$(getent passwd $user | cut -d : -f 6 | sed -e 's/\/$//')
-if [ "$(getent group $user | cut -d : -f 1)" != "$user" ]; then
- echo "$0: $tool does not appear to be a tool" >&2
- exit 1
-fi
-if [ "$home" = "" -o ! -d "$home/$public" ]; then
- echo "$tool does not have a $public" >&2
- exit 1
-fi
+def start_web_job(server):
+ """
+ Submits a job to the grid, running a particular server, for current user
-job=$(qstat -q "webgrid-$server" -j "$server-$tool" 2>&1 | grep job_number: |
sed -e 's/^.* *\(.*\)$/\1/')
-if [ "$job" != "" ]; then
- si=$(qstat -j "$job" 2>&1 | grep 'scheduling info': | sed -e 's/^.*:
*\(.*\)$/\1/')
-fi
+ :param server: Server type to start job as. Current options are lighttpd
and tomcat
+ """
+ command = ['qsub',
+ '-e', '%s/error.log' % HOME,
+ '-o', '%s/error.log' % HOME,
+ '-i', '/dev/null',
+ '-q', 'webgrid-%s' % server,
+ '-l', 'h_vmem=%s' % MEMLIMIT,
+ '-b', 'y',
+ '-N', '%s-%s' % (server, TOOL),
+ '/usr/local/bin/tool-%s' % server]
+ subprocess.check_call(command)
-waitdown() {
- wj="$1"
- while [ "$wj" = "$1" ]; do
- wj=$(qstat -j "$1" 2>&1 | grep job_number: | sed -e 's/^.*
*\(.*\)$/\1/')
- echo -n .
- sleep 1
- done
-}
+def stop_job(job_id):
+ """
+ Deletes a job with given job id from the grid
-case "$1" in
- start)
+ :param job_id: Job id to delete
+ """
+ command = ['qdel', job_id]
+ subprocess.check_call(command)
- echo -n "Starting webservice..."
- if [ "$job" != "" ]; then
- echo "Webservice already running."
- else
- if qsub -e $home/error.log -o $home/error.log -i /dev/null -q
"webgrid-$server" -l h_vmem=$memlimit -b y -N "$server-$tool"
/usr/local/bin/tool-$server >/dev/null 2>&1 ; then
- echo " started."
- else
- echo " failed."
- fi
- fi
- ;;
- restart)
- echo -n "Restarting webservice.."
- if [ "$job" != "" ]; then
- qdel -j "$job" >/dev/null 2>&1
- waitdown "$job"
- else
- echo -n .
- fi
- if qsub -e $home/error.log -o /dev/null -i /dev/null -q
"webgrid-$server" -l h_vmem=$memlimit -b y -N "$server-$tool"
/usr/local/bin/tool-$server >/dev/null 2>&1; then
- echo " restarted."
- else
- echo " failed."
- fi
- ;;
- status)
- if [ "$job" != "" ]; then
- if [ "$si" != "" ]; then
- echo "Your webservice is scheduled:"
- echo " $si"
- else
- echo "Your webservice is running (job $job)."
- fi
- exit 0
- else
- echo "Your webservice is not running."
- exit 1
- fi
- ;;
+def qstat_xml(*args):
+ """
+ Executes a qstat call and returns the output in XML format
- stop)
- if [ "$job" != "" ]; then
- echo -n "Stopping webservice.."
- qdel -j "$job" >/dev/null 2>&1
- waitdown "$job"
- echo " stopped."
- else
- echo "Webservice not running."
- fi
- ;;
+ :param args: Arguments to the qstat call
+ :return: String response in XML form of the qstat output
+ """
+ qstat_args = ['qstat'] + list(args) + ['-xml']
+ output = subprocess.check_output(qstat_args)
+ return output
- *)
- echo "usage: $(basename $0) start|restart|status|stop" >&2
- exit 1
- ;;
-esac
+
+def xpath_string(string, xpath):
+ """
+ Parses given string as XML, returns single string value
+ produced by the given xpath query
+
+ :param string: String to parse as XML
+ :param xpath: XPath query to run over the parsed XML
+ :return: Single string that is the result of the XPath query
+ """
+ xml = ET.fromstring(string)
+ return xml.findtext(xpath)
+
+def get_job_id(queue_name, job_name):
+ """
+ Gets job id of a particular job with a particular name in a particular
queue
+
+ :param queue_name: Queue name to look in
+ :param job_name: Job name to look for
+ :return: Job id if the job is found, None otherwise
+ """
+ output = qstat_xml('-q', queue_name, '-j', job_name)
+ # GE is stupid.
+ # Returns output like:
+ # <><ST_name>blah</ST_name></>
+ # If the job is not found.
+ if '<unknown_jobs' in output and '<>' in output:
+ return None
+ return xpath_string(output, './/JB_job_number')
+
+def wait_for_job(queue_name, job_name, up=True):
+ """
+ Waits for a job to be either up (or down), printing .s while waiting
+
+ :param queue_name: Queue name to look for the job in
+ :param job_name: Name of job to look for
+ :param up: True if we want to wait for the job to be up, false for down
+ :return returns the job id, if up=True
+ """
+ while True:
+ jid = get_job_id(queue_name, job_name)
+ if jid is None == up:
+ print '.',
+ else:
+ return jid
+
+# Setup constants that we would need later on
+PREFIX = read_file('/etc/wmflabs-project', 'tools').strip() # project name
+
+pwd_entry = pwd.getpwuid(os.getuid())
+USER = pwd_entry.pw_name
+HOME = pwd_entry.pw_dir
+TOOL = re.sub(r'^%s.' % PREFIX, '', USER) # Tool users are of form
PREFIX.TOOLNAME
+
+# Read memlimit customizations for individual tools, set by
+# admins for tools that require more than usual memory limits.
+MEMLIMIT = read_file(
+ os.path.join(
+ '/data/project/.system/config/',
+ '%s.web-memlimit' % TOOL
+ ), '4g'
+)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('server', help='Type of server to start',
+ choices=['lighttpd', 'tomcat'], default='lighttpd',
nargs='?')
+parser.add_argument('action', help='Action to perform',
+ choices=['stop', 'start', 'restart'])
+args = parser.parse_args()
+
+queue_name = 'webgrid-%s' % args.server # Queues are named
webgrid-{tomcat,lighttpd}
+job_name = '%s-%s' % (args.server, TOOL) # Format for job names. one tool can
have only one job running on webgrid
+
+job_id = get_job_id(queue_name, job_name)
+
+if args.action == 'start':
+ if job_id is not None:
+ print 'Your webservice is already running'
+ exit()
+ start_web_job(args.server)
+ print 'Starting web service',
+ wait_for_job(queue_name, job_name, True)
+elif args.action == 'stop':
+ if job_id is None:
+ print 'Your webservice is not running'
+ exit()
+ stop_job(job_id)
+ print 'Stopping web service'
+ wait_for_job(queue_name, job_name, False)
+elif args.action == 'restart':
+ if job_id is not None:
+ print 'Restarting'
+ stop_job(job_id)
+ wait_for_job(queue_name, job_name, False)
+ start_web_job(args.server)
+ wait_for_job(queue_name, job_name, True)
+ else:
+ print 'Webservice not running, starting'
+ start_web_job(args.server)
+ wait_for_job(queue_name, job_name, True)
--
To view, visit https://gerrit.wikimedia.org/r/176293
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8eb49476544900629837768f4539215f0dc897f8
Gerrit-PatchSet: 1
Gerrit-Project: labs/toollabs
Gerrit-Branch: master
Gerrit-Owner: Yuvipanda <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
