BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/116918
Change subject: Convert sync-wikiversions to python
......................................................................
Convert sync-wikiversions to python
Replace the sync-wikiversions script with a pure python version. Also
uses the newly created task.sync_wikiversions function in task.scap.
Change-Id: I2519b61a7e87a285f791732fcc92f7404f8d5649
---
M bin/sync-wikiversions
M docs/scripts.rst
M scap/__init__.py
M scap/cli.py
M scap/log.py
M scap/main.py
M scap/tasks.py
7 files changed, 87 insertions(+), 47 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/18/116918/1
diff --git a/bin/sync-wikiversions b/bin/sync-wikiversions
index 8eb34d0..caaf46f 100755
--- a/bin/sync-wikiversions
+++ b/bin/sync-wikiversions
@@ -1,30 +1,17 @@
-#!/bin/bash
-. /usr/local/lib/mw-deployment-vars.sh
-BINDIR="/usr/local/bin"
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Rebuild and sync wikiversions.cdb to the cluster.
+#
+# Copyright © 2014 Wikimedia Foundation and contributors
-if [ ! -S "$SSH_AUTH_SOCK" ]; then
- echo >&2 "SSH_AUTH_SOCK not set or not pointing to a socket."
- echo >&2 "Did you start your ssh-agent?"
- exit 1
-fi
+import os
+import sys
-if ! $MW_COMMON_SOURCE/multiversion/refreshWikiversionsCDB; then
- echo 'Error: could not rebuild the wikiversions.cdb file'
-fi
+# Add scap package to search path
+script = os.path.realpath(sys.argv[0])
+scap_src = os.path.dirname(os.path.dirname(script))
+sys.path.append(scap_src)
-# No use going any further if we can't complete the sync
-[ $(which dsh 2>/dev/null) ] || {
- echo >&2 "Error: no dsh on this host, aborting"
- exit 1
-}
-
-# Copy files to apaches
-echo -n 'Copying wikiversions dat and cdb files to apaches...'
-dsh -cM -g mediawiki-installation -o -oSetupTimeout=10 -- "sudo -u mwdeploy
rsync -l $MW_RSYNC_HOST::common/wikiversions.{json,cdb} $MW_COMMON"
-
-echo 'Finished'
-
-# Notify
-$BINDIR/dologmsg "!log $USER rebuilt wikiversions.cdb and synchronized
wikiversions files: $*"
-$BINDIR/deploy2graphite sync-wikiversions
-exit 0
+import scap
+scap.SyncWikiversions.run()
diff --git a/docs/scripts.rst b/docs/scripts.rst
index ed8baad..97c3b31 100644
--- a/docs/scripts.rst
+++ b/docs/scripts.rst
@@ -7,8 +7,8 @@
scap
====
**scap** is the driver script for syncing the MediaWiki versions and
-configuration files currently staged on ``tin.eqiad.wmnet`` to the rest of the
-production cluster.
+configuration files currently staged on the deploy server to the rest of the
+cluster.
.. program-output:: ../bin/scap --help
.. seealso::
@@ -27,6 +27,18 @@
* :func:`scap.tasks.sync_common`
+sync-wikiversions
+=================
+**sync-wikiversions** compiles wikiversions.json into a CDB database and then
+syncs both the JSON and CDB versions to the rest of the cluster.
+
+.. program-output:: ../bin/sync-common --help
+.. seealso::
+ * :func:`scap.SyncWikiversions`
+ * :func:`scap.tasks.compile_wikiversions_cdb`
+ * :func:`scap.tasks.sync_wikiversions`
+
+
mwversionsinuse
===============
**mwversionsinuse** examines wikiversions.json to find the current active
diff --git a/scap/__init__.py b/scap/__init__.py
index 4ddcf3c..192df59 100644
--- a/scap/__init__.py
+++ b/scap/__init__.py
@@ -9,6 +9,7 @@
from .main import MWVersionsInUse
from .main import Scap
from .main import SyncCommon
+from .main import SyncWikiversions
from . import log
@@ -16,8 +17,13 @@
'MWVersionsInUse',
'Scap',
'SyncCommon',
+ 'SyncWikiversions',
)
-any((MWVersionsInUse, Scap, SyncCommon)) # Ignore unused import warning
+any((
+ MWVersionsInUse,
+ Scap,
+ SyncCommon,
+ SyncWikiversions)) # Ignore unused import warning
log.setup_loggers()
diff --git a/scap/cli.py b/scap/cli.py
index a5a1254..93df25b 100644
--- a/scap/cli.py
+++ b/scap/cli.py
@@ -20,6 +20,7 @@
"""Base class for creating command line applications."""
program_name = None
_logger = None
+ _irc_logger = None
arguments = None
config = None
@@ -35,6 +36,12 @@
self._logger = logging.getLogger(self.program_name)
return self._logger
+ def announce(self, *args):
+ """Announce a message on IRC."""
+ if self._irc_logger is None:
+ self._irc_logger = logging.getLogger('scap.irc')
+ self._irc_logger.info(*args)
+
def _parse_arguments(self, argv):
"""Parse command line arguments.
diff --git a/scap/log.py b/scap/log.py
index 0b3111e..f399f5a 100644
--- a/scap/log.py
+++ b/scap/log.py
@@ -256,8 +256,8 @@
udp_handler.setFormatter(LogstashFormatter())
logging.root.addHandler(udp_handler)
- # Send 'scap' messages to irc relay
- irc_logger = logging.getLogger('scap')
+ # Send 'scap.announce' messages to irc relay
+ irc_logger = logging.getLogger('scap.announce')
irc_logger.addHandler(IRCSocketHandler(*IRC_LOG_ENDPOINT))
diff --git a/scap/main.py b/scap/main.py
index 91ed01b..b3c1e6e 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -51,6 +51,26 @@
return 0
+class SyncWikiversions(cli.Application):
+ """Rebuild and sync wikiversions.cdb to the cluster."""
+
+ def _process_arguments(self, args, extra_args):
+ args.message = ' '.join(args.message) or '(no message)'
+ return args, extra_args
+
+ @cli.argument('message', nargs='*', help='Log message for SAL')
+ def main(self, *extra_args):
+ assert 'SSH_AUTH_SOCK' in os.environ, \
+ '%s requires SSH agent forwarding' % self.program_name
+
+ mw_install_hosts = utils.read_dsh_hosts_file('mediawiki-installation')
+ tasks.sync_wikiversions(mw_install_hosts, self.config)
+
+ self.announce(
+ 'rebuilt wikiversions.cdb and synchronized wikiversions files: %s',
+ self.arguments.message)
+
+
class Scap(cli.Application):
"""Deploy MediaWiki to the cluster."""
@@ -68,9 +88,10 @@
assert 'SSH_AUTH_SOCK' in os.environ, \
'scap requires SSH agent forwarding'
- tasks.scap(self.arguments.message, self.config)
+ self.announce('Started scap: %s', self.arguments.message)
+ tasks.scap(self.config)
- self.logger.info('Finished scap: %s (duration: %s)',
+ self.announce('Finished scap: %s (duration: %s)',
self.arguments.message, self.human_duration)
return 0
@@ -85,13 +106,13 @@
return utils.human_duration(self.duration)
def _handle_keyboard_interrupt(self, ex):
- self.logger.warning('scap aborted: %s (duration: %s)',
+ self.announce('scap aborted: %s (duration: %s)',
self.arguments.message, self.human_duration)
return 1
def _handle_exception(self, ex):
self.logger.debug('Unhandled error:', exc_info=True)
- self.logger.error('scap failed: %s %s (duration: %s)',
+ self.announce('scap failed: %s %s (duration: %s)',
type(ex).__name__, ex, self.human_duration)
return 1
diff --git a/scap/tasks.py b/scap/tasks.py
index cb7f652..7c0af1c 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -135,7 +135,22 @@
subprocess.check_call(rsync)
-def scap(message, cfg):
+def sync_wikiversions(hosts, cfg):
+ """Rebuild and sync wikiversions.cdb to the cluster.
+
+ :param hosts: List of hosts to sync to
+ :param cfg: Global configuration
+ """
+ stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port']))
+ with log.Timer('sync_wikiversions', stats):
+ compile_wikiversions_cdb(cfg)
+ ssh.cluster_monitor(hosts,
+ 'sudo -u mwdeploy /usr/bin/rsync -l '
+ '%(master_rsync)s::common/wikiversions*.{json,cdb} '
+ '%(deploy_dir)s' % cfg)
+
+
+def scap(cfg):
"""Core business logic of scap process.
1. Validate php syntax of wmf-config and multiversion
@@ -147,15 +162,11 @@
7. Update wikiversions.cdb on localhost
8. Ask apaches to sync wikiversions.cdb
- :param message: Reason for running scap
:param cfg: Configuration settings
"""
- logger = logging.getLogger('scap')
stats = log.Stats(cfg['statsd_host'], int(cfg['statsd_port']))
with utils.lock('/var/lock/scap'):
- logger.info('Started scap: %s', message)
-
check_php_syntax('%(stage_dir)s/wmf-config' % cfg,
'%(stage_dir)s/multiversion' % cfg)
@@ -187,9 +198,5 @@
'/usr/local/bin/scap-rebuild-cdbs')
t.mark('scap-rebuild-cdbs')
- with log.Timer('syncing wikiversions.cdb', stats) as t:
- compile_wikiversions_cdb(cfg)
- ssh.cluster_monitor(mw_install_hosts,
- 'sudo -u mwdeploy /usr/bin/rsync -l '
- '%(master_rsync)s::common/wikiversions.{json,cdb} '
- '%(deploy_dir)s' % cfg)
+ # Update and sync wikiversions.cdb
+ sync_wikiversions(mw_install_hosts, cfg)
--
To view, visit https://gerrit.wikimedia.org/r/116918
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2519b61a7e87a285f791732fcc92f7404f8d5649
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
