Thcipriani has uploaded a new change for review.
https://gerrit.wikimedia.org/r/224374
Change subject: Add service deploy via scap
......................................................................
Add service deploy via scap
Preliminary rough concept for deploying services via git from scap.
The command would looks something like: `deploy <rev>` run from
/srv/deployment/restbase/deploy (or other service directory: like
git-deploy now) and would require a `.scaprc` file at the root of the
repository being deployed with the following information:
[global]
git_server: tin.eqiad.wmnet
git_repo: restbase/deploy
git_deploy_dir: /srv/deployment
dsh_targets: restbase-installation
[wmflabs]
git_server: staging-test-tin.staging.eqiad.wmflabs
Change-Id: I542bb9acc0f65360452fb4f0b0c92dec4894e5d8
---
A bin/deploy
A bin/deploy-local
M scap/__init__.py
M scap/config.py
M scap/main.py
M scap/tasks.py
M scap/utils.py
7 files changed, 157 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/scap
refs/changes/74/224374/1
diff --git a/bin/deploy b/bin/deploy
new file mode 100755
index 0000000..e048d76
--- /dev/null
+++ b/bin/deploy
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Sync MW source and config to cluster.
+#
+# Copyright © 2014 Wikimedia Foundation and contributors
+
+import os
+import sys
+
+# Add scap package to search path
+script = os.path.realpath(sys.argv[0])
+scap_src = os.path.dirname(os.path.dirname(script))
+sys.path.append(scap_src)
+
+import scap
+scap.Deploy.run()
diff --git a/bin/deploy-local b/bin/deploy-local
new file mode 100755
index 0000000..09e0ac1
--- /dev/null
+++ b/bin/deploy-local
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Sync MW source and config to cluster.
+#
+# Copyright © 2014 Wikimedia Foundation and contributors
+
+import os
+import sys
+
+# Add scap package to search path
+script = os.path.realpath(sys.argv[0])
+scap_src = os.path.dirname(os.path.dirname(script))
+sys.path.append(scap_src)
+
+import scap
+scap.DeployLocal.run()
diff --git a/scap/__init__.py b/scap/__init__.py
index 84af164..98607eb 100644
--- a/scap/__init__.py
+++ b/scap/__init__.py
@@ -21,6 +21,8 @@
SyncFile,
SyncWikiversions,
UpdateL10n,
+ DeployLocal,
+ Deploy,
)
__all__ = (
@@ -38,6 +40,8 @@
'SyncFile',
'SyncWikiversions',
'UpdateL10n',
+ 'DeployLocal',
+ 'Deploy',
)
any((
@@ -54,4 +58,7 @@
SyncDocroot,
SyncFile,
SyncWikiversions,
- UpdateL10n)) # Ignore unused import warning
+ UpdateL10n,
+ DeployLocal,
+ Deploy
+ )) # Ignore unused import warning
diff --git a/scap/config.py b/scap/config.py
index b4a4348..ba32e8d 100644
--- a/scap/config.py
+++ b/scap/config.py
@@ -30,6 +30,10 @@
'apache_pid_file': '/var/run/apache2/apache2.pid',
'pybal_interface': 'lo:LVS',
'dsh_targets': 'mediawiki-installation',
+ 'git_deploy_dir': '/srv/deployments',
+ 'git_repo_user': 'mwdeploy',
+ 'git_server': 'tin.eqiad.wmnet',
+ 'git_scheme': 'http',
}
@@ -68,11 +72,18 @@
parser.readfp(cfg_file)
else:
- parser.read([
+ cfg_paths = [
os.path.join(os.path.dirname(__file__), '..', 'scap.cfg'),
'/srv/scap/scap.cfg',
'/etc/scap.cfg',
- ])
+ ]
+
+ local_cfg = os.path.join(os.getcwd(), ".scaprc")
+
+ if os.access(local_cfg, os.R_OK):
+ cfg_paths += [local_cfg]
+
+ parser.read(cfg_paths)
fqdn = socket.getfqdn().split('.')
sections = ['global']
diff --git a/scap/main.py b/scap/main.py
index 70402b0..f938387 100644
--- a/scap/main.py
+++ b/scap/main.py
@@ -559,3 +559,67 @@
self.get_stats().increment('deploy.restart')
return exit_code
+
+
+class DeployLocal(cli.Application):
+ """Fetch new restbase code from gerrit"""
+
+ def main(self, *extra_args):
+ repo = self.config['git_repo']
+ repo_user = self.config['git_repo_user']
+ server = self.config['git_server']
+ rev = self.config['git_rev']
+
+ location =
os.path.normpath("{0}/{1}".format(self.config['git_deploy_dir'], repo))
+
+ if self.config['git_scheme'] == 'git' or self.config['git_scheme'] ==
'http':
+ scheme = 'http'
+
+ url = os.path.normpath('{0}/{1}'.format(server, repo))
+ url = "{0}://{1}".format(scheme, url)
+
+ # Fetch
+ tasks.git_fetch(location, url, repo_user)
+
+ # Checkout
+ tasks.git_co(location, rev, repo_user)
+
+
+class Deploy(cli.Application):
+
+ @cli.argument('rev', help='Revision to deploy')
+ def main(self, *extra_args):
+
+ in_deploy_dir = os.path.commonprefix([
+ os.getcwd(),
+ self.config['git_deploy_dir']
+ ]) == self.config['git_deploy_dir']
+
+ if not in_deploy_dir:
+ raise IOError(1,
+ 'Your path is not a part of the git deploy path',
+ self.config['git_deploy_dir'])
+
+ repo = self.config['git_repo']
+ targets = utils.read_dsh_hosts_file(self.config['dsh_targets'])
+ with utils.lock(self.config['lock_file']):
+ with log.Timer('deploy_' + repo):
+ deploy_rev = ssh.Job(targets, user=self.config['ssh_user'])
+ deploy_rev.command([
+ self.get_script_path('deploy-local'),
+ '-D git_repo:{}'.format(repo),
+ '-D git_repo_user:{}'.format(self.config['git_repo_user']),
+ '-D git_server:{}'.format(self.config['git_server']),
+ '-D git_rev:{}'.format(self.arguments.rev)
+ ])
+ deploy_rev.progress('deploy_' + repo)
+ succeeded, failed = deploy_rev.run()
+ if failed:
+ self.get_logger().warning(
+ '%d targets had deploy errors', failed)
+ self.soft_errors = True
+
+ if self.soft_errors:
+ return 1
+ else:
+ return 0
diff --git a/scap/tasks.py b/scap/tasks.py
index 2fa6cea..9dde01e 100644
--- a/scap/tasks.py
+++ b/scap/tasks.py
@@ -533,3 +533,27 @@
restart.command('sudo -u mwdeploy -n -- %s' %
os.path.join(cfg['bin_dir'], 'scap-hhvm-restart'))
return restart.progress('restart_hhvm').run(batch_size=batch_size)
+
+
+def git_fetch(location, repo, user="mwdeploy"):
+ """Fetch a git repo to a location as a user
+ """
+ if os.path.isdir("{}/.git".format(location)):
+ with utils.cd(location):
+ cmd = '/usr/bin/git fetch'
+ utils.sudo_check_call(user, cmd)
+ else:
+ cmd = '/usr/bin/git clone {0} {1}'.format(repo, location)
+ utils.sudo_check_call(user, cmd)
+
+
+def git_co(location, rev, user="mwdeploy", submodules=False):
+ """Checkout a git repo sha at a location as a user
+ """
+ with utils.cd(location):
+ cmd = '/usr/bin/git checkout --force --quiet {}'.format(rev)
+ utils.sudo_check_call(user, cmd)
+
+ if submodules is True:
+ cmd = '/usr/bin/git submodule update --init --recursive'
+ utils.sudo_check_call(user, cmd)
diff --git a/scap/utils.py b/scap/utils.py
index ff1a31d..24d47e6 100644
--- a/scap/utils.py
+++ b/scap/utils.py
@@ -260,6 +260,20 @@
os.unlink(filename)
[email protected]
+def cd(dirname):
+ """Context manager. Cds to dirname, moves back to previous dir on context
exit
+
+ :param dirname: directory into which it should change
+ """
+ old_path = os.getcwd()
+ try:
+ os.chdir(dirname)
+ yield
+ finally:
+ os.chdir(old_path)
+
+
def md5_file(path):
"""Compute the md5 checksum of a file's contents.
--
To view, visit https://gerrit.wikimedia.org/r/224374
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I542bb9acc0f65360452fb4f0b0c92dec4894e5d8
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/scap
Gerrit-Branch: master
Gerrit-Owner: Thcipriani <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
