Legoktm has uploaded a new change for review.
https://gerrit.wikimedia.org/r/189303
Change subject: [WIP] Initial commit
......................................................................
[WIP] Initial commit
Change-Id: I11e3ac7c3414009a6a4c9a4c97977597efd0e764
---
A NOTES
A app.py
A rebuild.py
A requirements.txt
A tardist.json
A tardist.py
A tox.ini
A worker.py
8 files changed, 315 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/tardist
refs/changes/03/189303/1
diff --git a/NOTES b/NOTES
new file mode 100644
index 0000000..90dcadc
--- /dev/null
+++ b/NOTES
@@ -0,0 +1,10 @@
+/ - tardist version info / link to docs?
+/update/extensions/MassMessage/master - queue to rebuild a tarball
+/list - returns a list of extensions that are supported
+/info/extensions/MassMessage - returns list of branches+links to tarballs, and
other metadata (license, description (localized?), etc...)
+/info/extensions - batch return of all extension info maybe?
+
+
+
+Need submodule domain whitelisting?
+
diff --git a/app.py b/app.py
new file mode 100644
index 0000000..1583410
--- /dev/null
+++ b/app.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+from flask import Flask, Response, request
+import json
+
+from tardist import tardist
+
+__version__ = '0.0.1'
+app = Flask(__name__)
+
+
+def respond(status='ok', **kwargs) -> Response:
+ """
+
+ :param status: 'ok' or False
+ :type status: str|bool
+ :return:
+ """
+ if request.args.get('pretty'):
+ json_kwargs = {'indent': 4, 'separators': (',', ': '), 'sort_keys':
True}
+ else:
+ json_kwargs = {}
+ return Response(
+ json.dumps(
+ dict(_status=status or 'error', **kwargs),
+ **json_kwargs
+ ),
+ content_type='application/json'
+ )
+
+
[email protected]('/')
[email protected]('/version')
+def version():
+ return respond(version=__version__)
+
+
[email protected]('/update/<type_>/<extension>')
[email protected]('/update/<type_>/<extension>/<branch>')
+def update(type_, extension, branch='*'):
+ repo = '%s/%s' % (type_, extension)
+ if repo not in tardist.get_all_repos():
+ return respond(status=False, error='Invalid repo provided')
+
+ tardist.push_update(repo, branch)
+ return respond(update='queued', repo=repo, branch=branch)
+
+
[email protected]('/list')
[email protected]('/list/<type_>')
+def list_(type_=''):
+ if type_ not in ['extensions', 'skins', '']:
+ return respond(status=False, error='Invalid type provided')
+
+ return respond(repos=[repo for repo in tardist.get_all_repos() if
repo.startswith(type_)])
+
+if __name__ == '__main__':
+ app.run(debug=True)
diff --git a/rebuild.py b/rebuild.py
new file mode 100644
index 0000000..9977de8
--- /dev/null
+++ b/rebuild.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import sys
+
+from tardist import tardist
+
+if len(sys.argv) > 1:
+ repos = sys.argv[1:]
+else:
+ repos = tardist.get_all_repos(cached=False)
+
+for repo in repos:
+ if len(repo.split('/')) > 2:
+ repo, branch = repo.rsplit('/', 1)
+ else:
+ branch = '*'
+ tardist.push_update(repo, branch)
+
+print('Queued %s updates.' % len(repos))
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..b8a0ee2
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+flask
+redis
+requests
diff --git a/tardist.json b/tardist.json
new file mode 100644
index 0000000..451e1dd
--- /dev/null
+++ b/tardist.json
@@ -0,0 +1,5 @@
+{
+ "GIT_URL": "https://gerrit.wikimedia.org/r/mediawiki/%s";,
+ "SRC_PATH": "/home/km/projects/tardist/src",
+ "DIST_PATH": "/home/km/projects/tardist/dist"
+}
diff --git a/tardist.py b/tardist.py
new file mode 100644
index 0000000..b8b7a4d
--- /dev/null
+++ b/tardist.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+
+import glob
+import json
+import logging
+import os
+import redis
+import requests
+import subprocess
+
+
+class cwd:
+ """
+ Context manager to easily change current directory
+ and then go back
+ """
+ def __init__(self, path):
+ self.path = path
+ self.cwd = ''
+
+ def __enter__(self):
+ self.cwd = os.getcwd()
+ os.chdir(self.path)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ os.chdir(self.cwd)
+
+
+class TarDist:
+ REPO_LIST_KEY = 'tardist-repo-list'
+ UPDATES_KEY = 'tardist-updates'
+
+ def __init__(self, conf):
+ self.conf = conf
+ self._redis = None
+ self._session = requests.Session()
+ self.logger = logging.getLogger(__file__)
+
+ @property
+ def redis(self) -> redis.Redis:
+ if not self._redis:
+ self._redis = redis.Redis()
+
+ return self._redis
+
+ def _make_gerrit_request(self, endpoint) -> dict:
+ """
+ Make a Gerrit API request
+
+ :param endpoint: API endpoint to hit: /projects/?...=...
+ :return:
+ """
+ url = 'https://gerrit.wikimedia.org/r' + endpoint
+ r = self._session.get(url)
+ if not r.ok:
+ self.logger.error('Gerrit request failed: %s: %s' % (endpoint,
r.status_code))
+ return {}
+ clean = r.text[4:]
+ return json.loads(clean)
+
+ def get_all_repos(self, cached=True) -> list:
+ data = False
+ if cached:
+ data = json.loads((self.redis.get(self.REPO_LIST_KEY) or
b'false').decode())
+ if not data:
+ data = sorted(list(self._get_all_repos()))
+ self.redis.set(self.REPO_LIST_KEY, json.dumps(data)) # Cache for an
hour
+ return data
+
+ def _get_all_repos(self) -> iter:
+ resp = self._make_gerrit_request('/projects/?p=mediawiki/')
+ for repo in resp:
+ if repo.startswith(('mediawiki/skins/', 'mediawiki/extensions/'))
and len(repo.split('/')) == 3:
+ yield repo[10:] # Trim "mediawiki/"
+
+ def shell_exec(self, args, **kwargs) -> str:
+ """
+ Shortcut wrapper to execute a shell command
+
+ >>> self.shell_exec(['ls', '-l'])
+ """
+ return subprocess.check_output(args, **kwargs).decode()
+
+ def push_update(self, repo, branch):
+ """
+ Queue an update for the repo on branch
+ :param repo: Full name of repo, "extensions/MassMessage"
+ :param branch: Branch name, "master". '*' means all branches
+ """
+ self.redis.sadd(self.UPDATES_KEY, repo + '/' + branch)
+
+ def get_update(self) -> dict:
+ """
+ Get a queued update to process
+
+ redis sets are not ordered, so a random entry
+ will be removed from the queue.
+ """
+ info = self.redis.spop(self.UPDATES_KEY)
+ if info:
+ split = info.decode().rsplit('/', 1)
+ return {'repo': split[0], 'branch': split[1]}
+ else:
+ return {}
+
+ def update_extension(self, repo, branch):
+ """
+ Fetch an extension's updates, and
+ create new tarballs if needed
+ """
+ print(repo, branch)
+ name = repo.split('/')[1] # Name without type prefix, "Vector"
+ full_path = os.path.join(self.conf['SRC_PATH'], repo)
+ self.logger.info('Starting update for %s' % repo)
+ if not os.path.exists(full_path):
+ with cwd(self.conf['SRC_PATH']):
+ self.logger.debug('Cloning %s' % repo)
+ self.shell_exec(['git', 'clone', self.conf['GIT_URL'] % repo,
repo])
+ os.chdir(full_path)
+ with cwd(full_path):
+ self.logger.info('Creating %s for %s' % (branch, repo))
+ # Update remotes
+ self.shell_exec(['git', 'fetch'])
+ try:
+ # Could fail if repo is empty
+ self.shell_exec(['git', 'reset', '--hard', 'origin/master'])
+ # Reset everything!
+ self.shell_exec(['git', 'clean', '-ffd'])
+ # Checkout the branch
+ self.shell_exec(['git', 'checkout', 'origin/%s' % branch])
+ except subprocess.CalledProcessError:
+ # Just a warning because this is expected for some extensions
+ self.logger.warning('could not checkout origin/%s' % branch)
+ return
+ # Reset everything, again.
+ self.shell_exec(['git', 'clean', '-ffd'])
+ # Sync submodules in case their urls have changed
+ self.shell_exec(['git', 'submodule', 'sync'])
+ # Update them, initializing new ones if needed
+ self.shell_exec(['git', 'submodule', 'update', '--init'])
+ # Gets short hash of HEAD
+ rev = self.shell_exec(['git', 'rev-parse', '--short',
'HEAD']).strip()
+ tarball_fname = '%s-%s-%s.tar.gz' % (name, branch, rev)
+ # Create a 'version' file with basic info about the tarball
+ with open('version', 'w') as f:
+ f.write('%s: %s\n' % (name, branch))
+ f.write(self.shell_exec(['date', '+%Y-%m-%dT%H:%M:%S']) +
'\n') # TODO: Do this in python
+ f.write(rev + '\n')
+ old_tarballs = glob.glob(os.path.join(self.conf['DIST_PATH'],
'%s-%s-*.tar.gz' % (name, branch)))
+ self.logger.debug('Deleting old tarballs...')
+ for old in old_tarballs:
+ # FIXME: Race condition, we should probably do this later on...
+ os.unlink(old)
+ with cwd(self.conf['SRC_PATH']):
+ # Finally, create the new tarball
+ with cwd(os.path.join(self.conf['SRC_PATH'], repo.rsplit('/',
1)[0])):
+ self.shell_exec(
+ ['tar', '--exclude', '.git', '-czPf',
os.path.join(self.conf['DIST_PATH'], tarball_fname), name]
+ )
+ self.logger.info('Finished update for %s' % repo)
+
+
+def _get_conf():
+ conf = False
+ for path in ['/etc/tardist.conf', os.path.join(os.path.dirname(__file__),
'tardist.json')]:
+ if os.path.exists(path):
+ with open(path) as f:
+ conf = json.load(f)
+
+ if conf is False:
+ print('tardist is not configured properly.')
+ quit()
+
+ return conf
+
+tardist = TarDist(_get_conf())
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..9a4162f
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,23 @@
+[tox]
+
+# Ensure 1.6+ is used to support 'skipsdist'
+minversion = 1.6
+
+# Do not run install command
+skipsdist = True
+
+# Environements to execute when invoking 'tox'
+envlist = flake8
+
+[testenv:flake8]
+commands = flake8
+deps = flake8
+base_python=python3.4
+
+[flake8]
+exclude = .tox
+max_line_length = 120
+
+[testenv:tests]
+commands = python nightly_test.py
+base_python=python3.4
diff --git a/worker.py b/worker.py
new file mode 100644
index 0000000..41f9962
--- /dev/null
+++ b/worker.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+import time
+
+from tardist import tardist
+
+
+def main(*args):
+ while True:
+ update = tardist.get_update()
+ if update:
+ tardist.update_extension(**update)
+ elif '--continuous' in args:
+ time.sleep(5)
+ else:
+ break
+
+
+if __name__ == '__main__':
+ main(*sys.argv)
--
To view, visit https://gerrit.wikimedia.org/r/189303
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I11e3ac7c3414009a6a4c9a4c97977597efd0e764
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/tardist
Gerrit-Branch: master
Gerrit-Owner: Legoktm <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
