jenkins-bot has submitted this change and it was merged. Change subject: build: Test with flake8 too, and make pass ......................................................................
build: Test with flake8 too, and make pass Change-Id: I84b7f1a4d9fabc85268d998711fdde5ed849a0e9 --- A .gitignore M forrestbot.py M gerrit_rest.py M pop3bot.py M tox.ini M utils.py 6 files changed, 119 insertions(+), 44 deletions(-) Approvals: Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5f9f2c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.tox +__pycache__ +*.pyc diff --git a/forrestbot.py b/forrestbot.py index cfc79d8..2f0566e 100644 --- a/forrestbot.py +++ b/forrestbot.py @@ -1,5 +1,5 @@ """ -See https://etherpad.wikimedia.org/p/T280 / https://phabricator.wikimedia.org/T280 +https://etherpad.wikimedia.org/p/T280 & https://phabricator.wikimedia.org/T280 """ __author__ = 'Merlijn van Deen' @@ -16,10 +16,10 @@ from utils import wmf_number phab = legophab.Phabricator( - config.PHAB_HOST, - config.PHAB_USER, - config.PHAB_CERT - ) + config.PHAB_HOST, + config.PHAB_USER, + config.PHAB_CERT +) gerrit = gerrit_rest.GerritREST("https://gerrit.wikimedia.org/r";) @@ -39,8 +39,9 @@ wmf_parts = newest_wmf.split("wmf") next_wmf = wmf_parts[0] + "wmf" + str(int(wmf_parts[1]) + 1) if repository != 'mediawiki/core': - # Only do REL1_XX branches for MediaWiki core, since WMF-deployed extensions - # are not the same as those that are bundled inside the tarball. + # Only do REL1_XX branches for MediaWiki core, since WMF-deployed + # extensions are not the same as those that are bundled inside the + # tarball. return [next_wmf] def REL_key(REL): @@ -48,7 +49,9 @@ return int(major), int(minor) marker = 'refs/heads/REL' - newest_REL = sorted([b for b in projbranches if marker in b], key=REL_key)[-1] + newest_REL = sorted( + [b for b in projbranches if marker in b], key=REL_key + )[-1] newest_REL = newest_REL.split(marker)[1] REL_parts = newest_REL.split("_") @@ -56,10 +59,14 @@ return [next_REL, next_wmf] + @functools.lru_cache() def get_repos_to_watch(): # This url will redirect like 3 times, but requests handles it nicely - r = requests.get('https://phabricator.wikimedia.org/diffusion/MREL/browse/master/make-wmf-branch/config.json?view=raw') + r = requests.get( + 'https://phabricator.wikimedia.org/diffusion/MREL/browse/master/' + + 'make-wmf-branch/config.json?view=raw' + ) conf = r.json() repos = ['mediawiki/core'] for ext in conf['extensions']: @@ -69,13 +76,19 @@ # Intentionally ignore special_extensions because they're special return repos + @functools.lru_cache() def get_slug_PHID(slug): - rq = list(phab.request('project.query', {'slugs': [slug]})['slugMap'].values()) + rq = list( + phab.request('project.query', {'slugs': [slug]})['slugMap'].values() + ) if rq: - logging.debug("Slug {slug} = PHID {phid}".format(slug=slug, phid=rq[0])) + logging.debug( + "Slug {slug} = PHID {phid}".format(slug=slug, phid=rq[0]) + ) return rq[0] raise Exception("No PHID found for slug #%s!" % slug) + def get_slug(branch): """ Slugify the branch name. @@ -98,6 +111,7 @@ logging.debug('Unknown branch type %s, returning None' % branch) return None + def slugify(taskbranches): """ Slugify all branches @@ -108,8 +122,10 @@ slugs = [get_slug(b) for b in taskbranches] return [s for s in slugs if s] + class SkipMailException(Exception): pass + def process_mail(mail): proj = mail.get('Gerrit-Project', '') @@ -120,7 +136,11 @@ try: taskbranches = [mail['Gerrit-Branch']] - task = mail.get('Bug', '') or mail.get('Closes', '') or mail.get('Task', '') + task = ( + mail.get('Bug', '') or + mail.get('Closes', '') or + mail.get('Task', '') + ) if not task: raise KeyError('No Task ID (Bug, Closes or Task)') except KeyError as e: @@ -147,8 +167,11 @@ # query projects -if __name__=="__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s: %(levelname)-8s – %(message)s') +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s: %(levelname)-8s – %(message)s' + ) # logger.info("Current master branches are: %r" % (get_master_branches(),)) @@ -161,17 +184,21 @@ actions = [] - for i,mail in enumerate(pop3bot.gerritmail_generator(mailbox)): + for i, mail in enumerate(pop3bot.gerritmail_generator(mailbox)): try: action = process_mail(mail) actions.append(action) - logger.info("{url}: merged in branch {branch}, Task {task}, needs slugs {slugs}".format(**action)) + logger.info( + "{url}: merged in branch {branch}, Task {task}," + + " needs slugs {slugs}".format(**action) + ) except SkipMailException as e: logger.debug("%s: skipping (%r)" % (mail['X-Gerrit-ChangeURL'], e)) pass # after parsing all entries, make sure we only do a single edit per Task. - # When detecting the bug header, strip arbitrary whitespace so bad headers like "Bug: T###" are ok + # When detecting the bug header, strip arbitrary whitespace so bad headers + # like "Bug: T###" are OK key = lambda x: int(x['task'].strip()[1:]) for task, acts in itertools.groupby(sorted(actions, key=key), key=key): acts = sorted(acts, key=lambda x: x['slugs']) @@ -183,13 +210,18 @@ for slug in act['slugs']: add_PHIDs.add(get_slug_PHID(slug)) - logger.info("Adding PHID {PHIDs} to T{task}.".format(PHIDs=add_PHIDs, task=task)) + logger.info("Adding PHID {PHIDs} to T{task}.".format( + PHIDs=add_PHIDs, task=task + )) # now we get the task to know what the existing tags are task_info = phab.request('maniphest.info', {'task_id': str(task)}) if not task_info: # Security bug? T101133 - logger.warning('Unable to get information about T{task}, maybe it is private?'.format(task=task)) + logger.warning( + 'Unable to get information about T{task}, maybe it is' + + ' private?'.format(task=task) + ) continue old_projs = set(task_info['projectPHIDs']) logger.debug("Existing PHIDs: {old_projs}".format(old_projs=old_projs)) @@ -200,6 +232,8 @@ } ) else: - logging.info("Skipping T{task}; no new projects to add".format(task=task)) + logging.info( + "Skipping T{task}; no new projects to add".format(task=task) + ) mailbox.quit() diff --git a/gerrit_rest.py b/gerrit_rest.py index 0f77c61..f1a6143 100644 --- a/gerrit_rest.py +++ b/gerrit_rest.py @@ -1,6 +1,8 @@ -import json, requests +import json +import requests requests.adapters.DEFAULT_RETRIES = 5 + class GerritREST(object): def __init__(self, url): @@ -16,13 +18,16 @@ self._session.headers.update({'Accept': 'application/json'}) def _request(self, name, **kwargs): - """ Make a request. Parameters: - * name - The name of the REST endpoint. This will be appended to the base URL. + """ Make a request. + + Parameters: + * name - The name of the REST endpoint. This will be appended to + the base URL. * any parameters taken by the REST endpoint (via kwargs) """ - kwargs = {k:v for (k,v) in kwargs.items() if v is not None} + kwargs = {k: v for (k, v) in kwargs.items() if v is not None} r = self._session.get(self._url + '/%s/' % name, params=kwargs) - realjson = r.text[5:] # strips anti-XSS prefix + realjson = r.text[5:] # strips anti-XSS prefix return json.loads(realjson) def __getattr__(self, name): @@ -33,27 +38,40 @@ return wrapper def changes(self, q="", n=25, o=[]): - """ Submits a request to the /changes/ REST API. Parameters: + """ Submits a request to the /changes/ REST API. + + Parameters: * q - the query string, * n - the maximum number of results to return - 25 by default, - * o - the list of options to pass. CURRENT_REVISION and CURRENT_FILES by default. + * o - the list of options to pass. CURRENT_REVISION and + CURRENT_FILES by default. - See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html - for more details. """ + See https://gerrit-review.googlesource.com/Documentation/ + rest-api-changes.html for more details. """ return self._request('changes', q=q, n=n, o=o) def branches(self, project, n=None, s=None, m=None, r=None): - """ Submits a request to the /project/{name}/branches/ REST API. Parameters: - * n - the maximum number of results to return - 25 by default, - * s - Skip the given number of branches from the beginning of the list. - * m - Limit the results to those projects that match the specified substring. - * r - Limit the results to those branches that match the specified regex. + """ Submits a request to the /project/{name}/branches/ REST API. - See https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html - for more details. """ + Parameters: + * n - the maximum number of results to return - 25 by default. + * s - Skip the given number of branches from the beginning of the + list. + * m - Limit the results to those projects that match the specified + substring. + * r - Limit the results to those branches that match the specified + regex. - return self._request('projects/{project}/branches'.format(project=project), n=n, s=s, m=m, r=r) + See https://gerrit-review.googlesource.com/Documentation/ + rest-api-changes.html for more details. """ + return self._request( + 'projects/{project}/branches'.format(project=project), + n=n, + s=s, + m=m, + r=r + ) # def accounts, def groups, def projects, etc. diff --git a/pop3bot.py b/pop3bot.py index 866547a..0b3959c 100644 --- a/pop3bot.py +++ b/pop3bot.py @@ -4,11 +4,12 @@ import logging logger = logging.getLogger('pop3bot') + def mkmailbox(debug=0): username = config.username password = config.password - mailbox = poplib.POP3_SSL('pop.googlemail.com', '995') + mailbox = poplib.POP3_SSL('pop.googlemail.com', '995') mailbox.set_debuglevel(debug) mailbox.user(username) @@ -16,13 +17,17 @@ return mailbox + def mail_generator(mailbox): """ RETRieves the contents of mails, yields those and DELEtes them before the next mail is RETRieved """ nmails, octets = mailbox.stat() - for i in range(1,nmails+1): - yield "\n".join([x.decode('utf-8', 'replace') for x in mailbox.retr(i)[1]]) + for i in range(1, nmails+1): + yield "\n".join( + [x.decode('utf-8', 'replace') for x in mailbox.retr(i)[1]] + ) mailbox.dele(i) + def message_generator(mailbox): p = email.parser.Parser() @@ -34,18 +39,27 @@ continue yield mail, payload.decode('utf-8', 'replace') + def gerritmail_generator(mailbox): for message, contents in message_generator(mailbox): if '[email protected]' not in message.get('From', ''): - logger.debug('Skipping message from %s' % message.get('From', '???')) + logger.debug( + 'Skipping message from %s' % message.get('From', '???') + ) continue - gerrit_data = dict((k,v) for (k,v) in message.items() if k.startswith('X-Gerrit')) + gerrit_data = dict( + (k, v) for (k, v) in message.items() + if k.startswith('X-Gerrit') + ) gerrit_data.update(dict( line.split(": ", 1) for line in contents.split('\n') if ( - line.startswith("Gerrit-") or line.startswith('Bug:') or line.startswith('Task:') or line.startswith('Closes:') + line.startswith("Gerrit-") or + line.startswith('Bug:') or + line.startswith('Task:') or + line.startswith('Closes:') ) and ( ": " in line ) @@ -57,6 +71,7 @@ import gerrit_rest g = gerrit_rest.GerritREST('https://gerrit.wikimedia.org/r') + def get_changeset(changeid, o=['CURRENT_REVISION', 'CURRENT_FILES']): matchingchanges = g.changes(changeid, n=1, o=o) if matchingchanges: diff --git a/tox.ini b/tox.ini index 7699cae..7aa5367 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,11 @@ [tox] +minversion = 1.6 skipsdist = True +envlist = flake8,py27,py35 [testenv] commands = python -m doctest utils.py + +[testenv:flake8] +deps = flake8 +commands = flake8 diff --git a/utils.py b/utils.py index cf455e6..ceaeca0 100644 --- a/utils.py +++ b/utils.py @@ -36,4 +36,3 @@ minor = '0' + minor return int(major + minor) - -- To view, visit https://gerrit.wikimedia.org/r/239446 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I84b7f1a4d9fabc85268d998711fdde5ed849a0e9 Gerrit-PatchSet: 1 Gerrit-Project: labs/tools/forrestbot Gerrit-Branch: master Gerrit-Owner: Jforrester <[email protected]> Gerrit-Reviewer: Merlijn van Deen <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
