jenkins-bot has submitted this change and it was merged. Change subject: Port wikibugs logging to forrestbot, flake8 fixes ......................................................................
Port wikibugs logging to forrestbot, flake8 fixes Take optional --logfile argument to specify log file location. Log files are private by default (might leak private information?) and log on DEBUG level, except for requests (INFO level) Also fix flake8 issues so jenkins stops complaining. Change-Id: Ieadf87faf001e2660d6788053a48d2b70b61114f --- M forrestbot.py M pop3bot.py A wblogging.py 3 files changed, 77 insertions(+), 10 deletions(-) Approvals: Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/forrestbot.py b/forrestbot.py index fabce93..64e6784 100644 --- a/forrestbot.py +++ b/forrestbot.py @@ -1,19 +1,28 @@ """ https://etherpad.wikimedia.org/p/T280 & https://phabricator.wikimedia.org/T280 """ -__author__ = 'Merlijn van Deen' +__author__ = 'Merlijn van Deen' # noqa import functools import itertools import requests import logging -logger = logging.getLogger('forrestbot') +from wblogging import LoggingSetupParser import gerrit_rest import phabricator as legophab import config from utils import wmf_number + +parser = LoggingSetupParser( + description="Process changesets and add release tags as required", +) +args = parser.parse_args() + +logging.getLogger('requests').setLevel(logging.INFO) +logger = logging.getLogger('forrestbot') + phab = legophab.Phabricator( config.PHAB_HOST, @@ -65,8 +74,8 @@ def get_repos_to_watch(): # This url will redirect like 3 times, but requests handles it nicely r = requests.get( - 'https://phabricator.wikimedia.org/diffusion/MREL/browse/master/' - + 'make-wmf-branch/config.json?view=raw' + 'https://phabricator.wikimedia.org/diffusion/MREL/browse/master/' + + 'make-wmf-branch/config.json?view=raw' ) conf = r.json() repos = ['mediawiki/core'] @@ -197,8 +206,8 @@ action = process_mail(mail) actions.append(action) logger.info( - ("{url}: merged in branch {branch}, Task {task}," - + " needs slugs {slugs}").format(**action) + ("{url}: merged in branch {branch}, Task {task}," + + " needs slugs {slugs}").format(**action) ) except SkipMailException as e: logger.debug("%s: skipping (%r)" % (mail['X-Gerrit-ChangeURL'], e)) @@ -207,7 +216,9 @@ # after parsing all entries, make sure we only do a single edit per Task. # When detecting the bug header, strip arbitrary whitespace so bad headers # like "Bug: T###" are OK - key = lambda x: int(x['task'].strip()[1:]) + def key(x): + return int(x['task'].strip()[1:]) + for task, acts in itertools.groupby(sorted(actions, key=key), key=key): acts = sorted(acts, key=lambda x: x['slugs']) @@ -227,8 +238,8 @@ if not task_info: # Security bug? T101133 logger.warning( - ('Unable to get information about T{task}, maybe it is' - + ' private?').format(task=task) + ('Unable to get information about T{task}, maybe it is' + + ' private?').format(task=task) ) continue old_projs = set(task_info['projectPHIDs']) diff --git a/pop3bot.py b/pop3bot.py index 0b3959c..25941d1 100644 --- a/pop3bot.py +++ b/pop3bot.py @@ -2,6 +2,8 @@ import email.parser import config import logging +import gerrit_rest + logger = logging.getLogger('pop3bot') @@ -68,7 +70,6 @@ if gerrit_data: yield gerrit_data -import gerrit_rest g = gerrit_rest.GerritREST('https://gerrit.wikimedia.org/r') diff --git a/wblogging.py b/wblogging.py new file mode 100644 index 0000000..b775632 --- /dev/null +++ b/wblogging.py @@ -0,0 +1,55 @@ +import os +import sys +import logging +import logging.handlers +import argparse + + +def private_open(file, flags, dir_fd=None): + return os.open(file, flags, mode=0o600, dir_fd=dir_fd) + + +class PrivateTimedRotatingFileHandler( + logging.handlers.TimedRotatingFileHandler): + def _open(self): + return open( + self.baseFilename, self.mode, + encoding=self.encoding, + opener=private_open + ) + + +class LoggingSetupParser(argparse.ArgumentParser): + def __init__(self, *args, **kwargs): + super(LoggingSetupParser, self).__init__(*args, **kwargs) + self.add_argument( + "--logfile", dest='logfile', default=None, + help="Log to this (rotated) log file; if not provided, " + "log to stdout", + ) + + def parse_args(self, *args, **kwargs): + result = super(LoggingSetupParser, self).parse_args(*args, **kwargs) + self.set_up_logging(result) + return result + + def set_up_logging(self, args): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + if args.logfile: + handler = PrivateTimedRotatingFileHandler( + args.logfile, + when='midnight', backupCount=7, + utc=True, + encoding='utf-8', + ) + print("Logging to %s" % args.logfile) + else: + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + handler.setFormatter(formatter) + logger.addHandler(handler) -- To view, visit https://gerrit.wikimedia.org/r/280668 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ieadf87faf001e2660d6788053a48d2b70b61114f Gerrit-PatchSet: 4 Gerrit-Project: labs/tools/forrestbot Gerrit-Branch: master Gerrit-Owner: Merlijn van Deen <[email protected]> Gerrit-Reviewer: Merlijn van Deen <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
