John Vandenberg has uploaded a new change for review. https://gerrit.wikimedia.org/r/246213
Change subject: [WIP] EverySiteBot ...................................................................... [WIP] EverySiteBot Run a bot on every site Used to create: https://meta.wikimedia.org/wiki/Talk:Interwiki_sorting_order#2015_usage Fails on anarchopedia (v1.14) and wikimediachapter:uk (certificate) Python 3 only atm. Several more re-ordering of imports needed before this can be merged. e.g. $ python3 pwb.py listpages -family:wikimediachapter -lang:all -get -save -page:'MediaWiki:Interwiki_config-sorting_order' Change-Id: I5d5a83e99bcf56a6759ac317831dc97c5f4a12ae --- M pywikibot/__init__.py M pywikibot/bot.py M pywikibot/config2.py M pywikibot/pagegenerators.py M scripts/listpages.py 5 files changed, 237 insertions(+), 50 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core refs/changes/13/246213/1 diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index ba3aae3..e9c35fb 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -32,6 +32,9 @@ critical, debug, error, exception, log, output, stdout, warning ) +# Error must exist, due to bot importing pagegenerators which imports i18n which needs Error +from pywikibot.exceptions import Error + from pywikibot import config2 as config from pywikibot.bot import ( input, input_choice, input_yn, inputChoice, handle_args, showHelp, ui, @@ -45,7 +48,7 @@ from pywikibot.data.api import UploadWarning as _UploadWarning from pywikibot.diff import PatchManager from pywikibot.exceptions import ( - Error, InvalidTitle, BadTitle, NoPage, NoMoveTarget, SectionError, + InvalidTitle, BadTitle, NoPage, NoMoveTarget, SectionError, SiteDefinitionError, NoSuchSite, UnknownSite, UnknownFamily, UnknownExtension, NoUsername, UserBlocked, diff --git a/pywikibot/bot.py b/pywikibot/bot.py index 80905bf..83a6057 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -63,7 +63,9 @@ # Note: all output goes thru python std library "logging" module import codecs +import collections import datetime +import itertools import json import logging import logging.handlers @@ -79,6 +81,24 @@ _logger = "bot" import pywikibot + +# comms.http needs this. +# Command line parsing and help +def calledModuleName(): + """Return the name of the module calling this function. + + This is required because the -help option loads the module's docstring + and because the module name will be used for the filename of the log. + + @rtype: unicode + """ + # get commandline arguments + called = pywikibot.argvu[0].strip() + if ".py" in called: # could end with .pyc, .pyw, etc. on some platforms + # clip off the '.py?' filename extension + called = called[:called.rindex('.py')] + return os.path.basename(called) + from pywikibot import backports from pywikibot import config @@ -96,12 +116,18 @@ debug, error, exception, log, output, stdout, warning, ) from pywikibot.logging import critical # noqa: unused -from pywikibot.tools import deprecated, deprecated_args, PY2, PYTHON_VERSION +from pywikibot.tools import ( + deprecated, deprecated_args, PY2, PYTHON_VERSION, + StringTypes, +) from pywikibot.tools._logging import ( LoggingFormatter as _LoggingFormatter, RotatingFileHandler, ) from pywikibot.tools.formatter import color_format + +# Loaded after others due to cyclic imports +from pywikibot.pagegenerators import GeneratorFactory if not PY2: unicode = str @@ -783,23 +809,6 @@ if self._current_match is None: raise ValueError('No current range') return self._current_match[3] - - -# Command line parsing and help -def calledModuleName(): - """Return the name of the module calling this function. - - This is required because the -help option loads the module's docstring - and because the module name will be used for the filename of the log. - - @rtype: unicode - """ - # get commandline arguments - called = pywikibot.argvu[0].strip() - if ".py" in called: # could end with .pyc, .pyw, etc. on some platforms - # clip off the '.py?' filename extension - called = called[:called.rindex('.py')] - return os.path.basename(called) def handle_args(args=None, do_help=True): @@ -1604,6 +1613,82 @@ self._site = page.site +class EverySiteBot(MultipleSitesBot): + + """A bot class working on every known site.""" + + def __init__(self, families=None, codes=None, **kwargs): + """Constructor.""" + if families is None or families == 'all': + self.families = sorted(config.family_files.keys()) + elif isinstance(families, StringTypes): + self.families = [families] + else: + assert isinstance(families, collections.Iterable) + self.families = families + + if codes is None or codes == 'all': + self.codes = None + elif isinstance(codes, basestring): + self.codes = [codes] + else: + assert isinstance(codes, collections.Iterable) + self.codes = codes + + super(EverySiteBot, self).__init__(**kwargs) + + def site_generator(self): + """Generate requested sites.""" + for family_name in self.families: + fam = pywikibot.family.Family.load(family_name) + + if self.codes: + codes = [code for code in self.codes if code in fam.langs] + else: + codes = sorted(fam.langs) + + if fam == 'wikimediachapter': + codes = [code for code in codes if code != 'uk'] + + pywikibot.warning('Processing family: %s ; codes: %s' + % (fam, codes)) + + for code in codes: + site = pywikibot.Site(code, fam) + pywikibot.warning('Processing site: %s' % site) + yield site + + def run(self): + """Process all sites in generator.""" + super(EverySiteBot, self).run() + + +class EverySitePageGeneratorBot(EverySiteBot): + + """A bot class working on every known site.""" + + def __init__(self, generator_args, families=None, codes=None, **kwargs): + """Constructor.""" + super(EverySitePageGeneratorBot, self).__init__(families, codes, **kwargs) + self.generator_args = generator_args + + @property + def generators(self): + for site in self.site_generator(): + generator_factory = GeneratorFactory(site) + gen = generator_factory.generator(self.generator_args) + assert gen + yield gen + + @property + def generator(self): + return itertools.chain.from_iterable(self.generators) + + def run(self): + """Process all sites in generator.""" + super(EverySitePageGeneratorBot, self).run() + + class CurrentPageBot(BaseBot): """ @@ -1655,6 +1740,19 @@ **kwargs) +class QuietCurrentPageBot(CurrentPageBot): + + def treat(self, page): + """Set page to current page and treat that page.""" + self._current_page = page + self.treat_page() + + @deprecated_args(comment='summary') + def userPut(self, page, *args, **kwargs): + self._current_page = page + return super(QuietCurrentPageBot, self).userPut(page, *args, **kwargs) + + class AutomaticTWSummaryBot(CurrentPageBot): """ diff --git a/pywikibot/config2.py b/pywikibot/config2.py index 4cac315..0abe960 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -357,7 +357,7 @@ def register_families_folder(folder_path): """Register all family class files contained in a directory.""" for file_name in os.listdir(folder_path): - if file_name.endswith("_family.py"): + if file_name.endswith("_family.py") and not file_name.startswith('anarchopedia'): family_name = file_name[:-len("_family.py")] register_family_file(family_name, os.path.join(folder_path, file_name)) diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index ecffb9a..0230713 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -331,6 +331,11 @@ @param site: Site for generator results. @type site: L{pywikibot.site.BaseSite} """ + self._site = site + self._reset() + + def _reset(self): + """Reset factory.""" self.gens = [] self._namespaces = [] self.step = None @@ -339,7 +344,6 @@ self.titlefilter_list = [] self.claimfilter_list = [] self.intersect = False - self._site = site @property def site(self): @@ -835,6 +839,25 @@ else: return False + def generator(self, args): + """ + Return a generator for only the provided arguments. + + @param args: a list of arguments supported by the factory + @type args: list of str + @return: combined generator + @rtype: generator or None + """ + self._reset() + for arg in args: + rv = self.handleArg(arg) + if not rv: + pywikibot.warning('Unknown argument \'%s\'' % arg) + + gen = self.getCombinedGenerator() + self._reset() + return gen + def AllpagesPageGenerator(start='!', namespace=0, includeredirects=True, site=None, step=None, total=None, content=False): diff --git a/scripts/listpages.py b/scripts/listpages.py index d2dc095..f343d47 100755 --- a/scripts/listpages.py +++ b/scripts/listpages.py @@ -74,7 +74,7 @@ ¶ms; """ # -# (C) Pywikibot team, 2008-2014 +# (C) Pywikibot team, 2008-2015 # # Distributed under the terms of the MIT license. # @@ -86,8 +86,15 @@ import os import pywikibot + from pywikibot import config2 as config -from pywikibot.pagegenerators import GeneratorFactory, parameterHelp +from pywikibot.bot import ( + EverySitePageGeneratorBot, + QuietCurrentPageBot, + ExistingPageBot +) + +from pywikibot.pagegenerators import parameterHelp docuReplacements = {'¶ms;': parameterHelp} @@ -156,6 +163,70 @@ return fmt.format(num=num, page=self) +class ListPageTitlesBot(EverySitePageGeneratorBot, QuietCurrentPageBot): + + def __init__(self, fmt='1', lang=None, *args, **kwargs): + """Constructor.""" + super(ListPageTitlesBot, self).__init__(*args, **kwargs) + self.fmt = fmt + self.lang = lang + + def treat_page(self): + """Print page.""" + page = self.current_page + fmt = self.fmt + i = self._treat_counter + outputlang = self.lang + + page_fmt = Formatter(page, outputlang) + pywikibot.stdout(page_fmt.output(num=i, fmt=fmt)) + + +class SavePagesBot(ExistingPageBot): + + def __init__(self, base_dir=None, encoding=None, *args, **kwargs): + """Constructor.""" + super(SavePagesBot, self).__init__(*args, **kwargs) + self.base_dir = base_dir + self.encoding = encoding or config.textfile_encoding + + def treat_page(self): + """Save page.""" + base_dir = self.base_dir + encoding = self.encoding + + page = self.current_page + + family_name = page.site.family.name + code = page.site.code + + site_dir = os.path.join(base_dir, family_name, code) + if not os.path.exists(site_dir): + os.makedirs(site_dir, mode=0o744) + + filename = os.path.join(site_dir, page.title(as_filename=True)) + pywikibot.output(u'Saving %s to %s' % (page.title(), filename)) + with open(filename, mode='wb') as f: + f.write(page.text.encode(encoding)) + + +class ListPagesBot(ListPageTitlesBot, SavePagesBot): + + def __init__(self, show_title=True, *args, **kwargs): + """Constructor.""" + super(ListPagesBot, self).__init__(*args, **kwargs) + self.show_title = show_title + + def treat_page(self): + if self.show_title: + ListPageTitlesBot.treat_page(self) + + pywikibot.output(self.current_page.text, toStdout=True) + + if self.base_dir: + SavePagesBot.treat_page(self) + + def main(*args): """ Process command line arguments and invoke bot. @@ -165,17 +236,18 @@ @param args: command line arguments @type args: list of unicode """ - gen = None + # gen = None notitle = False fmt = '1' - outputlang = None + # outputlang = None page_get = False base_dir = None encoding = config.textfile_encoding # Process global args and prepare generator args parser local_args = pywikibot.handle_args(args) - genFactory = GeneratorFactory() + + generator_args = [] for arg in local_args: if arg == '-notitle': @@ -183,8 +255,8 @@ elif arg.startswith('-format:'): fmt = arg[len('-format:'):] fmt = fmt.replace(u'\\03{{', u'\03{{') - elif arg.startswith('-outputlang:'): - outputlang = arg[len('-outputlang:'):] + # elif arg.startswith('-outputlang:'): + # outputlang = arg[len('-outputlang:'):] elif arg == '-get': page_get = True elif arg.startswith('-save'): @@ -192,7 +264,7 @@ elif arg.startswith('-encode:'): encoding = arg.partition(':')[2] else: - genFactory.handleArg(arg) + generator_args.append(arg) if base_dir: base_dir = os.path.expanduser(base_dir) @@ -214,28 +286,19 @@ % base_dir) base_dir = None - gen = genFactory.getCombinedGenerator() - if gen: - i = 0 - for i, page in enumerate(gen, start=1): - if not notitle: - page_fmt = Formatter(page, outputlang) - pywikibot.stdout(page_fmt.output(num=i, fmt=fmt)) - if page_get: - try: - pywikibot.output(page.text, toStdout=True) - except pywikibot.Error as err: - pywikibot.output(err) - if base_dir: - filename = os.path.join(base_dir, page.title(as_filename=True)) - pywikibot.output(u'Saving %s to %s' % (page.title(), filename)) - with open(filename, mode='wb') as f: - f.write(page.text.encode(encoding)) - pywikibot.output(u"%i page(s) found" % i) - return True + if generator_args and (not notitle or page_get or base_dir): + if page_get or base_dir: + bot = ListPagesBot(fmt=fmt, show_title=not notitle, + base_dir=base_dir, encoding=encoding, + families='all', codes='all', + generator_args=generator_args) + else: + bot = ListPageTitlesBot(fmt=fmt, + families='all', codes='all', + generator_args=generator_args) + bot.run() else: - pywikibot.bot.suggest_help(missing_generator=True) - return False + pywikibot.showHelp() if __name__ == "__main__": -- To view, visit https://gerrit.wikimedia.org/r/246213 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5d5a83e99bcf56a6759ac317831dc97c5f4a12ae Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
