John Vandenberg has uploaded a new change for review. https://gerrit.wikimedia.org/r/221637
Change subject: [WIP] Support custom families in pywikibot.families ...................................................................... [WIP] Support custom families in pywikibot.families Bug: T104130 Change-Id: I6a9786b91772f079c8aa3f7dbb34152dbaf41a86 --- R families/pywikibot/families/wikia/lyricwiki_family.py A families/setup_wikia.py M pywikibot/__init__.py M pywikibot/bot.py A pywikibot/config.py M pywikibot/config2.py D pywikibot/families/__init__.py M pywikibot/family.py A pywikibot/tools/py2.py M setup.py 10 files changed, 246 insertions(+), 77 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core refs/changes/37/221637/1 diff --git a/pywikibot/families/lyricwiki_family.py b/families/pywikibot/families/wikia/lyricwiki_family.py similarity index 100% rename from pywikibot/families/lyricwiki_family.py rename to families/pywikibot/families/wikia/lyricwiki_family.py diff --git a/families/setup_wikia.py b/families/setup_wikia.py new file mode 100644 index 0000000..b30b790 --- /dev/null +++ b/families/setup_wikia.py @@ -0,0 +1,25 @@ +from setuptools import setup + +setup( + name='PywikibotWikiaFamily', + version='0.1', + description='Wikia configuration for Pywikibot', + long_description='Wikia configuration for Pywikibot', + maintainer='The Pywikibot team', + maintainer_email='[email protected]', + license='MIT License', + packages=['pywikibot', 'pywikibot.families', 'pywikibot.families.wikia'], + install_requires='pywikibot', + url='https://www.mediawiki.org/wiki/Pywikibot', + classifiers=[ + 'License :: OSI Approved :: MIT License', + 'Development Status :: 4 - Beta', + 'Operating System :: OS Independent', + 'Intended Audience :: Developers', + 'Environment :: Console', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + ], + use_2to3=False, + zip_safe=False +) diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index b1d2cc3..22842d6 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -25,6 +25,10 @@ from warnings import warn +if __path__: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) + # Use pywikibot. prefix for all in-package imports; this is to prevent # confusion with similarly-named modules in version 1 framework, for users # who want to continue using both @@ -97,12 +101,12 @@ # ) + textlib_methods # pep257 also doesn't support __all__ multiple times in a document # so instead use this trick -globals()['__all__'] = globals()['__all__'] + textlib_methods +#globals()['__all__'] = globals()['__all__'] + textlib_methods for _name in textlib_methods: target = getattr(textlib, _name) wrapped_func = redirect_func(target) - globals()[_name] = wrapped_func + #globals()[_name] = wrapped_func deprecated = redirect_func(pywikibot.tools.deprecated) @@ -165,8 +169,8 @@ """ return self.strftime(self.ISO8601Format) - toISOformat = redirect_func(isoformat, old_name='toISOformat', - class_name='Timestamp') + #toISOformat = redirect_func(isoformat, old_name='toISOformat', + # class_name='Timestamp') def totimestampformat(self): """Convert object to a MediaWiki internal timestamp.""" @@ -624,10 +628,10 @@ # alias for backwards-compability -getSite = pywikibot.tools.redirect_func(Site, old_name='getSite') +#getSite = pywikibot.tools.redirect_func(Site, old_name='getSite') -from .page import ( +from pywikibot.page import ( Page, FilePage, Category, @@ -637,13 +641,13 @@ PropertyPage, Claim, ) -from .page import html2unicode, url2unicode, unicode2html +from pywikibot.page import html2unicode, url2unicode, unicode2html link_regex = re.compile(r'\[\[(?P<title>[^\]|[<>{}]*)(\|.*?)?\]\]') [email protected]("comment parameter for page saving method") +#@pywikibot.tools.deprecated("comment parameter for page saving method") def setAction(s): """Set a summary to use for changed page submissions.""" config.default_edit_summary = s @@ -748,17 +752,17 @@ _putthread.setName('Put-Thread') _putthread.setDaemon(True) -wrapper = pywikibot.tools.ModuleDeprecationWrapper(__name__) -wrapper._add_deprecated_attr('ImagePage', FilePage) -wrapper._add_deprecated_attr( - 'PageNotFound', pywikibot.exceptions.DeprecatedPageNotFoundError, - warning_message=('{0}.{1} is deprecated, and no longer ' - 'used by pywikibot; use http.fetch() instead.')) -wrapper._add_deprecated_attr( - 'UserActionRefuse', pywikibot.exceptions._EmailUserError, - warning_message='UserActionRefuse is deprecated; ' - 'use UserRightsError and/or NotEmailableError') -wrapper._add_deprecated_attr( - 'QuitKeyboardInterrupt', pywikibot.bot.QuitKeyboardInterrupt, - warning_message='pywikibot.QuitKeyboardInterrupt is deprecated; ' - 'use pywikibot.bot.QuitKeyboardInterrupt instead') +#wrapper = pywikibot.tools.ModuleDeprecationWrapper(__name__) +#wrapper._add_deprecated_attr('ImagePage', FilePage) +#wrapper._add_deprecated_attr( +# 'PageNotFound', pywikibot.exceptions.DeprecatedPageNotFoundError, +# warning_message=('{0}.{1} is deprecated, and no longer ' +# 'used by pywikibot; use http.fetch() instead.')) +#wrapper._add_deprecated_attr( +# 'UserActionRefuse', pywikibot.exceptions._EmailUserError, +# warning_message='UserActionRefuse is deprecated; ' +# 'use UserRightsError and/or NotEmailableError') +#wrapper._add_deprecated_attr( +# 'QuitKeyboardInterrupt', pywikibot.bot.QuitKeyboardInterrupt, +# warning_message='pywikibot.QuitKeyboardInterrupt is deprecated; ' +# 'use pywikibot.bot.QuitKeyboardInterrupt instead') diff --git a/pywikibot/bot.py b/pywikibot/bot.py index 722670e..f09c1b4 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -40,7 +40,7 @@ import pywikibot from pywikibot import backports -from pywikibot import config +from pywikibot import config2 as config from pywikibot import daemonize from pywikibot import version from pywikibot.bot_choice import ( # noqa: unused imports diff --git a/pywikibot/config.py b/pywikibot/config.py new file mode 120000 index 0000000..442fab7 --- /dev/null +++ b/pywikibot/config.py @@ -0,0 +1 @@ +config2.py \ No newline at end of file diff --git a/pywikibot/config2.py b/pywikibot/config2.py index 0acc717..7ec96e2 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -320,27 +320,7 @@ break family_files = {} - -def register_family_file(family_name, file_path): - """Register a single family class file.""" - usernames[family_name] = {} - sysopnames[family_name] = {} - disambiguation_comment[family_name] = {} - family_files[family_name] = file_path - - -def register_families_folder(folder_path): - """Register all family class files contained in a directory.""" - for file_name in os.listdir(folder_path): - if file_name.endswith("_family.py"): - family_name = file_name[:-len("_family.py")] - register_family_file(family_name, os.path.join(folder_path, file_name)) - - -# Get the names of all known families, and initialize with empty dictionaries. -# ‘families/’ is a subdirectory of the directory in which config2.py is found. -register_families_folder(os.path.join(os.path.dirname(__file__), 'families')) -register_family_file('wikiapiary', 'https://wikiapiary.com') +family_files['wikiapiary'] = 'https://wikiapiary.com' # Set to True to override the {{bots}} exclusion protocol (at your own risk!) ignore_bot_templates = False diff --git a/pywikibot/families/__init__.py b/pywikibot/families/__init__.py deleted file mode 100644 index ccb2305..0000000 --- a/pywikibot/families/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -"""Families package.""" -# -# (C) Pywikibot team, 2007 -# -# Distributed under the terms of the MIT license. -# -from __future__ import unicode_literals - -__version__ = '$Id$' diff --git a/pywikibot/family.py b/pywikibot/family.py index a4634fc..9a44b4c 100644 --- a/pywikibot/family.py +++ b/pywikibot/family.py @@ -10,12 +10,15 @@ __version__ = '$Id$' # -import sys -import logging -import re import collections import imp +import importlib +import logging +import os +import pkgutil +import re import string +import sys import warnings if sys.version_info[0] > 2: @@ -34,11 +37,102 @@ ) from pywikibot.exceptions import UnknownFamily, FamilyMaintenanceWarning +try: + import pywikibot_families +except ImportError: + pywikibot_families = None + logger = logging.getLogger("pywiki.wiki.family") # Legal characters for Family.name and Family.langs keys NAME_CHARACTERS = string.ascii_letters + string.digits CODE_CHARACTERS = string.ascii_lowercase + string.digits + '-' + +_internal_family_files = None + + +def get_families_files(folder_path=None): + """Register all family class files contained in a directory.""" + if folder_path is None: + global _internal_family_files + if _internal_family_files is not None: + return _internal_family_files + folder_path = os.path.join(os.path.dirname(__file__), 'families') + _internal_family_files = get_families_files(folder_path) + return _internal_family_files + + family_files = {} + if not os.path.exists(folder_path): + return family_files + + for file_name in os.listdir(folder_path): + path = os.path.join(folder_path, file_name) + if file_name.endswith("_family.py"): + family_name = file_name[:-len("_family.py")] + family_files[family_name] = path + elif os.path.isdir(path) and '__' not in file_name: + family_files.update(get_families_files(path)) + + return family_files + + +def package_subpackages(package): + """Get list of subpackages.""" + return [modname for importer, modname, ispkg + in pkgutil.walk_packages(path=package.__path__)] + + +def package_submodules(package): + """Get list of all submodules without loading the leaf modules.""" + modules = [] + prefix = package.__name__ + '.' + for importer, modname, ispkg in pkgutil.walk_packages( + path=package.__path__, + prefix=prefix, + onerror=lambda x: None): + modules.append(modname[len(prefix):]) + return modules + + +def _add_family_names(package, names, ignore_duplicates=False): + """Add names.""" + subpackages = package_subpackages(package) + modules = package_submodules(package) + for mod in modules: + base, dummy, rest = mod.partition('.') + assert base in subpackages + if not dummy: + continue + + assert '.' not in rest + if rest.endswith('_family'): + rest = rest[:-7] + if not ignore_duplicates: + assert rest not in names + + names.add(rest) + + +def family_names(): + """Obtain list of all families.""" + names = set() + + if pywikibot_families: + _add_family_names(pywikibot_families, names) + try: + import pywikibot.families + except ImportError: + pass + else: + _add_family_names(pywikibot.families, names, True) + + # Get the names of all known families by inspecting ‘families/’ which + # is a subdirectory of the directory in which family.py is found. + internal = get_families_files() + + names.update(internal.keys()) + + return names class Family(object): @@ -896,29 +990,47 @@ if fam in Family._families: return Family._families[fam] - if fam in config.family_files: - family_file = config.family_files[fam] - + if fam in pywikibot.config.family_files: + family_file = pywikibot.config.family_files[fam] if family_file.startswith('http://') or family_file.startswith('https://'): myfamily = AutoFamily(fam, family_file) Family._families[fam] = myfamily return Family._families[fam] - elif fam == 'lockwiki': - raise UnknownFamily( - "Family 'lockwiki' has been removed as it not a public wiki.\n" - "You may install your own family file for this wiki, and a " - "old family file may be found at:\n" - "http://git.wikimedia.org/commitdiff/pywikibot%2Fcore.git/dfdc0c9150fa8e09829bb9d236";) + raise UnknownFamily( + 'Family %s in pywikibot.config.family_files is not supported. ' + 'Create your own family package following instructions at ' + 'https://www.mediawiki.org/wiki/Manual:Pywikibot/custom_family' + % fam) + + if fam not in family_names(): + raise UnknownFamily( + 'Family name %s is not known. ' + 'Create your own family package following instructions at ' + 'https://www.mediawiki.org/wiki/Manual:Pywikibot/custom_family' + % fam) + + mod = None try: - # Ignore warnings due to dots in family names. - # TODO: use more specific filter, so that family classes can use - # RuntimeWarning's while loading. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", RuntimeWarning) - mod = imp.load_source(fam, config.family_files[fam]) - except (ImportError, KeyError): - raise UnknownFamily(u'Family %s does not exist' % fam) + mod = importlib.import_module('pywikibot.families.%s' % fam) + except ImportError: + try: + mod = importlib.import_module('pywikibot.families.%s_family' % fam) + except ImportError: + pass + + if not mod: + internal = get_families_files() + if fam not in internal: + raise UnknownFamily(u'Family %s does not exist' % fam) + try: + mod = imp.load_source(fam, internal[fam]) + except ImportError as e: + raise UnknownFamily('Family %s (%s) could not be loaded: %s' + % (fam, internal[fam], e)) + + assert mod + cls = mod.Family() if cls.name != fam: warn(u'Family name %s does not match family module name %s' diff --git a/pywikibot/tools/py2.py b/pywikibot/tools/py2.py new file mode 100644 index 0000000..f8fbe74 --- /dev/null +++ b/pywikibot/tools/py2.py @@ -0,0 +1,55 @@ +from __future__ import unicode_literals, absolute_import + +import imp +import os + +from pywikibot.tools import redirect_func + + +class ModuleContextDict(dict): + + """Execution context that updates a module.""" + + def __init__(self, module): + self.__module = module + + def __setitem__(self, item, value): + setattr(self.__module, item, value) + + def __getitem__(self, item): + if item == 'getattr': + return getattr + if item == 'object': + return object + if item == 'False': + return False + if item == 'True': + return True + + return getattr(self.__module, item) + + +def get_real_pywikibot(paths): + """Get the real path for pywikibot.""" + real_pywikibot_dirs = [p for p in paths if 'pywikibot-' in p] + for path in real_pywikibot_dirs: + if os.path.isdir(path): + if os.path.exists(os.path.join(path, 'backports.py')): + return path + + +def fix_base_import(module, paths, name): + """Update pywikibot module with real pywikibot module.""" + pywikibot_dir = get_real_pywikibot(paths) + assert pywikibot_dir + + pywikibot_init = pywikibot_dir + '/__init__.py' + + module.__dict__['redirect_func'] = redirect_func + + context = ModuleContextDict(module) + + with open(pywikibot_init, 'rb') as f: + exec(compile(f.read(), pywikibot_init, 'exec'), context) + + module.__file__ = pywikibot_init diff --git a/setup.py b/setup.py index 972c985..09d3b31 100644 --- a/setup.py +++ b/setup.py @@ -141,6 +141,10 @@ version = '2.0rc1.post2' github_url = 'https://github.com/wikimedia/pywikibot-core' +packages = ['pywikibot', 'pywikibot.families'] +packages.extend(package for package in find_packages() + if package.startswith('pywikibot.')) + setup( name=name, version=version, @@ -149,9 +153,7 @@ maintainer='The Pywikibot team', maintainer_email='[email protected]', license='MIT License', - packages=['pywikibot'] + [package - for package in find_packages() - if package.startswith('pywikibot.')], + packages=packages, install_requires=dependencies, dependency_links=dependency_links, extras_require=extra_deps, -- To view, visit https://gerrit.wikimedia.org/r/221637 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6a9786b91772f079c8aa3f7dbb34152dbaf41a86 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
