Hello community, here is the log from the commit of package python-wptools for openSUSE:Factory checked in at 2018-05-29 10:47:49 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-wptools (Old) and /work/SRC/openSUSE:Factory/.python-wptools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-wptools" Tue May 29 10:47:49 2018 rev:7 rq:612424 version:0.4.14 Changes: -------- --- /work/SRC/openSUSE:Factory/python-wptools/python-wptools.changes 2018-05-01 23:28:41.193489374 +0200 +++ /work/SRC/openSUSE:Factory/.python-wptools.new/python-wptools.changes 2018-05-29 10:47:50.242033307 +0200 @@ -1,0 +2,14 @@ +Fri May 25 16:34:44 UTC 2018 - [email protected] + +- Update to version 0.4.14: + + Moved expensive backlinks request to page.get_more() and other + minor fixes. +- Changes from version 0.4.13: + + Support getting backlinks with continuations, and more general + continuation support in wptools.core +- Minor rebase of python-wptools-avoid-reading-readme.patch. +- Drop %%check entirely, tests require network. +- Fix up sed command to remove hashbang: check for a valid + hashbang on the first line of file only and delete a match. + +------------------------------------------------------------------- Old: ---- wptools-0.4.12.tar.gz New: ---- wptools-0.4.14.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-wptools.spec ++++++ --- /var/tmp/diff_new_pack.IQammh/_old 2018-05-29 10:47:51.345992576 +0200 +++ /var/tmp/diff_new_pack.IQammh/_new 2018-05-29 10:47:51.345992576 +0200 @@ -19,7 +19,7 @@ %define pyname wptools %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-wptools -Version: 0.4.12 +Version: 0.4.14 Release: 0 Summary: Wikipedia tools (for Humans) License: MIT @@ -32,7 +32,6 @@ BuildRequires: %{python_module setuptools} BuildRequires: fdupes BuildRequires: python-rpm-macros -BuildRequires: python3-unittest2 Requires: python-certifi >= 2017.7.27.1 Requires: python-html2text >= 2016.9.19 Requires: python-lxml >= 3.8.0 @@ -54,7 +53,7 @@ %prep %setup -q -n wptools-%{version} %patch0 -p1 -sed -i '1d' scripts/wptool.py +sed -E -i "1{/^#!\/usr\/bin.*python/d}" scripts/wptool.py %build %python_build @@ -63,12 +62,6 @@ %python_install %python_expand %fdupes -s %{buildroot}%{$python_sitelib} -%check -# TESTS ONLY WORK WITH PY3 UNITTEST -%ifpython3 -%python_exec setup.py test -%endif - %files %{python_files} %doc HISTORY.rst LICENSE README.rst %python3_only %{_bindir}/wptool ++++++ python-wptools-avoid-reading-readme.patch ++++++ --- /var/tmp/diff_new_pack.IQammh/_old 2018-05-29 10:47:51.365991837 +0200 +++ /var/tmp/diff_new_pack.IQammh/_new 2018-05-29 10:47:51.369991690 +0200 @@ -13,7 +13,7 @@ @@ -12,7 +10,16 @@ setup( name='wptools', - version='0.4.12', + version='0.4.14', description='Wikipedia tools (for Humans)', - long_description=readme + '\n\n' + history, + long_description= ++++++ wptools-0.4.12.tar.gz -> wptools-0.4.14.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/HISTORY.rst new/wptools-0.4.14/HISTORY.rst --- old/wptools-0.4.12/HISTORY.rst 2018-04-30 00:33:58.000000000 +0200 +++ new/wptools-0.4.14/HISTORY.rst 2018-05-23 05:56:54.000000000 +0200 @@ -3,9 +3,11 @@ Release History --------------- -0.4.12 (2018-04-29) +0.4.14 (2018-05-22) +++++++++++++++++++ +* Put API warnings in data attribute (#120) +* Support getting backlinks w/continuations (#119, #122) * Always respect silent flag (#118) * Improved infobox parsing (#91, #109) * Added image (license, assessment) metadata (#104, #105) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/PKG-INFO new/wptools-0.4.14/PKG-INFO --- old/wptools-0.4.12/PKG-INFO 2018-04-30 00:38:59.000000000 +0200 +++ new/wptools-0.4.14/PKG-INFO 2018-05-23 06:11:08.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: wptools -Version: 0.4.12 +Version: 0.4.14 Summary: Wikipedia tools (for Humans) Home-page: https://github.com/siznax/wptools/ Author: Steve @siznax @@ -295,9 +295,11 @@ Release History --------------- - 0.4.12 (2018-04-29) + 0.4.14 (2018-05-22) +++++++++++++++++++ + * Put API warnings in data attribute (#120) + * Support getting backlinks w/continuations (#119, #122) * Always respect silent flag (#118) * Improved infobox parsing (#91, #109) * Added image (license, assessment) metadata (#104, #105) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/setup.py new/wptools-0.4.14/setup.py --- old/wptools-0.4.12/setup.py 2018-04-30 00:34:14.000000000 +0200 +++ new/wptools-0.4.14/setup.py 2018-05-23 05:57:02.000000000 +0200 @@ -10,7 +10,7 @@ setup( name='wptools', - version='0.4.12', + version='0.4.14', description='Wikipedia tools (for Humans)', long_description=readme + '\n\n' + history, url='https://github.com/siznax/wptools/', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/wptools/__init__.py new/wptools-0.4.14/wptools/__init__.py --- old/wptools-0.4.12/wptools/__init__.py 2018-04-30 00:34:32.000000000 +0200 +++ new/wptools-0.4.14/wptools/__init__.py 2018-05-23 05:57:11.000000000 +0200 @@ -18,7 +18,7 @@ __contact__ = "https://github.com/siznax/wptools"; __license__ = "MIT" __title__ = "wptools" -__version__ = "0.4.12" +__version__ = "0.4.14" from . import core from . import query diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/wptools/category.py new/wptools-0.4.14/wptools/category.py --- old/wptools-0.4.12/wptools/category.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.14/wptools/category.py 2018-05-08 02:04:57.000000000 +0200 @@ -94,10 +94,7 @@ if action == 'random': return qobj.random(namespace=14) elif action == 'category': - qry = qobj.category(title=title, pageid=pageid) - if self.data.get('cmcontinue'): - qry += "&cmcontinue=%s" % self.data['cmcontinue'] - return qry + return qobj.category(title, pageid, self._continue_params()) def _set_data(self, action): """ @@ -105,14 +102,7 @@ """ data = self._load_response(action) - try: - cmcontinue = data.get('continue').get('cmcontinue') - if cmcontinue: - self.data['cmcontinue'] = cmcontinue - del self.cache['category'] - except AttributeError: - if 'cmcontinue' in self.data: - del self.data['cmcontinue'] + self._handle_continuations(data, 'category') if action == 'category': members = data.get('query').get('categorymembers') @@ -151,9 +141,8 @@ self._get('category', show, proxy, timeout) - if self.data.get('cmcontinue'): - while self.data.get('cmcontinue'): - self._get('category', show, proxy, timeout) + while self.data.get('continue'): + self._get('category', show, proxy, timeout) return self diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/wptools/core.py new/wptools-0.4.14/wptools/core.py --- old/wptools-0.4.12/wptools/core.py 2018-04-10 04:16:35.000000000 +0200 +++ new/wptools-0.4.14/wptools/core.py 2018-05-23 06:05:21.000000000 +0200 @@ -21,7 +21,7 @@ """ REQUEST_DELAY = 0 - REQUEST_LIMIT = 25 + REQUEST_LIMIT = 50 cache = None data = None @@ -60,6 +60,83 @@ if kwargs.get('wiki'): self.params.update({'wiki': kwargs.get('wiki')}) + def _build_showstr(self, seed): + """ + Returns show() display string for data attribute + """ + output = ["%s (%s) data" % (seed, self.params['lang'])] + + output.append('{') + + maxwidth = WPToolsQuery.MAXWIDTH + + for item in sorted(self.data): + + if self.data[item] is None: + continue + + prefix = item + value = self.data[item] + + if isinstance(value, dict): + prefix = "%s: <dict(%d)>" % (prefix, len(value)) + value = ', '.join(value.keys()) + elif isinstance(value, int): + prefix = "%s:" % prefix + if 'pageid' not in prefix: + value = "{:,}".format(value) + elif isinstance(value, list): + prefix = "%s: <list(%d)>" % (prefix, len(value)) + value = ', '.join((safestr(x) for x in value if x)) + elif isinstance(value, tuple): + prefix = "%s: <tuple(%d)>" % (prefix, len(value)) + value = ', '.join((safestr(x) for x in value if x)) + elif utils.is_text(value): + value = value.strip().replace('\n', '') + if len(value) > (maxwidth - len(prefix)): + prefix = "%s: <str(%d)>" % (prefix, len(value)) + else: + prefix = "%s:" % prefix + + output.append(" %s %s" % (prefix, value)) + + output.append('}') + + return output + + def _continue_params(self): + """ + Returns query string fragment continue parameters + """ + if not self.data.get('continue'): + return + + params = [] + for item in self.data['continue']: + params.append("&%s=%s" % (item, self.data['continue'][item])) + + return ''.join(params) + + def _handle_continuations(self, response, cache_key): + """ + Select continue params and clear cache or last continue params + """ + rcontinue = response.get('continue') + listen = ['blcontinue', 'cmcontinue'] + cparams = {} + + if rcontinue: + for flag in listen: + if rcontinue.get(flag): + cparams[flag] = rcontinue.get(flag) + + if cparams: + self.data['continue'] = cparams + del self.cache[cache_key] + else: # no more continuations + if 'continue' in self.data: + del self.data['continue'] + def _get(self, action, show, proxy, timeout): """ make HTTP request and cache response @@ -122,8 +199,11 @@ except ValueError: raise ValueError(_query) - if data.get('warnings') and not self.flags.get('silent'): - utils.stderr("API warning: %s" % data.get('warnings')) + if data.get('warnings'): + if 'WARNINGS' in self.data: + self.data['WARNINGS'].update(data['warnings']) + else: + self.data['WARNINGS'] = data['warnings'] if data.get('error'): utils.stderr("API error: %s" % data.get('error')) @@ -196,6 +276,9 @@ if not self.data: return + if self.data.get('continue'): + return + ptitle = self.params.get('title') dtitle = self.data.get('title') pageid = self.params.get('pageid') @@ -204,45 +287,7 @@ if utils.is_text(seed): seed = seed.replace('_', ' ') - output = ["%s (%s) data" % (seed, self.params['lang'])] - - output.append('{') - - maxwidth = WPToolsQuery.MAXWIDTH - - for item in sorted(self.data): - - if self.data[item] is None: - continue - - prefix = item - value = self.data[item] - - if isinstance(value, dict): - prefix = "%s: <dict(%d)>" % (prefix, len(value)) - value = ', '.join(value.keys()) - elif isinstance(value, int): - prefix = "%s:" % prefix - if 'pageid' not in prefix: - value = "{:,}".format(value) - elif isinstance(value, list): - prefix = "%s: <list(%d)>" % (prefix, len(value)) - value = ', '.join((safestr(x) for x in value if x)) - elif isinstance(value, tuple): - prefix = "%s: <tuple(%d)>" % (prefix, len(value)) - value = ', '.join((safestr(x) for x in value if x)) - elif utils.is_text(value): - value = value.strip().replace('\n', '') - if len(value) > (maxwidth - len(prefix)): - prefix = "%s: <str(%d)>" % (prefix, len(value)) - else: - prefix = "%s:" % prefix - - output.append(" %s %s" % (prefix, value)) - - output.append('}') - - prettyprint(output) + prettyprint(self._build_showstr(seed)) def handle_wikidata_errors(data, query): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/wptools/page.py new/wptools-0.4.14/wptools/page.py --- old/wptools-0.4.12/wptools/page.py 2018-04-10 04:12:09.000000000 +0200 +++ new/wptools-0.4.14/wptools/page.py 2018-05-23 04:58:00.000000000 +0200 @@ -161,7 +161,7 @@ elif action == 'query': qstr = qobj.query(title, pageid) elif action == 'querymore': - qstr = qobj.querymore(title, pageid) + qstr = qobj.querymore(title, pageid, self._continue_params()) elif action == 'parse': qstr = qobj.parse(title, pageid) elif action == 'imageinfo': @@ -262,7 +262,7 @@ set image data from action=parse response """ image = infobox.get('image') - cover = infobox.get('Cover') + cover = infobox.get('Cover') or infobox.get('cover') if image or cover: if 'image' not in self.data: @@ -281,11 +281,19 @@ data = self._load_response(action) page = data['query']['pages'][0] + self._handle_continuations(data, 'querymore') + if action == 'query': self.data['random'] = data['query']['random'][0]["title"] - if 'redirects' in data['query']: - self.data['redirected'] = data['query']['redirects'] + backlinks = data['query'].get('backlinks') + if backlinks: + if 'backlinks' in self.data: + self.data['backlinks'].extend(backlinks) + else: + self.data['backlinks'] = backlinks + + self.data['redirected'] = data['query'].get('redirects') self._set_query_data_fast_1(page) # avoid pylint too-many-branches self._set_query_data_fast_2(page) @@ -441,7 +449,11 @@ """ calls get_imageinfo() if data image missing info """ - if self._missing_imageinfo() and not self.flags.get('defer_imageinfo'): + missing = self._missing_imageinfo() + deferred = self.flags.get('defer_imageinfo') + continuing = self.data.get('continue') + + if missing and not deferred and not continuing: self.get_imageinfo(show=False) def _update_params(self): @@ -631,6 +643,9 @@ self._get('querymore', show, proxy, timeout) + while self.data.get('continue'): + self._get('querymore', show, proxy, timeout) + return self def get_random(self, show=True, proxy=None, timeout=0): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/wptools/query.py new/wptools-0.4.14/wptools/query.py --- old/wptools-0.4.12/wptools/query.py 2018-02-07 00:15:11.000000000 +0100 +++ new/wptools-0.4.14/wptools/query.py 2018-05-23 04:59:43.000000000 +0200 @@ -79,11 +79,14 @@ QUERYMORE = Template(( "${WIKI}/w/api.php?action=query" + "&bllimit=500" + "&bltitle=${TITLES}" "&cllimit=500" "&clshow=!hidden" "&format=json" "&formatversion=2" "&imlimit=500" + "&list=backlinks" "&lllimit=500" "&pclimit=500" "&prop=categories|contributors|images|langlinks|pageviews" @@ -119,14 +122,14 @@ self.domain = domain_name(self.wiki) self.uri = self.wiki_uri(self.wiki) - def category(self, title, pageid=None, limit=500, namespace=None): + def category(self, title, pageid=None, cparams=None, namespace=None): """ Returns category query string """ query = self.LIST.substitute(WIKI=self.uri, LIST='categorymembers') + status = pageid or title - if limit: - query += "&cmlimit=%d" % limit + query += "&cmlimit=500" if namespace is not None: query += "&cmnamespace=%d" % namespace @@ -140,7 +143,11 @@ if pageid: query += "&cmpageid=%d" % pageid - self.set_status('categorymembers', pageid or title) + if cparams: + query += cparams + status += ' (%s)' % cparams + + self.set_status('categorymembers', status) return query @@ -199,6 +206,7 @@ """ query = self.QUERY.substitute(WIKI=self.uri, TITLES=safequote(titles) or pageids) + status = titles or pageids if pageids and not titles: query = query.replace('&titles=', '&pageids=') @@ -206,11 +214,11 @@ if self.variant: query += '&variant=' + self.variant - self.set_status('query', titles or pageids) + self.set_status('query', status) return query - def querymore(self, titles, pageids=None): + def querymore(self, titles, pageids=None, cparams=None): """ Returns MediaWiki action=query query string (for MORE) A much more expensive query for popular pages @@ -219,13 +227,19 @@ WIKI=self.uri, TITLES=safequote(titles) or pageids) + status = "%s" % (pageids or titles) + if pageids and not titles: query = query.replace('&titles=', '&pageids=') + if cparams: + query += cparams + status += " (%s)" % cparams + if self.variant: query += '&variant=' + self.variant - self.set_status('querymore', pageids or titles) + self.set_status('querymore', status) return query diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.12/wptools.egg-info/PKG-INFO new/wptools-0.4.14/wptools.egg-info/PKG-INFO --- old/wptools-0.4.12/wptools.egg-info/PKG-INFO 2018-04-30 00:38:59.000000000 +0200 +++ new/wptools-0.4.14/wptools.egg-info/PKG-INFO 2018-05-23 06:11:08.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: wptools -Version: 0.4.12 +Version: 0.4.14 Summary: Wikipedia tools (for Humans) Home-page: https://github.com/siznax/wptools/ Author: Steve @siznax @@ -295,9 +295,11 @@ Release History --------------- - 0.4.12 (2018-04-29) + 0.4.14 (2018-05-22) +++++++++++++++++++ + * Put API warnings in data attribute (#120) + * Support getting backlinks w/continuations (#119, #122) * Always respect silent flag (#118) * Improved infobox parsing (#91, #109) * Added image (license, assessment) metadata (#104, #105)
