Hello community, here is the log from the commit of package python-wptools for openSUSE:Factory checked in at 2018-10-23 20:40:30 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-wptools (Old) and /work/SRC/openSUSE:Factory/.python-wptools.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-wptools" Tue Oct 23 20:40:30 2018 rev:9 rq:643793 version:0.4.17 Changes: -------- --- /work/SRC/openSUSE:Factory/python-wptools/python-wptools.changes 2018-06-29 22:24:16.542579600 +0200 +++ /work/SRC/openSUSE:Factory/.python-wptools.new/python-wptools.changes 2018-10-23 20:41:47.856443435 +0200 @@ -1,0 +2,13 @@ +Mon Oct 22 14:09:27 UTC 2018 - [email protected] + +- Update to version 0.4.17: + * Support proxy authentication (gh#siznax/wptools#130). + * Support page links continuations (gh#siznax/wptools#129). + +------------------------------------------------------------------- +Wed Jul 18 07:27:54 UTC 2018 - [email protected] + +- Update to version 0.4.16: + + Support for alternate API endpoint. + +------------------------------------------------------------------- Old: ---- wptools-0.4.15.tar.gz New: ---- wptools-0.4.17.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-wptools.spec ++++++ --- /var/tmp/diff_new_pack.rVsjDs/_old 2018-10-23 20:41:49.340441662 +0200 +++ /var/tmp/diff_new_pack.rVsjDs/_new 2018-10-23 20:41:49.360441638 +0200 @@ -19,7 +19,7 @@ %define pyname wptools %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-wptools -Version: 0.4.15 +Version: 0.4.17 Release: 0 Summary: Wikipedia tools (for Humans) License: MIT ++++++ python-wptools-avoid-reading-readme.patch ++++++ --- /var/tmp/diff_new_pack.rVsjDs/_old 2018-10-23 20:41:49.380441614 +0200 +++ /var/tmp/diff_new_pack.rVsjDs/_new 2018-10-23 20:41:49.384441609 +0200 @@ -13,7 +13,7 @@ @@ -12,7 +10,16 @@ setup( name='wptools', - version='0.4.15', + version='0.4.17', description='Wikipedia tools (for Humans)', - long_description=readme + '\n\n' + history, + long_description= ++++++ wptools-0.4.15.tar.gz -> wptools-0.4.17.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/HISTORY.rst new/wptools-0.4.17/HISTORY.rst --- old/wptools-0.4.15/HISTORY.rst 2018-05-30 22:39:06.000000000 +0200 +++ new/wptools-0.4.17/HISTORY.rst 2018-08-17 01:52:21.000000000 +0200 @@ -3,9 +3,12 @@ Release History --------------- -0.4.15 (2018-05-30) +0.4.17 (2018-08-16) +++++++++++++++++++ +* Support proxy authentication (#130) +* Support page links continuations (#129) +* Support alternate API endpoints (#127) * Merged @mcepl update to package tests (#125) * Put API warnings in data attribute (#120) * Support getting backlinks w/continuations (#119, #122) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/PKG-INFO new/wptools-0.4.17/PKG-INFO --- old/wptools-0.4.15/PKG-INFO 2018-05-30 22:42:40.000000000 +0200 +++ new/wptools-0.4.17/PKG-INFO 2018-08-17 01:58:46.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: wptools -Version: 0.4.15 +Version: 0.4.17 Summary: Wikipedia tools (for Humans) Home-page: https://github.com/siznax/wptools/ Author: Steve @siznax @@ -295,9 +295,12 @@ Release History --------------- - 0.4.15 (2018-05-30) + 0.4.17 (2018-08-16) +++++++++++++++++++ + * Support proxy authentication (#130) + * Support page links continuations (#129) + * Support alternate API endpoints (#127) * Merged @mcepl update to package tests (#125) * Put API warnings in data attribute (#120) * Support getting backlinks w/continuations (#119, #122) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/setup.py new/wptools-0.4.17/setup.py --- old/wptools-0.4.15/setup.py 2018-05-30 22:35:29.000000000 +0200 +++ new/wptools-0.4.17/setup.py 2018-08-17 01:50:58.000000000 +0200 @@ -10,7 +10,7 @@ setup( name='wptools', - version='0.4.15', + version='0.4.17', description='Wikipedia tools (for Humans)', long_description=readme + '\n\n' + history, url='https://github.com/siznax/wptools/', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/category.py new/wptools-0.4.17/tests/category.py --- old/wptools-0.4.15/tests/category.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/category.py 2018-07-09 01:28:18.000000000 +0200 @@ -4,6 +4,6 @@ response = r"""{"batchcomplete":"","query":{"categorymembers":[{"pageid":53206947,"ns":0,"title":"Academic freedom in the Middle East"},{"pageid":52982000,"ns":0,"title":"Access to public information in Albania"},{"pageid":52922616,"ns":0,"title":"Access to public information in Armenia"},{"pageid":53000611,"ns":0,"title":"Access to public information in Bosnia and Herzegovina"},{"pageid":53125280,"ns":0,"title":"Access to public information in Bulgaria"},{"pageid":53106462,"ns":0,"title":"Access to public information in Croatia"},{"pageid":52049646,"ns":0,"title":"Access to public information in Europe"},{"pageid":52970153,"ns":0,"title":"Access to public information in Georgia"},{"pageid":53248779,"ns":0,"title":"Access to public information in Greece"},{"pageid":53033394,"ns":0,"title":"Access to public information in Kosovo"},{"pageid":52959127,"ns":0,"title":"Access to public information in Moldova"},{"pageid":53134476,"ns":0,"title":"Access to public information in Montenegro"},{"pageid":48302927,"ns":0,"title":"Access to public information in Serbia"},{"pageid":53209634,"ns":0,"title":"Access to public information in Slovenia"},{"pageid":53251063,"ns":0,"title":"Access to public information in the Republic of Macedonia"},{"pageid":53188837,"ns":0,"title":"Access to public information in Turkey"},{"pageid":1694303,"ns":0,"title":"Adultism"},{"pageid":22648231,"ns":0,"title":"Ageism"},{"pageid":19932294,"ns":0,"title":"Anti-homelessness legislation"},{"pageid":55084,"ns":0,"title":"Cultural genocide"},{"pageid":40331171,"ns":0,"title":"Casagrande case"},{"pageid":5047204,"ns":0,"title":"Right to education"},{"pageid":44594933,"ns":0,"title":"Equal Pay Day"},{"pageid":1108449,"ns":0,"title":"Equal pay for equal work"},{"pageid":5818292,"ns":0,"title":"Fetal rights"},{"pageid":18593011,"ns":0,"title":"Freedom from discrimination"},{"pageid":333994,"ns":0,"title":"Freedom of assembly"},{"pageid":219580,"ns":0,"title":"Freedom of association"},{"pageid":4319616,"ns":0,"title":"Freedom of education"},{"pageid":911519,"ns":0,"title":"Freedom of information"},{"pageid":1270497,"ns":0,"title":"Freedom of movement"},{"pageid":21401843,"ns":0,"title":"Freedom of speech"},{"pageid":557913,"ns":0,"title":"Freedom of thought"},{"pageid":300602,"ns":0,"title":"Internet access"},{"pageid":47275784,"ns":0,"title":"Intersex human rights"},{"pageid":499423,"ns":0,"title":"Legal aid"},{"pageid":1600241,"ns":0,"title":"Prisoners' rights"},{"pageid":7532615,"ns":0,"title":"Psychiatric survivors movement"},{"pageid":338825,"ns":0,"title":"Toleration"},{"pageid":660608,"ns":0,"title":"Right to a fair trial"},{"pageid":30105520,"ns":0,"title":"Right to an adequate standard of living"},{"pageid":35684112,"ns":0,"title":"Right to clothing"},{"pageid":370782,"ns":0,"title":"Right to die"},{"pageid":11224741,"ns":0,"title":"Right to equal protection"},{"pageid":8032176,"ns":0,"title":"Right to food"},{"pageid":35973887,"ns":0,"title":"Right to Food Guidelines"},{"pageid":3085490,"ns":0,"title":"Right to health"},{"pageid":21311955,"ns":0,"title":"Right to housing"},{"pageid":27913563,"ns":0,"title":"Right to Internet access"},{"pageid":219243,"ns":0,"title":"Right to keep and bear arms"},{"pageid":468580,"ns":0,"title":"Right to life"},{"pageid":8313678,"ns":0,"title":"Right to petition"},{"pageid":3119733,"ns":0,"title":"Right to property"},{"pageid":26207472,"ns":0,"title":"Right to science and culture"},{"pageid":30012163,"ns":0,"title":"Right to social security"},{"pageid":3062336,"ns":0,"title":"Human right to water and sanitation"},{"pageid":783318,"ns":0,"title":"Right to work"},{"pageid":5001094,"ns":0,"title":"Security of person"},{"pageid":29269,"ns":0,"title":"Self-determination"},{"pageid":19231063,"ns":0,"title":"Services for mental disorders"},{"pageid":9674519,"ns":0,"title":"Sex characteristics"},{"pageid":265564,"ns":0,"title":"Solitary confinement"},{"pageid":50066223,"ns":0,"title":"Solitary confinement of women"},{"pageid":2023819,"ns":0,"title":"Stateless nation"},{"pageid":4720758,"ns":0,"title":"Substantive rights"},{"pageid":53400430,"ns":0,"title":"Transparency of media ownership in Turkey"},{"pageid":177101,"ns":0,"title":"Patients' rights"},{"pageid":31802,"ns":0,"title":"Universal access to education"},{"pageid":29542150,"ns":14,"title":"Category:Child marriage"},{"pageid":3862359,"ns":14,"title":"Category:Children's rights"},{"pageid":14338109,"ns":14,"title":"Category:Consumer protection"},{"pageid":38779392,"ns":14,"title":"Category:Digital rights"},{"pageid":5119200,"ns":14,"title":"Category:Disability rights"},{"pageid":24946646,"ns":14,"title":"Category:Environmental protection"},{"pageid":33248265,"ns":14,"title":"Category:Freedom of assembly"},{"pageid":33188995,"ns":14,"title":"Category:Freedom of association"},{"pageid":1584861,"ns":14,"title":"Category:Freedom of expression"},{"pageid":53094272,"ns":14,"title":"Category:Freedom of movement"},{"pageid":15279591,"ns":14,"title":"Category:Habeas corpus"},{"pageid":47799573,"ns":14,"title":"Category:Right to housing"},{"pageid":19850720,"ns":14,"title":"Category:Human rights in Islam"},{"pageid":26106850,"ns":14,"title":"Category:Indigenous rights"},{"pageid":3534405,"ns":14,"title":"Category:Internet access"},{"pageid":17713681,"ns":14,"title":"Category:Labor rights"},{"pageid":25070788,"ns":14,"title":"Category:LGBT rights by issue"},{"pageid":9660914,"ns":14,"title":"Category:Linguistic rights"},{"pageid":2090360,"ns":14,"title":"Category:Men's rights"},{"pageid":1804476,"ns":14,"title":"Category:Privacy"},{"pageid":2235897,"ns":14,"title":"Category:Reproductive rights"},{"pageid":46195979,"ns":14,"title":"Category:Right to work"},{"pageid":48543091,"ns":14,"title":"Category:Victims' rights"},{"pageid":3431123,"ns":14,"title":"Category:Women's rights"}]}}""" -cache = {'query': query, - 'response': response, - 'info': {'content': 'TEST', 'status': 200}} +cache = {'query': query, + 'response': response, + 'info': {'content-type': 'TEST', 'status': 200}} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/category_cmcontinue.py new/wptools-0.4.17/tests/category_cmcontinue.py --- old/wptools-0.4.15/tests/category_cmcontinue.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/category_cmcontinue.py 2018-07-09 01:28:03.000000000 +0200 @@ -19,6 +19,6 @@ } }""" -cache = {'query': query, - 'response': response, - 'info': {'content': 'TEST', 'status': 200}} +cache = {'query': query, + 'response': response, + 'info': {'content-type': 'TEST', 'status': 200}} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/query_plcontinue.py new/wptools-0.4.17/tests/query_plcontinue.py --- old/wptools-0.4.15/tests/query_plcontinue.py 1970-01-01 01:00:00.000000000 +0100 +++ new/wptools-0.4.17/tests/query_plcontinue.py 2018-08-16 19:06:32.000000000 +0200 @@ -0,0 +1,38 @@ +# -*- coding:utf-8 -*- + +query = 'https://en.wikipedia.org/w/api.php?action=query&formatversion=2&pllimit=500&prop=links&list=random&titles=List_of_programming_languages' + +response = r""" +{ + "continue": { + "plcontinue": "144146|0|Perl_Data_Language", + "continue": "||" + }, + "query": { + "pages": [ + { + "pageid": 144146, + "ns": 0, + "title": "List of programming languages", + "links": [ + { + "ns": 0, + "title": "A++" + } + ] + } + ], + "random": [ + { + "id": 123, + "ns": 0, + "title": "RANDOM" + } + ] + } +} +""" + +cache = {'query': query, + 'response': response, + 'info': {'content-type': 'TEST', 'status': 200}} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/querymore_blcontinue.py new/wptools-0.4.17/tests/querymore_blcontinue.py --- old/wptools-0.4.15/tests/querymore_blcontinue.py 2018-05-23 05:36:11.000000000 +0200 +++ new/wptools-0.4.17/tests/querymore_blcontinue.py 2018-07-09 01:28:25.000000000 +0200 @@ -35,6 +35,6 @@ } """ -cache = {'query': query, - 'response': response, - 'info': {'content': 'TEST', 'status': 200}} +cache = {'query': query, + 'response': response, + 'info': {'content-type': 'TEST', 'status': 200}} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/rest.py new/wptools-0.4.17/tests/rest.py --- old/wptools-0.4.15/tests/rest.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/rest.py 2018-07-09 01:28:59.000000000 +0200 @@ -4,5 +4,5 @@ response = r"""{"items":["data-parsoid","graph","html","mobile-sections","mobile-sections-lead","mobile-sections-remaining","pdf","random","related","revision","segments","summary","title","wikitext"]}""" -info = {'content': 'TEST', 'status': 200} +info = {'content-type': 'TEST', 'status': 200} cache = {'query': query, 'response': response, 'info': info} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/rest_html.py new/wptools-0.4.17/tests/rest_html.py --- old/wptools-0.4.15/tests/rest_html.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/rest_html.py 2018-07-09 01:28:31.000000000 +0200 @@ -390,8 +390,8 @@ <span about="#mwt237" typeof="mw:Transclusion" data-mw='{"parts":[{"template":{"target":{"wt":"s-start","href":"./Template:S-start"},"params":{},"i":0}},"\n",{"template":{"target":{"wt":"s-bef","href":"./Template:S-bef"},"params":{"before":{"wt":"[[Anthony Read]]"}},"i":1}}," \n",{"template":{"target":{"wt":"s-ttl","href":"./Template:S-ttl"},"params":{"title":{"wt":"''[[Doctor Who]]'' script editor"},"years":{"wt":"1979–80"}},"i":2}}," \n",{"template":{"target":{"wt":"s-aft","href":"./Template:S-aft"},"params":{"after":{"wt":"[[Christopher H. Bidmead]]"}},"i":3}},"\n",{"template":{"target":{"wt":"s-end","href":"./Template:S-end"},"params":{},"i":4}}]}' id="mwA_k"> </span><table class="wikitable succession-box" style="margin:0.5em auto; font-size:95%;clear:both;" about="#mwt237"> <tbody><tr style="text-align:center;"> -<td style="width:30%;" rowspan="1">Preceded<span typeof="mw:Entity"> </span>by<br/><span style="font-weight: bold"><a rel="mw:WikiLink" href="./Anthony_Read" title="Anthony Read">Anthony Read</a></span></td> -<td style="width: 40%; text-align: center;" rowspan="1"><b><i><a rel="mw:WikiLink" href="./Doctor_Who" title="Doctor Who">Doctor Who</a></i> script editor</b><br/>1979–80 </td> +<td style="width:30%;" rowspan="1">Preceded<span typeof="mw:Entity"> </span>by<br/><span style="font-weight: bold"><a rel="mw:WikiLink" href="./Anthony_Read" title="Anthony Read">Anthony Read</a></span></td> +<td style="width: 40%; text-align: center;" rowspan="1"><b><i><a rel="mw:WikiLink" href="./Doctor_Who" title="Doctor Who">Doctor Who</a></i> script editor</b><br/>1979–80 </td> <td style="width: 30%; text-align: center;" rowspan="1"> Succeeded<span typeof="mw:Entity"> </span>by<br/><span style="font-weight: bold"><a rel="mw:WikiLink" href="./Christopher_H._Bidmead" title="Christopher H. Bidmead">Christopher H. Bidmead</a></span></td></tr> </tbody></table> @@ -615,7 +615,7 @@ <link rel="mw:PageProp/Category" href="./Category:Douglas_Adams#%20" id="mwBAA"/> <link rel="mw:PageProp/Category" href="./Category:1952_births" id="mwBAE"/> <link rel="mw:PageProp/Category" href="./Category:2001_deaths" id="mwBAI"/> -<link rel="mw:PageProp/Category" href="./Category:20th-century_atheists" id="mwBAM"/> +<link rel="mw:PageProp/Category" href="./Category:20th-century_atheists" id="mwBAM"/> <link rel="mw:PageProp/Category" href="./Category:21st-century_atheists" id="mwBAQ"/> <link rel="mw:PageProp/Category" href="./Category:Alumni_of_St_John's_College,_Cambridge" id="mwBAU"/> <link rel="mw:PageProp/Category" href="./Category:British_atheism_activists" id="mwBAY"/> @@ -642,5 +642,5 @@ <link rel="mw:PageProp/Category" href="./Category:20th-century_English_novelists" id="mwBBs"/> <link rel="mw:PageProp/Category" href="./Category:21st-century_British_novelists" id="mwBBw"/></body></html>""" -info = {'content': 'text/html', 'status': 200} +info = {'content-type': 'text/html', 'status': 200} cache = {'query': query, 'response': response, 'info': info} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/rest_lead.py new/wptools-0.4.17/tests/rest_lead.py --- old/wptools-0.4.15/tests/rest_lead.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/rest_lead.py 2018-07-09 01:28:37.000000000 +0200 @@ -4,5 +4,5 @@ response = r"""{"ns":0,"id":8091,"revision":"793736590","lastmodified":"2017-08-03T16:37:48Z","lastmodifier":{"name":"Quinton Feldberg","gender":"male"},"displaytitle":"Douglas Adams","normalizedtitle":"Douglas Adams","wikibase_item":"Q42","description":"English writer and humorist","protection":{},"editable":true,"languagecount":69,"image":{"file":"Douglas adams portrait cropped.jpg","urls":{"320":"//upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg","640":"//upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg","800":"//upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg","1024":"//upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg"}},"spoken":{"files":["File:Douglas_Adams_Part_1.ogg","File:Douglas_Adams_Part_2.ogg"]},"hatnotes":["For other people named Douglas Adams, see <a href=\"/wiki/Douglas_Adams_(disambiguation)\" title=\"Douglas Adams (disambiguation)\">Douglas Adams (disambiguation)</a>.","<strong>Warning:</strong> Page using <a href=\"/wiki/Template:Infobox_writer\" title=\"Template:Infobox writer\">Template:Infobox writer</a> with unknown parameter \"height\" (this message is shown only in preview)."],"sections":[{"id":0,"text":"<span><p><b>Douglas Noel Adams</b> (11 March 1952 – 11 May 2001) was an English <a href=\"/wiki/Author\" title=\"Author\">author</a>, <a href=\"/wiki/Scriptwriter\" title=\"Scriptwriter\" class=\"mw-redirect\">scriptwriter</a>, <a href=\"/wiki/Essayist\" title=\"Essayist\" class=\"mw-redirect\">essayist</a>, <a href=\"/wiki/List_of_humorists\" title=\"List of humorists\">humorist</a>, <a href=\"/wiki/Satirist\" title=\"Satirist\" class=\"mw-redirect\">satirist</a> and <a href=\"/wiki/Dramatist\" title=\"Dramatist\" class=\"mw-redirect\">dramatist</a>.</p></span><div class=\"hatnote navigation-not-searchable\">For other people named Douglas Adams, see <a href=\"/wiki/Douglas_Adams_(disambiguation)\" title=\"Douglas Adams (disambiguation)\">Douglas Adams (disambiguation)</a>.</div>\n<p class=\"skipped\">\n</p>\n<table class=\"infobox vcard\" style=\"width:22em\" id=\"mwBw\"><tbody><tr><th colspan=\"2\" style=\"text-align:center;font-size:125%;font-weight:bold\"><span class=\"fn\">Douglas Adams</span></th></tr><tr><td colspan=\"2\" style=\"text-align:center\">\n<span class=\"mw-default-size\"><a href=\"/wiki/File:Douglas_adams_portrait_cropped.jpg\" class=\"image\"><img src=\"//upload.wikimedia.org/wikipedia/commons/thumb/c/c0/Douglas_adams_portrait_cropped.jpg/220px-Douglas_adams_portrait_cropped.jpg\" data-file-type=\"bitmap\" height=\"255\" width=\"220\" srcset=\"//upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg 2x, //upload.wikimedia.org/wikipedia/commons/thumb/c/c0/Douglas_adams_portrait_cropped.jpg/330px-Douglas_adams_portrait_cropped.jpg 1.5x\"></a></span></td></tr><tr><th scope=\"row\" style=\"padding-top:0.225em;line-height:1.1em;padding-right:0.65em;\">Born</th><td style=\"line-height:1.4em;\">\nDouglas Noel Adams<br><span style=\"display:none\">(<span class=\"bday\">1952-03-11</span>)</span>11 March 1952<br><a href=\"/wiki/Cambridge\" title=\"Cambridge\">Cambridge</a>, England</td></tr><tr><th scope=\"row\" style=\"padding-top:0.225em;line-height:1.1em;padding-right:0.65em;\">Died</th><td style=\"line-height:1.4em;\">\n11 May 2001<span style=\"display:none\">(<span class=\"dday deathdate\">2001-05-11</span>)</span> (aged<span> </span>49)<br><a href=\"/wiki/Montecito,_California\" title=\"Montecito, California\">Montecito, California</a>, US</td></tr><tr><th scope=\"row\" style=\"padding-top:0.225em;line-height:1.1em;padding-right:0.65em;\">Resting place</th><td style=\"line-height:1.4em;\">\n<a href=\"/wiki/Highgate_Cemetery\" title=\"Highgate Cemetery\">Highgate Cemetery</a>, London, England</td></tr><tr><th scope=\"row\" style=\"padding-top:0.225em;line-height:1.1em;padding-right:0.65em;\">Occupation</th><td class=\"role\" style=\"line-height:1.4em;\">\nWriter</td></tr><tr><th scope=\"row\" style=\"padding-top:0.225em;line-height:1.1em;padding-right:0.65em;\">Alma<span> </span>mater</th><td style=\"line-height:1.4em;\">\n<a href=\"/wiki/St_John's_College,_Cambridge\" title=\"St John's College, Cambridge\">St John's College, Cambridge</a></td></tr><tr><th scope=\"row\" style=\"padding-top:0.225em;line-height:1.1em;padding-right:0.65em;\">Genre</th><td class=\"category\" style=\"line-height:1.4em;\">\nScience fiction, comedy, satire</td></tr><tr><th colspan=\"2\" style=\"text-align:center\">Website</th></tr><tr><td colspan=\"2\" style=\"text-align:center;line-height:1.4em;\">\n<span class=\"url\"><a rel=\"mw:ExtLink\" href=\"http://douglasadams.com/\"; class=\"external\">douglasadams<wbr>.com</a></span></td></tr></tbody></table><div class=\"hatnote\" style=\"color:red\"><strong>Warning:</strong> Page using <a href=\"/wiki/Template:Infobox_writer\" title=\"Template:Infobox writer\">Template:Infobox writer</a> with unknown parameter \"height\" (this message is shown only in preview).</div>\n\n\n\n\n<p>Adams is best known as the author of <i><a href=\"/wiki/The_Hitchhiker's_Guide_to_the_Galaxy\" title=\"The Hitchhiker's Guide to the Galaxy\">The Hitchhiker's Guide to the Galaxy</a></i>, which originated in 1978 as a BBC <a href=\"/wiki/The_Hitchhiker's_Guide_to_the_Galaxy_(radio_series)\" title=\"The Hitchhiker's Guide to the Galaxy (radio series)\">radio comedy</a> before developing into a \"trilogy\" of five books that sold more than 15<span> </span>million copies in his lifetime and generated a <a href=\"/wiki/The_Hitchhiker's_Guide_to_the_Galaxy_(TV_series)\" title=\"The Hitchhiker's Guide to the Galaxy (TV series)\">television series</a>, several stage plays, comics, a <a href=\"/wiki/The_Hitchhiker's_Guide_to_the_Galaxy_(computer_game)\" title=\"The Hitchhiker's Guide to the Galaxy (computer game)\" class=\"mw-redirect\">computer game</a>, and in 2005 a <a href=\"/wiki/The_Hitchhiker's_Guide_to_the_Galaxy_(film)\" title=\"The Hitchhiker's Guide to the Galaxy (film)\">feature film</a>. Adams's contribution to UK radio is commemorated in <a href=\"/wiki/Radio_Academy\" title=\"Radio Academy\">The Radio Academy</a>'s Hall of Fame.<span class=\"mw-ref\" id=\"cite_ref-radioacad_1-0\"><a href=\"#cite_note-radioacad-1\" style=\"counter-reset: mw-Ref 1;\"><span class=\"mw-reflink-text\">[1]</span></a></span></p>\n\n<p>Adams also wrote <i><a href=\"/wiki/Dirk_Gently's_Holistic_Detective_Agency\" title=\"Dirk Gently's Holistic Detective Agency\">Dirk Gently's Holistic Detective Agency</a></i> (1987) and <i><a href=\"/wiki/The_Long_Dark_Tea-Time_of_the_Soul\" title=\"The Long Dark Tea-Time of the Soul\">The Long Dark Tea-Time of the Soul</a></i> (1988), and co-wrote <i><a href=\"/wiki/The_Meaning_of_Liff\" title=\"The Meaning of Liff\">The Meaning of Liff</a></i> (1983), <i><a href=\"/wiki/The_Deeper_Meaning_of_Liff\" title=\"The Deeper Meaning of Liff\" class=\"mw-redirect\">The Deeper Meaning of Liff</a></i> (1990), <i><a href=\"/wiki/Last_Chance_to_See\" title=\"Last Chance to See\">Last Chance to See</a></i> (1990), and three stories for the television series <i><a href=\"/wiki/Doctor_Who\" title=\"Doctor Who\">Doctor Who</a></i>; he also served as <a href=\"/wiki/Script_editor\" title=\"Script editor\">script editor</a> for the show's seventeenth season in 1979. A posthumous collection of his works, including an unfinished novel, was published as <i><a href=\"/wiki/The_Salmon_of_Doubt\" title=\"The Salmon of Doubt\">The Salmon of Doubt</a></i> in 2002.</p>\n\n<p>Adams was known as an advocate for environmentalism and <a href=\"/wiki/Conservation_movement\" title=\"Conservation movement\">conservation</a>, as a lover of fast cars, cameras, <a href=\"/wiki/Technological_innovation\" title=\"Technological innovation\" class=\"mw-redirect\">technological innovation</a> and the <a href=\"/wiki/Apple_Macintosh\" title=\"Apple Macintosh\" class=\"mw-redirect\">Apple Macintosh</a>, and as a \"devout <a href=\"/wiki/Atheist\" title=\"Atheist\" class=\"mw-redirect\">atheist</a>\".</p>\n\n"},{"id":1,"toclevel":1,"anchor":"Early_life","line":"Early life"},{"id":2,"toclevel":2,"anchor":"Education","line":"Education"},{"id":3,"toclevel":1,"anchor":"Career","line":"Career"},{"id":4,"toclevel":2,"anchor":"Writing","line":"Writing"},{"id":5,"toclevel":3,"anchor":"The_Hitchhiker.27s_Guide_to_the_Galaxy","line":"<i>The Hitchhiker's Guide to the Galaxy</i>"},{"id":6,"toclevel":3,"anchor":"Dirk_Gently_series","line":"<i>Dirk Gently</i> series"},{"id":7,"toclevel":3,"anchor":"Doctor_Who","line":"<i>Doctor Who</i>"},{"id":8,"toclevel":2,"anchor":"Music","line":"Music"},{"id":9,"toclevel":3,"anchor":"Pink_Floyd","line":"Pink Floyd"},{"id":10,"toclevel":2,"anchor":"Computer_games_and_projects","line":"Computer games and projects"},{"id":11,"toclevel":1,"anchor":"Personal_beliefs_and_activism","line":"Personal beliefs and activism"},{"id":12,"toclevel":2,"anchor":"Atheism_and_views_on_religion","line":"Atheism and views on religion"},{"id":13,"toclevel":2,"anchor":"Environmental_activism","line":"Environmental activism"},{"id":14,"toclevel":2,"anchor":"Technology_and_innovation","line":"Technology and innovation"},{"id":15,"toclevel":1,"anchor":"Personal_life","line":"Personal life"},{"id":16,"toclevel":1,"anchor":"Death_and_legacy","line":"Death and legacy"},{"id":17,"toclevel":1,"anchor":"Awards_and_nominations","line":"Awards and nominations"},{"id":18,"toclevel":1,"anchor":"Works","line":"Works"},{"id":19,"toclevel":1,"anchor":"Writing_credits","line":"Writing credits"},{"id":20,"toclevel":1,"anchor":"Notes","line":"Notes"},{"id":21,"toclevel":1,"anchor":"References","line":"References"},{"id":22,"toclevel":1,"anchor":"Further_reading","line":"Further reading"},{"id":23,"toclevel":2,"anchor":"Articles","line":"Articles"},{"id":24,"toclevel":2,"anchor":"Other","line":"Other"},{"id":25,"toclevel":1,"anchor":"External_links","line":"External links"}]}""" -info = {'content': 'TEST', 'status': 200} +info = {'content-type': 'TEST', 'status': 200} cache = {'query': query, 'response': response, 'info': info} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/rest_page.py new/wptools-0.4.17/tests/rest_page.py --- old/wptools-0.4.15/tests/rest_page.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/rest_page.py 2018-07-09 01:28:44.000000000 +0200 @@ -4,5 +4,5 @@ response = r"""{"items":["data-parsoid","graph","html","mobile-sections","mobile-sections-lead","mobile-sections-remaining","pdf","random","related","segments","summary","title","wikitext"]}""" -info = {'content': 'TEST', 'status': 200} +info = {'content-type': 'TEST', 'status': 200} cache = {'query': query, 'response': response, 'info': info} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/rest_summary.py new/wptools-0.4.17/tests/rest_summary.py --- old/wptools-0.4.15/tests/rest_summary.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/tests/rest_summary.py 2018-07-09 01:28:51.000000000 +0200 @@ -4,5 +4,5 @@ response = r"""{"title":"Douglas Adams","displaytitle":"Douglas Adams","pageid":8091,"extract":"Douglas Noel Adams (11 March 1952 – 11 May 2001) was an English author, scriptwriter, essayist, humorist, satirist and dramatist.\nAdams is best known as the author of The Hitchhiker's Guide to the Galaxy, which originated in 1978 as a BBC radio comedy before developing into a \"trilogy\" of five books that sold more than 15 million copies in his lifetime and generated a television series, several stage plays, comics, a computer game, and in 2005 a feature film. Adams's contribution to UK radio is commemorated in The Radio Academy's Hall of Fame.\nAdams also wrote Dirk Gently's Holistic Detective Agency (1987) and The Long Dark Tea-Time of the Soul (1988), and co-wrote The Meaning of Liff (1983), The Deeper Meaning of Liff (1990), Last Chance to See (1990), and three stories for the television series Doctor Who; he also served as script editor for the show's seventeenth season in 1979.","extract_html":"<p><b>Douglas Noel Adams</b> (11 March 1952 – 11 May 2001) was an English author, scriptwriter, essayist, humorist, satirist and dramatist.</p>\n<p>Adams is best known as the author of <i>The Hitchhiker's Guide to the Galaxy</i>, which originated in 1978 as a BBC radio comedy before developing into a \"trilogy\" of five books that sold more than 15 million copies in his lifetime and generated a television series, several stage plays, comics, a computer game, and in 2005 a feature film. Adams's contribution to UK radio is commemorated in The Radio Academy's Hall of Fame.</p>\n<p>Adams also wrote <i>Dirk Gently's Holistic Detective Agency</i> (1987) and <i>The Long Dark Tea-Time of the Soul</i> (1988), and co-wrote <i>The Meaning of Liff</i> (1983), <i>The Deeper Meaning of Liff</i> (1990), <i>Last Chance to See</i> (1990), and three stories for the television series <i>Doctor Who</i>; he also served as script editor for the show's seventeenth season in 1979.","thumbnail":{"source":"https://upload.wikimedia.org/wikipedia/commons/thumb/c/c0/Douglas_adams_portrait_cropped.jpg/276px-Douglas_adams_portrait_cropped.jpg","width":276,"height":320,"original":"http://upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg"},"originalimage":{"source":"https://upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg","width":333,"height":386},"lang":"en","dir":"ltr","timestamp":"2017-08-03T16:37:48Z","description":"English writer and humorist"}""" -info = {'content': 'TEST', 'status': 200} +info = {'content-type': 'TEST', 'status': 200} cache = {'query': query, 'response': response, 'info': info} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/tests/test_basic.py new/wptools-0.4.17/tests/test_basic.py --- old/wptools-0.4.15/tests/test_basic.py 2018-05-23 05:47:28.000000000 +0200 +++ new/wptools-0.4.17/tests/test_basic.py 2018-08-16 23:52:39.000000000 +0200 @@ -23,6 +23,7 @@ from . import parse_91 from . import parse_109 from . import query +from . import query_plcontinue from . import querymore from . import querymore_blcontinue from . import random_query @@ -338,7 +339,7 @@ qstr = page._query('wikidata', qobj) self.assertTrue('action=wbgetentities' in qstr) - page.params.update({'endpoint': '/page/summary/'}) + page.params.update({'rest_endpoint': '/page/summary/'}) qstr = page._query('restbase', qobj) self.assertTrue('api/rest' in qstr) @@ -500,8 +501,25 @@ self.assertTrue('requests' not in page.data) + def test_page_get_query_continue(self): + page = wptools.page('TEST', skip=SKIP_FLAG, silent=SILENT_FLAG) + + page.cache['query'] = query.cache + page._set_data('query') + self.assertEqual(len(page.data['links']), 384) + + page.cache = {'query': query_plcontinue.cache} + page._set_data('query') + self.assertTrue('continue' in page.data) + self.assertEqual(page.data['continue'], + {'plcontinue': u'144146|0|Perl_Data_Language'}) + self.assertEqual(len(page.data['links']), 385) + + qry = page._query('query', wptools.query.WPToolsQuery()) + self.assertTrue(qry.endswith('&plcontinue=144146|0|Perl_Data_Language')) + def test_page_get_more_continue(self): - page = wptools.page('TEST', silent=SILENT_FLAG) + page = wptools.page('TEST', skip=SKIP_FLAG, silent=SILENT_FLAG) page.cache['querymore'] = querymore.cache page._set_data('querymore') @@ -514,7 +532,6 @@ self.assertEqual(len(page.data['backlinks']), 501) qry = page._query('querymore', wptools.query.WPToolsQuery()) - print("\n\n>>>\n%s\n<<<\n\n" % qry) self.assertTrue(qry.endswith('&blcontinue=0|2062757')) def test_page_get_imageinfo(self): @@ -552,7 +569,8 @@ self.assertTrue('requests' not in page.data) def test_page_get_restbase(self): - page = wptools.page('TEST', endpoint='summary', silent=SILENT_FLAG) + page = wptools.page('TEST', silent=SILENT_FLAG) + page.params.update({'rest_endpoint': 'summary'}) page.cache = {'restbase': rest_summary.cache} page._set_data('restbase') data = page.data @@ -637,6 +655,19 @@ self.assertEqual(qobj.domain, 'example.com') self.assertEqual(qobj.uri, 'http://example.com') + def test_query_init_endpoint(self): + qobj = wptools.query.WPToolsQuery(endpoint='ENDPOINT') + self.assertEqual(qobj.endpoint, 'ENDPOINT') + self.assertEqual(qobj.uri, 'https://en.wikipedia.org') + self.assertTrue('.orgENDPOINT?' in qobj.category('CAT')) + self.assertTrue('.orgENDPOINT?' in qobj.labels(['QID'])) + self.assertTrue('.orgENDPOINT?' in qobj.imageinfo(['FILE'])) + self.assertTrue('.orgENDPOINT?' in qobj.parse('TITLE')) + self.assertTrue('.orgENDPOINT?' in qobj.query('TITLE')) + self.assertTrue('.orgENDPOINT?' in qobj.querymore('TITLE')) + self.assertTrue('.orgENDPOINT?' in qobj.site('siteinfo')) + self.assertTrue('.orgENDPOINT?' in qobj.wikidata('TITLE')) + def test_query_init_lang(self): qobj = wptools.query.WPToolsQuery(lang='zz') self.assertEqual(qobj.lang, 'zz') @@ -786,8 +817,7 @@ self.assertTrue('requests' not in page.data) def test_get_restbase_page(self): - page = wptools.restbase(silent=SILENT_FLAG) - page.params.update({'endpoint': '/page/'}) + page = wptools.restbase('/page/', silent=SILENT_FLAG) page.cache['restbase'] = rest_page.cache page._set_data('restbase') self.assertTrue('requests' not in page.data) @@ -827,7 +857,7 @@ def test_get_restbase_summary(self): page = wptools.restbase('TEST', silent=SILENT_FLAG) - page.params.update({'endpoint': '/page/summary/'}) + page.params.update({'rest_endpoint': '/page/summary/'}) page.cache['restbase'] = rest_summary.cache page._set_data('restbase') @@ -858,7 +888,7 @@ info = wptools.request.curl_info(req.cobj) self.assertEqual(info.get('status'), 0) self.assertEqual(info.get('bytes'), 0.0) - self.assertEqual(info.get('content'), None) + self.assertEqual(info.get('content-type'), None) self.assertTrue('wptools' in info.get('user-agent')) def test_request_curl_setup(self): @@ -869,6 +899,15 @@ info = req.cobj.getinfo(wptools.request.pycurl.RESPONSE_CODE) self.assertEqual(info, 0) + def test_request_curl_proxy_setup(self): + req = wptools.request.WPToolsRequest(verbose=True) + test_proxy = { + 'PROXY': 'TEST_PROXY', + 'PORT': 0, + 'USERPWD': 'TEST_USER:PWD' + } + req.curl_setup(proxy=test_proxy) + def test_request_user_agent(self): agent = wptools.request.user_agent() self.assertTrue(agent.startswith('wptools')) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/__init__.py new/wptools-0.4.17/wptools/__init__.py --- old/wptools-0.4.15/wptools/__init__.py 2018-05-30 22:38:35.000000000 +0200 +++ new/wptools-0.4.17/wptools/__init__.py 2018-08-17 01:52:45.000000000 +0200 @@ -18,7 +18,7 @@ __contact__ = "https://github.com/siznax/wptools"; __license__ = "MIT" __title__ = "wptools" -__version__ = "0.4.15" +__version__ = "0.4.17" from . import core from . import query diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/category.py new/wptools-0.4.17/wptools/category.py --- old/wptools-0.4.15/wptools/category.py 2018-05-08 02:04:57.000000000 +0200 +++ new/wptools-0.4.17/wptools/category.py 2018-07-07 02:41:15.000000000 +0200 @@ -25,6 +25,7 @@ - [title]: <str> category title Optional keyword {params}: + - [endpoint]: <str> alternative API endpoint (default=/w/api.php) - [lang]: <str> Mediawiki language code (default=en) - [namespace]: <int> filter members (0=article, 14=category) - [pageid]: <int> category pageid @@ -41,6 +42,7 @@ """ super(WPToolsCategory, self).__init__(**kwargs) + endpoint = kwargs.get('endpoint') pageid = kwargs.get('pageid') namespace = kwargs.get('namespace') @@ -49,6 +51,9 @@ title = args[0] self.params.update({'title': title}) + if endpoint: + self.params.update({'endpoint': endpoint}) + if pageid: try: self.params.update({'pageid': int(pageid)}) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/core.py new/wptools-0.4.17/wptools/core.py --- old/wptools-0.4.15/wptools/core.py 2018-05-23 06:05:21.000000000 +0200 +++ new/wptools-0.4.17/wptools/core.py 2018-08-16 19:06:32.000000000 +0200 @@ -122,7 +122,7 @@ Select continue params and clear cache or last continue params """ rcontinue = response.get('continue') - listen = ['blcontinue', 'cmcontinue'] + listen = ['blcontinue', 'cmcontinue', 'plcontinue'] cparams = {} if rcontinue: @@ -168,7 +168,8 @@ # make the request qobj = WPToolsQuery(lang=self.params['lang'], variant=self.params.get('variant'), - wiki=self.params.get('wiki')) + wiki=self.params.get('wiki'), + endpoint=self.params.get('endpoint')) qstr = self._query(action, qobj) req = self._request(proxy, timeout) response = req.get(qstr, qobj.status) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/page.py new/wptools-0.4.17/wptools/page.py --- old/wptools-0.4.15/wptools/page.py 2018-05-23 04:58:00.000000000 +0200 +++ new/wptools-0.4.17/wptools/page.py 2018-08-16 19:06:32.000000000 +0200 @@ -44,6 +44,7 @@ Optional keyword {params}: - [boxterm]: <str> Infobox title name or substring + - [endpoint]: <str> alternative API endpoint (default=/w/api.php) - [lang]: <str> Mediawiki language code (default=en) - [pageid]: <int> Mediawiki pageid - [variant]: <str> Mediawiki language variant @@ -63,6 +64,10 @@ if boxterm: self.params.update({'boxterm': boxterm}) + endpoint = kwargs.get('endpoint') + if endpoint: + self.params.update({'endpoint': endpoint}) + pageid = kwargs.get('pageid') if pageid: self.params.update({'pageid': pageid}) @@ -118,6 +123,16 @@ self.__insert_image_info(title, _from, info) + def _extend_data(self, datapoint, new_data): + """ + extend or assign new data to datapoint + """ + if new_data: + try: + self.data[datapoint].extend(new_data) + except KeyError: + self.data[datapoint] = new_data + def _missing_imageinfo(self): """ returns list of image filenames that are missing info @@ -152,14 +167,13 @@ title = self.params.get('title') pageid = self.params.get('pageid') wikibase = self.params.get('wikibase') - endpoint = self.params.get('endpoint') qstr = None if action == 'random': qstr = qobj.random() elif action == 'query': - qstr = qobj.query(title, pageid) + qstr = qobj.query(title, pageid, self._continue_params()) elif action == 'querymore': qstr = qobj.querymore(title, pageid, self._continue_params()) elif action == 'parse': @@ -171,7 +185,7 @@ elif action == 'wikidata': qstr = qobj.wikidata(title, wikibase) elif action == 'restbase': - qstr = qobj.restbase(endpoint, title) + qstr = qobj.restbase(self.params.get('rest_endpoint'), title) if qstr is None: raise ValueError("Unknown action: %s" % action) @@ -281,17 +295,12 @@ data = self._load_response(action) page = data['query']['pages'][0] - self._handle_continuations(data, 'querymore') + self._handle_continuations(data, action) if action == 'query': self.data['random'] = data['query']['random'][0]["title"] - backlinks = data['query'].get('backlinks') - if backlinks: - if 'backlinks' in self.data: - self.data['backlinks'].extend(backlinks) - else: - self.data['backlinks'] = backlinks + self._extend_data('backlinks', data['query'].get('backlinks')) self.data['redirected'] = data['query'].get('redirects') @@ -325,18 +334,12 @@ if length: self.data['length'] = length - self.data['links'] = utils.get_links(page.get('links')) + self._extend_data('links', utils.get_links(page.get('links'))) - modified = page.get('touched') - if modified: - if 'modified' in self.data: - self.data['modified'].update({'page': modified}) - else: - self.data['modified'] = {'page': modified} + self._update_data('modified', 'page', page.get('touched')) pageprops = page.get('pageprops') if pageprops: - wikibase = pageprops.get('wikibase_item') if wikibase: self.data['wikibase'] = wikibase @@ -445,6 +448,16 @@ self.data.update({'pageid': pageid, 'title': title}) + def _update_data(self, datapoint, key, new_data): + """ + update or assign new data to datapoint + """ + if new_data: + try: + self.data[datapoint].update({key: new_data}) + except KeyError: + self.data[datapoint] = {key: new_data} + def _update_imageinfo(self): """ calls get_imageinfo() if data image missing info @@ -467,6 +480,14 @@ if self.data.get('wikibase'): self.params['wikibase'] = self.data.get('wikibase') + def skip_action(self, action): + """ + append action to skip flag + """ + if 'skip' not in self.flags: + self.flags['skip'] = [] + self.flags['skip'].append(action) + def get(self, show=True, proxy=None, timeout=0): """ Make Mediawiki, RESTBase, and Wikidata requests for page data @@ -501,14 +522,16 @@ self.get_parse(False, proxy, timeout) if not self.data.get('wikibase'): - if 'skip' not in self.flags: - self.flags['skip'] = [] - self.flags['skip'].append('wikidata') + self.skip_action('wikidata') self.get_wikidata(False, proxy, timeout) self.flags['defer_imageinfo'] = False + wiki = self.params.get('wiki') + if wiki and 'wikipedia.org' not in wiki: + self.skip_action('restbase') + self.get_restbase('/page/summary/', False, proxy, timeout) if show and not self.flags.get('silent'): @@ -614,6 +637,9 @@ self._get('query', show, proxy, timeout) + while self.data.get('continue'): + self._get('query', show, proxy, timeout) + return self def get_querymore(self, show=True, proxy=None, timeout=0): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/query.py new/wptools-0.4.17/wptools/query.py --- old/wptools-0.4.15/wptools/query.py 2018-05-23 04:59:43.000000000 +0200 +++ new/wptools-0.4.17/wptools/query.py 2018-08-16 19:06:32.000000000 +0200 @@ -30,11 +30,12 @@ WPToolsQuery class """ + DEFAULT_ENDPOINT = '/w/api.php' MAXWIDTH = 72 RPAD = 4 IMAGEINFO = Template(( - "${WIKI}/w/api.php?action=query" + "${WIKI}${ENDPOINT}?action=query" "&format=json" "&formatversion=2" "&iiprop=size|url|timestamp|extmetadata" @@ -42,13 +43,13 @@ "&titles=${FILES}")) LIST = Template(( - "${WIKI}/w/api.php?action=query" + "${WIKI}${ENDPOINT}?action=query" "&format=json" "&formatversion=2" "&list=${LIST}")) PARSE = Template(( - "${WIKI}/w/api.php?action=parse" + "${WIKI}${ENDPOINT}?action=parse" "&format=json" "&formatversion=2" "&contentmodel=text" @@ -60,7 +61,7 @@ "&page=${PAGE}")) QUERY = Template(( - "${WIKI}/w/api.php?action=query" + "${WIKI}${ENDPOINT}?action=query" "&exintro" "&format=json" "&formatversion=2" @@ -78,7 +79,7 @@ "&titles=${TITLES}")) QUERYMORE = Template(( - "${WIKI}/w/api.php?action=query" + "${WIKI}${ENDPOINT}?action=query" "&bllimit=500" "&bltitle=${TITLES}" "&cllimit=500" @@ -94,19 +95,20 @@ "&titles=${TITLES}")) WIKIDATA = Template(( - "${WIKI}/w/api.php?action=wbgetentities" + "${WIKI}${ENDPOINT}?action=wbgetentities" "&format=json" "&formatversion=2" "&languages=${LANG}" "&props=${PROPS}" "&redirects=yes")) + endpoint = None lang = None status = None variant = None wiki = None - def __init__(self, lang='en', variant=None, wiki=None): + def __init__(self, lang='en', variant=None, wiki=None, endpoint=None): """ Returns a WPToolsQuery object @@ -114,19 +116,24 @@ - [lang=en]: <str> Mediawiki language code - [variant=None]: <str> language variant - [wiki=None]: <str> alternative wiki site + - [endpoint=None]: <str> alternative API endoint """ self.lang = lang self.variant = variant self.wiki = wiki or "%s.wikipedia.org" % self.lang self.domain = domain_name(self.wiki) + self.endpoint = endpoint or self.DEFAULT_ENDPOINT self.uri = self.wiki_uri(self.wiki) def category(self, title, pageid=None, cparams=None, namespace=None): """ Returns category query string """ - query = self.LIST.substitute(WIKI=self.uri, LIST='categorymembers') + query = self.LIST.substitute( + WIKI=self.uri, + ENDPOINT=self.endpoint, + LIST='categorymembers') status = pageid or title query += "&cmlimit=500" @@ -163,6 +170,7 @@ query = self.WIKIDATA.substitute( WIKI=self.uri, + ENDPOINT=self.endpoint, LANG=self.variant or self.lang, PROPS='labels') @@ -181,14 +189,19 @@ self.set_status('imageinfo', files) - return self.IMAGEINFO.substitute(WIKI=self.uri, FILES=files) + return self.IMAGEINFO.substitute( + WIKI=self.uri, + ENDPOINT=self.endpoint, + FILES=files) def parse(self, title, pageid=None): """ Returns Mediawiki action=parse query string """ - qry = self.PARSE.substitute(WIKI=self.uri, - PAGE=safequote(title) or pageid) + qry = self.PARSE.substitute( + WIKI=self.uri, + ENDPOINT=self.endpoint, + PAGE=safequote(title) or pageid) if pageid and not title: qry = qry.replace('&page=', '&pageid=').replace('&redirects', '') @@ -200,17 +213,23 @@ return qry - def query(self, titles, pageids=None): + def query(self, titles, pageids=None, cparams=None): """ Returns MediaWiki action=query query string """ - query = self.QUERY.substitute(WIKI=self.uri, - TITLES=safequote(titles) or pageids) + query = self.QUERY.substitute( + WIKI=self.uri, + ENDPOINT=self.endpoint, + TITLES=safequote(titles) or pageids) status = titles or pageids if pageids and not titles: query = query.replace('&titles=', '&pageids=') + if cparams: + query += cparams + status += " (%s)" % cparams + if self.variant: query += '&variant=' + self.variant @@ -225,6 +244,7 @@ """ query = self.QUERYMORE.substitute( WIKI=self.uri, + ENDPOINT=self.endpoint, TITLES=safequote(titles) or pageids) status = "%s" % (pageids or titles) @@ -247,7 +267,10 @@ """ Returns query string for random page """ - query = self.LIST.substitute(WIKI=self.uri, LIST='random') + query = self.LIST.substitute( + WIKI=self.uri, + ENDPOINT=self.endpoint, + LIST='random') query += "&rnlimit=1&rnnamespace=%d" % namespace emoji = [ @@ -309,28 +332,31 @@ """ query = None viewdays = 7 + hostpath = self.uri + self.endpoint if action == 'siteinfo': - query = self.uri + ('/w/api.php?action=query' - '&meta=siteinfo|siteviews' - '&siprop=general|statistics' - '&list=mostviewed&pvimlimit=max') + query = hostpath + ( + '?action=query' + '&meta=siteinfo|siteviews' + '&siprop=general|statistics' + '&list=mostviewed&pvimlimit=max') query += '&pvisdays=%d' % viewdays # meta=siteviews self.set_status('query', 'siteinfo|siteviews|mostviewed') elif action == 'sitematrix': - query = self.uri + '/w/api.php?action=sitematrix' + query = hostpath + '?action=sitematrix' self.set_status('sitematrix', 'all') elif action == 'sitevisitors': - query = self.uri + ('/w/api.php?action=query' - '&meta=siteviews&pvismetric=uniques') + query = hostpath + ( + '?action=query' + '&meta=siteviews&pvismetric=uniques') query += '&pvisdays=%d' % viewdays # meta=siteviews self.set_status('query', 'siteviews:uniques') - query += '&format=json&formatversion=2' - if not query: raise ValueError("Could not form query") + query += '&format=json&formatversion=2' + return query def wiki_uri(self, wiki): @@ -350,6 +376,7 @@ query = self.WIKIDATA.substitute( WIKI=self.uri, + ENDPOINT=self.endpoint, LANG=self.variant or self.lang, PROPS="aliases|info|claims|descriptions|labels|sitelinks") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/request.py new/wptools-0.4.17/wptools/request.py --- old/wptools-0.4.15/wptools/request.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/wptools/request.py 2018-08-16 23:52:39.000000000 +0200 @@ -94,7 +94,17 @@ def curl_setup(self, proxy=None, timeout=0): """ - set curl options + Set curl options + - http://pycurl.io/docs/latest/ + - https://curl.haxx.se/libcurl/c/curl_easy_setopt.html + + Proxy options (str or dict): + proxy = 'https://example.com:80' # pycurl.PROXY + proxy = { + 'PROXY': <str> host, # pycurl.PROXY + 'PORT': <int> port, # pycurl.PROXYPORT + 'USERPWD': <str> user:pwd, # pycurl.PROXYUSERPWD + } """ crl = pycurl.Curl() @@ -102,8 +112,16 @@ crl.setopt(pycurl.FOLLOWLOCATION, True) crl.setopt(pycurl.CAINFO, certifi.where()) - if proxy: + if isinstance(proxy, str): crl.setopt(pycurl.PROXY, proxy) + if isinstance(proxy, dict): + if proxy.get('PROXY'): + crl.setopt(pycurl.PROXY, proxy['PROXY']) + if proxy.get('PORT'): + crl.setopt(pycurl.PROXYPORT, proxy['PORT']) + if proxy.get('USERPWD'): + crl.setopt(pycurl.PROXYUSERPWD, proxy['USERPWD']) + if timeout: # 0 = wait forever crl.setopt(pycurl.CONNECTTIMEOUT, timeout) crl.setopt(pycurl.TIMEOUT, timeout) @@ -120,9 +138,16 @@ kbps = crl.getinfo(crl.SPEED_DOWNLOAD) / 1000.0 url = crl.getinfo(crl.EFFECTIVE_URL) url = url.replace("&format=json", '').replace("&formatversion=2", '') + + # https://travis-ci.org/siznax/wptools/jobs/401109410 + try: + content_type = crl.getinfo(crl.CONTENT_TYPE) + except TypeError: + content_type = None + return {"url": url, "user-agent": user_agent(), - "content": crl.getinfo(crl.CONTENT_TYPE), + "content-type": content_type, "status": crl.getinfo(crl.RESPONSE_CODE), "bytes": crl.getinfo(crl.SIZE_DOWNLOAD), "seconds": "%5.3f" % crl.getinfo(crl.TOTAL_TIME), diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/restbase.py new/wptools-0.4.17/wptools/restbase.py --- old/wptools-0.4.15/wptools/restbase.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/wptools/restbase.py 2018-07-09 01:29:11.000000000 +0200 @@ -42,7 +42,7 @@ """ returns RESTBase response if appropriate """ - content = self.cache['restbase']['info']['content'] + content = self.cache['restbase']['info']['content-type'] if content.startswith('text/html'): html = self.cache['restbase']['response'] if isinstance(html, bytes): @@ -68,7 +68,8 @@ """ returns WPToolsQuery string from action """ - return qobj.restbase(self.params['endpoint'], self.params.get('title')) + return qobj.restbase(self.params['rest_endpoint'], + self.params.get('title')) def _set_data(self, action): """ @@ -170,7 +171,6 @@ Optional {params}: - [title]: <str> Mediawiki page title, file, category, etc. - - [endpoint]: the RESTBase entry point (default=/page/) - [lang]: <str> Mediawiki language code (default=en) Optional arguments: @@ -195,7 +195,7 @@ if endpoint != '/page/' and not self.params.get('title'): raise StandardError("endpoint %s needs a title" % endpoint) - self.params.update({'endpoint': endpoint}) + self.params.update({'rest_endpoint': endpoint}) self._get('restbase', show, proxy, timeout) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools/site.py new/wptools-0.4.17/wptools/site.py --- old/wptools-0.4.15/wptools/site.py 2018-02-06 23:25:53.000000000 +0100 +++ new/wptools-0.4.17/wptools/site.py 2018-07-07 02:41:15.000000000 +0200 @@ -26,6 +26,7 @@ Returns a WPToolsSite object. Optional keyword {params}: + - [endpoint]: <str> alternative API endpoint (default=/w/api.php) - [lang]: <str> Mediawiki language code (default=en) - [wiki]: <str> alternative wiki site (default=wikipedia.org) @@ -36,6 +37,10 @@ """ super(WPToolsSite, self).__init__(*args, **kwargs) + endpoint = kwargs.get('endpoint') + if endpoint: + self.params.update({'endpoint': endpoint}) + def _query(self, action, qobj): """ returns query string diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools.egg-info/PKG-INFO new/wptools-0.4.17/wptools.egg-info/PKG-INFO --- old/wptools-0.4.15/wptools.egg-info/PKG-INFO 2018-05-30 22:42:40.000000000 +0200 +++ new/wptools-0.4.17/wptools.egg-info/PKG-INFO 2018-08-17 01:58:46.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: wptools -Version: 0.4.15 +Version: 0.4.17 Summary: Wikipedia tools (for Humans) Home-page: https://github.com/siznax/wptools/ Author: Steve @siznax @@ -295,9 +295,12 @@ Release History --------------- - 0.4.15 (2018-05-30) + 0.4.17 (2018-08-16) +++++++++++++++++++ + * Support proxy authentication (#130) + * Support page links continuations (#129) + * Support alternate API endpoints (#127) * Merged @mcepl update to package tests (#125) * Put API warnings in data attribute (#120) * Support getting backlinks w/continuations (#119, #122) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/wptools-0.4.15/wptools.egg-info/SOURCES.txt new/wptools-0.4.17/wptools.egg-info/SOURCES.txt --- old/wptools-0.4.15/wptools.egg-info/SOURCES.txt 2018-05-30 22:42:40.000000000 +0200 +++ new/wptools-0.4.17/wptools.egg-info/SOURCES.txt 2018-08-17 01:58:46.000000000 +0200 @@ -23,6 +23,7 @@ tests/parse_91.py tests/parse_infobox.py tests/query.py +tests/query_plcontinue.py tests/querymore.py tests/querymore_blcontinue.py tests/random_query.py
![https://upload.wikimedia.org/wikipedia/commons/thumb/c/c0/Douglas_adams_portrait_cropped.jpg/276px-Douglas_adams_portrait_cropped.jpg","width":276,"height":320,"original":"http://upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg"},"originalimage":{"source":"https://upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg","width":333,"height":386},"lang":"en","dir":"ltr","timestamp":"2017-08-03T16:37:48Z","description":"English](https://upload.wikimedia.org/wikipedia/commons/thumb/c/c0/Douglas_adams_portrait_cropped.jpg/276px-Douglas_adams_portrait_cropped.jpg","width":276,"height":320,"original":"http://upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg"},"originalimage":{"source":"https://upload.wikimedia.org/wikipedia/commons/c/c0/Douglas_adams_portrait_cropped.jpg","width":333,"height":386},"lang":"en","dir":"ltr","timestamp":"2017-08-03T16:37:48Z","description":"English){kind=link}