jenkins-bot has submitted this change and it was merged.
Change subject: Performance improvement for interwiki.py
......................................................................
Performance improvement for interwiki.py
interwiki.py makes intensive use of 'pageprops' in order to detect
disambiguation pages. As of now, a separate request is issued for
each page being processed which slows down the processing very
significatly.
This change allows optionally requesting 'pageprops' in site.preloadpages
and modifies interwiki.py to make use of it.
Change-Id: I0b247bfb588caf67cf4799b6ee566e39552717ac
---
M pywikibot/site.py
M scripts/interwiki.py
2 files changed, 4 insertions(+), 2 deletions(-)
Approvals:
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 5e6dcc2..9656f6b 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -2964,7 +2964,7 @@
return page._redirtarget
def preloadpages(self, pagelist, groupsize=50, templates=False,
- langlinks=False):
+ langlinks=False, pageprops=False):
"""Return a generator to a list of preloaded pages.
Note that [at least in current implementation] pages may be iterated
@@ -2987,6 +2987,8 @@
props += '|templates'
if langlinks:
props += '|langlinks'
+ if pageprops:
+ props += '|pageprops'
rvgen = api.PropertyGenerator(props, site=self)
rvgen.set_maximum_items(-1) # suppress use of "rvlimit" parameter
if len(pageids) == len(sublist):
diff --git a/scripts/interwiki.py b/scripts/interwiki.py
index 258b7ca..950f8f1 100755
--- a/scripts/interwiki.py
+++ b/scripts/interwiki.py
@@ -2319,7 +2319,7 @@
pywikibot.output(u"NOTE: Nothing left to do 2")
return False
# Get the content of the assembled list in one blow
- gen = site.preloadpages(pageGroup, templates=True, langlinks=True)
+ gen = site.preloadpages(pageGroup, templates=True, langlinks=True,
pageprops=True)
for page in gen:
# we don't want to do anything with them now. The
# page contents will be read via the Subject class.
--
To view, visit https://gerrit.wikimedia.org/r/237130
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0b247bfb588caf67cf4799b6ee566e39552717ac
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Malafaya <[email protected]>
Gerrit-Reviewer: John Vandenberg <[email protected]>
Gerrit-Reviewer: Ladsgroup <[email protected]>
Gerrit-Reviewer: Malafaya <[email protected]>
Gerrit-Reviewer: Merlijn van Deen <[email protected]>
Gerrit-Reviewer: XZise <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
