Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1201010?usp=email )
Change subject: [proofreadpage] Cache page quality and existence in
_get_page_mappings
......................................................................
[proofreadpage] Cache page quality and existence in _get_page_mappings
Enhance _get_page_mappings() to store page quality levels (QL) and
existence information for each page linked to the index.
Changes:
* Add internal dicts _qls and _existences to cache page quality and existence.
* Set quality=1 and existence=False for non-existing pages.
* Extract quality level from 'qualityN' class for existing pages.
* reuse cached values for generators
* check_if_cached for IndexPage.page_gen
These additions allow downstream methods to access page quality and
existence status without re-parsing the index page and avoid additional
API requests. In benchmarks, page_gen() is about 20 times faster.
Patch submitted by Ignacio RodrÃguez (Ninovolador)
Bug: T409023
Change-Id: Ibe148a998938c87db1bfad3d9fdc58be81318103
---
M pywikibot/proofreadpage.py
1 file changed, 38 insertions(+), 9 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Verified; Looks good to me, approved
diff --git a/pywikibot/proofreadpage.py b/pywikibot/proofreadpage.py
index 99ba949..0d0bf9d 100644
--- a/pywikibot/proofreadpage.py
+++ b/pywikibot/proofreadpage.py
@@ -1182,7 +1182,13 @@
assert 'purged' in rawdata['purge'][0], error_message
def _get_page_mappings(self) -> None:
- """Associate label and number for each page linked to the index."""
+ """Associate label and number for each page linked to the index.
+
+ .. version-changed:: 11.4
+ Added internal caching of page quality levels (``_qls``)
+ and existence flags (``_existences``) for each page linked
+ to the index.
+ """
# Clean cache, if any.
self._page_from_numbers = {}
self._numbers_from_page: dict[pywikibot.page.Page, int] = {}
@@ -1190,6 +1196,8 @@
self._pages_from_label: dict[str, set[pywikibot.Page]] = {}
self._labels_from_page_number: dict[int, str] = {}
self._labels_from_page: dict[pywikibot.page.Page, str] = {}
+ self._qls: dict[int, str] = {}
+ self._existences: dict[int, bool] = {}
self._soup = _bs4_soup(self.get_parsed_page(True))
# Do not search for "new" here, to avoid to skip purging if links
# to non-existing pages are present.
@@ -1229,7 +1237,8 @@
class_ = a_tag.get('class')
href = a_tag.get('href')
- if 'new' in class_:
+ new = 'new' in class_
+ if new:
title = self._parse_redlink(href) # non-existing page
if title is None: # title not conforming to required format
continue
@@ -1267,6 +1276,15 @@
label, set()).add(page_cnt)
self._pages_from_label.setdefault(label, set()).add(page)
+ # Existences and QLs
+ self._existences[page_cnt] = not new
+ if new:
+ self._qls[page_cnt] = ProofreadPage.NOT_PROOFREAD
+ else:
+ for clss in class_:
+ if m := re.search(r'quality(\d)', clss):
+ self._qls[page_cnt] = int(m[1])
+
# Sanity check: all links to Page: ns must have been considered.
assert (set(self._labels_from_page)
== set(self._all_page_links.values()))
@@ -1283,6 +1301,7 @@
"""
return len(self._page_from_numbers)
+ @check_if_cached
@remove_last_args(['content']) # since 9.0.0
def page_gen(
self, start: int = 1,
@@ -1296,11 +1315,14 @@
.. version-changed:: 9.0
The *content* parameter was removed
+ .. version-changed:: 11.4
+ Use cached quality levels and page existence information to
+ reduce additional API requests and improve performance.
:param start: First page, defaults to 1
:param end: Num_pages if end is None
- :param filter_ql: Filters quality levels
- If None: all but 'Without Text'.
+ :param filter_ql: Filters quality levels. If None: all but
+ 'Without Text'.
:param only_existing: Yields only existing pages.
"""
if end is None:
@@ -1312,17 +1334,24 @@
# All but 'Without Text'
if filter_ql is None:
- filter_ql = list(self.site.proofread_levels)
+ filter_ql = set(self.site.proofread_levels)
filter_ql.remove(ProofreadPage.WITHOUT_TEXT)
+ else:
+ filter_ql = set(filter_ql)
- gen = (self.get_page(i) for i in range(start, end + 1))
+ gen = self.site.preloadpages(
+ self.get_page(i) for i in range(start, end + 1))
- gen = self.site.preloadpages(gen)
# Filter by QL.
- gen = (p for p in gen if p.ql in filter_ql)
+ gen = (
+ p for p in gen
+ if (number := self._numbers_from_page.get(p)) in self._qls
+ and self._qls[number] in filter_ql
+ )
# Yield only existing.
if only_existing:
- gen = (p for p in gen if p.exists())
+ gen = (p for p in gen
+ if self._existences.get(self._numbers_from_page[p]))
return gen
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1201010?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ibe148a998938c87db1bfad3d9fdc58be81318103
Gerrit-Change-Number: 1201010
Gerrit-PatchSet: 5
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
Gerrit-CC: Ignacio RodrÃguez <[email protected]>
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
