jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1328373?usp=email )
Change subject: pagegenerators: Traverse categories once
......................................................................
pagegenerators: Traverse categories once
Each requested filter creates a fresh page category iterator. This can
repeat API queries for a single page.
Track unmatched categories during one traversal and stop as soon as all
filters match.
Change-Id: I7600e772555d84f9300f432b64394707ead4bfd3
---
M pywikibot/pagegenerators/_filters.py
M tests/pagegenerators_tests.py
2 files changed, 37 insertions(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/pagegenerators/_filters.py
b/pywikibot/pagegenerators/_filters.py
index 2bfbf0a..190bb13 100644
--- a/pywikibot/pagegenerators/_filters.py
+++ b/pywikibot/pagegenerators/_filters.py
@@ -349,9 +349,18 @@
:param generator: A generator object
:param category_list: Categories used to filter generated pages
"""
+ required_categories: set[pywikibot.page.BasePage] = set(category_list)
for page in generator:
- if all(x in page.categories() for x in category_list):
+ remaining_categories = required_categories.copy()
+ if not remaining_categories:
yield page
+ continue
+
+ for category in page.categories():
+ remaining_categories.discard(category)
+ if not remaining_categories:
+ yield page
+ break
# name the generator methods
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index d0146a1..e4d6f5a 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -140,6 +140,33 @@
gen, ('Talk', 'Template'), site)
self.assertLength(tuple(gen), 10)
+ def test_category_filter_enumerates_once(self) -> None:
+ """Test that categories are enumerated once for multiple filters."""
+ categories = [
+ pywikibot.Category(self.site, f'Category:{name}')
+ for name in ('First', 'Second')
+ ]
+ matching_page = mock.Mock()
+ matching_page.categories.side_effect = lambda: iter(categories)
+ missing_page = mock.Mock()
+ missing_page.categories.side_effect = lambda: iter(categories[:1])
+
+ pages = list(pagegenerators.CategoryFilterPageGenerator(
+ [matching_page, missing_page], categories))
+
+ self.assertLength(pages, 1)
+ self.assertIs(pages[0], matching_page)
+ matching_page.categories.assert_called_once_with()
+ missing_page.categories.assert_called_once_with()
+
+ unfiltered_page = mock.Mock()
+ pages = list(pagegenerators.CategoryFilterPageGenerator(
+ [unfiltered_page], []))
+
+ self.assertLength(pages, 1)
+ self.assertIs(pages[0], unfiltered_page)
+ unfiltered_page.categories.assert_not_called()
+
def test_RegexFilterPageGenerator(self) -> None:
"""Test RegexFilterPageGenerator."""
self.assertFunction('RegexFilterPageGenerator')
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1328373?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I7600e772555d84f9300f432b64394707ead4bfd3
Gerrit-Change-Number: 1328373
Gerrit-PatchSet: 2
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
