jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1343146?usp=email )
Change subject: pagegenerators: Cache compiled ignore patterns
......................................................................
pagegenerators: Cache compiled ignore patterns
Compile page-title ignore patterns lazily once per site name.
Reuse the compiled patterns for every later page from the same site.
Change-Id: If9cf34f29a6ed5becb02fbbe5b8cef3d90a609f2
---
M pywikibot/pagegenerators/_filters.py
M tests/pagegenerators_tests.py
2 files changed, 35 insertions(+), 6 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators/_filters.py
b/pywikibot/pagegenerators/_filters.py
index 7163d94..87aa4b6 100644
--- a/pywikibot/pagegenerators/_filters.py
+++ b/pywikibot/pagegenerators/_filters.py
@@ -87,23 +87,32 @@
def PageTitleFilterPageGenerator(
generator: Iterable[pywikibot.page.BasePage],
- ignore_list: dict[str, dict[str, str]],
+ ignore_list: dict[str, dict[str, Sequence[str]]],
) -> Generator[pywikibot.page.BasePage]:
"""Yield only those pages are not listed in the ignore list.
:param ignore_list: Family names are mapped to dictionaries in which
language codes are mapped to lists of page titles. Each title must
- be a valid regex as they are compared using :py:obj:`re.search`.
+ be a valid regex.
"""
+ compiled_ignores: dict[str, tuple[re.Pattern[str], ...]] = {}
for page in generator:
+ site = page.site
+ sitename = site.sitename
try:
- site_ig_list = ignore_list[page.site.family.name][page.site.code]
+ compiled_site_ignores = compiled_ignores[sitename]
except KeyError:
- yield page
- continue
+ try:
+ site_ignores = ignore_list[site.family.name][site.code]
+ except KeyError:
+ yield page
+ continue
+ compiled_site_ignores = tuple(map(re.compile, site_ignores))
+ compiled_ignores[sitename] = compiled_site_ignores
page_title = page.title()
- if not any(re.search(ig, page_title) for ig in site_ig_list):
+ if not any(pattern.search(page_title)
+ for pattern in compiled_site_ignores):
yield page
continue
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 08ff461..f4b64be 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -8,6 +8,7 @@
import calendar
import datetime
+import re
import sys
import unittest
from collections import Counter
@@ -159,6 +160,25 @@
gen, ('Talk', 'Template'), site)
self.assertLength(tuple(gen), 10)
+ def test_PageTitleFilterPageGenerator_compiles_once(self) -> None:
+ """Test ignore patterns are compiled once per site."""
+ patterns = ['^Talk:', '/Meta$']
+ ignore_list = {'wikipedia': {'en': patterns}}
+ pages = list(pagegenerators.PagesFromTitlesGenerator(
+ self.titles, self.site))
+ expected = [page for page in pages
+ if not page.title().startswith('Talk:')
+ and not page.title().endswith('/Meta')]
+
+ with mock.patch(
+ 'pywikibot.pagegenerators._filters.re.compile', wraps=re.compile
+ ) as compile_pattern:
+ result = list(pagegenerators.PageTitleFilterPageGenerator(
+ pages, ignore_list))
+
+ self.assertEqual(result, expected)
+ self.assertEqual(compile_pattern.call_count, len(patterns))
+
def test_category_filter_enumerates_once(self) -> None:
"""Test that categories are enumerated once for multiple filters."""
categories = [
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1343146?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: If9cf34f29a6ed5becb02fbbe5b8cef3d90a609f2
Gerrit-Change-Number: 1343146
Gerrit-PatchSet: 5
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
