jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1358142?usp=email )
Change subject: solve_disambiguation: Stream referring pages lazily
......................................................................
solve_disambiguation: Stream referring pages lazily
Collecting all backlinks before yielding delays processing and retains
all referring pages in memory for heavily linked disambiguation pages.
Change-Id: Ie77699e82c3621a3b954004ef6d3386e97bd4cac
---
M scripts/solve_disambiguation.py
M tests/solve_disambiguation_tests.py
2 files changed, 86 insertions(+), 21 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/scripts/solve_disambiguation.py b/scripts/solve_disambiguation.py
index c60a061..6f9f676 100755
--- a/scripts/solve_disambiguation.py
+++ b/scripts/solve_disambiguation.py
@@ -82,7 +82,7 @@
import re
from collections.abc import Generator
from contextlib import suppress
-from itertools import chain
+from itertools import chain, islice
from pathlib import Path
import pywikibot
@@ -411,28 +411,31 @@
self.main_only = main_only
def __iter__(self) -> Generator[pywikibot.Page]:
- """Yield pages."""
- # TODO: start yielding before all referring pages have been found
- refs = list(self.page.getReferences(with_template_inclusion=False,
- namespaces=0 if self.main_only
- else None))
- pywikibot.info(f'Found {len(refs)} references.')
- # Remove ignorables
+ """Yield eligible pages, buffering only enough to check the minimum."""
site = self.page.site
- if site.family.name in ignore_title \
- and site.lang in ignore_title[site.family.name]:
- for ig in ignore_title[site.family.name][site.lang]:
- for i in range(len(refs) - 1, -1, -1):
- if re.match(ig, refs[i].title()):
- pywikibot.log('Ignoring page ' + refs[i].title())
- del refs[i]
- elif self.primaryIgnoreManager.isIgnored(refs[i]):
- del refs[i]
- if len(refs) < self.minimum:
+ patterns = ignore_title.get(site.family.name, {}).get(site.lang, [])
+
+ def eligible_pages():
+ for ref in self.page.getReferences(
+ with_template_inclusion=False,
+ namespaces=0 if self.main_only else None):
+ for pattern in patterns:
+ if re.match(pattern, ref.title()):
+ pywikibot.log('Ignoring page ' + ref.title())
+ break
+ else:
+ if not self.primaryIgnoreManager.isIgnored(ref):
+ yield ref
+
+ refs = eligible_pages()
+ buffered = list(islice(refs, max(0, self.minimum)))
+ if len(buffered) < self.minimum:
pywikibot.info(
- f'Found only {len(refs)} pages to work on; skipping.')
+ f'Found only {len(buffered)} pages to work on; skipping.')
return
- pywikibot.info(f'Will work on {len(refs)} pages.')
+ pywikibot.info('Working on referring pages.')
+ yield from buffered
+ del buffered
yield from refs
diff --git a/tests/solve_disambiguation_tests.py
b/tests/solve_disambiguation_tests.py
index 1afb19d..fff4b55 100755
--- a/tests/solve_disambiguation_tests.py
+++ b/tests/solve_disambiguation_tests.py
@@ -12,10 +12,72 @@
from unittest.mock import patch
import pywikibot
-from scripts.solve_disambiguation import DisambiguationRobot
+from scripts.solve_disambiguation import (
+ DisambiguationRobot,
+ ReferringPageGeneratorWithIgnore,
+)
from tests.aspects import TestCase
+class TestReferringPageGenerator(TestCase):
+
+ """Test streaming and the minimum eligible reference count."""
+
+ family = 'wikipedia'
+ code = 'de'
+ dry = True
+
+ def test_streaming(self) -> None:
+ """Yield the first reference without consuming the remaining ones."""
+ page = pywikibot.Page(self.site, 'Disambiguation')
+ ignored = pywikibot.Page(self.site, 'Primary')
+ ref = pywikibot.Page(self.site, 'Article')
+
+ def references():
+ yield ignored
+ yield ref
+ self.fail(
+ 'References were consumed before yielding the first page')
+
+ with patch('scripts.solve_disambiguation.ignore_title', {}), \
+ patch.object(page, 'getReferences',
+ return_value=references()) as get_refs, \
+ patch('scripts.solve_disambiguation.PrimaryIgnoreManager'
+ '.isIgnored', side_effect=lambda page: page == ignored):
+ gen = iter(ReferringPageGeneratorWithIgnore(page, main_only=True))
+ self.assertEqual(next(gen), ref)
+ get_refs.assert_called_once_with(with_template_inclusion=False,
+ namespaces=0)
+ gen.close()
+
+ def test_minimum_after_filtering(self) -> None:
+ """Buffer eligible pages only, and skip if the minimum is unmet."""
+ page = pywikibot.Page(self.site, 'Disambiguation')
+ refs = [
+ pywikibot.Page(self.site, title)
+ for title in ('Ignored', 'Primary', 'First', 'Second', 'Third')]
+ consumed = []
+
+ def references():
+ for ref in refs:
+ consumed.append(ref)
+ yield ref
+
+ with patch('scripts.solve_disambiguation.ignore_title',
+ {'wikipedia': {'de': ['Ignored']}}), \
+ patch.object(page, 'getReferences',
+ side_effect=lambda **kwargs: references()), \
+ patch('scripts.solve_disambiguation.PrimaryIgnoreManager'
+ '.isIgnored', side_effect=lambda ref: ref == refs[1]):
+ gen = iter(ReferringPageGeneratorWithIgnore(page, minimum=2))
+ self.assertEqual(next(gen), refs[2])
+ self.assertEqual(consumed, refs[:4])
+ self.assertEqual(list(gen), refs[3:])
+
+ self.assertEqual(
+ list(ReferringPageGeneratorWithIgnore(page, minimum=4)), [])
+
+
class TestDisambiguationRobot(TestCase):
"""Test the disambiguation bot."""
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1358142?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ie77699e82c3621a3b954004ef6d3386e97bd4cac
Gerrit-Change-Number: 1358142
Gerrit-PatchSet: 6
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
