Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1353445?usp=email )
Change subject: Cache site language links and their removal regex
......................................................................
Cache site language links and their removal regex
Reuse sorted language codes across calls to validLanguageLinks(), while
returning a fresh list and refreshing the cache when namespaces change.
Cache the language-link removal regex by its code tuple with a bounded
cache. Preserve public method overrides and use set membership when
applying interwiki sort priorities.
Change-Id: I2015a3f0fe098b73c188aecf9a85c9e19b83e8de
---
M pywikibot/site/_basesite.py
M pywikibot/textlib.py
M tests/textlib_tests.py
3 files changed, 46 insertions(+), 7 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/pywikibot/site/_basesite.py b/pywikibot/site/_basesite.py
index 908b6e6..5096e66 100644
--- a/pywikibot/site/_basesite.py
+++ b/pywikibot/site/_basesite.py
@@ -276,8 +276,15 @@
def validLanguageLinks(self): # noqa: N802
"""Return list of language codes to be used in interwiki links."""
- return [lang for lang in sorted(self.codes)
- if self.namespaces.lookup_normalized_name(lang) is None]
+ namespaces = self.namespaces
+ cached_links = getattr(self, '_valid_language_links', None)
+ if cached_links is None or cached_links[0] is not namespaces:
+ links = tuple(
+ lang for lang in sorted(self.codes)
+ if namespaces.lookup_normalized_name(lang) is None)
+ cached_links = (namespaces, links)
+ self._valid_language_links = cached_links
+ return list(cached_links[1])
def _interwiki_urls(self, only_article_suffixes: bool = False):
base_path = self.path()
diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py
index 2b6fe7c..44cbcae 100644
--- a/pywikibot/textlib.py
+++ b/pywikibot/textlib.py
@@ -13,6 +13,7 @@
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from contextlib import closing, suppress
from dataclasses import dataclass
+from functools import lru_cache
from html.parser import HTMLParser
from typing import NamedTuple
from urllib.parse import quote
@@ -1475,6 +1476,14 @@
return result
+@lru_cache
+def _language_links_regex(languages: tuple[str, ...]) -> re.Pattern[str]:
+ """Cache the removal regex for a site's language codes."""
+ codes = '|'.join(languages)
+ return re.compile(rf'\[\[({codes})\s?:[^\[\]\n]*\]\][\s]*',
+ re.IGNORECASE)
+
+
def removeLanguageLinks(text: str, site=None, marker: str = '') -> str:
"""Return text with all inter-language links removed.
@@ -1492,12 +1501,11 @@
site = pywikibot.Site()
# This regular expression will find every interwiki link, plus trailing
# whitespace.
- languages = '|'.join(site.validLanguageLinks()
- + list(site.family.obsolete))
+ languages = (tuple(site.validLanguageLinks())
+ + tuple(site.family.obsolete))
if not languages:
return text
- interwikiR = re.compile(rf'\[\[({languages})\s?:[^\[\]\n]*\]\][\s]*',
- re.IGNORECASE)
+ interwikiR = _language_links_regex(languages)
text = replaceExcept(text, interwikiR, '',
['comment', 'math', 'nowiki', 'pre',
'syntaxhighlight'],
@@ -1701,7 +1709,7 @@
if putfirst:
# In this case I might have to change the order
firstsites = []
- validlanglinks = insite.validLanguageLinks()
+ validlanglinks = frozenset(insite.validLanguageLinks())
for code in putfirst:
if code in validlanglinks:
site = insite.getSite(code=code)
diff --git a/tests/textlib_tests.py b/tests/textlib_tests.py
index 127fa0e..2e0a127 100755
--- a/tests/textlib_tests.py
+++ b/tests/textlib_tests.py
@@ -1568,6 +1568,30 @@
self._template_not_case_sensitive)
+class TestRemoveLanguageLinks(TestCase):
+
+ """Test language link removal with cached language data."""
+
+ net = False
+
+ def test_namespace_replacement(self) -> None:
+ """Refresh language data when the namespace mapping is replaced."""
+ site = pywikibot.site.BaseSite('en', 'wikipedia')
+ retained = '<nowiki>[[yo:Hidden]]</nowiki>\n[[:yo:Inline]]'
+ text = retained + '\n[[yo:Page]]'
+ self.assertEqual(textlib.removeLanguageLinks(text, site), retained)
+
+ links = site.validLanguageLinks()
+ links.remove('yo')
+ self.assertIn('yo', site.validLanguageLinks())
+
+ namespaces = pywikibot.site.Namespace.builtin_namespaces()
+ namespaces[100] = pywikibot.site.Namespace(100, 'Yo')
+ site._namespaces = pywikibot.site.NamespacesDict(namespaces)
+ self.assertNotIn('yo', site.validLanguageLinks())
+ self.assertEqual(textlib.removeLanguageLinks(text, site), text)
+
+
class TestGetLanguageLinks(SiteAttributeTestCase):
"""Test :py:obj:`textlib.getLanguageLinks` function."""
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1353445?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I2015a3f0fe098b73c188aecf9a85c9e19b83e8de
Gerrit-Change-Number: 1353445
Gerrit-PatchSet: 2
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
