jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1346493?usp=email )
Change subject: [IMPR] Unify negative page generator options
......................................................................
[IMPR] Unify negative page generator options
Allow using ':not' suffix for negative filter options (e.g.
-titleregex:not, -grep:not, -onlyif:not) to match -ns:not syntax,
while maintaining backward compatibility for legacy forms.
Bug: T219282
Change-Id: Ib64f014a9a7d8956cddec2ae81a7e64a0d71380d
---
M pywikibot/pagegenerators/__init__.py
M pywikibot/pagegenerators/_factory.py
M tests/pagegenerators_tests.py
3 files changed, 250 insertions(+), 5 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/pagegenerators/__init__.py
b/pywikibot/pagegenerators/__init__.py
index 7a2c9c0..9f66519 100644
--- a/pywikibot/pagegenerators/__init__.py
+++ b/pywikibot/pagegenerators/__init__.py
@@ -468,6 +468,18 @@
Case insensitive regular expressions will be used and
dot matches any character, including a newline.
+ You may use a preleading "not" to return the page only
+ if the regular expression does not match.
+
+ Examples::
+
+ -grep:not:regexpr
+
+ .. note:: If a regular expression pattern starts with
+ literal ``not:``, wrap it in a group (e.g.
+ ``-grep:(?:not:foo)``) to avoid triggering the ``:not``
+ negation modifier.
+
-grepnot Like -grep, but return the page only if the regular
expression does not match.
@@ -531,6 +543,13 @@
page will be returned only if all claims are present.
Argument can be also given as "-onlyif:expression".
+ You may use a preleading "not" to return the item only
+ if the claim does not match.
+
+ Examples::
+
+ -onlyif:not:expression
+
-onlyifnot A claim the page must not contain, otherwise the
item won't be returned. For usage and examples, see
`-onlyif` above.
@@ -558,6 +577,18 @@
Case insensitive regular expressions will be used and
dot matches any character.
+ You may use a preleading "not" to return the page only
+ if the regular expression does not match.
+
+ Examples::
+
+ -titleregex:not:regexpr
+
+ .. note:: If a regular expression pattern starts with
+ literal ``not:``, wrap it in a group (e.g.
+ ``-titleregex:(?:not:foo)``) to avoid triggering the
+ ``:not`` negation modifier.
+
-titleregexnot Like -titleregex, but return the page only if the regular
expression does not match.
""" # noqa: N816
diff --git a/pywikibot/pagegenerators/_factory.py
b/pywikibot/pagegenerators/_factory.py
index 31439be..a455a99 100644
--- a/pywikibot/pagegenerators/_factory.py
+++ b/pywikibot/pagegenerators/_factory.py
@@ -888,7 +888,11 @@
return True
def _handle_titleregexnot(self, value: str) -> Literal[True]:
- """Handle `-titleregexnot` argument."""
+ """Handle `-titleregexnot` and `-titleregex:not` argument.
+
+ .. version-changed:: 11.9
+ Support ``-titleregex:not`` syntax.
+ """
if not value:
value = pywikibot.input(
'All pages except which ones?')
@@ -903,7 +907,11 @@
return True
def _handle_grepnot(self, value: str) -> Literal[True]:
- """Handle `-grepnot` argument."""
+ """Handle `-grepnot` and `-grep:not` argument.
+
+ .. version-changed:: 11.9
+ Support ``-grep:not`` syntax.
+ """
if not value:
value = pywikibot.input('Which pattern do you want to skip?')
self.articlenotfilter_list.append(value)
@@ -930,7 +938,11 @@
return self._onlyif_onlyifnot_handler(value, False)
def _handle_onlyifnot(self, value: str) -> Literal[True]:
- """Handle `-onlyifnot` argument."""
+ """Handle `-onlyifnot` and `-onlyif:not` argument.
+
+ .. version-changed:: 11.9
+ Support ``-onlyif:not`` syntax.
+ """
return self._onlyif_onlyifnot_handler(value, True)
def _onlyif_onlyifnot_handler(self, value: str, ifnot: bool
@@ -1052,6 +1064,10 @@
specified.
+ .. version-changed:: 11.9
+ Support ``:not`` suffix for negative filter options (e.g.
+ ``-titleregex:not``, ``-grep:not``, ``-onlyif:not``).
+
:param arg: Pywikibot argument consisting of -name:value
:return: True if the argument supplied was recognised by the factory
"""
@@ -1067,10 +1083,19 @@
value = None
opt = arg[1:]
- if opt in self.disabled_options:
+ base_opt = opt
+ if value and (value == 'not' or value.startswith('not:')):
+ not_opt = opt + 'not'
+ if hasattr(self, '_handle_' + not_opt):
+ opt = not_opt
+ value = value[4:] if value.startswith('not:') else None
+
+ if opt in self.disabled_options or base_opt in self.disabled_options:
return False
- if self.enabled_options and opt not in self.enabled_options:
+ if (self.enabled_options
+ and opt not in self.enabled_options
+ and base_opt not in self.enabled_options):
return False
handler = getattr(self, '_handle_' + opt, None)
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 5f7707c..08ff461 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -969,6 +969,145 @@
self.assertFalse(gf.handle_arg('-ì'))
self.assertFalse(gf.handle_arg('ì'))
+ def test_titleregex_filter(self) -> None:
+ """Test -titleregex, -titleregex:not, and -titleregexnot arguments."""
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ self.assertTrue(gf.handle_arg('-titleregex:foo'))
+ self.assertEqual(gf.titlefilter_list, ['foo'])
+ self.assertTrue(gf.handle_arg('-titleregex:not:bar'))
+ self.assertEqual(gf.titlenotfilter_list, ['bar'])
+ self.assertTrue(gf.handle_arg('-titleregexnot:baz'))
+ self.assertEqual(gf.titlenotfilter_list, ['bar', 'baz'])
+ # Positive filter with pattern starting with literal 'not:'
+ self.assertTrue(gf._handle_titleregex('not:foo'))
+ self.assertEqual(gf.titlefilter_list, ['foo', 'not:foo'])
+ with mock.patch('pywikibot.input', return_value='not:prompted'):
+ self.assertTrue(gf.handle_arg('-titleregex'))
+ self.assertEqual(gf.titlefilter_list,
+ ['foo', 'not:foo', 'not:prompted'])
+ # Interactive -titleregex:not
+ with mock.patch('pywikibot.input', return_value='qux'):
+ self.assertTrue(gf.handle_arg('-titleregex:not'))
+ self.assertEqual(gf.titlenotfilter_list, ['bar', 'baz', 'qux'])
+
+ def test_grep_filter(self) -> None:
+ """Test -grep, -grep:not, and -grepnot arguments."""
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ self.assertTrue(gf.handle_arg('-grep:foo'))
+ self.assertEqual(gf.articlefilter_list, ['foo'])
+ self.assertTrue(gf.handle_arg('-grep:not:bar'))
+ self.assertEqual(gf.articlenotfilter_list, ['bar'])
+ self.assertTrue(gf.handle_arg('-grepnot:baz'))
+ self.assertEqual(gf.articlenotfilter_list, ['bar', 'baz'])
+ # Positive filter with pattern starting with literal 'not:'
+ self.assertTrue(gf._handle_grep('not:foo'))
+ self.assertEqual(gf.articlefilter_list, ['foo', 'not:foo'])
+ with mock.patch('pywikibot.input', return_value='not:prompted'):
+ self.assertTrue(gf.handle_arg('-grep'))
+ self.assertEqual(gf.articlefilter_list,
+ ['foo', 'not:foo', 'not:prompted'])
+ # Interactive -grep:not
+ with mock.patch('pywikibot.input', return_value='qux'):
+ self.assertTrue(gf.handle_arg('-grep:not'))
+ self.assertEqual(gf.articlenotfilter_list, ['bar', 'baz', 'qux'])
+
+ def test_onlyif_filter(self) -> None:
+ """Test -onlyif, -onlyif:not, and -onlyifnot arguments."""
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ self.assertTrue(gf.handle_arg('-onlyif:P1=Q2'))
+ self.assertEqual(gf.claimfilter_list, [('P1', 'Q2', {}, False)])
+ self.assertTrue(gf.handle_arg('-onlyif:not:P3=Q4'))
+ self.assertEqual(gf.claimfilter_list, [
+ ('P1', 'Q2', {}, False),
+ ('P3', 'Q4', {}, True),
+ ])
+ self.assertTrue(gf.handle_arg('-onlyifnot:P5=Q6'))
+ self.assertEqual(gf.claimfilter_list, [
+ ('P1', 'Q2', {}, False),
+ ('P3', 'Q4', {}, True),
+ ('P5', 'Q6', {}, True),
+ ])
+ # Positive filter with claim starting with literal 'not:'
+ self.assertTrue(gf._handle_onlyif('not:P7=Q8'))
+ self.assertEqual(gf.claimfilter_list, [
+ ('P1', 'Q2', {}, False),
+ ('P3', 'Q4', {}, True),
+ ('P5', 'Q6', {}, True),
+ ('not:P7', 'Q8', {}, False),
+ ])
+ # Interactive -onlyif:not
+ with mock.patch('pywikibot.input', return_value='P9=Q10'):
+ self.assertTrue(gf.handle_arg('-onlyif:not'))
+ self.assertEqual(gf.claimfilter_list, [
+ ('P1', 'Q2', {}, False),
+ ('P3', 'Q4', {}, True),
+ ('P5', 'Q6', {}, True),
+ ('not:P7', 'Q8', {}, False),
+ ('P9', 'Q10', {}, True),
+ ])
+
+ def test_grep_filter_generator(self) -> None:
+ """Test -grep, -grep:not, and -grepnot content filtering."""
+ page1 = mock.MagicMock()
+ page2 = mock.MagicMock()
+ page1.text = 'The quick brown fox'
+ page2.text = 'jumps over the lazy dog'
+
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2]]
+ gf.nopreload = True
+ self.assertTrue(gf.handle_arg('-grep:fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page1])
+
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2]]
+ gf.nopreload = True
+ self.assertTrue(gf.handle_arg('-grep:not:fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page2])
+
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2]]
+ gf.nopreload = True
+ self.assertTrue(gf.handle_arg('-grepnot:fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page2])
+
+ # Positive filter with pattern starting with literal 'not:'
+ page3 = mock.MagicMock()
+ page3.text = 'This has not:fox in it'
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2, page3]]
+ gf.nopreload = True
+ self.assertTrue(gf._handle_grep('not:fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page3])
+
+ def test_titleregex_filter_generator(self) -> None:
+ """Test -titleregex and -titleregex:not content filtering."""
+ page1 = mock.MagicMock()
+ page2 = mock.MagicMock()
+ page3 = mock.MagicMock()
+ page1.title.return_value = 'The quick brown Fox'
+ page2.title.return_value = 'jumps over the lazy dog'
+ page3.title.return_value = 'not:fox'
+
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2]]
+ gf.nopreload = True
+ self.assertTrue(gf.handle_arg('-titleregex:Fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page1])
+
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2]]
+ gf.nopreload = True
+ self.assertTrue(gf.handle_arg('-titleregex:not:Fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page2])
+
+ # Positive filter with pattern starting with literal 'not:'
+ gf = pagegenerators.GeneratorFactory(site=self.get_site())
+ gf.gens = [[page1, page2, page3]]
+ gf.nopreload = True
+ self.assertTrue(gf._handle_titleregex('not:fox'))
+ self.assertEqual(list(gf.getCombinedGenerator()), [page3])
+
class TestItemClaimFilterPageGenerator(WikidataTestCase):
@@ -1227,6 +1366,46 @@
self.assertLessEqual(len(pages), 10)
self.assertPagesInNamespaces(pages, 1)
+ def test_titleregex_not_default(self) -> None:
+ """Test allpages generator with titleregex:not filter."""
+ gf = pagegenerators.GeneratorFactory()
+ self.assertTrue(gf.handle_arg('-start'))
+ # matches titles with less than 11 characters
+ self.assertTrue(gf.handle_arg('-titleregex:not:.{11,}'))
+ gf.handle_arg('-limit:10')
+ gen = gf.getCombinedGenerator()
+ self.assertIsNotNone(gen)
+ pages = list(gen)
+ self.assertLessEqual(len(pages), 10)
+ for page in pages:
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertNotRegex(page.title().lower(), '.{11,}')
+
+ def test_titleregex_not_ns_after(self) -> None:
+ """Test allpages generator with titleregex:not and namespace filter."""
+ gf = pagegenerators.GeneratorFactory()
+ self.assertTrue(gf.handle_arg('-start'))
+ self.assertTrue(gf.handle_arg('-titleregex:not:zzzz'))
+ gf.handle_arg('-ns:1')
+ gf.handle_arg('-limit:10')
+ gen = gf.getCombinedGenerator()
+ pages = list(gen)
+ self.assertLessEqual(len(pages), 10)
+ self.assertPagesInNamespaces(pages, 1)
+
+ def test_titleregex_not_ns_before(self) -> None:
+ """Test allpages generator with namespace and titleregex:not filter."""
+ gf = pagegenerators.GeneratorFactory()
+ self.assertTrue(gf.handle_arg('-start'))
+ gf.handle_arg('-ns:1')
+ self.assertTrue(gf.handle_arg('-titleregex:not:zzzz'))
+ gf.handle_arg('-limit:10')
+ gen = gf.getCombinedGenerator()
+ self.assertIsNotNone(gen)
+ pages = list(gen)
+ self.assertLessEqual(len(pages), 10)
+ self.assertPagesInNamespaces(pages, 1)
+
def test_allpages_with_two_ns(self) -> None:
"""Test that allpages fails with two ns as parameter."""
gf = pagegenerators.GeneratorFactory()
@@ -1672,6 +1851,16 @@
self.assertIsNotNone(gen)
self.assertIsEmpty(set(gen))
+ def test_onlyif_not(self) -> None:
+ """Test -onlyif:not without qualifiers."""
+ gf = pagegenerators.GeneratorFactory(site=self.site)
+ gf.handle_arg('-page:Q15745378')
+ gf.handle_arg('-onlyif:not:P1476=International Journal of Minerals\\, '
+ 'Metallurgy\\, and Materials')
+ gen = gf.getCombinedGenerator()
+ self.assertIsNotNone(gen)
+ self.assertIsEmpty(set(gen))
+
def test_onlyif_qualifiers(self) -> None:
"""Test -onlyif with qualifiers."""
gf = pagegenerators.GeneratorFactory(site=self.site)
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1346493?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ib64f014a9a7d8956cddec2ae81a7e64a0d71380d
Gerrit-Change-Number: 1346493
Gerrit-PatchSet: 3
Gerrit-Owner: ShubhamSChauhan <[email protected]>
Gerrit-Reviewer: ShubhamSChauhan <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
Gerrit-CC: Mahveotm <[email protected]>
Gerrit-CC: Thiagollipe <[email protected]>
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
