John Vandenberg has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169519
Change subject: ListPopGenerator
......................................................................
ListPopGenerator
Scripts may be written to dynamically populate a list of pages to work on.
Ideally this list would be created using a generator, however that can
require a large rewrite in the flow control.
To simplify the porting of scripts, this changeset adds classes
ListPopGenerator and ListPopPreloadingGenerator classes that allow the page
list to be extended during iteration.
Implemented in makecat.
Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088
---
M pywikibot/pagegenerators.py
M pywikibot/tools.py
M scripts/makecat.py
M tests/pagegenerators_tests.py
4 files changed, 66 insertions(+), 21 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/19/169519/1
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index eb8930a..959c749 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -28,7 +28,7 @@
import time
import pywikibot
from pywikibot import date, config, i18n
-from pywikibot.tools import deprecated_args
+from pywikibot.tools import deprecated_args, ListPopGenerator
from pywikibot.comms import http
import pywikibot.data.wikidataquery as wdquery
@@ -1143,6 +1143,18 @@
yield i
+def ListPopPreloadingGenerator(generator, step=50):
+ assert(isinstance(generator, ListPopGenerator))
+
+ while True:
+ page_count = min(len(generator), step)
+ if not page_count:
+ raise StopIteration
+
+ for page in PreloadingGenerator(generator, page_count):
+ yield page
+
+
def PreloadingItemGenerator(generator, step=50):
"""
Yield preloaded pages taken from another generator.
diff --git a/pywikibot/tools.py b/pywikibot/tools.py
index 6064d86..7002fae 100644
--- a/pywikibot/tools.py
+++ b/pywikibot/tools.py
@@ -320,6 +320,21 @@
EMPTY_DEFAULT = EmptyDefault()
+
+class ListPopGenerator(list):
+
+ """A generator that allows items to be added during generating."""
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ if len(self):
+ return self.pop()
+ else:
+ raise StopIteration
+
+
# Decorators
#
# Decorator functions without parameters are _invoked_ differently from
diff --git a/scripts/makecat.py b/scripts/makecat.py
index e523cff..50ed13d 100644
--- a/scripts/makecat.py
+++ b/scripts/makecat.py
@@ -44,6 +44,7 @@
import codecs
import pywikibot
from pywikibot import date, pagegenerators, i18n, textlib
+from pywikibot.tools import ListPopGenerator
def isdate(s):
@@ -197,7 +198,7 @@
removeparent = True
main = True
workingcatname = ''
- tocheck = []
+ tocheck = ListPopGenerator()
for arg in pywikibot.handleArgs():
if arg.startswith('-nodate'):
skipdates = True
@@ -271,26 +272,14 @@
answer = workingcatname
pywikibot.output(u'' + answer)
pl = pywikibot.Page(mysite, answer)
- tocheck = []
checked[pl] = pl
include(pl)
- loaded = 0
- while tocheck:
- if loaded == 0:
- if len(tocheck) < 50:
- loaded = len(tocheck)
- else:
- loaded = 50
- tocheck = [x for x in
pagegenerators.PreloadingGenerator(tocheck[:loaded])]
- if not checkbroken:
- if not tocheck[0].exists():
- pass
- else:
- asktoadd(tocheck[0])
- else:
- asktoadd(tocheck[0])
- tocheck = tocheck[1:]
- loaded -= 1
+
+ gen = pagegenerators.ListPopPreloadingGenerator(tocheck)
+
+ for page in gen:
+ if checkbroken or page.exists():
+ asktoadd(page)
finally:
try:
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 44a99cd..7a84baa 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -11,7 +11,12 @@
import pywikibot
from pywikibot import pagegenerators
-from tests.aspects import unittest, TestCase, WikidataTestCase
+from tests.aspects import (
+ unittest,
+ TestCase,
+ WikidataTestCase,
+ DefaultSiteTestCase,
+)
class TestPageGenerators(TestCase):
@@ -154,6 +159,30 @@
self.assertEqual(len(set(item['revid'] for item in items)), 4)
+class TestListPreloadingGenerator(DefaultSiteTestCase):
+
+ """Test preloading generator on lists."""
+
+ def test_array_based_preloading(self):
+ """Test TestPreloadingGenerator with pages added to array."""
+ mainpage = self.get_mainpage()
+
+ pages = pywikibot.tools.ListPopGenerator([mainpage])
+ gen = pagegenerators.ListPopPreloadingGenerator(pages)
+ pages_out = list()
+ for page in gen:
+ pages_out.append(page)
+ # Add a page to the generator
+ if page.namespace() == 0:
+ pages.extend([page.toggleTalkPage()])
+
+ self.assertTrue(all(isinstance(page, pywikibot.Page) for page in
pages_out))
+ self.assertEqual(len(pages_out), 2)
+ self.assertEqual(pages_out[1].namespace(), 1)
+ self.assertIn(mainpage, pages_out)
+ self.assertIn(mainpage.toggleTalkPage(), pages_out)
+
+
class TestPreloadingItemGenerator(WikidataTestCase):
"""Test preloading item generator."""
--
To view, visit https://gerrit.wikimedia.org/r/169519
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
