Mpaa has uploaded a new change for review.
https://gerrit.wikimedia.org/r/243489
Change subject: proofreadpage.py: add ProofreadPage.index property and other
methods
......................................................................
proofreadpage.py: add ProofreadPage.index property and other methods
Add:
- ProofreadPage.index property to get Index page containing the page
- IndexPage.page_generator() to load pages related to an Index page in
specified page range (filters are available for quality level and
page existance)
- IndexPage.get_number() to get page number of a page
Rename:
- IndexPage.get_page_from_number() to get_page()
Added and cleaned up docstrings.
Added related tests.
Change-Id: I9dab8c2e75dc27fe87500eac3202f14553525a82
---
M pywikibot/proofreadpage.py
M tests/proofreadpage_tests.py
2 files changed, 160 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/89/243489/1
diff --git a/pywikibot/proofreadpage.py b/pywikibot/proofreadpage.py
index edc0060..602c95b 100644
--- a/pywikibot/proofreadpage.py
+++ b/pywikibot/proofreadpage.py
@@ -28,6 +28,8 @@
import pywikibot
+from pywikibot.pagegenerators import PreloadingGenerator
+
class FullHeader(object):
@@ -70,6 +72,11 @@
PROOFREAD = 3
VALIDATED = 4
+ # Indicate if page is linked to no/several Index pages.
+ UNLINKED = 0
+ LINKED = 1
+ MULTILINKED = 2
+
open_tag = '<noinclude>'
close_tag = '</noinclude>'
p_open = re.compile(r'<noinclude>')
@@ -89,6 +96,48 @@
if self.namespace() != site.proofread_page_ns:
raise ValueError('Page %s must belong to %s namespace'
% (self.title(), ns))
+
+ @property
+ def index(self):
+ """Get the Index page which contains ProofreadPage.
+
+ To force reload, delete index and call it again.
+
+ Returns:
+ None: if ProofreadPage is linked to no or several Index pages.
+ IndexPage: if ProofreadPage is linked to one Index page.
+ """
+ if not hasattr(self, '_index'):
+ index_ns = self.site.proofread_index_ns
+ what_links_here = list(self.getReferences(namespaces=index_ns))
+ if not what_links_here:
+ self._index = (self.UNLINKED, None)
+ elif len(what_links_here) > 1:
+ self._index = (self.MULTILINKED, what_links_here)
+ else:
+ self._index = (self.LINKED, IndexPage(what_links_here[0]))
+
+ status, page = self._index
+ if status == self.UNLINKED:
+ pywikibot.warning('Page %s is not linked to any Index page'
+ % self)
+ elif status == self.MULTILINKED:
+ pywikibot.warning('Page %s is linked to several Index pages: %s'
+ % (self, page))
+ else:
+ return page
+
+ @index.setter
+ def index(self, value):
+ if not isinstance(value, IndexPage):
+ raise ValueError('value %s must be a IndexPage object.'
+ % value)
+ self._index = (self.LINKED, value)
+
+ @index.deleter
+ def index(self):
+ if hasattr(self, "_index"):
+ del self._index
def decompose(fn):
"""Decorator.
@@ -347,6 +396,13 @@
on de wikisource).
page label is the label associated with a page in the Index page.
+ This class provides methods to get pages contained in Index page,
+ and relative page numbers and labels by means of several helper
+ functions.
+
+ It also providesa generator to pages contained in Index page, with
+ possibility to define range, filter by quality levels and page
existance.
+
Raises UnknownExtension if source Site has no ProofreadPage Extension.
"""
# Check if BeautifulSoup is imported.
@@ -393,6 +449,7 @@
title = a_tag.get('title')
page = ProofreadPage(self.site, title)
+ page.index = self # set index property for page
if page not in self._all_page_links:
raise pywikibot.Error('Page %s not recognised.' % page)
@@ -434,6 +491,41 @@
self._get_page_mappings()
return len(self._page_from_numbers)
+ def page_generator(self, start=1, end=None, filter_ql=None,
+ only_existing=False, preload=True):
+ """Return a page generator which yields pages contained in Index page.
+
+ @param start: first page
+ @type start: int
+ @param end: last page (num_pages if end is greater than num_pages)
+ @type end: int
+ @param filter_ql: filters quality levels (default: all but Without
Text).
+ @type filter_ql: list of ints (corresponding to ql constants
+ defined in ProofreadPage).
+ @param only_exist: yields only existing pages.
+ @type only_exist: bool
+ """
+ # start >= 1
+ start = max(1, start)
+ if end is None:
+ end = self.num_pages
+ else:
+ # start <= end <= num_pages
+ end = min(max(start, end), self.num_pages)
+
+ if filter_ql is None:
+ filter_ql = [1, 2, 3, 4]
+
+ gen = (self.get_page(i) for i in range(start, end + 1))
+ if preload:
+ gen = PreloadingGenerator(gen)
+
+ gen = (p for p in gen if p.ql in filter_ql)
+ if only_existing:
+ gen = (p for p in gen if p.exists())
+
+ return gen
+
def get_label_from_page(self, page):
"""Return 'page label' for page.
@@ -466,7 +558,7 @@
try:
return self._labels_from_page_number[page_number]
except KeyError:
- raise KeyError('Page number ".../%s" not range.'
+ raise KeyError('Page number ".../%s" not in range.'
% page_number)
def _get_from_label(self, mapping_dict, label):
@@ -503,7 +595,7 @@
"""
return self._get_from_label(self._pages_from_label, label)
- def get_page_from_number(self, page_number):
+ def get_page(self, page_number):
"""Return a page object from page number.
@param page_number: int
@@ -517,3 +609,18 @@
return self._page_from_numbers[page_number]
except KeyError:
raise KeyError('Invalid page number: %s.' % page_number)
+
+ def get_number(self, page):
+ """Return a page number from page object.
+
+ @param page: page object
+ @return: page_number: int
+ @rtype: int
+ """
+ if not self._page_from_numbers:
+ self._get_page_mappings()
+
+ try:
+ return self._numbers_from_page[page]
+ except KeyError:
+ raise KeyError('Invalid page: %s.' % page)
diff --git a/tests/proofreadpage_tests.py b/tests/proofreadpage_tests.py
index 8a1715f..487bf04 100644
--- a/tests/proofreadpage_tests.py
+++ b/tests/proofreadpage_tests.py
@@ -86,6 +86,7 @@
valid = {
'title': 'Page:Popular Science Monthly Volume 1.djvu/12',
+ 'index': 'Index:Popular Science Monthly Volume 1.djvu',
'ql': 4,
'user': 'T. Mazzei',
'header': u"{{rh|2|''THE POPULAR SCIENCE MONTHLY.''}}",
@@ -94,6 +95,10 @@
existing_invalid = {
'title': 'Main Page',
+ }
+
+ existing_unlinked = {
+ 'title': 'Page:Pywikibot unlinked test page',
}
not_existing_invalid = {
@@ -200,6 +205,18 @@
page_text = page._page_to_json()
self.assertEqual(json.loads(page_text), json.loads(loaded_text))
+
+ def test_index(self):
+ """Test index property."""
+ # Page with Index.
+ page = ProofreadPage(self.site, self.valid['title'])
+ index_page = IndexPage(self.site, self.valid['index'])
+ self.assertEqual(page.index, index_page)
+
+ # Page without Index.
+ page = ProofreadPage(self.site, self.existing_unlinked['title'])
+ self.assertEqual(page.index, None)
+ self.assertEqual(page._index, (page.UNLINKED, None))
@require_modules('bs4')
@@ -382,19 +399,15 @@
# Error if label does not exists.
self.assertRaises(KeyError, index_page.get_page_from_label, 'dummy
label')
- # Test consistency of page <-> numbers mapping on last page_set and
- # num_set used.
- for p in page_set:
- n = index_page._numbers_from_page[p]
- self.assertEqual(index_page._page_from_numbers[n], p)
+ # Test get_page.
for n in num_set:
- p = index_page._page_from_numbers[n]
- self.assertEqual(index_page._numbers_from_page[p], n)
+ p = index_page.get_page(n)
+ self.assertEqual(index_page.get_number(p), n)
- # Test get_page_from_number.
- for n in num_set:
- p = index_page.get_page_from_number(n)
- self.assertEqual(index_page._numbers_from_page[p], n)
+ # Test get_number.
+ for p in page_set:
+ n = index_page.get_number(p)
+ self.assertEqual(index_page.get_page(n), p)
def test_page_number_mapping(self, key):
"""Test consistency of page <-> mapping."""
@@ -415,6 +428,33 @@
# Error if page does not exists.
self.assertRaises(KeyError, index_page.get_label_from_page, None)
+ def test_index_property(self, key):
+ """Test index property."""
+ data = self.sites[key]
+ index_page = IndexPage(self.site, self.sites[key]['index'])
+
+ # Get page_set to test (reuse pages used for label tests
+ # (label is not used here).
+ for label, page_set in data['get_page']:
+ for page in page_set:
+ self.assertEqual(page.index, index_page)
+ # test deleter
+ del page.index
+ self.assertFalse(hasattr(page, '_index'))
+ # test setter
+ page.index = index_page
+ self.assertEqual(page.index, index_page)
+
+ def test_page_generator(self, key):
+ """Test Index page generator."""
+ index_page = IndexPage(self.site, self.sites[key]['index'])
+
+ gen = index_page.page_generator(-1, -1, filter_ql=range(5))
+ self.assertEqual(len(list(gen)), 1)
+
+ gen = index_page.page_generator(end=10, filter_ql=range(5))
+ self.assertEqual(len(list(gen)), 10)
+
if __name__ == '__main__':
try:
--
To view, visit https://gerrit.wikimedia.org/r/243489
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9dab8c2e75dc27fe87500eac3202f14553525a82
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Mpaa <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
