Revision: 6960
Author: russblau
Date: 2009-06-14 17:28:36 +0000 (Sun, 14 Jun 2009)
Log Message:
-----------
add "step" and "total" parameters to applicable page generators; deprecate
inconsistent parameters
Modified Paths:
--------------
branches/rewrite/pywikibot/pagegenerators.py
Modified: branches/rewrite/pywikibot/pagegenerators.py
===================================================================
--- branches/rewrite/pywikibot/pagegenerators.py 2009-06-13 03:31:53 UTC
(rev 6959)
+++ branches/rewrite/pywikibot/pagegenerators.py 2009-06-14 17:28:36 UTC
(rev 6960)
@@ -19,7 +19,8 @@
__version__ = '$Id$'
import pywikibot
-from pywikibot import config2 as config
+from pywikibot import config
+from pywikibot import deprecate_arg
import itertools
import Queue
@@ -408,13 +409,16 @@
def AllpagesPageGenerator(start ='!', namespace=0, includeredirects=True,
- site=None):
+ site=None, step=None, total=None):
"""
Iterate Page objects for all titles in a single namespace.
If includeredirects is False, redirects are not included. If
includeredirects equals the string 'only', only redirects are added.
-
+
+ @param step: Maximum number of pages to retrieve per API query
+ @param total: Maxmum number of pages to retrieve in total
+
"""
if site is None:
site = pywikibot.getSite()
@@ -426,11 +430,11 @@
else:
filterredir = False
return site.allpages(start=start, namespace=namespace,
- filterredir=filterredir)
+ filterredir=filterredir, step=step, total=total)
def PrefixingPageGenerator(prefix, namespace=None, includeredirects=True,
- site=None):
+ site=None, step=None, total=None):
if site is None:
site = pywikibot.Site()
prefixlink = pywikibot.Link(prefix, site)
@@ -445,26 +449,26 @@
else:
filterredir = False
return site.allpages(prefix=title, namespace=namespace,
- filterredir=filterredir)
+ filterredir=filterredir, step=step, total=total)
-
-def NewpagesPageGenerator(number=100, get_redirect=False, repeat=False,
- site=None):
+...@deprecate_arg("number", "total")
+def NewpagesPageGenerator(get_redirect=False, repeat=False, site=None,
+ step=None, total=None):
# API does not (yet) have a newpages function, so this tries to duplicate
# it by filtering the recentchanges output
# defaults to namespace 0 because that's how Special:Newpages defaults
if site is None:
site = pywikibot.Site()
- return site.recentchanges(limit=number, showredirects=get_redirect,
- changetype="new", namespaces=0)
+ return site.recentchanges(showRedirects=get_redirect, changetype="new",
+ namespaces=0, step=step, total=total)
-def FileLinksGenerator(referredImagePage):
- return referredImagePage.usingPages()
+def FileLinksGenerator(referredImagePage, step=None, total=None):
+ return referredImagePage.usingPages(step=step, total=total)
-def ImagesPageGenerator(pageWithImages):
- return pageWithImages.imagelinks()
+def ImagesPageGenerator(pageWithImages, step=None, total=None):
+ return pageWithImages.imagelinks(step=step, total=total)
def InterwikiPageGenerator(page):
@@ -473,23 +477,26 @@
yield pywikibot.Page(link)
-def LanguageLinksPageGenerator(page):
+def LanguageLinksPageGenerator(page, step=None, total=None):
"""Iterator over all interwiki language links on a page."""
- for link in page.langlinks():
+ for link in page.iterlanglinks(step=step, total=total):
yield pywikibot.Page(link)
def ReferringPageGenerator(referredPage, followRedirects=False,
withTemplateInclusion=True,
- onlyTemplateInclusion=False):
+ onlyTemplateInclusion=False,
+ step=None, total=None):
'''Yields all pages referring to a specific page.'''
return referredPage.getReferences(
follow_redirects=followRedirects,
withTemplateInclusion=withTemplateInclusion,
- onlyTemplateInclusion=onlyTemplateInclusion)
+ onlyTemplateInclusion=onlyTemplateInclusion,
+ step=step, total=total)
-def CategorizedPageGenerator(category, recurse=False, start=None):
+def CategorizedPageGenerator(category, recurse=False, start=None,
+ step=None, total=None):
'''Yield all pages in a specific category.
If recurse is True, pages in subcategories are included as well; if
@@ -500,12 +507,13 @@
alphabetically are included.
''' # TODO: page generator could be modified to use cmstartsortkey ...
- for a in category.articles(recurse=recurse):
+ for a in category.articles(recurse=recurse, step=step, total=total):
if start is None or a.title(withNamespace=False) >= start:
yield a
-def SubCategoriesPageGenerator(category, recurse=False, start=None):
+def SubCategoriesPageGenerator(category, recurse=False, start=None,
+ step=None, total=None):
'''Yields all subcategories in a specific category.
If recurse is True, pages in subcategories are included as well; if
@@ -516,14 +524,14 @@
start alphabetically are included.
''' # TODO: page generator could be modified to use cmstartsortkey ...
- for s in category.subcategories(recurse=recurse):
+ for s in category.subcategories(recurse=recurse, step=step, total=total):
if start is None or s.title(withNamespace=False) >= start:
yield s
-def LinkedPageGenerator(linkingPage):
+def LinkedPageGenerator(linkingPage, step=None, total=None):
"""Yields all pages linked from a specific page."""
- return linkingPage.linkedPages()
+ return linkingPage.linkedPages(step=step, total=total)
def TextfilePageGenerator(filename=None, site=None):
@@ -562,16 +570,21 @@
yield pywikibot.Page(pywikibot.Link(title, site))
-def UserContributionsGenerator(username, number=250, namespaces=None,
- site=None):
- """Yields number unique pages edited by user:username
+...@deprecate_arg("number", "total")
+def UserContributionsGenerator(username, namespaces=None, site=None,
+ step=None, total=None):
+ """Yield unique pages edited by user:username
- namespaces : list of namespace numbers to fetch contribs from
+ @param namespaces: list of namespace numbers to fetch contribs from
"""
if site is None:
site = pywikibot.Site()
- return site.usercontribs(user=username, limit=number,
namespaces=namespaces)
+ return DuplicateFilterPageGenerator(
+ pywikibot.Page(pywikibot.Link(contrib["title"], source=site))
+ for contrib in site.usercontribs(user=username, namespaces=namespaces,
+ step=step, total=total)
+ )
def NamespaceFilterPageGenerator(generator, namespaces, site=None):
@@ -582,6 +595,9 @@
The namespace list can contain both integers (namespace numbers) and
strings/unicode strings (namespace names).
+ NOTE: API-based generators that have a "namespaces" parameter perform
+ namespace filtering more efficiently than this generator.
+
"""
if site is None:
site = pywikibot.Site()
@@ -652,7 +668,9 @@
yield page.toggleTalkPage()
-def PreloadingGenerator(generator, pageNumber=50, lookahead=10):
+...@deprecate_arg("pageNumber", "step")
+...@deprecate_arg("lookahead", None)
+def PreloadingGenerator(generator, step=50):
"""Yield preloaded pages taken from another generator."""
# pages may be on more than one site, for example if an interwiki
@@ -662,14 +680,14 @@
for page in generator:
site = page.site()
sites.setdefault(site, []).append(page)
- if len(sites[site]) >= pageNumber:
+ if len(sites[site]) >= step:
group = sites[site]
sites[site] = []
- for i in site.preloadpages(group, pageNumber):
+ for i in site.preloadpages(group, step):
yield i
for site in sites:
if sites[site]:
- for i in site.preloadpages(sites[site], pageNumber):
+ for i in site.preloadpages(sites[site], step):
yield i
#TODO below
_______________________________________________
Pywikipedia-svn mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikipedia-svn
