Revision: 8388
Author: multichill
Date: 2010-08-07 16:08:55 +0000 (Sat, 07 Aug 2010)
Log Message:
-----------
Implemented 3 different threads
1. Fetcher thread gets all information
2. User thread does the user interaction
3. Uploader thread does the actual uploading
This speeds up the bot a lot.
Modified Paths:
--------------
trunk/pywikipedia/imagecopy_enwp.py
Modified: trunk/pywikipedia/imagecopy_enwp.py
===================================================================
--- trunk/pywikipedia/imagecopy_enwp.py 2010-08-07 13:37:23 UTC (rev 8387)
+++ trunk/pywikipedia/imagecopy_enwp.py 2010-08-07 16:08:55 UTC (rev 8388)
@@ -51,6 +51,7 @@
import os, sys, re, codecs
import urllib, httplib, urllib2
import webbrowser
+from Queue import Queue
import time, threading
import wikipedia, config, socket
import pagegenerators, add_text
@@ -267,255 +268,252 @@
return (self.filename, self.description, self.date, self.source,
self.author, self.licensetemplate, self.categories, self.skip)
-def doiskip(imagepage):
+class imageFetcher(threading.Thread):
'''
- Skip this image or not.
- Returns True if the image is on the skip list, otherwise False
+ Tries to fetch information for all images in the generator
'''
- for template in imagepage.templates():
- if template in skipTemplates:
- wikipedia.output(u'Found ' + template + u' which is on the
template skip list')
- return True
- return False
+ def __init__ ( self, pagegenerator, prefetchQueue):
+ self.pagegenerator = pagegenerator
+ self.prefetchQueue = prefetchQueue
+ imagerecat.initLists()
+ threading.Thread.__init__ ( self )
-def getNewFields(imagepage):
- '''
- Build a new description based on the imagepage
- '''
- if u'{{Information' in imagepage.get() or u'{{information' in
imagepage.get():
- (description, date, source, author) =
getNewFieldsFromInformation(imagepage)
- else:
- (description, date, source, author) =
getNewFieldsFromFreetext(imagepage)
+ def run(self):
+ for page in self.pagegenerator:
+ self.processImage(page)
+ self.prefetchQueue.put(None)
+ wikipedia.output(u'Fetched all images.')
+ return True
- licensetemplate = getNewLicensetemplate(imagepage)
- categories = getNewCategories(imagepage)
- return (description, date, source, author, licensetemplate, categories)
+ def processImage(self, page):
+ '''
+ Work on a single image
+ '''
+ if page.exists() and (page.namespace() == 6) and (not
page.isRedirectPage()):
+ imagepage = wikipedia.ImagePage(page.site(), page.title())
-def getNewFieldsFromInformation(imagepage):
- '''
- '''
- description = u''
- date = u''
- source = u''
- author = u''
- permission = u''
- other_versions = u''
- text = imagepage.get()
- # Need to add the permission field
- # Need to use pywikipedia template parser code
- regexes
=[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}',
-
u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}',
- ]
+ #First do autoskip.
+ if self.doiskip(imagepage):
+ wikipedia.output(u'Skipping %s : Got a template on the skip
list.' % page.title())
+ return False
+ text = imagepage.get()
+ foundMatch = False
+ for (regex, replacement) in licenseTemplates:
+ match = re.search(regex, text, re.IGNORECASE)
+ if match:
+ foundMatch = True
+ if not foundMatch:
+ wikipedia.output(u'Skipping %s : No suitable license template
was found.' % page.title())
+ return False
+ self.prefetchQueue.put(self.getNewFields(imagepage))
- for regex in regexes:
- match =re.search(regex, text, re.IGNORECASE|re.DOTALL)
- if match:
- description = convertLinks(match.group(u'description').strip(),
imagepage.site())
-
- date = match.group(u'date').strip()
- if date == u'':
- date = getUploadDate(imagepage)
+ def doiskip(self, imagepage):
+ '''
+ Skip this image or not.
+ Returns True if the image is on the skip list, otherwise False
+ '''
+ for template in imagepage.templates():
+ if template in skipTemplates:
+ wikipedia.output(u'Found ' + template + u' which is on the
template skip list')
+ return True
+ return False
- source = getSource(imagepage,
source=convertLinks(match.group(u'source').strip(), imagepage.site()))
+ def getNewFields(self, imagepage):
+ '''
+ Build a new description based on the imagepage
+ '''
+ if u'{{Information' in imagepage.get() or u'{{information' in
imagepage.get():
+ (description, date, source, author) =
self.getNewFieldsFromInformation(imagepage)
+ else:
+ (description, date, source, author) =
self.getNewFieldsFromFreetext(imagepage)
- author = convertLinks(match.group(u'author').strip(),
imagepage.site())
- if author == u'':
- author = getAuthorText(imagepage)
-
- if u'permission' in match.groupdict():
- permission = convertLinks(match.group(u'permission').strip(),
imagepage.site())
- if u'other_versions' in match.groupdict():
- other_versions =
convertLinks(match.group(u'other_versions').strip(), imagepage.site())
- # Return the stuff we found
- return (description, date, source, author)
-
- #We didn't find anything, return the empty strings
- return (description, date, source, author)
+ licensetemplate = self.getNewLicensetemplate(imagepage)
+ categories = self.getNewCategories(imagepage)
+ return (imagepage, description, date, source, author, licensetemplate,
categories)
-def getNewFieldsFromFreetext(imagepage):
- '''
- '''
- text = imagepage.get()
- #text = re.sub(u'== Summary ==', u'', text, re.IGNORECASE)
- #text = re.sub(u'== Licensing ==', u'', text, re.IGNORECASE)
- #text = re.sub(u'\{\{(self|self2)\|[^\}]+\}\}', u'', text, re.IGNORECASE)
+ def getNewFieldsFromInformation(self, imagepage):
+ '''
+ Try to extract fields from the current information template for the
new information template.
+ '''
+ description = u''
+ date = u''
+ source = u''
+ author = u''
+ permission = u''
+ other_versions = u''
+ text = imagepage.get()
+ # Need to add the permission field
+ # Need to use pywikipedia template parser code
+ regexes
=[u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*permission.*=(?P<permission>[^\}]*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}',
+
u'\{\{Information[\s\r\n]*\|[\s\r\n]*description[\s\r\n]*=(?P<description>.*)\|[\s\r\n]*source[\s\r\n]*=(?P<source>.*)\|[\s\r\n]*date[\s\r\n]*=(?P<date>.*)\|[\s\r\n]*author[\s\r\n]*=(?P<author>.*)\|[\s\r\n]*other_versions.*=(?P<other_versions>[^\}]*)\}\}',
+ ]
+
+ for regex in regexes:
+ match =re.search(regex, text, re.IGNORECASE|re.DOTALL)
+ if match:
+ description =
self.convertLinks(match.group(u'description').strip(), imagepage.site())
+
+ date = match.group(u'date').strip()
+ if date == u'':
+ date = self.getUploadDate(imagepage)
- for toRemove in sourceGarbage:
- text = re.sub(toRemove, u'', text, re.IGNORECASE)
-
- for (regex, repl) in licenseTemplates:
- text = re.sub(regex, u'', text, re.IGNORECASE)
+ source = self.getSource(imagepage,
source=self.convertLinks(match.group(u'source').strip(), imagepage.site()))
- text = wikipedia.removeCategoryLinks(text, imagepage.site()).strip()
+ author = self.convertLinks(match.group(u'author').strip(),
imagepage.site())
+ if author == u'':
+ author = self.getAuthorText(imagepage)
+
+ if u'permission' in match.groupdict():
+ permission =
self.convertLinks(match.group(u'permission').strip(), imagepage.site())
+ if u'other_versions' in match.groupdict():
+ other_versions =
self.convertLinks(match.group(u'other_versions').strip(), imagepage.site())
+ # Return the stuff we found
+ return (description, date, source, author)
- description = convertLinks(text.strip(), imagepage.site())
- date = getUploadDate(imagepage)
- source = getSource(imagepage)
- author = getAuthorText(imagepage)
- return (description, date, source, author)
+ #We didn't find anything, return the empty strings
+ return (description, date, source, author)
-def getUploadDate(imagepage):
- # Get the original upload date
- uploadtime = imagepage.getFileVersionHistory()[-1][0]
- uploadDatetime = datetime.strptime(uploadtime, u'%Y-%m-%dT%H:%M:%SZ')
- return u'{{Date|' + str(uploadDatetime.year) + u'|' +
str(uploadDatetime.month) + u'|' + str(uploadDatetime.day) + u'}} (original
upload date)'
+ def getNewFieldsFromFreetext(self, imagepage):
+ '''
+ Try to extract fields from free text for the new information template.
+ '''
+ text = imagepage.get()
+ #text = re.sub(u'== Summary ==', u'', text, re.IGNORECASE)
+ #text = re.sub(u'== Licensing ==', u'', text, re.IGNORECASE)
+ #text = re.sub(u'\{\{(self|self2)\|[^\}]+\}\}', u'', text,
re.IGNORECASE)
-def getSource(imagepage, source=u''):
- site = imagepage.site()
- lang = site.language()
- family = site.family.name
- if source==u'':
- source=u'{{Own}}'
+ for toRemove in sourceGarbage:
+ text = re.sub(toRemove, u'', text, re.IGNORECASE)
- return source.strip() + u'<BR />Transferred from
[http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang,
u'family' : family}
+ for (regex, repl) in licenseTemplates:
+ text = re.sub(regex, u'', text, re.IGNORECASE)
-def getAuthorText(imagepage):
- site = imagepage.site()
- lang = site.language()
- family = site.family.name
-
- firstuploader = getAuthor(imagepage)
- #FIXME : Make other sites than Wikipedia work
- return u'[[:%(lang)s:User:%(firstuploader)s|%(firstuploader)s]] at
[http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang,
u'family' : family , u'firstuploader' : firstuploader}
+ text = wikipedia.removeCategoryLinks(text, imagepage.site()).strip()
+
+ description = self.convertLinks(text.strip(), imagepage.site())
+ date = self.getUploadDate(imagepage)
+ source = self.getSource(imagepage)
+ author = self.getAuthorText(imagepage)
+ return (description, date, source, author)
-def getAuthor(imagepage):
- return imagepage.getFileVersionHistory()[-1][1].strip()
+ def getUploadDate(self, imagepage):
+ '''
+ Get the original upload date to put in the date field of the new
information template. If we really have nothing better.
+ '''
+ uploadtime = imagepage.getFileVersionHistory()[-1][0]
+ uploadDatetime = datetime.strptime(uploadtime, u'%Y-%m-%dT%H:%M:%SZ')
+ return u'{{Date|' + str(uploadDatetime.year) + u'|' +
str(uploadDatetime.month) + u'|' + str(uploadDatetime.day) + u'}} (original
upload date)'
-def convertLinks(text, sourceSite):
- lang = sourceSite.language()
- family = sourceSite.family.name
- conversions =[(u'\[\[([^\[\]\|]+)\|([^\[\]\|]+)\]\]',
u'[[:%(lang)s:\\1|\\2]]'),
- (u'\[\[([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\1]]'),
- ]
-
- for (regex, replacement) in conversions:
- text = re.sub(regex, replacement % {u'lang' : lang, u'family' :
family}, text)
+ def getSource(self, imagepage, source=u''):
+ '''
+ Get the text to put in the source field of the new information
template.
+ '''
+ site = imagepage.site()
+ lang = site.language()
+ family = site.family.name
+ if source==u'':
+ source=u'{{Own}}'
+
+ return source.strip() + u'<BR />Transferred from
[http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang,
u'family' : family}
- return text
+ def getAuthorText(self, imagepage):
+ '''
+ Get the original uploader to put in the author field of the new
information template.
+ '''
+ site = imagepage.site()
+ lang = site.language()
+ family = site.family.name
+
+ firstuploader = self.getAuthor(imagepage)
+ return u'[[:%(lang)s:User:%(firstuploader)s|%(firstuploader)s]] at
[http://%(lang)s.%(family)s.org %(lang)s.%(family)s]' % {u'lang' : lang,
u'family' : family , u'firstuploader' : firstuploader}
-def getNewLicensetemplate(imagepage):
- '''
- '''
- text = imagepage.get()
-
- site = imagepage.site()
- lang = site.language()
- family = site.family.name
+ def getAuthor(self, imagepage):
+ '''
+ Get the first uploader.
+ '''
+ return imagepage.getFileVersionHistory()[-1][1].strip()
- result = u''
-
- for (regex, replacement) in licenseTemplates:
- match = re.search(regex, text, re.IGNORECASE)
- if match:
- result = re.sub(regex, replacement, match.group(0), re.IGNORECASE)
- return result % {u'author' : getAuthor(imagepage),
- u'lang' : lang,
- u'family' : family}
+ def convertLinks(self, text, sourceSite):
+ '''
+ Convert links from the current wiki to Commons.
+ '''
+ lang = sourceSite.language()
+ family = sourceSite.family.name
+ conversions =[(u'\[\[([^\[\]\|]+)\|([^\[\]\|]+)\]\]',
u'[[:%(lang)s:\\1|\\2]]'),
+ (u'\[\[([^\[\]\|]+)\]\]', u'[[:%(lang)s:\\1|\\1]]'),
+ ]
- return result
-
-def getNewCategories(imagepage):
- '''
- Get a categories for the image
- Dont forget to filter
- '''
- result = u''
- (commonshelperCats, usage, galleries) =
imagerecat.getCommonshelperCats(imagepage)
- newcats = imagerecat.applyAllFilters(commonshelperCats)
- for newcat in newcats:
- result = result + u'[[Category:' + newcat + u']] '
- return result
+ for (regex, replacement) in conversions:
+ text = re.sub(regex, replacement % {u'lang' : lang, u'family' :
family}, text)
-def getOriginalUploadLog(imagepage):
- filehistory = imagepage.getFileVersionHistory()
- filehistory.reverse()
+ return text
- site = imagepage.site()
- lang = site.language()
- family = site.family.name
-
- sourceimage =
imagepage.site().get_address(imagepage.title()).replace(u'&redirect=no&useskin=monobook',
u'')
-
- result = u'== {{Original upload log}} ==\n'
- result = result + u'The original description page is/was
[http://%(lang)s.%(family)s.org%(sourceimage)s here]. All following user names
refer to %(lang)s.%(family)s.\n' % {u'lang' : lang, u'family' : family ,
u'sourceimage' : sourceimage}
- for (timestamp, username, resolution, size, comment) in filehistory:
- date = datetime.strptime(timestamp,
u'%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d %H:%M')
- result = result + u'* %(date)s
[[:%(lang)s:user:%(username)s|%(username)s]] %(resolution)s (%(size)s bytes)
\'\'<nowiki>%(comment)s</nowiki>\'\'\n' % {
- u'lang' : lang,
- u'family' : family ,
- u'date' : date,
- u'username' : username,
- u'resolution': resolution,
- u'size': size,
- u'comment' : comment}
+ def getNewLicensetemplate(self, imagepage):
+ '''
+ Get a license template to put on the image to be uploaded
+ '''
+ text = imagepage.get()
- return result
+ site = imagepage.site()
+ lang = site.language()
+ family = site.family.name
-
+ result = u''
-def buildNewImageDescription(imagepage, description, date, source, author,
licensetemplate, categories, checkTemplate):
- '''
- Build a new information template
- '''
-
- site = imagepage.site()
- lang = site.language()
- family = site.family.name
-
- cid = u''
- if checkTemplate:
- cid = cid +
u'\n{{BotMoveToCommons|%(lang)s.%(family)s|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}\n'
% {u'lang' : lang, u'family' : family}
- cid = cid + u'== {{int:filedesc}} ==\n'
- cid = cid + u'{{Information\n'
- cid = cid + u'|description={{%(lang)s|1=' % {u'lang' : lang, u'family' :
family}
- cid = cid + description + u'}}\n'
- cid = cid + u'|date=' + date + u'\n'
- cid = cid + u'|source=' + source + u'\n'
- cid = cid + u'|author=' + author + u'\n'
- cid = cid + u'|permission=\n'
- cid = cid + u'|other_versions=\n'
- cid = cid + u'}}\n'
- cid = cid + u'== {{int:license}} ==\n'
- cid = cid + licensetemplate + u'\n'
- cid = cid + u'\n'
- cid = cid + getOriginalUploadLog(imagepage)
- cid = cid + u'__NOTOC__\n'
- if categories.strip()==u'':
- cid = cid + u'{{Subst:Unc}}'
- else:
- cid = cid + categories
- return cid
-
-
-def processImage(page, checkTemplate):
- skip = False
- if page.exists() and (page.namespace() == 6) and (not
page.isRedirectPage()):
- imagepage = wikipedia.ImagePage(page.site(), page.title())
-
- #First do autoskip.
- if doiskip(imagepage):
- wikipedia.output(u'Skipping %s : Got a template on the skip list.'
% page.title())
- return False
-
- text = imagepage.get()
- foundMatch = False
for (regex, replacement) in licenseTemplates:
match = re.search(regex, text, re.IGNORECASE)
if match:
- foundMatch = True
- if not foundMatch:
- wikipedia.output(u'Skipping %s : No suitable license template was
found.' % page.title())
- return False
+ result = re.sub(regex, replacement, match.group(0),
re.IGNORECASE)
+ return result % {u'author' : self.getAuthor(imagepage),
+ u'lang' : lang,
+ u'family' : family}
+
+ return result
- (description, date, source, author, licensetemplate, categories) =
getNewFields(imagepage)
+ def getNewCategories(self, imagepage):
+ '''
+ Get a categories for the image
+ Dont forget to filter
+ '''
+ result = u''
+ (commonshelperCats, usage, galleries) =
imagerecat.getCommonshelperCats(imagepage)
+ newcats = imagerecat.applyAllFilters(commonshelperCats)
+ for newcat in newcats:
+ result = result + u'[[Category:' + newcat + u']] '
+ return result
+class userInteraction(threading.Thread):
+ '''
+ Prompt all images to the user.
+ '''
+ def __init__ ( self, prefetchQueue, uploadQueue):
+ self.prefetchQueue = prefetchQueue
+ self.uploadQueue = uploadQueue
+ threading.Thread.__init__ ( self )
+
+ def run(self):
while True:
+ fields = self.prefetchQueue.get()
+ if fields:
+ self.processImage(fields)
+ else:
+ break
+ self.uploadQueue.put(None)
+ wikipedia.output(u'User worked on all images.')
+ return True
+
+ def processImage(self, fields):
+ '''
+ Work on a single image
+ '''
+ (imagepage, description, date, source, author, licensetemplate,
categories) = fields
+ while True:
# Do the Tkdialog to accept/reject and change te name
(filename, description, date, source, author, licensetemplate,
categories, skip)=Tkdialog(imagepage, description, date, source, author,
licensetemplate, categories).getnewmetadata()
if skip:
- wikipedia.output(u'Skipping %s : User pressed skip.' %
page.title())
+ wikipedia.output(u'Skipping %s : User pressed skip.' %
imagepage.title())
return False
# Check if the image already exists
@@ -524,13 +522,115 @@
break
else:
wikipedia.output('Image already exists, pick another name or
skip this image')
- # We dont overwrite images, pick another name, go to the start
of the loop
-
- cid = buildNewImageDescription(imagepage, description, date, source,
author, licensetemplate, categories, checkTemplate)
+ # We dont overwrite images, pick another name, go to the start
of the loop
+
+ self.uploadQueue.put((imagepage, filename, description, date, source,
author, licensetemplate, categories))
+
+
+class uploader(threading.Thread):
+ '''
+ Upload all images
+ '''
+ def __init__ ( self, uploadQueue):
+ self.uploadQueue = uploadQueue
+ self.checktemplate = True
+ threading.Thread.__init__ ( self )
+
+ def run(self):
+ while True: #Change later
+ fields = self.uploadQueue.get()
+ if fields:
+ self.processImage(fields)
+ else:
+ break
+ return True
+
+ def nochecktemplate(self):
+ '''
+ Don't want to add {{BotMoveToCommons}}
+ '''
+ self.checktemplate = False
+ return
+
+ def processImage(self, fields):
+ '''
+ Work on a single image
+ '''
+ (imagepage, filename, description, date, source, author,
licensetemplate, categories) = fields
+ cid = self.buildNewImageDescription(imagepage, description, date,
source, author, licensetemplate, categories)
wikipedia.output(cid)
bot = UploadRobot(url=imagepage.fileUrl(), description=cid,
useFilename=filename, keepFilename=True, verifyDescription=False, ignoreWarning
= True, targetSite = wikipedia.getSite('commons', 'commons'))
bot.run()
+ self.tagNowcommons(imagepage, filename)
+ self.replaceUsage(imagepage, filename)
+
+
+ def buildNewImageDescription(self, imagepage, description, date, source,
author, licensetemplate, categories):
+ '''
+ Build a new information template
+ '''
+
+ site = imagepage.site()
+ lang = site.language()
+ family = site.family.name
+
+ cid = u''
+ if self.checktemplate:
+ cid = cid +
u'\n{{BotMoveToCommons|%(lang)s.%(family)s|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}|day={{subst:CURRENTDAY}}}}\n'
% {u'lang' : lang, u'family' : family}
+ cid = cid + u'== {{int:filedesc}} ==\n'
+ cid = cid + u'{{Information\n'
+ cid = cid + u'|description={{%(lang)s|1=' % {u'lang' : lang, u'family'
: family}
+ cid = cid + description + u'}}\n'
+ cid = cid + u'|date=' + date + u'\n'
+ cid = cid + u'|source=' + source + u'\n'
+ cid = cid + u'|author=' + author + u'\n'
+ cid = cid + u'|permission=\n'
+ cid = cid + u'|other_versions=\n'
+ cid = cid + u'}}\n'
+ cid = cid + u'== {{int:license}} ==\n'
+ cid = cid + licensetemplate + u'\n'
+ cid = cid + u'\n'
+ cid = cid + self.getOriginalUploadLog(imagepage)
+ cid = cid + u'__NOTOC__\n'
+ if categories.strip()==u'':
+ cid = cid + u'{{Subst:Unc}}'
+ else:
+ cid = cid + categories
+ return cid
+
+ def getOriginalUploadLog(self, imagepage):
+ '''
+ Get the original upload log to put at the bottom of the image
description page at Commons.
+ '''
+ filehistory = imagepage.getFileVersionHistory()
+ filehistory.reverse()
+
+ site = imagepage.site()
+ lang = site.language()
+ family = site.family.name
+
+ sourceimage =
imagepage.site().get_address(imagepage.title()).replace(u'&redirect=no&useskin=monobook',
u'')
+
+ result = u'== {{Original upload log}} ==\n'
+ result = result + u'The original description page is/was
[http://%(lang)s.%(family)s.org%(sourceimage)s here]. All following user names
refer to %(lang)s.%(family)s.\n' % {u'lang' : lang, u'family' : family ,
u'sourceimage' : sourceimage}
+ for (timestamp, username, resolution, size, comment) in filehistory:
+ date = datetime.strptime(timestamp,
u'%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d %H:%M')
+ result = result + u'* %(date)s
[[:%(lang)s:user:%(username)s|%(username)s]] %(resolution)s (%(size)s bytes)
\'\'<nowiki>%(comment)s</nowiki>\'\'\n' % {
+ u'lang' : lang,
+ u'family' : family ,
+ u'date' : date,
+ u'username' : username,
+ u'resolution': resolution,
+ u'size': size,
+ u'comment' : comment}
+
+ return result
+
+ def tagNowcommons(self, imagepage, filename):
+ '''
+ Tagged the imag which has been moved to Commons for deletion.
+ '''
if wikipedia.Page(wikipedia.getSite('commons', 'commons'), u'File:' +
filename).exists():
#Get a fresh copy, force to get the page so we dont run into edit
conflicts
imtxt=imagepage.get(force=True)
@@ -554,27 +654,32 @@
wikipedia.showDiff(imagepage.get(), imtxt + addTemplate)
imagepage.put(imtxt + addTemplate, comment = commentText)
+ def replaceUsage(self, imagepage, filename):
+ '''
+ If the image is uploaded under a different name, replace all usage.
+ '''
+ if imagepage.titleWithoutNamespace() != filename:
gen = pagegenerators.FileLinksGenerator(imagepage)
preloadingGen = pagegenerators.PreloadingGenerator(gen)
-
- #If the image is uploaded under a different name, replace all
instances
- if imagepage.titleWithoutNamespace() != filename:
- if imagepage.site().language() in imageMoveMessage:
- moveSummary =
imageMoveMessage[imagepage.site().language()] %
(imagepage.titleWithoutNamespace(), filename)
- else:
- moveSummary = imageMoveMessage['_default'] %
(imagepage.titleWithoutNamespace(), filename)
- imagebot = ImageRobot(generator = preloadingGen, oldImage =
imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary,
always = True, loose = True)
- imagebot.run()
+
+ if imagepage.site().language() in imageMoveMessage:
+ moveSummary = imageMoveMessage[imagepage.site().language()] %
(imagepage.titleWithoutNamespace(), filename)
+ else:
+ moveSummary = imageMoveMessage['_default'] %
(imagepage.titleWithoutNamespace(), filename)
+ imagebot = ImageRobot(generator = preloadingGen, oldImage =
imagepage.titleWithoutNamespace(), newImage = filename, summary = moveSummary,
always = True, loose = True)
+ imagebot.run()
+def main(args):
+ wikipedia.output(u'WARNING: This is an experimental bot')
+ wikipedia.output(u'WARNING: It will only work on self published work
images')
+ wikipedia.output(u'WARNING: This bot is still full of bugs')
+ wikipedia.output(u'WARNING: Use at your own risk!')
-def main(args):
generator = None;
- #newname = "";
- imagepage = None;
always = False
checkTemplate = True
- imagerecat.initLists()
+
# Load a lot of default generators
genFactory = pagegenerators.GeneratorFactory()
@@ -590,16 +695,25 @@
pregenerator = pagegenerators.PreloadingGenerator(generator)
- for page in pregenerator:
- processImage(page, checkTemplate)
+ prefetchQueue = Queue(maxsize=50)
+ uploadQueue = Queue(maxsize=200)
+ imageFetcherThread = imageFetcher(pregenerator, prefetchQueue)
+ userInteractionThread = userInteraction(prefetchQueue, uploadQueue)
+ uploaderThread = uploader(uploadQueue)
- wikipedia.output(u'Still ' + str(threading.activeCount()) + u' active
threads, lets wait')
- for openthread in threading.enumerate():
- if openthread != threading.currentThread():
- openthread.join()
- wikipedia.output(u'All threads are done')
+ imageFetcherThread.daemon=False
+ userInteractionThread.daemon=False
+ uploaderThread.daemon=False
+
+ if not checkTemplate:
+ uploaderThread.nochecktemplate()
+
+ fetchDone = imageFetcherThread.start()
+ userDone = userInteractionThread.start()
+ uploadDone = uploaderThread.start()
+
if __name__ == "__main__":
try:
main(sys.argv[1:])
_______________________________________________
Pywikipedia-svn mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikipedia-svn
