Revision: 8576
Author: xqt
Date: 2010-09-19 17:10:15 +0000 (Sun, 19 Sep 2010)
Log Message:
-----------
update cc from trunk
Modified Paths:
--------------
branches/rewrite/scripts/cosmetic_changes.py
Modified: branches/rewrite/scripts/cosmetic_changes.py
===================================================================
--- branches/rewrite/scripts/cosmetic_changes.py 2010-09-19 16:42:57 UTC
(rev 8575)
+++ branches/rewrite/scripts/cosmetic_changes.py 2010-09-19 17:10:15 UTC
(rev 8576)
@@ -32,7 +32,8 @@
all of them, but be careful if you do.
"""
__version__ = '$Id$'
-import pywikibot, isbn
+import pywikibot
+import isbn
from pywikibot import pagegenerators
import sys
import re
@@ -49,6 +50,7 @@
# Summary message when using this module as a stand-alone script
msg_standalone = {
+ 'commons': u'Bot: [[Commons talk:Tools/pywiki file description
cleanup|desc page fmt]]',
'als':u'Bötli: chleineri Änderige',
'ar': u'روبوت: تغييرات تجميلية',
'be-x-old': u'Робат: касмэтычныя зьмены',
@@ -111,6 +113,7 @@
# Summary message that will be appended to the normal message when
# cosmetic changes are made on the fly
msg_append = {
+ 'commons': u'; [[Commons talk:Tools/pywiki file description cleanup|desc
page fmt]]',
'als':u'; chleineri Änderige',
'ar': u'; تغييرات تجميلية',
'be-x-old': u'; касмэтычныя зьмены',
@@ -170,33 +173,97 @@
'zh': u'; 細部更改',
}
+nn_iw_msg = u'<!--interwiki (no, sv, da first; then other languages
alphabetically by name)-->'
+
+# This is from interwiki.py;
+# move it to family file and implement global instances
+moved_links = {
+ 'ca' : (u'ús de la plantilla', u'/ús'),
+ 'cs' : (u'dokumentace', u'/doc'),
+ 'de' : (u'dokumentation', u'/Meta'),
+ 'en' : ([u'documentation',
+ u'template documentation',
+ u'template doc',
+ u'doc',
+ u'documentation, template'], u'/doc'),
+ 'es' : ([u'documentación', u'documentación de plantilla'], u'/doc'),
+ 'fr' : (u'/documentation', u'/Documentation'),
+ 'hu' : (u'sablondokumentáció', u'/doc'),
+ 'id' : (u'template doc', u'/doc'),
+ 'ja' : (u'documentation', u'/doc'),
+ 'ka' : (u'თარგის ინფო', u'/ინფო'),
+ 'ko' : (u'documentation', u'/설명문서'),
+ 'ms' : (u'documentation', u'/doc'),
+ 'pl' : (u'dokumentacja', u'/opis'),
+ 'pt' : ([u'documentação', u'/doc'], u'/doc'),
+ 'ro' : (u'documentaţie', u'/doc'),
+ 'ru' : (u'doc', u'/doc'),
+ 'sv' : (u'dokumentation', u'/dok'),
+ 'vi' : (u'documentation', u'/doc'),
+ 'zh' : ([u'documentation', u'doc'], u'/doc'),
+}
+
+# Template which should be replaced or removed.
+# Use a list with two entries. The first entry will be replaced by the second.
+# Examples:
+# For removing {{Foo}}, the list must be:
+# (u'Foo', None),
+#
+# The following also works:
+# (u'Foo', ''),
+#
+# For replacing {{Foo}} with {{Bar}} the list must be:
+# (u'Foo', u'Bar'),
+#
+# This also removes all template parameters of {{Foo}}
+# For replacing {{Foo}} with {{Bar}} but keep the template
+# parameters in its original order, please use:
+# (u'Foo', u'Bar\g<parameters>'),
+
+deprecatedTemplates = {
+ 'wikipedia': {
+ 'de': [
+ (u'Belege', u'Belege fehlen\g<parameters>'),
+ (u'Quelle', u'Belege fehlen\g<parameters>'),
+ (u'Quellen', u'Belege fehlen\g<parameters>'),
+ (u'Quellen fehlen', u'Belege fehlen\g<parameters>'),
+ ],
+ }
+}
+
class CosmeticChangesToolkit:
- def __init__(self, site, debug=False, redirect=False, namespace=None):
+ def __init__(self, site, debug=False, redirect=False, namespace=None,
pageTitle=None):
self.site = site
self.debug = debug
self.redirect = redirect
self.namespace = namespace
self.template = (self.namespace == 10)
self.talkpage = self.namespace >= 0 and self.namespace % 2 == 1
+ self.title = pageTitle
def change(self, text):
"""
Given a wiki source code text, return the cleaned up version.
"""
oldText = text
+ if self.site.sitename()== u'commons:commons' and self.namespace == 6:
+ text = self.commonsfiledesc(text)
text = self.fixSelfInterwiki(text)
- text = self.standardizeInterwiki(text)
- text = self.standardizeCategories(text)
+ text = self.standardizePageFooter(text)
text = self.cleanUpLinks(text)
text = self.cleanUpSectionHeaders(text)
text = self.putSpacesInLists(text)
text = self.translateAndCapitalizeNamespaces(text)
+ text = self.replaceDeprecatedTemplates(text)
text = self.resolveHtmlEntities(text)
text = self.validXhtml(text)
text = self.removeUselessSpaces(text)
text = self.removeNonBreakingSpaceBeforePercent(text)
text = self.fixSyntaxSave(text)
text = self.fixHtml(text)
+ text = self.fixStyle(text)
+ text = self.fixTypo(text)
+ text = self.fixArabicLetters(text)
try:
text = isbn.hyphenateIsbnNumbers(text)
except isbn.InvalidIsbnException, error:
@@ -210,21 +277,13 @@
Interwiki links to the site itself are displayed like local links.
Remove their language code prefix.
"""
- interwikiR = re.compile(r'\[\[%s\s?:([^\[\]\n]*)\]\]' % self.site.lang)
- text = interwikiR.sub(r'[[\1]]', text)
- return text
-
- def standardizeInterwiki(self, text):
- """
- Makes sure that interwiki links are put to the correct position and
- into the right order.
- """
if not self.talkpage and pywikibot.calledModuleName() <> 'interwiki':
- interwikiLinks = pywikibot.getLanguageLinks(text, insite =
self.site)
- text = pywikibot.replaceLanguageLinks(text, interwikiLinks, site =
self.site, template = self.template)
+ interwikiR = re.compile(r'\[\[%s\s?:([^\[\]\n]*)\]\]' %
self.site.lang)
+ text = interwikiR.sub(r'[[\1]]', text)
return text
- def standardizeCategories(self, text):
+
+ def standardizePageFooter(self, text):
"""
Makes sure that categories are put to the correct position, but
does not sort them.
@@ -256,7 +315,12 @@
continue
namespaces = list(family.namespace(self.site.lang, nsNumber, all =
True))
thisNs = namespaces.pop(0)
-
+ if nsNumber == 6 and family.name == 'wikipedia' and \
+ self.site.lang in ('en', 'fr'):
+ # do not change "Image" on en-wiki and fr-wiki
+ for image in [u'Image', u'image']:
+ if image in namespaces:
+ namespaces.remove(image)
# skip main (article) namespace
if thisNs and namespaces:
text = pywikibot.replaceExcept(text, r'\[\[\s*(' +
'|'.join(namespaces) + ') *:(?P<nameAndLabel>.*?)\]\]', r'[[' + thisNs +
':\g<nameAndLabel>]]', exceptions)
@@ -435,10 +499,26 @@
and French Wikipedia. It might be that it is not wanted on other wikis.
If there are any complaints, please file a bug report.
"""
- if not self.redirect:
- text = pywikibot.replaceExcept(text,
r'(?m)^(?P<bullet>[:;]*(\*+|#+)[:;\*#]*)(?P<char>[^\s\*#:;].+?)', '\g<bullet>
\g<char>', ['comment', 'math', 'nowiki', 'pre'])
+ exceptions = ['comment', 'math', 'nowiki', 'pre', 'source', 'timeline']
+ if not self.redirect and pywikibot.calledModuleName() <>
'capitalize_redirects':
+ text = pywikibot.replaceExcept(text,
r'(?m)^(?P<bullet>[:;]*(\*+|#+)[:;\*#]*)(?P<char>[^\s\*#:;].+?)', '\g<bullet>
\g<char>', exceptions)
return text
+ def replaceDeprecatedTemplates(self, text):
+ exceptions = ['comment', 'math', 'nowiki', 'pre']
+ if self.site.family.name in deprecatedTemplates and self.site.lang in
deprecatedTemplates[self.site.family.name]:
+ for template in
deprecatedTemplates[self.site.family.name][self.site.lang]:
+ old = template[0]
+ new = template[1]
+ if new == None:
+ new = ''
+ else:
+ new = '{{'+new+'}}'
+ if not self.site.nocapitalize:
+ old = '[' + old[0].upper() + old[0].lower() + ']' + old[1:]
+ text = pywikibot.replaceExcept(text, r'\{\{([mM][sS][gG]:)?' +
old + '(?P<parameters>\|[^}]+|)}}', new, exceptions)
+ return text
+
#from fixes.py
def fixSyntaxSave(self, text):
exceptions = ['nowiki', 'comment', 'math', 'pre', 'source',
'startspace']
@@ -469,11 +549,131 @@
# horizontal line with attributes; can't be done with wiki syntax
# so we only make it XHTML compliant
text = pywikibot.replaceExcept(text, r'(?i)<hr ([^>/]+?)>', r'<hr \1
/>', exceptions)
+ # a header where only spaces are in the same line
+ for level in range(1, 7):
+ equals = '\\1%s \\2 %s\\3' % ("="*level, "="*level)
+ text = pywikibot.replaceExcept(text,
+ r'(?i)([\r\n]) *<h%d> *([^<]+?)
*</h%d> *([\r\n])'%(level, level),
+ r'%s'%equals, exceptions)
+ #remove empty <ref/>-tag
+ text = pywikibot.replaceExcept(text, r'(?i)<ref\s*/>', r'', exceptions)
# TODO: maybe we can make the bot replace <p> tags with \r\n's.
return text
+ def fixStyle(self, text):
+ exceptions = ['nowiki', 'comment', 'math', 'pre', 'source',
'startspace']
+ # convert prettytable to wikitable class
+ if self.site.language in ('de', 'en'):
+ text = pywikibot.replaceExcept(text,
ur'(class="[^"]*)prettytable([^"]*")', ur'\1wikitable\2', exceptions)
+ return text
+
+ def fixTypo(self, text):
+ exceptions = ['nowiki', 'comment', 'math', 'pre', 'source',
'startspace', 'gallery', 'hyperlink', 'interwiki', 'link']
+ # change <number> ccm -> <number> cm³
+ text = pywikibot.replaceExcept(text, ur'(\d)\s* ccm',
ur'\1 cm³', exceptions)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s*ccm', ur'\1 cm³',
exceptions)
+ # Solve wrong Nº sign with °C or °F
+ # additional exception requested on fr-wiki for this stuff
+ pattern = re.compile(u'«.*?»', re.UNICODE)
+ exceptions.append(pattern)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s* [º°]([CF])',
ur'\1 °\2', exceptions)
+ text = pywikibot.replaceExcept(text, ur'(\d)\s*[º°]([CF])',
ur'\1 °\2', exceptions)
+ text = pywikibot.replaceExcept(text, ur'º([CF])', ur'°\1', exceptions)
+ return text
+
+ def fixArabicLetters(self, text):
+ if self.site.lang=='ckb':
+ exceptions = [
+ 'gallery',
+ 'hyperlink',
+ 'interwiki',
+ # but changes letters inside wikilinks
+ #'link',
+ 'math',
+ 'pre',
+ 'template',
+ 'timeline',
+ 'ref',
+ 'source',
+ 'startspace',
+ 'inputbox',
+ ]
+ # do not change inside file links
+ namespaces = list(self.site.namespace(6, all = True))
+ pattern = re.compile(u'\[\[(' + '|'.join(namespaces) +
'):.+?\..+?\]\]', re.UNICODE)
+ exceptions.append(pattern)
+ text = pywikibot.replaceExcept(text, u',', u'،', exceptions)
+ text = pywikibot.replaceExcept(text, ur'ه([.،_<\]\s])', ur'ە\1',
exceptions)
+ text = pywikibot.replaceExcept(text, u'ه', u'ە', exceptions)
+ text = pywikibot.replaceExcept(text, u'ه', u'ھ', exceptions)
+ text = pywikibot.replaceExcept(text, u'ك', u'ک', exceptions)
+ text = pywikibot.replaceExcept(text, ur'[ىي]', u'ی', exceptions)
+ # replace persian digits
+ for i in range(0,10):
+ text = pywikibot.replaceExcept(text, u'۰۱۲۳۴۵۶۷۸۹'[i],
u'٠١٢٣٤٥٦٧٨٩'[i], exceptions)
+ # do not change digits in class, style and table params
+ pattern = re.compile(u'=".*?"', re.UNICODE)
+ exceptions.append(pattern)
+ # do not change digits inside html-tags
+ pattern = re.compile(u'<[/]*?[^</]+?[/]*?>', re.UNICODE)
+ exceptions.append(pattern)
+ for i in range(0,10):
+ text = pywikibot.replaceExcept(text, str(i), u'٠١٢٣٤٥٦٧٨٩'[i],
exceptions)
+ return text
+
+ # Retrieved from
"http://commons.wikimedia.org/wiki/Commons:Tools/pywiki_file_description_cleanup";
+ def commonsfiledesc(self, text):
+ # section headers to {{int:}} versions
+ exceptions = ['comment', 'includeonly', 'math', 'noinclude', 'nowiki',
+ 'pre', 'source', 'ref', 'timeline']
+ text = pywikibot.replaceExcept(text,
+ r"([\r\n]|^)\=\= *Summary *\=\=",
+ r"\1== {{int:filedesc}} ==",
+ exceptions, True)
+ text = pywikibot.replaceExcept(
+ text,
+ r"([\r\n])\=\= *\[\[Commons:Copyright tags\|Licensing\]\]: *\=\=",
+ r"\1== {{int:license}} ==", exceptions, True)
+ text = pywikibot.replaceExcept(
+ text,
+ r"([\r\n])\=\= *(Licensing|License
information|{{int:license-header}}) *\=\=",
+ r"\1== {{int:license}} ==", exceptions, True)
+
+ # frequent field values to {{int:}} versions
+ text = pywikibot.replaceExcept(
+ text,
+ r'([\r\n]\|[Ss]ource *\= *)(?:[Oo]wn work by uploader|[Oo]wn
work|[Ee]igene [Aa]rbeit) *([\r\n])',
+ r'\1{{own}}\2', exceptions, True)
+ text = pywikibot.replaceExcept(
+ text,
+ r'(\| *Permission *\=) *(?:[Ss]ee below|[Ss]iehe unten) *([\r\n])',
+ r'\1\2', exceptions, True)
+
+ # added to transwikied pages
+ text = pywikibot.replaceExcept(text, r'__NOTOC__', '', exceptions,
True)
+
+ # tracker element for js upload form
+ text = pywikibot.replaceExcept(
+ text,
+ r'<!-- *{{ImageUpload\|(?:full|basic)}} *-->',
+ '', exceptions[1:], True)
+ text = pywikibot.replaceExcept(text,
r'{{ImageUpload\|(?:basic|full)}}',
+ '', exceptions, True)
+
+ # duplicated section headers
+ text = pywikibot.replaceExcept(
+ text,
+ r'([\r\n]|^)\=\= *{{int:filedesc}} *\=\=(?:[\r\n ]*)\=\=
*{{int:filedesc}} *\=\=',
+ r'\1== {{int:filedesc}} ==', exceptions, True)
+ text = pywikibot.replaceExcept(
+ text,
+ r'([\r\n]|^)\=\= *{{int:license}} *\=\=(?:[\r\n ]*)\=\=
*{{int:license}} *\=\=',
+ r'\1== {{int:license}} ==', exceptions, True)
+ return text
+
class CosmeticChangesBot:
- def __init__(self, generator, acceptall = False, comment=u'Robot: Cosmetic
changes'):
+ def __init__(self, generator, acceptall = False,
+ comment=u'Robot: Cosmetic changes'):
self.generator = generator
self.acceptall = acceptall
self.comment = comment
@@ -483,13 +683,17 @@
try:
# Show the title of the page we're working on.
# Highlight the title in purple.
- pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" %
page.title())
- ccToolkit = CosmeticChangesToolkit(page.site, debug = True,
namespace = page.namespace())
+ pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
+ % page.title())
+ ccToolkit = CosmeticChangesToolkit(page.site, debug=True,
+ namespace=page.namespace(),
+ pageTitle=page.title())
changedText = ccToolkit.change(page.get())
if changedText.strip() != page.get().strip():
if not self.acceptall:
- choice = pywikibot.inputChoice(u'Do you want to accept
these changes?',
- ['Yes', 'No', 'All',
'Quit'], ['y', 'N', 'a', 'q'], 'N')
+ choice = pywikibot.inputChoice(
+ u'Do you want to accept these changes?',
+ ['Yes', 'No', 'All', 'Quit'], ['y', 'N', 'a', 'q'],
'N')
if choice == 'a':
self.acceptall = True
elif choice == 'q':
@@ -498,15 +702,19 @@
if self.acceptall or choice == 'y':
page.put(changedText, comment=self.comment)
else:
- pywikibot.output('No changes were necessary in %s' %
page.title())
+ pywikibot.output('No changes were necessary in %s'
+ % page.title())
except pywikibot.NoPage:
- pywikibot.output("Page %s does not exist?!" % page.aslink())
+ pywikibot.output("Page %s does not exist?!"
+ % page.title(asLink=True))
except pywikibot.IsRedirectPage:
- pywikibot.output("Page %s is a redirect; skipping." %
page.aslink())
+ pywikibot.output("Page %s is a redirect; skipping."
+ % page.title(asLink=True))
except pywikibot.LockedPage:
- pywikibot.output("Page %s is locked?!" % page.aslink())
+ pywikibot.output("Page %s is locked?!" % page.title(asLink=True))
except pywikibot.EditConflict:
- pywikibot.output("An edit conflict has occured at %s." %
page.aslink())
+ pywikibot.output("An edit conflict has occured at %s."
+ % page.title(asLink=True))
def run(self):
try:
@@ -540,16 +748,6 @@
if editSummary == '':
# Load default summary message.
editSummary = pywikibot.translate(pywikibot.getSite(), msg_standalone)
-
- # Disabled this check. Although the point is still valid, there
- # is now a warning and a prompt (see below).
- #if pywikibot.getSite() == pywikibot.getSite('nl','wikipedia'):
- #print "Deze bot is op WikipediaNL niet gewenst."
- #print "Het toevoegen van cosmetic changes bij andere wijzigingen is
toegestaan,"
- #print "maar cosmetic_changes als stand-alone bot niet."
- #print "Zoek alstublieft een nuttig gebruik voor uw bot."
- #sys.exit()
-
if pageTitle:
site = pywikibot.getSite()
gen = iter([pywikibot.Page(pywikibot.Link(t, site)) for t in
pageTitle])
@@ -558,11 +756,14 @@
if not gen:
pywikibot.showHelp()
elif not always:
- answer = pywikibot.inputChoice(warning + '\nDo you really want to
continue?', ['yes', 'no'], ['y', 'N'], 'N')
+ answer = pywikibot.inputChoice(
+ warning + '\nDo you really want to continue?',
+ ['yes', 'no'], ['y', 'N'], 'N')
if answer == 'y':
preloadingGen = pagegenerators.PreloadingGenerator(gen)
- bot = CosmeticChangesBot(preloadingGen, acceptall=always,
comment=editSummary)
+ bot = CosmeticChangesBot(preloadingGen, acceptall=always,
+ comment=editSummary)
bot.run()
if __name__ == "__main__":
_______________________________________________
Pywikipedia-svn mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikipedia-svn
