Lokal Profil has uploaded a new change for review.
https://gerrit.wikimedia.org/r/281282
Change subject: De-flaking scripts and including them in tox
......................................................................
De-flaking scripts and including them in tox
Change-Id: I014851bcde70a9674b6a0486db8882fdeece1db3
---
M erfgoedbot/add_coord_to_articles.py
M erfgoedbot/images_of_monuments_without_id.py
M erfgoedbot/top_streets.py
M erfgoedbot/update_id_dump.py
M tox.ini
5 files changed, 77 insertions(+), 57 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/labs/tools/heritage
refs/changes/82/281282/1
diff --git a/erfgoedbot/add_coord_to_articles.py
b/erfgoedbot/add_coord_to_articles.py
index 27325e2..8f36cd3 100644
--- a/erfgoedbot/add_coord_to_articles.py
+++ b/erfgoedbot/add_coord_to_articles.py
@@ -19,19 +19,20 @@
import os
import monuments_config as mconfig
import pywikibot
-import re, MySQLdb
+import re
+import MySQLdb
-#coordinate templates for different language wikipedias
+# coordinate templates for different language wikipedias
wikiData = {
- ('et') : {
- 'coordTemplate' : 'Coordinate',
- #coordTemplateSyntax % (lat, lon, countrycode.upper() )
- 'coordTemplateSyntax' :
u'{{Coordinate|NS=%f|EW=%f|type=landmark|region=%s}}'
+ ('et'): {
+ 'coordTemplate': 'Coordinate',
+ # coordTemplateSyntax % (lat, lon, countrycode.upper() )
+ 'coordTemplateSyntax':
u'{{Coordinate|NS=%f|EW=%f|type=landmark|region=%s}}'
},
- ('fr') : {
- 'coordTemplate' : 'coord',
-
#{{coord|52.51626|13.3777|type:landmark_region:DE|format=dms|display=title}}
- 'coordTemplateSyntax' :
u'{{coord|%f|%f|type:landmark_region=%s|format=dms|display=title}}'
+ ('fr'): {
+ 'coordTemplate': 'coord',
+ #
{{coord|52.51626|13.3777|type:landmark_region:DE|format=dms|display=title}}
+ 'coordTemplateSyntax':
u'{{coord|%f|%f|type:landmark_region=%s|format=dms|display=title}}'
}
}
@@ -49,16 +50,16 @@
# classes
class Monument:
-#Constructor with default arguments
- def __init__(self, id = None):
- self.id = id
- self.name = u''
- self.country = u''
- self.wikilang = u''
- self.article = u''
- self.lat = None
- self.lon = None
- self.source = u''
+ # Constructor with default arguments
+ def __init__(self, id=None):
+ self.id = id
+ self.name = u''
+ self.country = u''
+ self.wikilang = u''
+ self.article = u''
+ self.lat = None
+ self.lon = None
+ self.source = u''
# functions
@@ -67,11 +68,13 @@
'''
Connect to the monuments mysql database
'''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
+ conn = MySQLdb.connect(
+ host=mconfig.db_server, db=mconfig.db,
read_default_file=os.path.expanduser("~/.my.cnf"),
use_unicode=True, charset='utf8')
cursor = conn.cursor()
return (conn, cursor)
+
def connectWikiDatabase(lang):
'''
@@ -80,14 +83,17 @@
if (lang):
hostName = lang + 'wiki.labsdb'
dbName = lang + 'wiki_p'
- #coordDbName = 'u_dispenser_p'
- conn = MySQLdb.connect(host=hostName, db=dbName,
+ # coordDbName = 'u_dispenser_p'
+ conn = MySQLdb.connect(
+ host=hostName, db=dbName,
read_default_file=os.path.expanduser("~/.my.cnf"),
use_unicode=True, charset='utf8')
cursor = conn.cursor()
return (conn, cursor)
-def processCountry(countrycode, lang, countryconfig, coordconfig, connMon,
cursorMon):
+
+def processCountry(countrycode, lang, countryconfig, coordconfig,
+ connMon, cursorMon):
'''
Work on a single country.
'''
@@ -142,7 +148,6 @@
addCoords(countrycode, lang, aMonument, coordconfig)
-
def getMonumentsWithCoordinates(countrycode, lang, cursor):
'''
Get monuments with coordinates from monuments database for a certain
country/language combination.
@@ -152,7 +157,7 @@
WHERE lat<>0 AND lon<>0 AND country=%s AND lang=%s"""
cursor.execute(query, (countrycode, lang))
- #result = cursor.fetchall ()
+ # result = cursor.fetchall ()
while True:
try:
row = cursor.fetchone()
@@ -165,6 +170,7 @@
break
return result
+
def hasCoordinates(pageId, lang, cursor):
'''
@@ -179,7 +185,7 @@
WHERE (gc_from = %s AND gc_primary = 1)
LIMIT 1"""
# FIXME escape & sanitize coordTable and pageId
- cursor.execute(query % (coordTable, int(pageId) ))
+ cursor.execute(query % (coordTable, int(pageId)))
if (cursor.rowcount > 0):
return True
@@ -188,19 +194,21 @@
else:
return False
-def getPageId(pageName, conn, cursor, pageNamespace = WP_ARTICLE_NS,
followRedirect = False):
+
+def getPageId(pageName, conn, cursor,
+ pageNamespace=WP_ARTICLE_NS, followRedirect=False):
'''
get Wikipedia pagename pageId
'''
- #underscores
+ # underscores
pageName = pageName.replace(u' ', u'_')
retStatus = ''
pageId = ''
redirNs = ''
redirTitle = u''
- #FIXME page_titles like 'Château_de_Bercy' won't work, but titles like
'Käru' do ??
+ # FIXME page_titles like 'Château_de_Bercy' won't work, but titles like
'Käru' do ??
query = """SELECT page_id, page_is_redirect FROM page
WHERE page_namespace = %s AND page_title = %s"""
cursor.execute(query, (pageNamespace, pageName))
@@ -223,6 +231,7 @@
retStatus = 'OK'
return (retStatus, pageId, redirNs, redirTitle)
+
def getRedirPageNsTitle(pageId, cursor):
'''
@@ -257,12 +266,12 @@
page = pywikibot.Page(site, monument.article)
try:
text = page.get()
- except pywikibot.NoPage: # First except, prevent empty pages
+ except pywikibot.NoPage: # First except, prevent empty pages
return False
- except pywikibot.IsRedirectPage: # second except, prevent redirect
+ except pywikibot.IsRedirectPage: # second except, prevent redirect
pywikibot.output(u'%s is a redirect!' % monument.article)
return False
- except pywikibot.Error: # third exception, take the problem and print
+ except pywikibot.Error: # third exception, take the problem and print
pywikibot.output(u"Some error, skipping..")
return False
@@ -271,7 +280,8 @@
newtext = text
replCount = 1
- coordText = coordTemplateSyntax % (monument.lat, monument.lon,
countrycode.upper() )
+ coordText = coordTemplateSyntax % (monument.lat, monument.lon,
+ countrycode.upper())
localCatName = pywikibot.getSite().namespace(WP_CATEGORY_NS)
catStart = r'\[\[(' + localCatName + '|Category):'
catStartPlain = u'[[' + localCatName + ':'
@@ -288,7 +298,7 @@
wikilist = matchWikipage.group(1)
comment = u'Adding template %s based on [[%s]], # %s' %
(coordTemplate, wikilist, monument.id)
pywikibot.showDiff(text, newtext)
- modPage = pywikibot.input(u'Modify page: %s ([y]/n) ?' %
(monument.article) )
+ modPage = pywikibot.input(u'Modify page: %s ([y]/n) ?' %
(monument.article))
if (modPage.lower == 'y' or modPage == ''):
page.put(newtext, comment)
return True
@@ -297,6 +307,7 @@
else:
return False
+
def main():
countrycode = u''
connMon = None
diff --git a/erfgoedbot/images_of_monuments_without_id.py
b/erfgoedbot/images_of_monuments_without_id.py
index d9f4ae5..8b9b76a 100644
--- a/erfgoedbot/images_of_monuments_without_id.py
+++ b/erfgoedbot/images_of_monuments_without_id.py
@@ -2,8 +2,8 @@
# -*- coding: utf-8 -*-
'''
-Add monument-ID-templates to images on Commons -- based on the image usage in
the lists -- and
- make a galleries of monuments without an id at Commons
+Add monument-ID-templates to images on Commons -- based on the image usage in
+ the lists -- and make a galleries of monuments without an id at Commons
Usage:
# loop thtough all countries
@@ -17,7 +17,6 @@
import pywikibot
import config
import MySQLdb
-##import re, imagerecat, pagegenerators, catlib
def connectDatabase():
@@ -75,7 +74,7 @@
text = u'<gallery>\n'
for image in withoutTemplate:
- if not image in ignoreList:
+ if image not in ignoreList:
# An image is in the category and is in the list of used images
if withPhoto.get(image):
added = addCommonsTemplate(
@@ -92,7 +91,9 @@
for image in withPhoto:
# Skip images which already have the templates and the ones in without
# templates to prevent duplicates
- if not image in ignoreList and not image in withTemplate and not image
in withoutTemplate:
+ if image not in ignoreList and \
+ image not in withTemplate and \
+ image not in withoutTemplate:
added = addCommonsTemplate(
image, commonsTemplate, withPhoto.get(image))
if not added:
diff --git a/erfgoedbot/top_streets.py b/erfgoedbot/top_streets.py
index 0f76e22..7026c3c 100644
--- a/erfgoedbot/top_streets.py
+++ b/erfgoedbot/top_streets.py
@@ -12,21 +12,25 @@
import MySQLdb
from collections import Counter
+
def connectDatabase():
'''
Connect to the monuments mysql database, if it fails, go down in flames.
This database is utf-8 encoded.
'''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db, user =
config.db_username, passwd = config.db_password, use_unicode=True,
charset='utf8')
+ conn = MySQLdb.connect(
+ host=mconfig.db_server, db=mconfig.db, user=config.db_username,
+ passwd=config.db_password, use_unicode=True, charset='utf8')
cursor = conn.cursor()
return (conn, cursor)
+
def getAddresses(countrycode, lang, municipality, conn, cursor):
'''
Get a list of addresses of a municipality in a country in a certain
language
'''
result = []
- query = u"""SELECT address FROM monuments_all WHERE country=%s AND lang=%s
AND municipality=%s ORDER BY address ASC""";
+ query = u"""SELECT address FROM monuments_all WHERE country=%s AND lang=%s
AND municipality=%s ORDER BY address ASC"""
cursor.execute(query, (countrycode, lang, municipality))
while True:
@@ -39,11 +43,12 @@
return result
-def printTopStreets (addresses, minimum):
+
+def printTopStreets(addresses, minimum):
'''
Print the top streets with a minimum number of hits
'''
- streets = Counter() #collections.Counter
+ streets = Counter() # collections.Counter
for address in addresses:
address = address.replace(u'{{sorteer|', u'')
temp = u''
@@ -69,14 +74,16 @@
filteredStreets.append(topStreet1)
break
- pywikibot.output(u'Filtered out the following. These are probably street
parts:')
+ pywikibot.output(u'Filtered out the following. These are probably '
+ u'street parts:')
for street in streets.most_common():
if street[1] < minimum:
break
if street[0] in filteredStreets:
pywikibot.output(u'* %s - %s' % street)
- pywikibot.output(u'Found the following entries which are probably real
streets:')
+ pywikibot.output(u'Found the following entries which are probably '
+ u'real streets:')
for street in streets.most_common():
if street[1] < minimum:
break
@@ -95,17 +102,18 @@
(conn, cursor) = connectDatabase()
for arg in pywikibot.handleArgs():
- if arg.startswith('-countrycode:'):
- countrycode = arg [len('-countrycode:'):]
- elif arg.startswith('-municipality:'):
- municipality = arg [len('-municipality:'):]
- elif arg.startswith('-minimum:'):
- minimum = int(arg [len('-minimum:'):])
+ option, sep, value = arg.partition(':')
+ if option == '-countrycode':
+ countrycode = value
+ elif option == '-municipality':
+ municipality = value
+ elif option == '-minimum':
+ minimum = int(value)
if countrycode and municipality:
lang = pywikibot.getSite().language()
addresses = getAddresses(countrycode, lang, municipality, conn, cursor)
- printTopStreets (addresses, minimum)
+ printTopStreets(addresses, minimum)
else:
print u'Usage'
diff --git a/erfgoedbot/update_id_dump.py b/erfgoedbot/update_id_dump.py
index 98dd424..9ca8d91 100755
--- a/erfgoedbot/update_id_dump.py
+++ b/erfgoedbot/update_id_dump.py
@@ -1,7 +1,7 @@
#!/usr/bin/python
-#$ -l h_rt=0:30:00
-#$ -j y
-#$ -o $HOME/erfgoedbot/update_id_dump.out
+# $ -l h_rt=0:30:00
+# $ -j y
+# $ -o $HOME/erfgoedbot/update_id_dump.out
# -*- coding: utf-8 -*-
'''
Update the id_dump table from some wiki page(s)
diff --git a/tox.ini b/tox.ini
index ea10587..573f8aa 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,7 +14,7 @@
commands = flake8
[flake8]
-exclude =
.venv,.tox,add_coord_to_articles.py,top_streets.py,add_object_location_monuments.py,images_of_monuments_without_id.py,update_id_dump.py
+exclude = .venv,.tox
ignore = E501,F841
[nosetests]
--
To view, visit https://gerrit.wikimedia.org/r/281282
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I014851bcde70a9674b6a0486db8882fdeece1db3
Gerrit-PatchSet: 1
Gerrit-Project: labs/tools/heritage
Gerrit-Branch: master
Gerrit-Owner: Lokal Profil <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
