jenkins-bot has submitted this change and it was merged.
Change subject: Restructure logging
......................................................................
Restructure logging
The output file for the normal database update run is currently 1,328,686 lines
out of which 1,282,521 are "Found unknown field on page". Setting logging levels
is the first step in making the logs useful for spotting actual errors.
pywikibot.output() has been replaced throughout by one of error(),
warning(), log() or debug(). Where unsure about the severity I have opted for
a more severe logging level, this can be revised later if needed.
When executed with the -log flag the warning() and error() will also output
to stdout whereas log() will only be outputted to the log file. For debug()
the execution must include the -debug flag.
Note that this has only been applied to python files called by
bin/update_monuments.sh
Bug: T138633
Change-Id: Ifc5e0ccf747033d10a87fbdadc408caf304ae665
---
M bin/update_monuments.sh
M erfgoedbot/categorize_images.py
M erfgoedbot/missing_commonscat_links.py
M erfgoedbot/unused_monument_images.py
M erfgoedbot/update_database.py
5 files changed, 83 insertions(+), 61 deletions(-)
Approvals:
Jean-Frédéric: Looks good to me, approved
jenkins-bot: Verified
diff --git a/bin/update_monuments.sh b/bin/update_monuments.sh
index e9f2792..287875e 100755
--- a/bin/update_monuments.sh
+++ b/bin/update_monuments.sh
@@ -13,7 +13,7 @@
# Load any config changes into the source tables
echo_time "Load changes to monuments_config..."
-/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/monument_tables.py
+/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/monument_tables.py -log
# Recreate the source tables
echo_time "Recreating the source tables..."
@@ -24,7 +24,7 @@
# Update all of the source tables
echo_time "Full source database update..."
-/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/update_database.py -fullupdate
+/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/update_database.py -fullupdate -log
# Update the all monuments table
echo_time "Update monuments_all table..."
@@ -40,7 +40,7 @@
# Make statistics
echo_time "Make statistics..."
-/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/database_statistics.py
+/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/database_statistics.py -log
# Make more detailed statistics
echo_time "Make more detailed statistics..."
@@ -48,11 +48,11 @@
# Update the list of unused monuments
echo_time "Update unused images list..."
-/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/unused_monument_images.py
+/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/unused_monument_images.py -log
# Make a list of missing commonscat links
echo_time "Make a list of missing commonscat links..."
-/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/missing_commonscat_links.py
+/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/missing_commonscat_links.py -log
# Dump database to a file so people can play around with it
cd ./public_html || exit
@@ -76,6 +76,6 @@
# Categorize some images
echo_time "Categorize images..."
-/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/categorize_images.py
+/data/project/heritage/pywikibot/pwb.py
/data/project/heritage/erfgoedbot/categorize_images.py -log
echo_time "Done with the update!"
diff --git a/erfgoedbot/categorize_images.py b/erfgoedbot/categorize_images.py
index 80bbbb6..47ca122 100644
--- a/erfgoedbot/categorize_images.py
+++ b/erfgoedbot/categorize_images.py
@@ -24,6 +24,8 @@
import re
import MySQLdb
+_logger = "categorize_images"
+
class
NoMonumentIdentifierFoundException(pywikibot.exceptions.PageRelatedError):
message = u"No Monument Identifier could be found for %s"
@@ -195,30 +197,31 @@
def categorizeImage(countrycode, lang, commonsTemplateName,
commonsCategoryBase, commonsCatTemplates, page, conn, cursor):
- pywikibot.output(u'Working on: %s' % page.title())
+ pywikibot.log(u'Working on: %s' % page.title())
site = pywikibot.Site(u'commons', u'commons')
commonsTemplate = pywikibot.Page(site, 'Template:%s' % commonsTemplateName)
currentcats = list(page.categories())
if commonsCategoryBase not in currentcats:
- pywikibot.output(u'%s category not found at: %s. Someone probably
already categorized it.' % (
+ pywikibot.log(u'%s category not found at: %s. Someone probably already
categorized it.' % (
commonsCategoryBase, page.title()))
return False
if u'Wikipedia image placeholders for cultural heritage monuments' in
currentcats:
- pywikibot.output(u'%s in %s is a placeholder, skipping it.' % (
+ pywikibot.log(u'%s in %s is a placeholder, skipping it.' % (
page.title(), commonsCategoryBase))
return False
templates = page.templates()
if commonsTemplate not in templates:
- pywikibot.output(u'%s template not found at: %s' %
- (commonsTemplate, page.title()))
+ pywikibot.log(u'%s template not found at: %s' % (
+ commonsTemplate, page.title()))
return False
try:
monumentId = get_monument_id(page, commonsTemplate)
except NoMonumentIdentifierFoundException:
- pywikibot.output(u'Didn\'t find a valid monument identifier')
+ pywikibot.warning(u'Didn\'t find a valid monument identifier at: %s' %
(
+ page.title(),))
return False
monData = getMonData(countrycode, lang, monumentId, conn, cursor)
@@ -227,11 +230,12 @@
monumentId = int(monumentId)
monData = getMonData(countrycode, lang, monumentId, conn, cursor)
except ValueError:
- pywikibot.output(
- u'Can\'t convert %s to an integer' % (monumentId,))
+ pywikibot.debug(
+ u'Can\'t convert %s to an integer' % (monumentId,), _logger)
if not monData:
- pywikibot.output(
+ # Triage as log since there are plenty of valid reasons for this
+ pywikibot.log(
u'Monument with id %s not in monuments database' % (monumentId, ))
return False
@@ -243,7 +247,7 @@
commonsTemplateName, monumentId)
replace_default_cat_with_new_categories_in_image(page,
commonsCategoryBase, newcats, comment, verbose=True)
else:
- pywikibot.output(u'Categories not found for %s' % page.title())
+ pywikibot.log(u'Categories not found for %s' % page.title())
def get_monument_id(page, commonsTemplate):
@@ -254,7 +258,9 @@
try:
monumentId = params[0]
except ValueError:
- pywikibot.output(u'Unable to extract a valid id')
+ pywikibot.warning(
+ u'Unable to extract a valid id for %s on %s' % (
+ template, page.title()))
break
if not monumentId:
raise NoMonumentIdentifierFoundException(page)
@@ -273,10 +279,12 @@
cat = pywikibot.Category(commons_site, monumentCommonscat)
newcats.append(cat)
except ValueError:
- pywikibot.output(u'The Commonscat field for %s contains an invalid
category %s' % (
- monumentId, monumentCommonscat))
+ pywikibot.warning(
+ u'The Commonscat field for %s contains an invalid category %s'
% (
+ monumentId, monumentCommonscat))
except pywikibot.exceptions.InvalidTitle:
- pywikibot.output(u'Incorrect category title %s' %
(monumentCommonscat,))
+ pywikibot.warning(
+ u'Incorrect category title %s' % (monumentCommonscat,))
# Option two is to use the article about the monument and see if it has
# Commonscat links
if not newcats:
@@ -296,12 +304,14 @@
newcats.append(
getCategoryFromCommonscat(monumentArticle,
commonsCatTemplates))
except pywikibot.SectionError:
- pywikibot.output(u'Incorrect redirect at %s' %
- (monumentArticle.title(),))
+ pywikibot.warning(u'Incorrect redirect at %s' % (
+ monumentArticle.title(),))
except pywikibot.exceptions.InvalidTitle:
- pywikibot.output(u'Incorrect article title %s' %
(monumentArticleTitle,))
+ pywikibot.warning(u'Incorrect article title %s' % (
+ monumentArticleTitle,))
except pywikibot.exceptions.Error as e:
- pywikibot.output(u'Error occured with monument %s: %s' %
(monumentId, str(e)))
+ pywikibot.error(u'Error occured with monument %s: %s' % (
+ monumentId, str(e)))
# Option three is to see if the list contains Commonscat links (whole list)
if not newcats:
monumentList = getList(lang, monumentSource)
@@ -313,7 +323,8 @@
try:
newcats = getCategories(monumentList, commonsCatTemplates)
except pywikibot.exceptions.Error as e:
- pywikibot.output(u'Error occured with monument %s: %s' %
(monumentId, str(e)))
+ pywikibot.error(u'Error occured with monument %s: %s' % (
+ monumentId, str(e)))
return newcats
@@ -331,7 +342,7 @@
page.put(final_text, comment)
return True
except pywikibot.EditConflict:
- pywikibot.output(
+ pywikibot.log(
u'Got an edit conflict. Someone else beat me to it at %s' %
page.title())
return False
@@ -378,7 +389,7 @@
row = cursor.fetchone()
return row
except TypeError:
- pywikibot.output(u'Didn\'t find anything for id %s' % (monumentId,))
+ pywikibot.warning(u'Didn\'t find anything for id %s' % (monumentId,))
return False
@@ -524,7 +535,7 @@
if (not commonsCatTemplates):
# No commonsCatTemplates found, just skip.
- pywikibot.output(
+ pywikibot.warning(
u'Language: %s has no commonsCatTemplates set!' % lang)
return False
@@ -638,10 +649,10 @@
if countrycode:
lang = pywikibot.Site().language()
if not mconfig.countries.get((countrycode, lang)):
- pywikibot.output(
+ pywikibot.warning(
u'I have no config for countrycode "%s" in language "%s"' %
(countrycode, lang))
return False
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' % (countrycode,
lang))
commonsCatTemplates = getCommonscatTemplates(lang)
# print commonsCatTemplates
@@ -650,7 +661,7 @@
else:
statistics = []
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
commonsCatTemplates = getCommonscatTemplates(lang)
result = processCountry(
diff --git a/erfgoedbot/missing_commonscat_links.py
b/erfgoedbot/missing_commonscat_links.py
index 8796bed..36753da 100644
--- a/erfgoedbot/missing_commonscat_links.py
+++ b/erfgoedbot/missing_commonscat_links.py
@@ -17,6 +17,8 @@
import re
import MySQLdb
+_logger = "missing_commonscat"
+
def connectDatabase():
'''
@@ -64,9 +66,8 @@
commonscats = getMonumentCommonscats(
commonsTrackerCategory, conn2, cursor2)
- pywikibot.output(u'withoutCommonscat %s elements' %
- (len(withoutCommonscat),))
- pywikibot.output(u'commonscats %s elements' % (len(commonscats),))
+ pywikibot.log(u'withoutCommonscat %s elements' % (len(withoutCommonscat),))
+ pywikibot.log(u'commonscats %s elements' % (len(commonscats),))
# People can add a /header template for with more info
text = u'{{#ifexist:{{FULLPAGENAME}}/header | {{/header}} }}\n'
@@ -102,7 +103,7 @@
categoryName, 'utf-8'), unicode(categoryName,
'utf-8').replace(u'_', u' '), monumentId, wikiSourceList)
totalCategories = totalCategories + 1
except ValueError:
- pywikibot.output(u'Got value error for %s' % (monumentId,))
+ pywikibot.warning(u'Got value error for %s' % (monumentId,))
# text = text + u'</gallery>'
@@ -119,7 +120,7 @@
site = pywikibot.Site(lang, u'wikipedia')
page = pywikibot.Page(site, missingCommonscatPage)
- pywikibot.output(text)
+ pywikibot.debug(text, _logger)
page.put(text, comment)
return totalCategories
@@ -211,7 +212,7 @@
site, u'Commons:Monuments database/Missing commonscat
links/Statistics')
comment = u'Updating missing commonscat links statistics. Total missing
links: %s' % totalCategories
- pywikibot.output(text)
+ pywikibot.debug(text, _logger)
page.put(newtext=text, comment=comment)
@@ -231,17 +232,17 @@
if countrycode:
lang = pywikibot.Site().language()
if not mconfig.countries.get((countrycode, lang)):
- pywikibot.output(
+ pywikibot.warning(
u'I have no config for countrycode "%s" in language "%s"' %
(countrycode, lang))
return False
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' % (countrycode,
lang))
processCountry(countrycode, lang, mconfig.countries.get(
(countrycode, lang)), conn, cursor, conn2, cursor2)
else:
totals = {}
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
totals[(countrycode, lang)] = processCountry(
countrycode, lang, countryconfig, conn, cursor, conn2, cursor2)
diff --git a/erfgoedbot/unused_monument_images.py
b/erfgoedbot/unused_monument_images.py
index 700ebb9..61e2b7b 100644
--- a/erfgoedbot/unused_monument_images.py
+++ b/erfgoedbot/unused_monument_images.py
@@ -17,6 +17,8 @@
import re
import MySQLdb
+_logger = "unused_images"
+
def connectDatabase():
'''
@@ -55,8 +57,8 @@
withoutPhoto = getMonumentsWithoutPhoto(countrycode, lang, conn, cursor)
photos = getMonumentPhotos(commonsTrackerCategory, conn2, cursor2)
- pywikibot.output(u'withoutPhoto %s elements' % (len(withoutPhoto),))
- pywikibot.output(u'photos %s elements' % (len(photos),))
+ pywikibot.log(u'withoutPhoto %s elements' % (len(withoutPhoto),))
+ pywikibot.log(u'photos %s elements' % (len(photos),))
# People can add a /header template for with more info
text = u'{{#ifexist:{{FULLPAGENAME}}/header | {{/header}} }}\n'
@@ -91,7 +93,9 @@
try:
wikiSourceList = m.group(1)
except AttributeError:
- pywikibot.output(u'Could not find wikiSourceList for %s
(%s)' % (monumentId, withoutPhoto.get(monumentId)))
+ pywikibot.warning(
+ u'Could not find wikiSourceList for %s (%s)' % (
+ monumentId, withoutPhoto.get(monumentId)))
continue
imageName = photos.get(catSortKey)
# pywikibot.output(u'Key %s returned a result' % (monumentId,))
@@ -103,7 +107,7 @@
unicode(imageName, 'utf-8'), wikiSourceList,
monumentId)
totalImages = totalImages + 1
except ValueError:
- pywikibot.output(u'Got value error for %s' % (monumentId,))
+ pywikibot.warning(u'Got value error for %s' % (monumentId,))
text = text + u'</gallery>'
@@ -120,7 +124,7 @@
site = pywikibot.Site(lang, project)
page = pywikibot.Page(site, unusedImagesPage)
- pywikibot.output(text)
+ pywikibot.debug(text, _logger)
page.put(text, comment, minorEdit=False)
return totalImages
@@ -203,7 +207,7 @@
site, u'Commons:Monuments database/Unused images/Statistics')
comment = u'Updating unused image statistics. Total unused images: %s' %
totalImages
- pywikibot.output(text)
+ pywikibot.debug(text, _logger)
page.put(newtext=text, comment=comment)
@@ -223,17 +227,17 @@
if countrycode:
lang = pywikibot.Site().language()
if not mconfig.countries.get((countrycode, lang)):
- pywikibot.output(
+ pywikibot.warning(
u'I have no config for countrycode "%s" in language "%s"' %
(countrycode, lang))
return False
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' % (countrycode,
lang))
processCountry(countrycode, lang, mconfig.countries.get(
(countrycode, lang)), conn, cursor, conn2, cursor2)
else:
totals = {}
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
totals[(countrycode, lang)] = processCountry(
countrycode, lang, countryconfig, conn, cursor, conn2, cursor2)
diff --git a/erfgoedbot/update_database.py b/erfgoedbot/update_database.py
index a4b1f9d..e747969 100755
--- a/erfgoedbot/update_database.py
+++ b/erfgoedbot/update_database.py
@@ -25,6 +25,8 @@
CH1903Converter
)
+_logger = "update_database"
+
def connectDatabase():
'''
@@ -38,11 +40,11 @@
def reportDataError(errorMsg, wikiPage, exceptWord, comment=''):
-
+ """Report data error to the talk page of the list."""
if not comment:
comment = errorMsg
- pywikibot.output(errorMsg)
+ pywikibot.debug(errorMsg, _logger)
talkPage = wikiPage.toggleTalkPage()
try:
content = talkPage.get()
@@ -384,8 +386,10 @@
contents[field] = value
else:
# FIXME: Include more information where it went wrong
- pywikibot.output(
- u'Found unknown field on page %s : (%s: %s)' % (title,
field, value))
+ pywikibot.debug(
+ u'Found unknown field on page %s : (%s: %s)' % (
+ title, field, value),
+ _logger)
if field in unknownFields:
unknownFields[field] = unknownFields.get(field) + 1
else:
@@ -411,7 +415,7 @@
updateMonument(
contents, source, countryconfig, conn, cursor, sourcePage)
else:
- pywikibot.output(u"No primkey available on %s" % title)
+ pywikibot.warning(u"No primkey available on %s" % title)
return unknownFields
@@ -520,28 +524,30 @@
if countrycode:
lang = pywikibot.Site().language()
if not mconfig.countries.get((countrycode, lang)):
- pywikibot.output(
+ pywikibot.warning(
u'I have no config for countrycode "%s" in language "%s"' %
(countrycode, lang))
return False
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' % (countrycode,
lang))
try:
processCountry(
mconfig.countries.get((countrycode, lang)), conn, cursor,
fullUpdate, daysBack)
except Exception, e:
- pywikibot.output("Unknown error occurred when processing country
%s in lang %s" % (countrycode, lang))
- pywikibot.output(str(e))
+ pywikibot.error(
+ u"Unknown error occurred when processing country %s in lang
%s\n%s" %
+ (countrycode, lang, str(e)))
else:
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
- pywikibot.output(
+ pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
try:
processCountry(countryconfig, conn, cursor, fullUpdate,
daysBack)
except Exception, e:
- pywikibot.output("Unknown error occurred when processing
country %s in lang %s" % (countrycode, lang))
- pywikibot.output(str(e))
+ pywikibot.error(
+ u"Unknown error occurred when processing country %s in
lang %s\n%s" %
+ (countrycode, lang, str(e)))
continue
'''
--
To view, visit https://gerrit.wikimedia.org/r/296885
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ifc5e0ccf747033d10a87fbdadc408caf304ae665
Gerrit-PatchSet: 1
Gerrit-Project: labs/tools/heritage
Gerrit-Branch: master
Gerrit-Owner: Lokal Profil <[email protected]>
Gerrit-Reviewer: Jean-Frédéric <[email protected]>
Gerrit-Reviewer: Multichill <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
