Lokal Profil has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/379140 )
Change subject: Make all erfgoedbot scripts respect the skipping mechanisms.
......................................................................
Make all erfgoedbot scripts respect the skipping mechanisms.
The skipping mechanisms (introduced through the merging of the
Wikidata branch) consist of the `skip` field in the config and
the `-skip_wd` command line argument.
Also: set nl-wd_(nl) to skip
Change-Id: I68f345cf09e555e2ea37c4db0197ccc11954abc1
---
M erfgoedbot/add_coord_to_articles.py
M erfgoedbot/add_object_location_monuments.py
M erfgoedbot/categorize_images.py
M erfgoedbot/images_of_monuments_without_id.py
M erfgoedbot/missing_commonscat_links.py
M erfgoedbot/monuments_config/nl-wd_nl.json
M erfgoedbot/populate_image_table.py
M erfgoedbot/unused_monument_images.py
M erfgoedbot/update_id_dump.py
9 files changed, 65 insertions(+), 12 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/labs/tools/heritage
refs/changes/40/379140/1
diff --git a/erfgoedbot/add_coord_to_articles.py
b/erfgoedbot/add_coord_to_articles.py
index 5a9950e..58fa56e 100644
--- a/erfgoedbot/add_coord_to_articles.py
+++ b/erfgoedbot/add_coord_to_articles.py
@@ -291,6 +291,7 @@
def main():
countrycode = u''
lang = u''
+ skip_wd = False
connMon = None
cursorMon = None
@@ -302,10 +303,12 @@
countrycode = value
elif option == '-langcode':
lang = value
+ elif option == '-skip_wd':
+ skip_wd = True
else:
raise Exception(
- u'Bad parameters. Expected "-countrycode", "-langcode" or '
- u'pywikibot args. Found "{}"'.format(option))
+ u'Bad parameters. Expected "-countrycode", "-langcode", '
+ u'"-skip_wd" or pywikibot args. Found "{}"'.format(option))
if countrycode and lang:
if not mconfig.countries.get((countrycode, lang)):
@@ -318,6 +321,9 @@
u'be used together.')
else:
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
pywikibot.output(u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
processCountry(countrycode, lang, countryconfig,
wikiData.get(lang), connMon, cursorMon)
diff --git a/erfgoedbot/add_object_location_monuments.py
b/erfgoedbot/add_object_location_monuments.py
index 25816f4..ae780d9 100644
--- a/erfgoedbot/add_object_location_monuments.py
+++ b/erfgoedbot/add_object_location_monuments.py
@@ -206,6 +206,7 @@
def main():
countrycode = u''
lang = u''
+ skip_wd = False
local_write = None
# Connect database, we need that
@@ -218,12 +219,15 @@
countrycode = value
elif option == '-langcode':
lang = value
+ elif option == '-skip_wd':
+ skip_wd = True
elif option == '-local_write':
local_write = value
else:
raise Exception(
u'Bad parameters. Expected "-countrycode", "-langcode", '
- u'"-local_write" or pywikibot args. Found "{}"'.format(option))
+ u'"-skip_wd", "-local_write" or pywikibot args. '
+ u'Found "{}"'.format(option))
pywikibot.setSite(pywikibot.getSite(u'commons', u'commons'))
@@ -241,6 +245,9 @@
u'be used together.')
else:
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
if not countryconfig.get('autoGeocode'):
pywikibot.output(
u'"%s" in language "%s" is not supported in auto geocode
mode (yet).' % (countrycode, lang))
diff --git a/erfgoedbot/categorize_images.py b/erfgoedbot/categorize_images.py
index 06c7b71..746dc59 100644
--- a/erfgoedbot/categorize_images.py
+++ b/erfgoedbot/categorize_images.py
@@ -531,6 +531,7 @@
countrycode = u''
lang = u''
overridecat = u''
+ skip_wd = False
local_write = None
conn = None
cursor = None
@@ -545,13 +546,15 @@
lang = value
elif option == '-overridecat':
overridecat = value
+ elif option == '-skip_wd':
+ skip_wd = True
elif option == '-local_write':
local_write = value
else:
raise Exception(
u'Bad parameters. Expected "-countrycode", "-langcode", '
- u'"-overridecat", "-local_write" or pywikibot args. '
- u'Found "{}"'.format(option))
+ u'"-overridecat", "-skip_wd", "-local_write" or pywikibot '
+ u'args. Found "{}"'.format(option))
if countrycode and lang:
if not mconfig.countries.get((countrycode, lang)):
@@ -572,6 +575,9 @@
else:
statistics = []
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
if (countrycode, lang) in SKIP_LIST:
pywikibot.log(
diff --git a/erfgoedbot/images_of_monuments_without_id.py
b/erfgoedbot/images_of_monuments_without_id.py
index e521682..3a61d04 100644
--- a/erfgoedbot/images_of_monuments_without_id.py
+++ b/erfgoedbot/images_of_monuments_without_id.py
@@ -195,6 +195,7 @@
def main():
countrycode = u''
lang = u''
+ skip_wd = False
local_write = None
conn = None
cursor = None
@@ -208,12 +209,15 @@
countrycode = value
elif option == '-langcode':
lang = value
+ elif option == '-skip_wd':
+ skip_wd = True
elif option == '-local_write':
local_write = value
else:
raise Exception(
u'Bad parameters. Expected "-countrycode", "-langcode", '
- u'"-local_write" or pywikibot args. Found "{}"'.format(option))
+ u'"-skip_wd", "-local_write" or pywikibot args. '
+ u'Found "{}"'.format(option))
if countrycode and lang:
if not mconfig.countries.get((countrycode, lang)):
@@ -229,6 +233,9 @@
u'be used together.')
else:
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
pywikibot.output(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
processCountry(
diff --git a/erfgoedbot/missing_commonscat_links.py
b/erfgoedbot/missing_commonscat_links.py
index 39705cb..ca342d7 100644
--- a/erfgoedbot/missing_commonscat_links.py
+++ b/erfgoedbot/missing_commonscat_links.py
@@ -207,6 +207,7 @@
def main():
countrycode = u''
lang = u''
+ skip_wd = False
local_write = None
conn = None
cursor = None
@@ -220,12 +221,15 @@
countrycode = value
elif option == '-langcode':
lang = value
+ elif option == '-skip_wd':
+ skip_wd = True
elif option == '-local_write':
local_write = value
else:
raise Exception(
u'Bad parameters. Expected "-countrycode", "-langcode", '
- u'"-local_write" or pywikibot args. Found "{}"'.format(option))
+ u'"-skip_wd", "-local_write" or pywikibot args. '
+ u'Found "{}"'.format(option))
if countrycode and lang:
if not mconfig.countries.get((countrycode, lang)):
@@ -242,6 +246,9 @@
else:
totals = {}
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
totals[(countrycode, lang)] = processCountry(
diff --git a/erfgoedbot/monuments_config/nl-wd_nl.json
b/erfgoedbot/monuments_config/nl-wd_nl.json
index 61d8216..e39351e 100644
--- a/erfgoedbot/monuments_config/nl-wd_nl.json
+++ b/erfgoedbot/monuments_config/nl-wd_nl.json
@@ -3,6 +3,7 @@
"lang": "nl",
"description": "Rijksmonumenten in the Netherlands in Dutch",
"type": "sparql",
+ "skip": true,
"project": "wikipedia",
"table": "monuments_nl-wd_(nl)",
"commonsTemplate": "Rijksmonument",
diff --git a/erfgoedbot/populate_image_table.py
b/erfgoedbot/populate_image_table.py
index 0ca7b66..4514fc4 100644
--- a/erfgoedbot/populate_image_table.py
+++ b/erfgoedbot/populate_image_table.py
@@ -44,10 +44,13 @@
pass
-def getSources(countrycode=u''):
+def getSources(countrycode=u'', skip_wd=False):
"""Get a dictionary of sources to go harvest."""
sources = {}
for (icountrycode, lang), countryconfig in
mconfig.get_countries().iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
if not countrycode or (countrycode and countrycode == icountrycode):
if icountrycode not in sources:
if countryconfig.get('commonsTemplate') and
countryconfig.get('commonsTrackerCategory'):
@@ -220,12 +223,15 @@
def main():
countrycode = u''
+ skip_wd = False
local_write = None
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
if option == '-countrycode':
countrycode = value
+ elif option == '-skip_wd':
+ skip_wd = True
elif option == '-local_write':
local_write = value
else:
@@ -245,7 +251,7 @@
else:
pywikibot.output(u'Working on all countrycodes')
- sources = getSources()
+ sources = getSources(skip_wd=skip_wd)
if not sources:
pywikibot.output(
u'No sources found, something went completely wrong')
diff --git a/erfgoedbot/unused_monument_images.py
b/erfgoedbot/unused_monument_images.py
index 07676d5..fffef99 100644
--- a/erfgoedbot/unused_monument_images.py
+++ b/erfgoedbot/unused_monument_images.py
@@ -197,6 +197,7 @@
def main():
countrycode = u''
lang = u''
+ skip_wd = False
local_write = None
conn = None
cursor = None
@@ -210,12 +211,15 @@
countrycode = value
elif option == '-langcode':
lang = value
+ elif option == u'-skip_wd':
+ skip_wd = True
elif option == '-local_write':
local_write = value
else:
raise Exception(
u'Bad parameters. Expected "-countrycode", "-langcode", '
- u'"-local_write" or pywikibot args. Found "{}"'.format(option))
+ u'"-skip_wd", "-local_write" or pywikibot args. '
+ u'Found "{}"'.format(option))
if countrycode and lang:
if not mconfig.countries.get((countrycode, lang)):
@@ -232,6 +236,9 @@
else:
totals = {}
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
totals[(countrycode, lang)] = processCountry(
diff --git a/erfgoedbot/update_id_dump.py b/erfgoedbot/update_id_dump.py
index b4b16b2..d89a5c5 100755
--- a/erfgoedbot/update_id_dump.py
+++ b/erfgoedbot/update_id_dump.py
@@ -125,6 +125,7 @@
countrycode = u''
lang = u''
+ skip_wd = False
conn = None
cursor = None
(conn, cursor) = connect_to_monuments_database()
@@ -135,10 +136,12 @@
countrycode = value
elif option == '-langcode':
lang = value
+ elif option == u'-skip_wd':
+ skip_wd = True
else:
raise Exception(
- u'Bad parameters. Expected "-countrycode", "-langcode" or '
- u'pywikibot args. Found "{}"'.format(option))
+ u'Bad parameters. Expected "-countrycode", "-langcode", '
+ u'"-skip_wd" or pywikibot args. Found "{}"'.format(option))
query = u"""TRUNCATE table `id_dump`"""
cursor.execute(query)
@@ -157,6 +160,9 @@
u'be used together.')
else:
for (countrycode, lang), countryconfig in
mconfig.countries.iteritems():
+ if (countryconfig.get('skip') or
+ (skip_wd and (countryconfig.get('type') == 'sparql'))):
+ continue
pywikibot.log(
u'Working on countrycode "%s" in language "%s"' %
(countrycode, lang))
processCountry(countrycode, lang, countryconfig, conn, cursor)
--
To view, visit https://gerrit.wikimedia.org/r/379140
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I68f345cf09e555e2ea37c4db0197ccc11954abc1
Gerrit-PatchSet: 1
Gerrit-Project: labs/tools/heritage
Gerrit-Branch: master
Gerrit-Owner: Lokal Profil <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
