Jean-Frédéric has uploaded a new change for review.
https://gerrit.wikimedia.org/r/303427
Change subject: Centralize methods to connect to database in one file
......................................................................
Centralize methods to connect to database in one file
The methods to connect to the databases (monuments and commons)
were repeated in every single file.
This centralizes in one for easier maintenance and reduce dependencies.
It also paves the way for using dependency injection
to avoid a hard dependency on MySQL.
Also rename the methods to clarify their purposes.
Change-Id: I10c6d1733b0a1c02ea3eaeaaab93dac4fb8cf19d
---
M erfgoedbot/add_coord_to_articles.py
M erfgoedbot/add_object_location_monuments.py
M erfgoedbot/categorize_images.py
A erfgoedbot/database_connection.py
M erfgoedbot/database_statistics.py
M erfgoedbot/images_of_monuments_without_id.py
M erfgoedbot/missing_commonscat_links.py
M erfgoedbot/populate_image_table.py
M erfgoedbot/top_streets.py
M erfgoedbot/unused_monument_images.py
M erfgoedbot/update_database.py
M erfgoedbot/update_id_dump.py
12 files changed, 77 insertions(+), 242 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/labs/tools/heritage
refs/changes/27/303427/1
diff --git a/erfgoedbot/add_coord_to_articles.py
b/erfgoedbot/add_coord_to_articles.py
index 1a649b4..3e75b55 100644
--- a/erfgoedbot/add_coord_to_articles.py
+++ b/erfgoedbot/add_coord_to_articles.py
@@ -15,12 +15,12 @@
python add_coord_to_articles.py -countrycode:XX -lang:YY
'''
-
-import os
-import monuments_config as mconfig
-import pywikibot
import re
-import MySQLdb
+
+import pywikibot
+
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database,
connect_to_commons_database
# coordinate templates for different language wikipedias
wikiData = {
@@ -64,33 +64,6 @@
# functions
-def connectMonDatabase():
- '''
- Connect to the monuments mysql database
- '''
- conn = MySQLdb.connect(
- host=mconfig.db_server, db=mconfig.db,
- read_default_file=os.path.expanduser("~/.my.cnf"),
- use_unicode=True, charset='utf8')
- cursor = conn.cursor()
- return (conn, cursor)
-
-
-def connectWikiDatabase(lang):
- '''
- Connect to the wiki database
- '''
- if (lang):
- hostName = lang + 'wiki.labsdb'
- dbName = lang + 'wiki_p'
- # coordDbName = 'u_dispenser_p'
- conn = MySQLdb.connect(
- host=hostName, db=dbName,
- read_default_file=os.path.expanduser("~/.my.cnf"),
- use_unicode=True, charset='utf8')
- cursor = conn.cursor()
- return (conn, cursor)
-
def processCountry(countrycode, lang, countryconfig, coordconfig,
connMon, cursorMon):
@@ -102,7 +75,7 @@
pywikibot.output(u'Language: %s has no coordTemplate set!' % lang)
return False
- (connWiki, cursorWiki) = connectWikiDatabase(lang)
+ (connWiki, cursorWiki) = connect_to_commons_database(lang)
withCoordinates = getMonumentsWithCoordinates(countrycode, lang, cursorMon)
@@ -313,7 +286,7 @@
connMon = None
cursorMon = None
- (connMon, cursorMon) = connectMonDatabase()
+ (connMon, cursorMon) = connect_to_monuments_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/add_object_location_monuments.py
b/erfgoedbot/add_object_location_monuments.py
index ed3e931..99c790b 100644
--- a/erfgoedbot/add_object_location_monuments.py
+++ b/erfgoedbot/add_object_location_monuments.py
@@ -5,31 +5,11 @@
Bot to add {{Object location dec}} to monuments. Location is based on
information from the monuments database.
'''
-import monuments_config as mconfig
import pywikibot
-from pywikibot import config
from pywikibot import pagegenerators
-import MySQLdb
-
-def connectDatabase():
- '''
- Connect to the monuments mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
user=config.db_username,
- passwd=config.db_password, use_unicode=True,
charset='utf8')
- cursor = conn.cursor()
- return (conn, cursor)
-
-
-def connectDatabase2():
- '''
- Connect to the commons mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect('commonswiki.labsdb', db='commonswiki_p',
- user=config.db_username, passwd=config.db_password,
use_unicode=True, charset='latin1')
- cursor = conn.cursor()
- return (conn, cursor)
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database,
connect_to_commons_database
def locateCountry(countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2):
@@ -220,8 +200,8 @@
countrycode = u''
# Connect database, we need that
- (conn, cursor) = connectDatabase()
- (conn2, cursor2) = connectDatabase2()
+ (conn, cursor) = connect_to_monuments_database()
+ (conn2, cursor2) = connect_to_commons_database()
generator = None
genFactory = pagegenerators.GeneratorFactory()
diff --git a/erfgoedbot/categorize_images.py b/erfgoedbot/categorize_images.py
index a3f8e39..bee10e5 100644
--- a/erfgoedbot/categorize_images.py
+++ b/erfgoedbot/categorize_images.py
@@ -16,13 +16,14 @@
python categorize_images.py -countrycode:ee -lang:et
'''
-import monuments_config as mconfig
+import re
+
import pywikibot
-from pywikibot import config
from pywikibot import pagegenerators
from pywikibot import textlib
-import re
-import MySQLdb
+
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database
_logger = "categorize_images"
@@ -183,17 +184,6 @@
u'Template category', u'Wikipedia category'
u'分类重定向', u'追蹤分類', u'共享資源', u'追蹤分類'],
}
-
-
-def connectDatabase():
- '''
- Connect to the mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
user=config.db_username,
- passwd=config.db_password, use_unicode=True,
charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
def categorizeImage(countrycode, lang, commonsTemplateName,
commonsCategoryBase, commonsCatTemplates, page, conn, cursor):
@@ -628,7 +618,7 @@
conn = None
cursor = None
# Connect database, we need that
- (conn, cursor) = connectDatabase()
+ (conn, cursor) = connect_to_monuments_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/database_connection.py
b/erfgoedbot/database_connection.py
new file mode 100644
index 0000000..7899b38
--- /dev/null
+++ b/erfgoedbot/database_connection.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+import MySQLdb
+import monuments_config as mconfig
+from pywikibot import config
+
+
+def connect_to_monuments_database():
+ """Connect to the mysql monuments database, if it fails, go down in
flames."""
+ conn = MySQLdb.connect(
+ host=mconfig.db_server, db=mconfig.db, user=config.db_username,
+ passwd=config.db_password, use_unicode=True, charset='utf8')
+ conn.ping(True)
+ cursor = conn.cursor()
+ return (conn, cursor)
+
+
+def connect_to_commons_database():
+ '''
+ Connect to the commons mysql database, if it fails, go down in flames
+ '''
+ conn = MySQLdb.connect('commonswiki.labsdb', db='commonswiki_p',
+ user=config.db_username, passwd=config.db_password,
use_unicode=True, charset='latin1')
+ cursor = conn.cursor()
+ return (conn, cursor)
diff --git a/erfgoedbot/database_statistics.py
b/erfgoedbot/database_statistics.py
index 8739089..8068849 100755
--- a/erfgoedbot/database_statistics.py
+++ b/erfgoedbot/database_statistics.py
@@ -5,22 +5,9 @@
FIXME: Too much code duplication. Should probably just have one list of the
possible fields for the functions to work on.
'''
-import monuments_config as mconfig
import pywikibot
-import MySQLdb
-from pywikibot import config
-
-def connectDatabase():
- '''
- Connect to the mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
- user=config.db_username, passwd=config.db_password,
- use_unicode=True, charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
+from database_connection import connect_to_monuments_database
def getCount(query, cursor):
@@ -331,7 +318,7 @@
conn = None
cursor = None
- (conn, cursor) = connectDatabase()
+ (conn, cursor) = connect_to_monuments_database()
statistics = {}
diff --git a/erfgoedbot/images_of_monuments_without_id.py
b/erfgoedbot/images_of_monuments_without_id.py
index 4f472b2..dfaa6fe 100644
--- a/erfgoedbot/images_of_monuments_without_id.py
+++ b/erfgoedbot/images_of_monuments_without_id.py
@@ -13,33 +13,10 @@
'''
-import monuments_config as mconfig
import pywikibot
-from pywikibot import config
-import MySQLdb
-
-def connectDatabase():
- '''
- Connect to the monuments mysql database, if it fails, go down in flames.
- This database is utf-8 encoded.
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
user=config.db_username,
- passwd=config.db_password, use_unicode=True,
charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
-
-
-def connectDatabase2():
- '''
- Connect to the commons mysql database, if it fails, go down in flames
- This database is latin1 encoded.
- '''
- conn = MySQLdb.connect('commonswiki.labsdb', db='commonswiki_p',
- user=config.db_username, passwd=config.db_password,
use_unicode=True, charset='latin1')
- cursor = conn.cursor()
- return (conn, cursor)
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database,
connect_to_commons_database
def processCountry(countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2):
@@ -212,8 +189,8 @@
conn = None
cursor = None
# Connect database, we need that
- (conn, cursor) = connectDatabase()
- (conn2, cursor2) = connectDatabase2()
+ (conn, cursor) = connect_to_monuments_database()
+ (conn2, cursor2) = connect_to_commons_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/missing_commonscat_links.py
b/erfgoedbot/missing_commonscat_links.py
index b55805f..a7982d1 100644
--- a/erfgoedbot/missing_commonscat_links.py
+++ b/erfgoedbot/missing_commonscat_links.py
@@ -11,36 +11,14 @@
python missing_commonscat_links.py -countrycode:XX -lang:YY
'''
-import monuments_config as mconfig
-import pywikibot
-from pywikibot import config
import re
-import MySQLdb
+
+import pywikibot
+
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database,
connect_to_commons_database
_logger = "missing_commonscat"
-
-
-def connectDatabase():
- '''
- Connect to the p_erfoed_p mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
- user=config.db_username, passwd=config.db_password,
- use_unicode=True, charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
-
-
-def connectDatabase2():
- '''
- Connect to the commons mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect('commonswiki.labsdb', db='commonswiki_p',
- user=config.db_username, passwd=config.db_password,
- use_unicode=True, charset='latin1')
- cursor = conn.cursor()
- return (conn, cursor)
def processCountry(countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2):
@@ -221,8 +199,8 @@
conn = None
cursor = None
# Connect database, we need that
- (conn, cursor) = connectDatabase()
- (conn2, cursor2) = connectDatabase2()
+ (conn, cursor) = connect_to_monuments_database()
+ (conn2, cursor2) = connect_to_commons_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/populate_image_table.py
b/erfgoedbot/populate_image_table.py
index f1c1865..76f7a1b 100644
--- a/erfgoedbot/populate_image_table.py
+++ b/erfgoedbot/populate_image_table.py
@@ -28,32 +28,11 @@
'''
import warnings
-import monuments_config as mconfig
+
import pywikibot
-from pywikibot import config
-import MySQLdb
-
-def connectDatabase():
- '''
- Connect to the p_erfgoed_p mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
user=config.db_username,
- passwd=config.db_password, use_unicode=True,
charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
-
-
-def connectDatabase2():
- '''
- Connect to the commons mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect('commonswiki.labsdb', db='commonswiki_p',
- user=config.db_username, passwd=config.db_password,
use_unicode=True, charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database,
connect_to_commons_database
def getSources(countrycode=u''):
@@ -196,8 +175,8 @@
conn = None
cursor = None
# Connect database, we need that
- (conn, cursor) = connectDatabase()
- (conn2, cursor2) = connectDatabase2()
+ (conn, cursor) = connect_to_monuments_database()
+ (conn2, cursor2) = connect_to_commons_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/top_streets.py b/erfgoedbot/top_streets.py
index 7f71200..815f454 100644
--- a/erfgoedbot/top_streets.py
+++ b/erfgoedbot/top_streets.py
@@ -6,23 +6,11 @@
* -municipality : The name of the municipality (as it is in the database)
* -minimum : (optional) The minimum of hits before we show the item
'''
-import monuments_config as mconfig
-import pywikibot
-from pywikibot import config
-import MySQLdb
from collections import Counter
+import pywikibot
-def connectDatabase():
- '''
- Connect to the monuments mysql database, if it fails, go down in flames.
- This database is utf-8 encoded.
- '''
- conn = MySQLdb.connect(
- host=mconfig.db_server, db=mconfig.db, user=config.db_username,
- passwd=config.db_password, use_unicode=True, charset='utf8')
- cursor = conn.cursor()
- return (conn, cursor)
+from database_connection import connect_to_monuments_database
def getAddresses(countrycode, lang, municipality, conn, cursor):
@@ -99,7 +87,7 @@
conn = None
cursor = None
# Connect database, we need that
- (conn, cursor) = connectDatabase()
+ (conn, cursor) = connect_to_monuments_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/unused_monument_images.py
b/erfgoedbot/unused_monument_images.py
index 110a334..29dec72 100644
--- a/erfgoedbot/unused_monument_images.py
+++ b/erfgoedbot/unused_monument_images.py
@@ -11,34 +11,14 @@
python unused_monument_images.py -countrycode:XX -lang:YY
'''
-import monuments_config as mconfig
-import pywikibot
-from pywikibot import config
import re
-import MySQLdb
+
+import pywikibot
+
+import monuments_config as mconfig
+from database_connection import connect_to_monuments_database,
connect_to_commons_database
_logger = "unused_images"
-
-
-def connectDatabase():
- '''
- Connect to the rijksmonumenten mysql database, if it fails, go down in
flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
user=config.db_username,
- passwd=config.db_password, use_unicode=True,
charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
-
-
-def connectDatabase2():
- '''
- Connect to the commons mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect('commonswiki.labsdb', db='commonswiki_p',
- user=config.db_username, passwd=config.db_password,
use_unicode=True, charset='latin1')
- cursor = conn.cursor()
- return (conn, cursor)
def processCountry(countrycode, lang, countryconfig, conn, cursor, conn2,
cursor2):
@@ -216,8 +196,8 @@
conn = None
cursor = None
# Connect database, we need that
- (conn, cursor) = connectDatabase()
- (conn2, cursor2) = connectDatabase2()
+ (conn, cursor) = connect_to_monuments_database()
+ (conn2, cursor2) = connect_to_commons_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/update_database.py b/erfgoedbot/update_database.py
index 246ad89..9fb2c35 100755
--- a/erfgoedbot/update_database.py
+++ b/erfgoedbot/update_database.py
@@ -13,11 +13,11 @@
'''
import warnings
import datetime
-import monuments_config as mconfig
+
import pywikibot
-import MySQLdb
-from pywikibot import config
from pywikibot import pagegenerators
+
+import monuments_config as mconfig
from converters import (
extractWikilink,
extract_elements_from_template_param,
@@ -32,18 +32,9 @@
check_integer,
check_lat_with_lon
)
+from database_connection import connect_to_monuments_database
_logger = "update_database"
-
-
-def connectDatabase():
- """Connect to the mysql database, if it fails, go down in flames."""
- conn = MySQLdb.connect(
- host=mconfig.db_server, db=mconfig.db, user=config.db_username,
- passwd=config.db_password, use_unicode=True, charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
def run_check(check, fieldValue, monumentKey, countryconfig, sourcePage):
@@ -395,7 +386,7 @@
daysBack = 2 # Default 2 days. Runs every night so can miss one night.
conn = None
cursor = None
- (conn, cursor) = connectDatabase()
+ (conn, cursor) = connect_to_monuments_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
diff --git a/erfgoedbot/update_id_dump.py b/erfgoedbot/update_id_dump.py
index 3733f65..e77ba2b 100755
--- a/erfgoedbot/update_id_dump.py
+++ b/erfgoedbot/update_id_dump.py
@@ -12,26 +12,14 @@
'''
-import monuments_config as mconfig
import pywikibot
-import MySQLdb
-from pywikibot import config
from pywikibot import pagegenerators
+import monuments_config as mconfig
from converters import (
extract_elements_from_template_param
)
-
-
-def connectDatabase():
- '''
- Connect to the mysql database, if it fails, go down in flames
- '''
- conn = MySQLdb.connect(host=mconfig.db_server, db=mconfig.db,
user=config.db_username,
- passwd=config.db_password, use_unicode=True,
charset='utf8')
- conn.ping(True)
- cursor = conn.cursor()
- return (conn, cursor)
+from database_connection import connect_to_monuments_database
def updateMonument(countrycode, lang, identifier, source, countryconfig, conn,
cursor):
@@ -134,7 +122,7 @@
countrycode = u''
conn = None
cursor = None
- (conn, cursor) = connectDatabase()
+ (conn, cursor) = connect_to_monuments_database()
for arg in pywikibot.handleArgs():
option, sep, value = arg.partition(':')
--
To view, visit https://gerrit.wikimedia.org/r/303427
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I10c6d1733b0a1c02ea3eaeaaab93dac4fb8cf19d
Gerrit-PatchSet: 1
Gerrit-Project: labs/tools/heritage
Gerrit-Branch: master
Gerrit-Owner: Jean-Frédéric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
