Adamw has uploaded a new change for review.
https://gerrit.wikimedia.org/r/91578
Change subject: logging in dedupe
......................................................................
logging in dedupe
Change-Id: I1df80f9618af370c090d71c2eff8f977295ff94c
---
M database/db.py
M dedupe/contact_cache.py
M dedupe/quick_autoreview.py
M dedupe/review_queue.py
4 files changed, 21 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/tools
refs/changes/78/91578/1
diff --git a/database/db.py b/database/db.py
index 9db6663..02616b3 100644
--- a/database/db.py
+++ b/database/db.py
@@ -4,6 +4,7 @@
import MySQLdb as Dbi
import atexit
+from process.logging import Logger as log
from process.globals import config
class Connection(object):
@@ -18,7 +19,10 @@
cursor = self.db_conn.cursor(cursorclass=Dbi.cursors.DictCursor)
if self.debug:
- print sql, params
+ if params:
+ log.debug(str(sql) + " % " + repr(params))
+ else:
+ log.debug(str(sql))
if params:
cursor.execute(sql, params)
diff --git a/dedupe/contact_cache.py b/dedupe/contact_cache.py
index 8fc353d..6475875 100644
--- a/dedupe/contact_cache.py
+++ b/dedupe/contact_cache.py
@@ -1,5 +1,6 @@
'''Optimized retrieval and in-memory storage of a small amount of information
across many contacts.'''
+from process.logging import Logger as log
from process.globals import config
from database import db
@@ -75,6 +76,7 @@
def buildQuery(self):
query = super(PagedGroup, self).buildQuery()
+ log.info("Limiting batch contact retrieval to {num}
records.".format(num=self.pagesize))
query.limit = self.pagesize
query.offset = self.offset
return query
diff --git a/dedupe/quick_autoreview.py b/dedupe/quick_autoreview.py
index 2e8fa6a..7f75ac0 100755
--- a/dedupe/quick_autoreview.py
+++ b/dedupe/quick_autoreview.py
@@ -2,6 +2,7 @@
'''Find low-hanging dupe fruits and mark them for the manual review queue'''
+from process.logging import Logger as log
from process.globals import load_config
load_config("dedupe")
from process.globals import config
@@ -62,7 +63,15 @@
ReviewQueue.tag(contact['id'], QuickAutoreview.QUICK_REVIEWED)
+ if not self.contactCache.contacts:
+ log.warn("Searched an empty batch of contacts!")
+ else:
+ last_seen = self.contactCache.contacts[-1]['id']
+ log.info("End of batch. Last contact scanned was ID
{id}".format(id=last_seen))
+
+
if __name__ == '__main__':
+ log.info("Begin quick_autoreview deduper")
lock.begin()
job = QuickAutoreview()
@@ -70,3 +79,4 @@
ReviewQueue.commit()
lock.end()
+ log.info("End quick_autoreview deduper")
diff --git a/dedupe/review_queue.py b/dedupe/review_queue.py
index b71355a..5f721b1 100644
--- a/dedupe/review_queue.py
+++ b/dedupe/review_queue.py
@@ -1,3 +1,4 @@
+from process.logging import Logger as log
from process.globals import config
from database import db
@@ -7,6 +8,7 @@
@staticmethod
def addMatch(job_id, oldId, newId, action, match):
+ log.info("Found a match: {old} -> {new} : {match}".format(old=oldId,
new=newId, match=match))
db.get_db(config.drupal_schema).execute("""
INSERT INTO donor_review_queue
SET
@@ -35,7 +37,9 @@
@staticmethod
def commit():
+ log.info("Committing tags...")
for tag, contacts in ReviewQueue.cached_tags.items():
+ log.info("Bulk tagging {num} contacts with tag
<{tag}>".format(num=len(contacts), tag=tag.name))
ReviewQueue.tag_many(contacts, tag)
@staticmethod
--
To view, visit https://gerrit.wikimedia.org/r/91578
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1df80f9618af370c090d71c2eff8f977295ff94c
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Adamw <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
