Awight has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/315638

Change subject: WIP Tests for the bulk export
......................................................................

WIP Tests for the bulk export

Actually works!  Lots of TODOs, most of all you'll need to allow access to
mysql db "test" for user "test" with no password.  This might be a default
already.  We'll want to pass in test configuration instead.

This skeletal example can be expanded with more interesting tests.

Bug: T131770
Bug: T145012
Change-Id: I697104bce3209a714a4124648fc058eba4cd235b
---
M database/db.py
A silverpop_export/tests/minimal_schema.sql
A silverpop_export/tests/test_update.py
M silverpop_export/update.py
M silverpop_export/update_table.sql
5 files changed, 230 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/tools 
refs/changes/38/315638/1

diff --git a/database/db.py b/database/db.py
index 9340e27..82fe2ec 100644
--- a/database/db.py
+++ b/database/db.py
@@ -159,6 +159,40 @@
     return db_conn[schema]
 
 
+def load_queries(script_path):
+    '''
+    Helper to parse queries out of a SQL file.  Requires that each statement
+    ends in /.*;\s*$/
+    '''
+    config = process.globals.get_config()
+
+    qbuf = []
+    queries = []
+    f = open(script_path, 'r')
+    for line in f:
+        line = line.rstrip()
+        if line:
+            qbuf.append(line)
+            if line.endswith(';'):
+                query = "\n".join(qbuf)
+                queries.append(query)
+
+                qbuf = []
+
+    f.close()
+    return queries
+
+
+def run_script(script_path):
+    '''
+    Load and run a SQL file using the current database connection.
+    '''
+
+    queries = load_queries(script_path)
+    for statement in queries:
+        get_db().execute(statement)
+
+
 def close_all():
     for conn in db_conn.values():
         conn.close()
diff --git a/silverpop_export/tests/minimal_schema.sql 
b/silverpop_export/tests/minimal_schema.sql
new file mode 100644
index 0000000..966a64c
--- /dev/null
+++ b/silverpop_export/tests/minimal_schema.sql
@@ -0,0 +1,114 @@
+-- TODO: Do something SQLy to make *sure* we're not in a real database.
+
+drop table if exists civicrm_email;
+create table civicrm_email (
+    id int(10) unsigned auto_increment primary key,
+    contact_id int(10) unsigned,
+    email varchar(254) COLLATE utf8_unicode_ci,
+    is_primary tinyint(4) default '1',
+    on_hold tinyint(4) default '0',
+    key UI_email (email)
+);
+
+drop table if exists civicrm_contact;
+create table civicrm_contact (
+    id int(10) unsigned auto_increment primary key,
+    do_not_email tinyint(4) default '0',
+    do_not_phone tinyint(4) default '0',
+    do_not_mail tinyint(4) default '0',
+    do_not_sms tinyint(4) default '0',
+    do_not_trade tinyint(4) default '1',
+    is_opt_out tinyint(4) default '0',
+    preferred_language varchar(32) COLLATE utf8_unicode_ci,
+    first_name varchar(64) COLLATE utf8_unicode_ci,
+    middle_name varchar(64) COLLATE utf8_unicode_ci,
+    last_name varchar(64) COLLATE utf8_unicode_ci,
+    is_deleted tinyint(4) default '0'
+);
+
+drop table if exists wmf_donor;
+create table wmf_donor (
+    id int(10) unsigned,
+    entity_id int(10) unsigned,
+    do_not_solicit tinyint(4),
+    is_2006_donor tinyint(4),
+    is_2007_donor tinyint(4),
+    is_2008_donor tinyint(4),
+    is_2009_donor tinyint(4),
+    is_2010_donor tinyint(4),
+    is_2011_donor tinyint(4),
+    is_2012_donor tinyint(4),
+    is_2013_donor tinyint(4),
+    is_2014_donor tinyint(4),
+    is_2015_donor tinyint(4),
+    is_2016_donor tinyint(4),
+    is_2017_donor tinyint(4),
+    is_2018_donor tinyint(4),
+    is_2019_donor tinyint(4),
+    is_2020_donor tinyint(4),
+    is_2021_donor tinyint(4),
+    is_2022_donor tinyint(4),
+    is_2023_donor tinyint(4),
+    is_2024_donor tinyint(4),
+    is_2025_donor tinyint(4),
+    last_donation_date datetime,
+    last_donation_currency varchar(255) COLLATE utf8_unicode_ci,
+    last_donation_amount decimal(20,2),
+    last_donation_usd decimal(20,2),
+    lifetime_usd_total decimal(20,2)
+);
+
+drop table if exists civicrm_contribution;
+create table civicrm_contribution (
+    id int(10) unsigned,
+    contact_id int(10) unsigned,
+    receive_date datetime,
+    total_amount decimal(20,2),
+    trxn_id varchar(255) COLLATE utf8_unicode_ci,
+    contribution_status_id int(10) unsigned
+);
+
+drop table if exists civicrm_address;
+create table civicrm_address (
+    contact_id int(10) unsigned,
+    is_primary tinyint(4),
+    city varchar(64) COLLATE utf8_unicode_ci,
+    postal_code varchar(64) COLLATE utf8_unicode_ci,
+    country_id int(10) unsigned,
+    state_province_id int(10) unsigned
+);
+
+drop table if exists civicrm_country;
+create table civicrm_country (
+    id int(10) unsigned,
+    iso_code char(2) COLLATE utf8_unicode_ci
+);
+
+drop table if exists civicrm_state_province;
+create table civicrm_state_province (
+  id int(10) unsigned,
+  name varchar(64) COLLATE utf8_unicode_ci
+);
+
+drop table if exists wmf_contribution_extra;
+create table wmf_contribution_extra (
+    entity_id int(10) unsigned,
+    original_amount decimal(20,2),
+    original_currency varchar(255) COLLATE utf8_unicode_ci
+);
+
+drop table if exists contribution_tracking;
+create table contribution_tracking (
+    contribution_id int(10) unsigned,
+    country varchar(2)
+);
+
+drop table if exists log_civicrm_email;
+create table log_civicrm_email (
+    email varchar(254) COLLATE utf8_unicode_ci
+);
+
+drop table if exists civicrm_uf_match;
+create table civicrm_uf_match (
+    uf_name varchar(128) COLLATE utf8_unicode_ci
+);
diff --git a/silverpop_export/tests/test_update.py 
b/silverpop_export/tests/test_update.py
new file mode 100644
index 0000000..45f6fa7
--- /dev/null
+++ b/silverpop_export/tests/test_update.py
@@ -0,0 +1,76 @@
+import mock
+import os
+
+import database.db
+import silverpop_export.update
+
+conn = None
+
+
+def setup():
+    global conn
+    # FIXME: parameterize test configuration
+    conn = database.db.Connection(host="localhost", db="test", user="test", 
debug=True)
+    conn.execute("set default_storage_engine=memory")
+
+
+def test_test_setup():
+    '''
+    Set up the civcrm and export databases and run the update with no data.
+    '''
+    run_update_with_fixtures(fixture_queries=[])
+
+
+def test_duplicate():
+    '''
+    Test that we export one record for a duplicate contact.
+    '''
+
+    run_update_with_fixtures(fixture_queries=["""
+    insert into civicrm_email (contact_id, email, is_primary, on_hold) values
+        (1, 'person1@localhost', 1, 0),
+        (2, 'person1@localhost', 1, 0);
+    """, """
+    insert into civicrm_contact (id) values
+        (1),
+        (2);
+    """])
+
+    cursor = conn.db_conn.cursor()
+    cursor.execute("select count(*) from silverpop_export")
+    assert cursor.fetchone() == (1,)
+
+
+def run_update_with_fixtures(fixture_path=None, fixture_queries=None):
+    with mock.patch("database.db.Connection") as MockConnection:
+
+        # Always return our test database connection.
+        MockConnection.return_value = conn
+
+        with mock.patch("process.globals.get_config") as MockConfig:
+            # Point all config at our test database.
+            MockConfig().civicrm_db.db = "test"
+            MockConfig().drupal_db.db = "test"
+            MockConfig().silverpop_db.db = "test"
+            MockConfig().log_civicrm_db.db = "test"
+
+            # Create fixtures
+            tests_dir = os.path.dirname(__file__)
+            script_path = os.path.join(tests_dir, "minimal_schema.sql")
+            database.db.run_script(script_path)
+
+            parent_dir = os.path.dirname(os.path.dirname(__file__))
+            script_path = os.path.join(parent_dir, 
"silverpop_countrylangs.sql")
+            database.db.run_script(script_path)
+
+            if fixture_path:
+                database.db.run_script(fixture_path)
+
+            if fixture_queries:
+                for statement in fixture_queries:
+                    conn.execute(statement)
+
+            # Run the bulk update.
+            # FIXME: Implementation should provide this as a single function.
+            update_queries = 
silverpop_export.update.load_queries('update_table.sql')
+            silverpop_export.update.run_queries(conn, update_queries)
diff --git a/silverpop_export/update.py b/silverpop_export/update.py
index d660dcf..e038e1d 100644
--- a/silverpop_export/update.py
+++ b/silverpop_export/update.py
@@ -12,6 +12,10 @@
 
 
 def load_queries(file):
+    # TODO: Reuse database.db.load_queries
+
+    config = process.globals.get_config()
+
     prefix = "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;-- 
Silverpop Export Script, %s" % file
     script_path = os.path.dirname(__file__)
     qbuf = [prefix]
diff --git a/silverpop_export/update_table.sql 
b/silverpop_export/update_table.sql
index 9773cd4..eed1b26 100755
--- a/silverpop_export/update_table.sql
+++ b/silverpop_export/update_table.sql
@@ -214,6 +214,7 @@
 
 -- Join on civicrm address where we do not already have a geolocated
 -- address from contribution tracking
+-- FIXME: needs addr.is_primary = 1
 UPDATE silverpop_export_staging ex
   JOIN civicrm.civicrm_address addr ON ex.contact_id = addr.contact_id
   JOIN civicrm.civicrm_country ctry ON addr.country_id = ctry.id
@@ -230,6 +231,7 @@
 -- And now updated by civicrm address where we have a country but no
 -- city from contribution tracking.  The countries must match.
 -- (11 minutes)
+-- FIXME: We need addr.is_primary = 1
 UPDATE silverpop_export_staging ex
   JOIN civicrm.civicrm_address addr ON ex.contact_id = addr.contact_id
   JOIN civicrm.civicrm_country ctry

-- 
To view, visit https://gerrit.wikimedia.org/r/315638
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I697104bce3209a714a4124648fc058eba4cd235b
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Awight <awi...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to