Awight has uploaded a new change for review.
https://gerrit.wikimedia.org/r/325360
Change subject: Split silverpop export from upload
......................................................................
Split silverpop export from upload
Bug: T152352
Change-Id: Ifd50947a9753be9fea35bfd64ca021d2da8909db
---
M silverpop_export/export.py
M silverpop_export/update.py
A silverpop_export/upload.py
3 files changed, 50 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/tools
refs/changes/60/325360/1
diff --git a/silverpop_export/export.py b/silverpop_export/export.py
index ec53199..b063759 100644
--- a/silverpop_export/export.py
+++ b/silverpop_export/export.py
@@ -10,11 +10,13 @@
from database.db import Connection as DbConnection, Query as DbQuery
import process.lock as lock
-from sftp.client import Client as SftpClient
import unicode_csv_writer
-def export_and_upload():
+def export_all():
+ """
+ Dump database contents to CSVs.
+ """
log.info("Begin Silverpop Export")
config = process.globals.get_config()
@@ -31,7 +33,6 @@
export_data(output_path=updatefile)
export_unsubscribes(output_path=unsubfile)
- upload([updatefile, unsubfile])
rotate_files()
log.info("End Silverpop Export")
@@ -105,14 +106,6 @@
)
-def upload(files=None):
- log.info("Uploading to silverpop")
- sftpc = SftpClient()
- for path in files:
- log.info("Putting file %s" % path)
- sftpc.put(path, os.path.basename(path))
-
-
def rotate_files():
config = process.globals.get_config()
@@ -147,6 +140,6 @@
lock.begin()
- export_and_upload()
+ export_all()
lock.end()
diff --git a/silverpop_export/update.py b/silverpop_export/update.py
index 693d9e1..d2bebd7 100644
--- a/silverpop_export/update.py
+++ b/silverpop_export/update.py
@@ -40,6 +40,9 @@
def run_queries(db, queries):
+ """
+ Build silverpop_export database from CiviCRM.
+ """
i = 1
for query in queries:
no_prefix = query[query.index("\n") + 1:]
@@ -63,7 +66,7 @@
log.info("Starting update query run")
run_queries(db, update_queries)
- export.export_and_upload()
+ export.export_all()
lock.end()
log.info("End Silverpop Export")
diff --git a/silverpop_export/upload.py b/silverpop_export/upload.py
new file mode 100644
index 0000000..950a3d4
--- /dev/null
+++ b/silverpop_export/upload.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import glob
+import os
+
+import process
+from process.logging import Logger as log
+from sftp.client import Client as SftpClient
+import process.lock as lock
+
+
+def upload_most_recent():
+ """
+ Send recently exported CSVs to Silverpop.
+ """
+ config = process.globals.get_config()
+ updatesglob = os.path.join(config.working_path, "DatabaseUpdate-*.csv")
+ unsubglob = os.path.join(config.working_path, "Unsubscribes-*.csv")
+ # Find most recently created export files.
+ updatefile = max(glob.iglob(updatesglob), key=os.path.getctime)
+ unsubfile = max(glob.iglob(unsubglob), key=os.path.getctime)
+
+ upload([updatefile, unsubfile])
+
+
+def upload(files=None):
+ log.info("Uploading to silverpop")
+ sftpc = SftpClient()
+ for path in files:
+ log.info("Putting file %s" % path)
+ sftpc.put(path, os.path.basename(path))
+
+
+if __name__ == '__main__':
+ process.globals.load_config('silverpop_export')
+
+ lock.begin()
+
+ upload_most_recent()
+
+ lock.end()
--
To view, visit https://gerrit.wikimedia.org/r/325360
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifd50947a9753be9fea35bfd64ca021d2da8909db
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Awight <awi...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
