Adamw has uploaded a new change for review.
https://gerrit.wikimedia.org/r/102041
Change subject: WIP paypal nightly auditing
......................................................................
WIP paypal nightly auditing
Change-Id: I3ecb4aff2de708f4288312e7caaf4a4403fc8230
---
A audit/paypal/SarFile.py
A audit/paypal/TrrFile.py
A audit/paypal/__init__.py
A audit/paypal/download_nightly.py
A audit/paypal/parse_nightly.py
A audit/paypal/ppreport.py
A audit/paypal/unicode_csv_reader.py
A sftp/__init__.py
A sftp/client.py
9 files changed, 216 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/tools
refs/changes/41/102041/1
diff --git a/audit/paypal/SarFile.py b/audit/paypal/SarFile.py
new file mode 100644
index 0000000..71a4c04
--- /dev/null
+++ b/audit/paypal/SarFile.py
@@ -0,0 +1,11 @@
+class SarFile(object):
+ @staticmethod
+ def handle(path):
+ obj = SarFile(path)
+ obj.parse()
+
+ def __init__(self, path):
+ self.path = path
+
+ def parse(self):
+ print "foo!"
diff --git a/audit/paypal/TrrFile.py b/audit/paypal/TrrFile.py
new file mode 100644
index 0000000..744fe71
--- /dev/null
+++ b/audit/paypal/TrrFile.py
@@ -0,0 +1,18 @@
+import ppreport
+
+class TrrFile(object):
+ VERSION=4
+
+ @staticmethod
+ def handle(path):
+ obj = TrrFile(path)
+ obj.parse()
+
+ def __init__(self, path):
+ self.path = path
+
+ def parse(self):
+ ppreport.read(self.path, self.VERSION, self.parse_line)
+
+ def parse_line(self, row):
+ print row
diff --git a/audit/paypal/__init__.py b/audit/paypal/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/audit/paypal/__init__.py
diff --git a/audit/paypal/download_nightly.py b/audit/paypal/download_nightly.py
new file mode 100755
index 0000000..836b836
--- /dev/null
+++ b/audit/paypal/download_nightly.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+import os, os.path
+
+from process.logging import Logger as log
+
+from process.globals import load_config
+load_config("paypal-audit")
+
+from process.globals import config
+import process.lock as lock
+import sftp.client
+
+remote_root = "ppreports/outgoing"
+
+def walk_files(paths):
+ result = []
+
+ if not hasattr(paths, 'extend'):
+ paths = [paths]
+
+ for root in paths:
+ for dirpath, dirnames, filenames in os.walk(root):
+ result += filenames
+
+ return result
+
+if __name__ == '__main__':
+ log.info("Begin PayPal nightly audit download")
+ lock.begin()
+
+ local_paths = [
+ config.incoming_path,
+ config.archive_path,
+ ]
+ local_files = walk_files(local_paths)
+
+ remote = sftp.client.Client()
+ remote_files = remote.ls(remote_root)
+
+ for filename in remote_files:
+ if filename in local_files:
+ log.info("Skipping already downloaded file
{filename}".format(filename=filename))
+ continue
+
+ log.info("Downloading file {filename}".format(filename=filename))
+ dest_path = os.path.join(config.incoming_path, filename)
+ remote.get(os.path.join(remote_root, filename), dest_path)
+
+ lock.end()
+ log.info("End PayPal nightly audit download")
diff --git a/audit/paypal/parse_nightly.py b/audit/paypal/parse_nightly.py
new file mode 100755
index 0000000..bbd9925
--- /dev/null
+++ b/audit/paypal/parse_nightly.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+import os, os.path
+import re
+from process.logging import Logger as log
+from process.globals import load_config
+load_config("paypal-audit")
+from process.globals import config
+import process.lock as lock
+
+import TrrFile, SarFile
+
+filename_re = r'''^(?x)
+ (?P<type>[A-Z]{3})-
+ (?P<date>[0-9]{8})[.]
+ (?P<sequence>[0-9]{2})[.]
+ (?P<version>[0-9]{3})[.]
+ CSV
+ $
+'''
+
+handlers = {
+ 'TRR': TrrFile.TrrFile.handle,
+ 'SAR': SarFile.SarFile.handle,
+}
+
+def parse(path):
+ matches = re.match(filename_re, filename)
+ if matches:
+ filetype = matches.group('type')
+ if filetype in handlers:
+ log.info("Parsing file using {type} handler".format(type=filetype))
+ handlers[filetype](path)
+ else:
+ log.info("No handler for type {type}".format(type=filetype))
+ else:
+ log.error("Could not parse filename:
{filename}".format(filename=filename))
+
+def archive(path):
+ filename = os.path.basename(path)
+ dest_path = os.path.join(config.archive_path, filename)
+
+ log.info("Archiving {orig} to {new}".format(orig=path, new=dest_path))
+ os.rename(path, dest_path)
+
+if __name__ == '__main__':
+ lock.begin()
+
+ for filename in os.listdir(config.incoming_path):
+ path = os.path.join(config.incoming_path, filename)
+ if os.path.isfile(path):
+ parse(path)
+ #DEBUG archive(path)
+
+ lock.end()
diff --git a/audit/paypal/ppreport.py b/audit/paypal/ppreport.py
new file mode 100644
index 0000000..46913af
--- /dev/null
+++ b/audit/paypal/ppreport.py
@@ -0,0 +1,34 @@
+import io
+
+from unicode_csv_reader import unicode_csv_reader
+
+dialect = dict(
+ delimiter=',',
+ quotechar='"'
+)
+
+def read(path, version, callback):
+ with io.open(path, 'r', encoding='utf-16') as csvfile:
+ plainreader = unicode_csv_reader(csvfile, **dialect)
+ for row in plainreader:
+ if row[0] == 'RH':
+ if int(row[4]) != version:
+ raise RuntimeError("This file uses an unexpected format
revision: {version}".format(version=row[4]))
+ elif row[0] == 'FH':
+ pass
+ elif row[0] == 'SH':
+ pass
+ elif row[0] == 'CH':
+ column_headers = ['Row Type'] + row[1:]
+ break
+ else:
+ raise RuntimeError("Unexpected row type:
{type}".format(type=row[0]))
+
+ for line in plainreader:
+ row = dict(zip(column_headers, line))
+ if row['Row Type'] == 'SB':
+ callback(row)
+ elif row['Row Type'] in ('SF', 'SC', 'RF', 'RC', 'FF'):
+ pass
+ else:
+ raise RuntimeError("Section ended and crazy stuff began:
{type}".format(type=row['Row Type']))
diff --git a/audit/paypal/unicode_csv_reader.py
b/audit/paypal/unicode_csv_reader.py
new file mode 100644
index 0000000..de1d964
--- /dev/null
+++ b/audit/paypal/unicode_csv_reader.py
@@ -0,0 +1,14 @@
+import csv
+
+# from http://docs.python.org/2/library/csv.html
+def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
+ # csv.py doesn't do Unicode; encode temporarily as UTF-8:
+ csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
+ dialect=dialect, **kwargs)
+ for row in csv_reader:
+ # decode UTF-8 back to Unicode, cell by cell:
+ yield [unicode(cell, 'utf-8') for cell in row]
+
+def utf_8_encoder(unicode_csv_data):
+ for line in unicode_csv_data:
+ yield line.encode('utf-8')
diff --git a/sftp/__init__.py b/sftp/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/sftp/__init__.py
diff --git a/sftp/client.py b/sftp/client.py
new file mode 100644
index 0000000..71dce41
--- /dev/null
+++ b/sftp/client.py
@@ -0,0 +1,33 @@
+import os, os.path
+import base64
+import paramiko
+
+from process.logging import Logger as log
+from process.globals import config
+
+class Client(object):
+ def __init__(self):
+ self.host_public_key =
paramiko.RSAKey(data=base64.decodestring(config.sftp.host_key))
+
+ self.connect()
+
+ def __del__(self):
+ self.client.close()
+
+ def connect(self):
+ log.info("Connecting to {host}".format(host=config.sftp.host))
+ transport = paramiko.Transport((config.sftp.host, 22))
+ transport.connect(username=config.sftp.username,
password=config.sftp.password, hostkey=self.host_public_key)
+ self.client = paramiko.SFTPClient.from_transport(transport)
+
+ def ls(self, path):
+ return self.client.listdir(path)
+
+ def get(self, filename, dest_path):
+ try:
+ self.client.get(filename, dest_path)
+ except:
+ if os.path.exists(dest_path):
+ log.info("Removing corrupted download:
{path}".format(path=dest_path))
+ os.unlink(dest_path)
+ raise
--
To view, visit https://gerrit.wikimedia.org/r/102041
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3ecb4aff2de708f4288312e7caaf4a4403fc8230
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Adamw <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
