Ejegg has submitted this change and it was merged.
Change subject: Cheap fix to handle multipart audit files
......................................................................
Cheap fix to handle multipart audit files
The big catch is that the second file does not include column headers.
Change-Id: Ib4963bc63226a8b1303010b2730dd32d5361e1b6
---
M audit/paypal/SarFile.py
M audit/paypal/TrrFile.py
M audit/paypal/ppreport.py
3 files changed, 103 insertions(+), 21 deletions(-)
Approvals:
Ejegg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/audit/paypal/SarFile.py b/audit/paypal/SarFile.py
index bb3ab79..f5f77ef 100644
--- a/audit/paypal/SarFile.py
+++ b/audit/paypal/SarFile.py
@@ -12,6 +12,33 @@
class SarFile(object):
VERSION=2
stomp = None
+ column_headers = [
+ "Column Type",
+ "Subscription ID",
+ "Subscription Action Type",
+ "Subscription Currency",
+ "Subscription Creation Date",
+ "Subscription Period 1",
+ "Period 1 Amount",
+ "Subscription Period 2",
+ "Period 2 Amount",
+ "Subscription Period 3",
+ "Period 3 Amount",
+ "Recurring",
+ "Recurrence number",
+ "Subscription Payer PayPal Account ID",
+ "Subscription Payer email address",
+ "Subscription Payer Name",
+ "Subscription Payer Business Name",
+ "Shipping Address Line1",
+ "Shipping Address City",
+ "Shipping Address State",
+ "Shipping Address Zip",
+ "Shipping Address Country",
+ "Subscription Description",
+ "Subscription Memo",
+ "Subscription Custom Field",
+ ]
@staticmethod
def handle(path):
@@ -23,7 +50,7 @@
self.crm = Civicrm(config.civicrm_db)
def parse(self):
- ppreport.read(self.path, self.VERSION, self.parse_line)
+ ppreport.read(self.path, self.VERSION, self.parse_line,
self.column_headers)
def parse_line(self, row):
required_fields = [
diff --git a/audit/paypal/TrrFile.py b/audit/paypal/TrrFile.py
index d32ec97..a81edf2 100644
--- a/audit/paypal/TrrFile.py
+++ b/audit/paypal/TrrFile.py
@@ -14,6 +14,64 @@
class TrrFile(object):
VERSION = [4, 8]
stomp = None
+ # FIXME: these are version 8 headers, we would fail on multi-part v4
files...
+ column_headers = [
+ "Column Type",
+ "Transaction ID",
+ "Invoice ID",
+ "PayPal Reference ID",
+ "PayPal Reference ID Type",
+ "Transaction Event Code",
+ "Transaction Initiation Date",
+ "Transaction Completion Date",
+ "Transaction Debit or Credit",
+ "Gross Transaction Amount",
+ "Gross Transaction Currency",
+ "Fee Debit or Credit",
+ "Fee Amount",
+ "Fee Currency",
+ "Transactional Status",
+ "Insurance Amount",
+ "Sales Tax Amount",
+ "Shipping Amount",
+ "Transaction Subject",
+ "Transaction Note",
+ "Payer's Account ID",
+ "Payer Address Status",
+ "Item Name",
+ "Item ID",
+ "Option 1 Name",
+ "Option 1 Value",
+ "Option 2 Name",
+ "Option 2 Value",
+ "Auction Site",
+ "Auction Buyer ID",
+ "Auction Closing Date",
+ "Shipping Address Line1",
+ "Shipping Address Line2",
+ "Shipping Address City",
+ "Shipping Address State",
+ "Shipping Address Zip",
+ "Shipping Address Country",
+ "Shipping Method",
+ "Custom Field",
+ "Billing Address Line1",
+ "Billing Address Line2",
+ "Billing Address City",
+ "Billing Address State",
+ "Billing Address Zip",
+ "Billing Address Country",
+ "Consumer ID",
+ "First Name",
+ "Last Name",
+ "Consumer Business Name",
+ "Card Type",
+ "Payment Source",
+ "Shipping Name",
+ "Authorization Review Status",
+ "Protection Eligibility",
+ "Payment Tracking ID",
+ ]
@staticmethod
def handle(path):
@@ -25,7 +83,8 @@
self.crm = Civicrm(config.civicrm_db)
def parse(self):
- ppreport.read(self.path, self.VERSION, self.parse_line)
+ # FIXME: encapsulation issues
+ ppreport.read(self.path, self.VERSION, self.parse_line,
self.column_headers)
def parse_line(self, row):
if row['Billing Address Line1']:
diff --git a/audit/paypal/ppreport.py b/audit/paypal/ppreport.py
index d83865c..78d7090 100644
--- a/audit/paypal/ppreport.py
+++ b/audit/paypal/ppreport.py
@@ -9,13 +9,13 @@
quotechar='"'
)
-def read(path, version, callback):
+def read(path, version, callback, column_headers):
try:
- read_encoded(path, version, callback, encoding='utf-16')
+ read_encoded(path, version, callback, column_headers,
encoding='utf-16')
except UnicodeError:
- read_encoded(path, version, callback, encoding='utf-8-sig')
+ read_encoded(path, version, callback, column_headers,
encoding='utf-8-sig')
-def read_encoded(path, version, callback, encoding):
+def read_encoded(path, version, callback, column_headers, encoding):
# Coerce to a list
if not hasattr(version, 'extend'):
version = [version]
@@ -24,28 +24,24 @@
plainreader = unicode_csv_reader(csvfile, **dialect)
for row in plainreader:
- if row[0] == 'RH':
+ column_type = row[0]
+ if column_type == 'RH':
if int(row[4]) not in version:
raise RuntimeError("This file uses an unexpected format
revision: {version}".format(version=row[4]))
- elif row[0] == 'FH':
+ elif column_type == 'FH':
pass
- elif row[0] == 'SH':
+ elif column_type == 'SH':
start_date, end_date = row[1:3]
log.info("Report file covers date range {start} to
{end}".format(start=start_date, end=end_date))
- elif row[0] == 'CH':
+ elif column_type == 'CH':
column_headers = ['Column Type'] + row[1:]
- break
- else:
- raise RuntimeError("Unexpected row type:
{type}".format(type=row[0]))
-
- for line in plainreader:
- row = dict(zip(column_headers, line))
- if row['Column Type'] == 'SB':
+ elif column_type == 'SB':
+ record = dict(zip(column_headers, row))
try:
- callback(row)
+ callback(record)
except:
- FailMailer.mail('BAD_AUDIT_LINE', data=row,
print_exception=True)
- elif row['Column Type'] in ('SF', 'SC', 'RF', 'RC', 'FF'):
+ FailMailer.mail('BAD_AUDIT_LINE', data=record,
print_exception=True)
+ elif column_type in ('SF', 'SC', 'RF', 'RC', 'FF'):
pass
else:
- raise RuntimeError("Section ended and crazy stuff began:
{type}".format(type=row['Column Type']))
+ raise RuntimeError("Unknown column type:
{type}".format(type=column_type))
--
To view, visit https://gerrit.wikimedia.org/r/178003
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib4963bc63226a8b1303010b2730dd32d5361e1b6
Gerrit-PatchSet: 2
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Awight <[email protected]>
Gerrit-Reviewer: AndyRussG <[email protected]>
Gerrit-Reviewer: Awight <[email protected]>
Gerrit-Reviewer: Ejegg <[email protected]>
Gerrit-Reviewer: Katie Horn <[email protected]>
Gerrit-Reviewer: Ssmith <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
