Awight has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/348119 )
Change subject: Pretend stomp never happened.
......................................................................
Pretend stomp never happened.
Change-Id: Ia19890d61d4ec7cf3f59ad4ca751f37931a15cb3
---
M audit/paypal/README
M audit/paypal/config.yaml.example
D audit/paypal/history.py
M process/version_stamp.py
D queue/stomp_wrap.py
D queue/tests/test_stomp_wrap.py
M requirements.txt
7 files changed, 4 insertions(+), 280 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/tools
refs/changes/19/348119/1
diff --git a/audit/paypal/README b/audit/paypal/README
index 2f692c0..6de7b54 100644
--- a/audit/paypal/README
+++ b/audit/paypal/README
@@ -1,3 +1,3 @@
-apt-get install python-paramiko python-yaml python-stompy python-mysqldb
+pip install -r requirements.txt
cp config.yaml.example /etc/fundraising/paypal-audit.yaml
diff --git a/audit/paypal/config.yaml.example b/audit/paypal/config.yaml.example
index 100275d..4cb806b 100644
--- a/audit/paypal/config.yaml.example
+++ b/audit/paypal/config.yaml.example
@@ -20,14 +20,6 @@
host_key: |-
ssh-rsa
AAAAB3NzaC1yc2EAAAADAQABAAABAQDHDBjNezkG40FluT09yGTiNH2zzB3PnvfzX0Dj+A1UiV2buDNF0fOlGxu/vMCrnUHnuxjHlJ2ieBRwMl/0NiFd4uL/HtmsyFkT/EbUjZF9Ky2u4TVHHVRuZYnipidCtJ5RABBRwjToeg60C185CJpHGciFTwwGKDrZDZaGbMpKRZB7z0Nt3/Wuk8pO3fvbUGaIC4pAPe2ljyIPp/LuEE4QNlER8Hexs8XHKbo+a22vb0/hrcV38saLFZfm0O62sYkJ8t8bFDN1k/3c+2/0TfMZc6qbWA6sD872zcqaRHM2BtntWn9TuZP36/SclC0k3h1S0M5lQ2Mp5TAKqXtwgCnl
-stomp:
- server: crm.dev
- port: 61613
- queues:
- refund: /queue/refund
- donations: /queue/donations
- recurring: /queue/recurring
-
redis:
server: localhost
port: 6379
diff --git a/audit/paypal/history.py b/audit/paypal/history.py
deleted file mode 100755
index 6d46b52..0000000
--- a/audit/paypal/history.py
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env python
-
-from ConfigParser import SafeConfigParser
-from optparse import OptionParser
-from queue.stomp_wrap import Stomp
-import json
-import csv
-import atexit
-import re
-import gzip
-import locale
-import dateutil.parser
-from civicrm.civicrm import Civicrm
-
-config = None
-messaging = None
-options = None
-civi = None
-log_file = None
-
-
-def main():
- global config, messaging, options, civi
- parser = OptionParser(usage="usage: %prog [options]")
- parser.add_option("-c", "--config", dest='configFile',
default=["paypal-audit.cfg"], action='append', help='Path to configuration
file')
- parser.add_option("-f", "--auditFile", dest='auditFile', default=None,
help='CSV of transaction history')
- parser.add_option('-l', "--logFile", dest='logFile', default="audit.log",
help='Destination logfile. New messages will be appended.')
- parser.add_option("-n", "--no-effect", dest='noEffect', default=False,
action="store_true", help="Dummy no-effect mode")
- (options, args) = parser.parse_args()
-
- path = options.auditFile
- if re.search(r'[.]gz$', path):
- f = gzip.open(path, "rb")
- else:
- f = open(path, "rU")
- infile = csv.DictReader(f)
-
- config = SafeConfigParser()
- config.read(options.configFile)
-
- if options.noEffect:
- log("*** Dummy mode! Not injecting stomp messages ***")
-
- messaging = Stomp(config)
- civi = Civicrm(config.items('Db'))
-
- locale.setlocale(locale.LC_NUMERIC, "")
-
- # fix spurious whitespace around column header names
- infile.fieldnames = [name.strip() for name in infile.fieldnames]
-
- ignore_types = [
- "Authorization",
- "Cancelled Fee",
- # currency conversion is an explanation of amounts which appear
elsewhere
- "Currency Conversion",
- # TODO: handle in IPN
- "Temporary Hold",
- # seems to be the cancellation of a temporary hold
- "Update to Reversal",
- "Website Payments Pro API Solution",
- ]
-
- audit_dispatch = {
- "Reversal": handle_refund,
- "Chargeback Settlement": handle_refund,
- "Refund": handle_refund,
-
- "Subscription Payment Received": handle_payment,
- "Web Accept Payment Received": handle_payment,
- "Shopping Cart Payment Received": handle_payment,
- "Virtual Debt Card Credit Received": handle_payment,
- "Payment Received": handle_payment,
- "Update to eCheck Received": handle_payment,
- }
-
- for line in infile:
- if line['Type'] in ignore_types:
- log("Ignoring %s of type %s" % (line['Transaction ID'],
line['Type']))
- continue
- if line['Type'] in audit_dispatch:
- audit_dispatch[line['Type']](line)
- else:
- handle_unknown(line)
-
-
-def handle_unknown(line):
- log("Unhandled transaction, type \"%s\": %s" % (line['Type'],
json.dumps(line)))
-
-
-def handle_refund(line):
- global config, messaging, civi
-
- if line['Status'] != "Completed":
- return handle_unknown(line)
-
- txn_id = line['Transaction ID']
-
- # Construct the STOMP message
- msg = normalize_refund_msg(line)
-
- if not civi.transaction_exists(line['Reference Txn ID']):
- log("Refund missing parent: %s" % (json.dumps(msg), ))
- elif not civi.transaction_exists(txn_id):
- log("Queueing refund %s" % (txn_id, ))
- messaging.send("refund", msg)
- else:
- log("Refund already exists: %s" % (txn_id, ))
-
-
-def handle_payment(line):
- global config, messaging, civi
-
- if line['Status'] != "Completed":
- return handle_unknown(line)
-
- txn_id = line['Transaction ID']
-
- # Construct the STOMP message
- msg = normalize_msg(line)
-
- if not civi.transaction_exists(txn_id):
- log("Queueing payment %s" % (txn_id, ))
- messaging.send("payment", msg)
- else:
- log("Payment already exists: %s" % (txn_id, ))
-
-
-def normalize_msg(line):
- timestamp = dateutil.parser.parse(
- line['Date'] + " " + line['Time'] + " " + line['Time Zone'],
- ).strftime("%s")
-
- names = line['Name'].split(" ")
-
- return {
- 'date': timestamp,
- 'email': line['From Email Address'],
-
- 'first_name': names[0],
- 'last_name': " ".join(names[1:]),
-
- 'street_address': line['Address Line 1'],
- 'supplemental_address_1': line['Address Line 2/District/Neighborhood'],
- 'city': line['Town/City'],
- 'state_province':
line['State/Province/Region/County/Territory/Prefecture/Republic'],
- 'country': line['Country'],
- 'postal_code': line['Zip/Postal Code'],
-
- 'comment': line['Note'],
- # somthing with: line['Subscription Number'],
-
- 'gross_currency': line['Currency'],
- 'gross': round(locale.atof(line['Gross']), 2),
- 'fee': round(locale.atof(line['Fee']), 2),
- 'net': round(locale.atof(line['Net']), 2),
- 'gateway': "paypal",
- 'gateway_txn_id': line['Transaction ID'],
- }
-
-
-def normalize_refund_msg(line):
- msg = normalize_msg(line)
-
- refund_type = "unknown"
- if line['Type'] == "Refund":
- refund_type = "refund"
- elif line['Type'] == "Chargeback Settlement":
- refund_type = "chargeback"
- elif line['Type'] == "Reversal":
- refund_type = "reversal"
-
- msg.update({
- 'gross': 0 - msg['gross'],
- 'fee': 0 - msg['fee'],
- 'net': 0 - msg['net'],
- 'type': refund_type,
- 'gateway_refund_id': line['Transaction ID'],
- 'gateway_parent_id': line['Reference Txn ID'],
- })
-
- return msg
-
-
-def log(msg):
- global options, log_file
- if not log_file:
- log_file = open(options.logFile, 'a')
- atexit.register(file.close, log_file)
- log_file.write(msg + "\n")
-
-
-if __name__ == "__main__":
- main()
diff --git a/process/version_stamp.py b/process/version_stamp.py
index 0da23f6..e41da51 100644
--- a/process/version_stamp.py
+++ b/process/version_stamp.py
@@ -8,9 +8,9 @@
if not cached_revision:
toolsRootDir =
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- stompPath = os.path.join(toolsRootDir, '.version-stamp')
- if os.path.exists(stompPath):
- cached_revision = file(stompPath, "r").read().strip()
+ stampPath = os.path.join(toolsRootDir, '.version-stamp')
+ if os.path.exists(stampPath):
+ cached_revision = file(stampPath, "r").read().strip()
else:
cached_revision = 'unknown'
return cached_revision
diff --git a/queue/stomp_wrap.py b/queue/stomp_wrap.py
deleted file mode 100644
index d972a58..0000000
--- a/queue/stomp_wrap.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from process.globals import config
-from process.logging import Logger as log
-import process.version_stamp
-
-import os
-import os.path
-import sys
-import json
-import socket
-import time
-from stompy import Stomp as DistStomp
-
-
-class Stomp(object):
- conn = None
-
- def __init__(self):
- if not config.no_effect:
- self.conn = DistStomp(config.stomp.server, config.stomp.port)
- self.conn.connect()
-
- def __del__(self):
- if self.conn:
- self.conn.disconnect()
-
- # Let the STOMP library catch up
- time.sleep(1)
-
- def send(self, queue_key, body):
- if config.no_effect:
- log.info("not queueing message. " + json.dumps(body))
- return
-
- self.conn.send(self.create_message(queue_key, body))
-
- def create_message(self, queue_key, body):
- msg = {
- 'destination': config.stomp.queues[queue_key],
- 'persistent': 'true',
- }
- msg.update(Stomp.source_meta())
-
- if 'gateway' in body and 'gateway_txn_id' in body:
- msg['correlation-id'] = '{gw}-{id}'.format(gw=body['gateway'],
id=body['gateway_txn_id'])
-
- msg.update({'body': json.dumps(body)})
-
- return msg
-
- @staticmethod
- def source_meta():
- return {
- 'source_name': os.path.basename(sys.argv[0]),
- # FIXME: the controlling script should pass its own source_type
- 'source_type': 'audit',
- 'source_run_id': os.getpid(),
- 'source_version': process.version_stamp.source_revision(),
- 'source_enqueued_time': time.time(),
- 'source_host': socket.gethostname(),
- }
diff --git a/queue/tests/test_stomp_wrap.py b/queue/tests/test_stomp_wrap.py
deleted file mode 100644
index 3bf2d20..0000000
--- a/queue/tests/test_stomp_wrap.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import time
-
-from queue.stomp_wrap import Stomp
-
-
-def test_source_meta():
- meta = Stomp.source_meta()
- assert meta['source_name'] is not None
- assert 'audit' == meta['source_type']
- assert int(meta['source_run_id']) > 0
- assert meta['source_version'] is not None
- assert meta['source_enqueued_time'] >= (time.time() - 60)
- assert meta['source_host'] is not None
diff --git a/requirements.txt b/requirements.txt
index 4725ace..555ae92 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,4 +10,3 @@
redis
# scipy - just use the package.
simplemediawiki
-stompy
--
To view, visit https://gerrit.wikimedia.org/r/348119
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia19890d61d4ec7cf3f59ad4ca751f37931a15cb3
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Awight <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
