Rush has submitted this change and it was merged.
Change subject: phab email pipe cleanup and allow maint
......................................................................
phab email pipe cleanup and allow maint
* If maint is true it rejects mail bound for phab.
* Cleaned out env variable to reflect upstream
best practice
* Updated settings in puppet role
Change-Id: Ia86643c00b3c27db5ecd01b4ee005de80330c018
---
M manifests/role/phabricator.pp
M modules/phabricator/files/phab_epipe.py
M modules/phabricator/manifests/mailrelay.pp
3 files changed, 122 insertions(+), 80 deletions(-)
Approvals:
Rush: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/phabricator.pp b/manifests/role/phabricator.pp
index 9b07fcd..8df17da 100644
--- a/manifests/role/phabricator.pp
+++ b/manifests/role/phabricator.pp
@@ -89,12 +89,14 @@
include phabricator::monitoring
class { '::phabricator::mailrelay':
- default => { security => 'default'},
+ default => { security => 'users',
+ maint => 'false',
+ save => 'false',
+ },
address_routing => { testproj => 'demoproject'},
phab_bot => { root_dir => '/srv/phab/phabricator/',
- env => 'default',
username => 'emailbot',
- host => "http://${domain}";,
+ host => "https://${domain}/api/";,
certificate => $emailbotcert,
},
}
diff --git a/modules/phabricator/files/phab_epipe.py
b/modules/phabricator/files/phab_epipe.py
index 3022a25..f3995f2 100644
--- a/modules/phabricator/files/phab_epipe.py
+++ b/modules/phabricator/files/phab_epipe.py
@@ -10,22 +10,30 @@
* direct comments allows non-phabricator recognized emails to attach comments
to a ticket for a specific project from allowed domains
--> custom attachments to be handled later by direct attach script
-
Looks for configuration file: /etc/phab_epipe.conf
+ [default]
+ security = users
+ # if 'true' will reject all email to phab
+ maint = false
+ # saves every message overwriting /tmp/rawmail
+ save = false
+ # saves particular messages from a comma
+ # separated list of users
+ debug = <emails_to_dump_for_debug>
+
[address_routing]
- <dest_email_address> = <project_id_to_route_for_task_creation>
+ <dest_email_address> = <project_name/security>
[direct_comments_allowed]
<project_name> = <comma_separated_list_of_allowed_email_domains>
[phab_bot]
root_dir = <path_on_disk_to_phabricator
- env = <phab_env>
username = <phab_user>
certificate = <phab_user_cert>
host = <phab_host>
+
"""
import os
import re
@@ -63,9 +71,12 @@
pass
+class EmailStatusError(Exception):
+ pass
+
+
def main():
- save = '-s' in sys.argv
log = lambda m: output(m, verbose='-v' in sys.argv)
# if stdin is empty we bail to avoid a loop
@@ -97,13 +108,6 @@
:param name: str of project name
"""
return phab.project.query(names=[name])
-
- def get_proj_phid_by_id(pid):
- """return project name by project id
- :param pid: int project id
- :returns: string
- """
- return phab.project.query(ids=[pid])['data'].keys()[0]
def external_user_comment(task, text):
"""update a task with a comment by id
@@ -137,7 +141,6 @@
block = "**`%s`** created via email on `%s`\n\n" % (sender, date)
block += '\n'.join(['> ' + s for s in body.splitlines()])
project_info = get_proj_by_name(project)
-
if not project_info['data']:
raise EmailParsingError("project %s does not exist" % (project))
@@ -165,12 +168,90 @@
upload = phab.file.upload(name=name, data_base64=data)
return phab.file.info(phid=upload.response).response
+ def extract_payload_and_upload(payload):
+ """ extract content and upload from MIME object
+ :param payload: MIME payload object
+ :return: str of created phab oject
+ """
+ name = None
+ # Some attachments have the field but it is None type
+ if not name and 'Content-Description' in payload:
+ name = payload['Content-Description']
+ if not name and 'Content-Disposition' in payload:
+ namex = re.search('name=\"(.*)\"', payload['Content-Type'])
+ if namex:
+ name = namex.group(1)
+ if not name and 'Content-Disposition' in payload:
+ fnamex = re.search('filename=(.*)', payload['Content-Disposition'])
+ if fnamex:
+ name = fnamex.group(1)
+
+ if name:
+ data = payload.get_payload()
+ upload = upload_file(name, data)
+ return upload['objectName']
+
+ def extract_body_and_attached(msg):
+ """ get body and payload objects
+ :param msg: email msg type
+ :return: str and list
+ """
+ attached = []
+ # https://docs.python.org/2/library/email.message.html
+ if msg.is_multipart():
+ for payload in msg.get_payload():
+ log('content type: %s' % (payload.get_content_type(),))
+ if payload.get_content_type() == 'text/plain':
+ body = payload.get_payload()
+ elif payload.get_content_type() == 'multipart/alternative':
+ msgparts = payload.get_payload()
+ if not isinstance(msgparts, list):
+ body = 'unknown body format'
+ continue
+ else:
+ for e in msgparts:
+ known_body_types = ['text/plain']
+ # text/plain; charset="utf-8"
+ # Inconsistent content-types so we
+ # try to match only basic string
+ type = e['Content-Type'].split(';')[0]
+ if type in known_body_types:
+ body = e.get_payload()
+ break
+ else:
+ err = 'Unknown email body format: from %s to %s'
+ dest = str(dest_addresses)
+ raise EmailParsingError(error % (src_address,
+ dest))
+
+ else:
+ log('attaching file')
+ attached.append(payload)
+
+ else:
+ body = msg.get_payload()
+ return body, attached
+
+ # If maint is true then reject all email interaction
+ if defaults['maint'].lower() == 'true':
+ raise EmailStatusError('Email interaction is currently disabled.')
+
+ if '-s' in sys.argv:
+ save = '/tmp/maildump'
+ elif 'save' in defaults:
+ save = defaults['save']
+ else:
+ save = False
+
+ if 'debug' in defaults:
+ defaults['debug'] = defaults['debug'].split(',')
+
+ # Reading in the message
stdin = sys.stdin.read()
if save:
- dump = '/tmp/rawmail'
- log('saving raw email to %s' % dump)
- with open(dump, 'w') as r:
+ log('saving raw email to %s' % save)
+ with open(save, 'w') as r:
r.write(stdin)
# ['Received', 'DKIM-Signature', 'X-Google-DKIM-Signature',
@@ -182,62 +263,9 @@
src_name, src_addy = parse_from_string(src_address)
dest_addresses = msg['to'].split(',')
- attached = []
- # https://docs.python.org/2/library/email.message.html
- if msg.is_multipart():
- for payload in msg.get_payload():
- log('content type: %s' % (payload.get_content_type(),))
- if payload.get_content_type() == 'text/plain':
- body = payload.get_payload()
- elif payload.get_content_type() == 'multipart/alternative':
- msgparts = payload.get_payload()
- if not isinstance(msgparts, list):
- body = 'unknown body format'
- continue
- else:
- for e in msgparts:
- known_body_types = ['text/plain']
- # text/plain; charset="utf-8"
- # Inconsistent content-types so we
- # try to match only basic string
- type = e['Content-Type'].split(';')[0]
- if type in known_body_types:
- body = e.get_payload()
- break
- else:
- err = 'Unknown email body format: from %s to %s'
- raise EmailParsingError(error % (src_address,
- str(dest_addresses)))
-
- else:
- log('attaching file')
- attached.append(payload)
-
- else:
- body = msg.get_payload()
-
- uploads = []
- for payload in attached:
-
- name = None
- # Some attachments have the field but it is None type
- if not name and 'Content-Description' in payload:
- name = payload['Content-Description']
-
- if not name and 'Content-Disposition' in payload:
- namex = re.search('name=\"(.*)\"', payload['Content-Type'])
- if namex:
- name = namex.group(1)
-
- if not name and 'Content-Disposition' in payload:
- fnamex = re.search('filename=(.*)', payload['Content-Disposition'])
- if fnamex:
- name = fnamex.group(1)
-
- if name:
- data = payload.get_payload()
- upload = upload_file(name, data)
- uploads.append(upload['objectName'])
+ if 'debug' in defaults and src_address in defaults['debug']:
+ with open('/tmp/%s' % (src_address + '.txt'), 'w') as r:
+ r.write(stdin)
# does this email have a direct to task addresss
dtask = extract_direct_task(dest_addresses)
@@ -248,6 +276,11 @@
if dtask:
log('found direct task: %s' % (dtask,))
+ body, attached = extract_body_and_attached(msg)
+ uploads = []
+ for payload in attached:
+ uploads.append(extract_payload_and_upload(payload))
+
# extra check for address validity for this process
if src_address.count('@') > 1:
error = 'invalid source address: %s' % (src_address,)
@@ -281,6 +314,13 @@
elif routed_addresses:
route_address = routed_addresses[0]
+ body, attached = extract_body_and_attached(msg)
+
+ # XXX: handle secure uploads for routed tasks
+ # uploads = []
+ # for payload in attached:
+ # uploads.append(extract_payload_and_upload(payload))
+
log('found routed address: %s' % (route_address,))
if '/' in address_routing[route_address]:
project, security = address_routing[route_address].split('/')
@@ -295,11 +335,9 @@
project,
security))
else:
- log('found phab default')
handler = os.path.join(parser.get(parser_mode, 'root_dir'),
"scripts/mail/mail_handler.php")
- process = subprocess.Popen([handler, parser.get(parser_mode, 'env')],
- stdin=subprocess.PIPE)
+ process = subprocess.Popen([handler], stdin=subprocess.PIPE)
process.communicate(stdin)
@@ -312,6 +350,9 @@
# output to the sender, in order to not reveal internal
# errors we only respond directly for defined errors
# otherwise swallow, return generic, and go for syslog
+ except EmailStatusError as e:
+ print "%s\n\n%s" % (e, contact)
+ exit(1)
except EmailParsingError as e:
syslog.syslog("EmailParsingError: %s" % (str(e)))
print "%s\n\n%s" % (e, contact)
diff --git a/modules/phabricator/manifests/mailrelay.pp
b/modules/phabricator/manifests/mailrelay.pp
index 1d855c6..eda838c 100644
--- a/modules/phabricator/manifests/mailrelay.pp
+++ b/modules/phabricator/manifests/mailrelay.pp
@@ -19,10 +19,9 @@
# === Examples
#
# class { '::phabricator::mailrelay':
-# address_routing => { maint-announce => 3},
+# address_routing => { maint-announce => ops-maint-announce},
# direct_comments_allowed => { testproj => 'cisco.com,gmail.com'},
# phab_bot => { root_dir => '/srv/phab/phabricator/',
-# env => 'default',
# username => 'phabot',
# host => 'http://myhost/api/',
# certificate => $certificate,
--
To view, visit https://gerrit.wikimedia.org/r/172915
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia86643c00b3c27db5ecd01b4ee005de80330c018
Gerrit-PatchSet: 6
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush <[email protected]>
Gerrit-Reviewer: Rush <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
