Modified: subversion/branches/move-tracking-2/tools/dist/security/mailer.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/tools/dist/security/mailer.py?rev=1697327&r1=1697326&r2=1697327&view=diff ============================================================================== --- subversion/branches/move-tracking-2/tools/dist/security/mailer.py (original) +++ subversion/branches/move-tracking-2/tools/dist/security/mailer.py Mon Aug 24 08:23:06 2015 @@ -23,6 +23,11 @@ Generator of signed advisory mails from __future__ import absolute_import +import re +import uuid +import hashlib +import smtplib +import textwrap import email.utils from email.mime.multipart import MIMEMultipart @@ -33,15 +38,21 @@ try: except ImportError: import security._gnupg as gnupg +import security.parser + class Mailer(object): """ Constructs signed PGP/MIME advisory mails. """ - def __init__(self, notification): + def __init__(self, notification, sender, message_template, + release_date, dist_revision, *release_versions): assert len(notification) > 0 + self.__sender = sender self.__notification = notification + self.__message_content = self.__message_content( + message_template, release_date, dist_revision, release_versions) def __subject(self): """ @@ -78,6 +89,79 @@ class Mailer(object): return template.format(**kwargs) + def __message_content(self, message_template, + release_date, dist_revision, release_versions): + """ + Construct the message from the notification mail template. + """ + + # Construct the replacement arguments for the notification template + culprits = set() + advisories = [] + base_version_keys = self.__notification.base_version_keys() + for metadata in self.__notification: + culprits |= metadata.culprit + advisories.append( + ' * {}\n {}'.format(metadata.tracking_id, metadata.title)) + release_version_keys = set(security.parser.Patch.split_version(n) + for n in release_versions) + + multi = (len(self.__notification) > 1) + kwargs = dict(multiple=(multi and 'multiple ' or 'a '), + alert=(multi and 'alerts' or 'alert'), + culprits=self.__culprits(culprits), + advisories='\n'.join(advisories), + release_date=release_date.strftime('%d %B %Y'), + release_day=release_date.strftime('%d %B'), + base_versions = self.__versions(base_version_keys), + release_versions = self.__versions(release_version_keys), + dist_revision=str(dist_revision)) + + # Parse, interpolate and rewrap the notification template + wrapped = [] + content = security.parser.Text(message_template) + for line in content.text.format(**kwargs).split('\n'): + if len(line) > 0 and not line[0].isspace(): + for part in textwrap.wrap(line, + break_long_words=False, + break_on_hyphens=False): + wrapped.append(part) + else: + wrapped.append(line) + return security.parser.Text(None, '\n'.join(wrapped).encode('utf-8')) + + def __versions(self, versions): + """ + Return a textual representation of the set of VERSIONS + suitable for inclusion in a notification mail. + """ + + text = tuple(security.parser.Patch.join_version(n) + for n in sorted(versions)) + assert len(text) > 0 + if len(text) == 1: + return text[0] + elif len(text) == 2: + return ' and '.join(text) + else: + return ', '.join(text[:-1]) + ' and ' + text[-1] + + def __culprits(self, culprits): + """ + Return a textual representation of the set of CULPRITS + suitable for inclusion in a notification mail. + """ + + if self.__notification.Metadata.CULPRIT_CLIENT in culprits: + if self.__notification.Metadata.CULPRIT_SERVER in culprits: + return 'clients and servers' + else: + return 'clients' + elif self.__notification.Metadata.CULPRIT_SERVER in culprits: + return 'servers' + else: + raise ValueError('Unknown culprit ' + repr(culprits)) + def __attachments(self): filenames = set() @@ -109,6 +193,60 @@ class Mailer(object): + ' Patch for Subversion ' + patch.base_version) yield attachment(filename, description, 'base64', patch.base64) + def generate_message(self): + message = SignedMessage( + self.__message_content, + self.__attachments()) + message['From'] = self.__sender + message['Reply-To'] = self.__sender + message['To'] = self.__sender # Will be replaced later + message['Subject'] = self.__subject() + message['Date'] = email.utils.formatdate() + + # Try to make the message-id refer to the sender's domain + address = email.utils.parseaddr(self.__sender)[1] + if not address: + domain = None + else: + domain = address.split('@')[1] + if not domain: + domain = None + + idstring = uuid.uuid1().hex + try: + msgid = email.utils.make_msgid(idstring, domain=domain) + except TypeError: + # The domain keyword was added in Python 3.2 + msgid = email.utils.make_msgid(idstring) + message["Message-ID"] = msgid + return message + + def send_mail(self, message, username, password, recipients=None, + host='mail-relay.apache.org', starttls=True, port=None): + if not port and starttls: + port = 587 + server = smtplib.SMTP(host, port) + if starttls: + server.starttls() + if username and password: + server.login(username, password) + + def send(message): + server.sendmail("From: " + message['From'], + "To: " + message['To'], + message.as_string()) + + if recipients is None: + # Test mode, send message back to originator to checck + # that contents and signature are OK. + message.replace_header('To', message['From']) + send(message) + else: + for recipient in recipients: + message.replace_header('To', recipient) + send(message) + server.quit() + class SignedMessage(MIMEMultipart): """ @@ -132,7 +270,7 @@ class SignedMessage(MIMEMultipart): payload, gpgbinary, gnupghome, use_agent, keyring, keyid) self.set_param('protocol', 'application/pgp-signature') - self.set_param('micalg', 'pgp-sha512') ####!!! + self.set_param('micalg', 'pgp-sha512') ####!!! GET THIS FROM KEY! self.preamble = 'This is an OpenPGP/MIME signed message.' self.attach(payload) self.attach(signature) @@ -162,9 +300,13 @@ class SignedMessage(MIMEMultipart): a MIME attachment. """ + # RFC3156 section 5 says line endings in the signed message + # must be canonical <CR><LF>. + cleartext = re.sub(r'\r?\n', '\r\n', payload.as_string()) + gpg = gnupg.GPG(gpgbinary=gpgbinary, gnupghome=gnupghome, use_agent=use_agent, keyring=keyring) - signature = gpg.sign(payload.as_string(), + signature = gpg.sign(cleartext, keyid=keyid, detach=True, clearsign=False) sig = MIMEText('') sig.set_type('application/pgp-signature')
Modified: subversion/branches/move-tracking-2/tools/dist/security/parser.py URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/tools/dist/security/parser.py?rev=1697327&r1=1697326&r2=1697327&view=diff ============================================================================== --- subversion/branches/move-tracking-2/tools/dist/security/parser.py (original) +++ subversion/branches/move-tracking-2/tools/dist/security/parser.py Mon Aug 24 08:23:06 2015 @@ -25,6 +25,7 @@ from __future__ import absolute_import import os +import re import ast import base64 import quopri @@ -49,15 +50,15 @@ class Notification(object): CULPRIT_SERVER = 'server' CULPRIT_CLIENT = 'client' - __culprits = ((CULPRIT_SERVER, CULPRIT_CLIENT, + __CULPRITS = ((CULPRIT_SERVER, CULPRIT_CLIENT, (CULPRIT_SERVER, CULPRIT_CLIENT), (CULPRIT_CLIENT, CULPRIT_SERVER))) def __init__(self, basedir, tracking_id, title, culprit, advisory, patches): - if culprit not in self.__culprits: + if culprit not in self.__CULPRITS: raise ValueError('Culprit should be one of: ' - + ', '.join(repr(x) for x in self.__culprits)) + + ', '.join(repr(x) for x in self.__CULPRITS)) if not isinstance(culprit, tuple): culprit = (culprit,) @@ -69,9 +70,7 @@ class Notification(object): for base_version, patchfile in patches.items(): patch = Patch(base_version, os.path.join(basedir, patchfile)) self.__patches.append(patch) - self.__patches.sort(reverse=True, - key=lambda x: tuple( - int(q) for q in x.base_version.split('.'))) + self.__patches.sort(reverse=True, key=lambda x: x.base_version_key) @property def tracking_id(self): @@ -99,8 +98,13 @@ class Notification(object): Create the security notification for all TRACKING_IDS. The advisories and patches for each tracking ID must be in the appropreiately named subdirectory of ROOTDIR. + + The notification text assumes that RELEASE_VERSIONS will + be published on RELEASE_DATE and that the tarballs are + available in DIST_REVISION of the dist repository. """ + assert(len(tracking_ids) > 0) self.__advisories = [] for tid in tracking_ids: self.__advisories.append(self.__parse_advisory(rootdir, tid)) @@ -112,6 +116,10 @@ class Notification(object): return len(self.__advisories) def __parse_advisory(self, rootdir, tracking_id): + """ + Parse a single advisory named TRACKING_ID in ROOTDIR. + """ + basedir = os.path.join(rootdir, tracking_id) with open(os.path.join(basedir, 'metadata'), 'rt') as md: metadata = ast.literal_eval(md.read()) @@ -122,31 +130,48 @@ class Notification(object): metadata['advisory'], metadata['patches']) + def base_version_keys(self): + """ + Return the set of base-version keys of all the patches. + """ + + base_version_keys = set() + for metadata in self: + for patch in metadata.patches: + base_version_keys.add(patch.base_version_key) + return base_version_keys -class __Part(object): - def __init__(self, path): - self.__text = self.__load_file(path) - def __load_file(self, path): +class __Part(object): + def __init__(self, path, text=None): """ - Load a file at PATH into memory as an array of lines. - if self.TEXTMODE is True, strip whitespace from the end of + Create a text object with contents from the file at PATH. + If self.TEXTMODE is True, strip whitespace from the end of all lines and strip empty lines from the end of the file. + + Alternatively, if PATH is None, set the contents to TEXT, + which must be convertible to bytes. """ - text = [] + assert (path is None) is not (text is None) + if path: + self.__text = self.__load_file(path) + else: + self.__text = bytes(text) + + def __load_file(self, path): with open(path, 'rb') as src: + if not self.TEXTMODE: + return src.read() + + text = [] for line in src: - if self.TEXTMODE: - line = line.rstrip() + b'\n' - text.append(line) + text.append(line.rstrip() + b'\n') - # Strip trailing empty lines in text mode - if self.TEXTMODE: + # Strip trailing empty lines in text mode while len(text) and not text[-1]: del text[-1] - - return b''.join(text) + return b''.join(text) @property def text(self): @@ -204,11 +229,52 @@ class Patch(__Part): def __init__(self, base_version, path): super(Patch, self).__init__(path) self.__base_version = base_version + self.__base_version_key = self.split_version(base_version) @property def base_version(self): return self.__base_version @property + def base_version_key(self): + return self.__base_version_key + + @property def quoted_printable(self): raise NotImplementedError('Quoted-printable patches? Really?') + + + __SPLIT_VERSION_RX = re.compile(r'^(\d+)(?:\.(\d+))?(?:\.(\d+))?(.+)?$') + + @classmethod + def split_version(cls, version): + """ + Splits a version number in the form n.n.n-tag into a tuple + of its components. + """ + def splitv(version): + for s in cls.__SPLIT_VERSION_RX.match(version).groups(): + if s is None: + continue + try: + n = int(s) + except ValueError: + n = s + yield n + return tuple(splitv(version)) + + @classmethod + def join_version(cls, version_tuple): + """ + Joins a version number tuple returned by Patch.split_version + into a string. + """ + + def joinv(version_tuple): + prev = None + for n in version_tuple: + if isinstance(n, int) and prev is not None: + yield '.' + prev = n + yield str(n) + return ''.join(joinv(version_tuple))
