------------------------------------------------------------
revno: 6529
committer: Barry Warsaw <[EMAIL PROTECTED]>
branch nick: 3.0
timestamp: Thu 2007-07-12 00:12:45 -0400
message:
Convert the Scrubber test to a doctest, and fix Scrubber.py, but otherwise
don't modernize the Scrubber handler.
The is the last of the handler test conversions until we figure out what to do
with the Approve handler. In a unified user database the semantics of this
are unclear.
added:
Mailman/docs/scrubber.txt
modified:
Mailman/Handlers/Scrubber.py
Mailman/testing/test_handlers.py
=== added file 'Mailman/docs/scrubber.txt'
--- a/Mailman/docs/scrubber.txt 1970-01-01 00:00:00 +0000
+++ b/Mailman/docs/scrubber.txt 2007-07-12 04:12:45 +0000
@@ -0,0 +1,217 @@
+The scrubber
+============
+
+The scrubber is an integral part of Mailman, both in the normal delivery of
+messages and in components such as the archiver. Its primary purpose is to
+scrub attachments from messages so that binary goop doesn't end up in an
+archive message.
+
+ >>> from Mailman.Handlers.Scrubber import process, save_attachment
+ >>> from Mailman.Message import Message
+ >>> from Mailman.configuration import config
+ >>> from Mailman.database import flush
+ >>> from email import message_from_string
+ >>> mlist = config.list_manager.create('[EMAIL PROTECTED]')
+ >>> mlist.preferred_language = 'en'
+ >>> flush()
+
+Helper functions for getting the attachment data.
+
+ >>> import os, re
+ >>> def read_attachment(filename, remove=True):
+ ... path = os.path.join(mlist.archive_dir(), filename)
+ ... fp = open(path)
+ ... try:
+ ... data = fp.read()
+ ... finally:
+ ... fp.close()
+ ... if remove:
+ ... os.unlink(path)
+ ... return data
+
+ >>> from urlparse import urlparse
+ >>> def read_url_from_message(msg):
+ ... url = None
+ ... for line in msg.get_payload().splitlines():
+ ... mo = re.match('URL: <(?P<url>[^>]+)>', line)
+ ... if mo:
+ ... url = mo.group('url')
+ ... break
+ ... path = '/'.join(urlparse(url).path.split('/')[3:])
+ ... return read_attachment(path)
+
+
+Saving attachments
+------------------
+
+The Scrubber handler exposes a function called save_attachments() which can be
+used to strip various types of attachments and store them in the archive
+directory. This is a public interface used by components outside the normal
+processing pipeline.
+
+Site administrators can decide whether the scrubber should use the attachment
+filename suggested in the message's Content-Disposition: header or not. If
+enabled, the filename will be used when this header attribute is present (yes,
+this is an unfortunate double negative).
+
+ >>> config.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME = False
+ >>> msg = message_from_string("""\
+ ... Content-Type: image/gif; name="xtest.gif"
+ ... Content-Transfer-Encoding: base64
+ ... Content-Disposition: attachment; filename="xtest.gif"
+ ...
+ ... R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw==
+ ... """, Message)
+ >>> save_attachment(mlist, msg, '')
+ '<http://www.example.com/pipermail/[EMAIL PROTECTED]//xtest.gif>'
+ >>> data = read_attachment('xtest.gif')
+ >>> data[:6]
+ 'GIF87a'
+ >>> len(data)
+ 34
+
+Saving the attachment does not alter the original message.
+
+ >>> print msg.as_string()
+ Content-Type: image/gif; name="xtest.gif"
+ Content-Transfer-Encoding: base64
+ Content-Disposition: attachment; filename="xtest.gif"
+ <BLANKLINE>
+ R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw==
+
+The site administrator can also configure Mailman to ignore the
+Content-Disposition: filename. This is the default for reasons described in
+the Defaults.py.in file.
+
+ >>> config.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME = True
+ >>> msg = message_from_string("""\
+ ... Content-Type: image/gif; name="xtest.gif"
+ ... Content-Transfer-Encoding: base64
+ ... Content-Disposition: attachment; filename="xtest.gif"
+ ...
+ ... R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw==
+ ... """, Message)
+ >>> save_attachment(mlist, msg, '')
+ '<http://www.example.com/pipermail/[EMAIL PROTECTED]/.../attachment.gif>'
+ >>> data = read_attachment('xtest.gif')
+ Traceback (most recent call last):
+ IOError: [Errno ...] No such file or directory:
+ '.../archives/private/[EMAIL PROTECTED]/xtest.gif'
+ >>> data = read_attachment('attachment.gif')
+ >>> data[:6]
+ 'GIF87a'
+ >>> len(data)
+ 34
+
+
+Scrubbing image attachments
+---------------------------
+
+When scrubbing image attachments, the original message is modified to include
+a reference to the attachment file as available through the on-line archive.
+
+ >>> msg = message_from_string("""\
+ ... MIME-Version: 1.0
+ ... Content-Type: multipart/mixed; boundary="BOUNDARY"
+ ...
+ ... --BOUNDARY
+ ... Content-type: text/plain; charset=us-ascii
+ ...
+ ... This is a message.
+ ... --BOUNDARY
+ ... Content-Type: image/gif; name="xtest.gif"
+ ... Content-Transfer-Encoding: base64
+ ... Content-Disposition: attachment; filename="xtest.gif"
+ ...
+ ... R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw==
+ ... --BOUNDARY--
+ ... """, Message)
+ >>> msgdata = {}
+
+The Scrubber.process() function is different than other handler process
+functions in that it returns the scrubbed message.
+
+ >>> scrubbed_msg = process(mlist, msg, msgdata)
+ >>> scrubbed_msg is msg
+ True
+ >>> print scrubbed_msg.as_string()
+ MIME-Version: 1.0
+ Message-ID: ...
+ Content-Type: text/plain; charset="us-ascii"
+ Content-Transfer-Encoding: 7bit
+ <BLANKLINE>
+ This is a message.
+ -------------- next part --------------
+ A non-text attachment was scrubbed...
+ Name: xtest.gif
+ Type: image/gif
+ Size: 34 bytes
+ Desc: not available
+ URL: <http://www.example.com/pipermail/[EMAIL
PROTECTED]/attachments/.../attachment.gif>
+ <BLANKLINE>
+
+This is the same as the transformed message originally passed in.
+
+ >>> print msg.as_string()
+ MIME-Version: 1.0
+ Message-ID: ...
+ Content-Type: text/plain; charset="us-ascii"
+ Content-Transfer-Encoding: 7bit
+ <BLANKLINE>
+ This is a message.
+ -------------- next part --------------
+ A non-text attachment was scrubbed...
+ Name: xtest.gif
+ Type: image/gif
+ Size: 34 bytes
+ Desc: not available
+ URL: <http://www.example.com/pipermail/[EMAIL
PROTECTED]/attachments/.../attachment.gif>
+ <BLANKLINE>
+ >>> msgdata
+ {}
+
+The URL will point to the attachment sitting in the archive.
+
+ >>> data = read_url_from_message(msg)
+ >>> data[:6]
+ 'GIF87a'
+ >>> len(data)
+ 34
+
+
+Scrubbing text attachments
+--------------------------
+
+Similar to image attachments, text attachments will also be scrubbed, but the
+placeholder will be slightly different.
+
+ >>> msg = message_from_string("""\
+ ... MIME-Version: 1.0
+ ... Content-Type: multipart/mixed; boundary="BOUNDARY"
+ ...
+ ... --BOUNDARY
+ ... Content-type: text/plain; charset=us-ascii; format=flowed; delsp=no
+ ...
+ ... This is a message.
+ ... --BOUNDARY
+ ... Content-type: text/plain; name="xtext.txt"
+ ... Content-Disposition: attachment; filename="xtext.txt"
+ ...
+ ... This is a text attachment.
+ ... --BOUNDARY--
+ ... """, Message)
+ >>> scrubbed_msg = process(mlist, msg, {})
+ >>> print scrubbed_msg.as_string()
+ MIME-Version: 1.0
+ Message-ID: ...
+ Content-Transfer-Encoding: 7bit
+ Content-Type: text/plain; charset="us-ascii"; format="flowed"; delsp="no"
+ <BLANKLINE>
+ This is a message.
+ -------------- next part --------------
+ An embedded and charset-unspecified text was scrubbed...
+ Name: xtext.txt
+ URL: <http://www.example.com/pipermail/[EMAIL
PROTECTED]/attachments/.../attachment.txt>
+ <BLANKLINE>
+ >>> read_url_from_message(msg)
+ 'This is a text attachment.'
=== modified file 'Mailman/Handlers/Scrubber.py'
--- a/Mailman/Handlers/Scrubber.py 2007-06-22 18:53:02 +0000
+++ b/Mailman/Handlers/Scrubber.py 2007-07-12 04:12:45 +0000
@@ -345,12 +345,8 @@
charsets.append(partcharset)
# Now join the text and set the payload
sep = _('-------------- next part --------------\n')
- # The i18n separator is in the list's charset. Coerce to unicode.
- try:
- sep = unicode(sep, lcset, 'replace')
- except (UnicodeError, LookupError, ValueError):
- # This shouldn't occur.
- pass
+ assert isinstance(sep, unicode), (
+ 'Expected a unicode separator, got %s' % type(sep))
rept = sep.join(text)
# Replace entire message with text and scrubbed notice.
# Try with message charsets and utf-8
@@ -373,16 +369,21 @@
def makedirs(dir):
- # Create all the directories to store this attachment in
+ # Create all the directories to store this attachment in and try to make
+ # sure that the permissions of the directories are set correctly.
try:
os.makedirs(dir, 02775)
- # Unfortunately, FreeBSD seems to be broken in that it doesn't honor
- # the mode arg of mkdir().
- def twiddle(arg, dirname, names):
- os.chmod(dirname, 02775)
- os.path.walk(dir, twiddle, None)
except OSError, e:
- if e.errno <> errno.EEXIST: raise
+ if e.errno == errno.EEXIST:
+ return
+ # Some systems such as FreeBSD ignore mkdir's mode, so walk the just
+ # created directories and try to set the mode, ignoring any OSErrors that
+ # occur here.
+ for dirpath, dirnames, filenames in os.walk(dir):
+ try:
+ os.chmod(dirpath, 02775)
+ except OSError:
+ pass
=== modified file 'Mailman/testing/test_handlers.py'
--- a/Mailman/testing/test_handlers.py 2007-07-11 11:06:34 +0000
+++ b/Mailman/testing/test_handlers.py 2007-07-12 04:12:45 +0000
@@ -17,30 +17,17 @@
"""Unit tests for the various Mailman/Handlers/*.py modules."""
-import os
-import sha
-import time
import email
-import errno
-import cPickle
import unittest
-from email.Generator import Generator
-
from Mailman import Errors
from Mailman import Message
-from Mailman import Version
from Mailman import passwords
from Mailman.MailList import MailList
-from Mailman.Queue.Switchboard import Switchboard
from Mailman.configuration import config
from Mailman.testing.base import TestBase
-from Mailman.Handlers import Acknowledge
-from Mailman.Handlers import AfterDelivery
from Mailman.Handlers import Approve
-from Mailman.Handlers import Moderate
-from Mailman.Handlers import Scrubber
# Don't test handlers such as SMTPDirect and Sendmail here
@@ -126,100 +113,7 @@
-class TestScrubber(TestBase):
- def test_save_attachment(self):
- mlist = self._mlist
- msg = email.message_from_string("""\
-Content-Type: image/gif; name="xtest.gif"
-Content-Transfer-Encoding: base64
-Content-Disposition: attachment; filename="xtest.gif"
-
-R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw==
-""")
- Scrubber.save_attachment(mlist, msg, '')
- f = open(os.path.join(mlist.archive_dir(), 'attachment.gif'))
- img = f.read()
- self.assertEqual(img.startswith('GIF87a'), True)
- self.assertEqual(len(img), 34)
-
- def _saved_file(self, s):
- # a convenient function to get the saved attachment file
- for i in s.splitlines():
- if i.startswith('URL: '):
- f = i.replace(
- 'URL: <' + self._mlist.GetBaseArchiveURL() + '/' , '')
- f = os.path.join(self._mlist.archive_dir(), f.rstrip('>'))
- return f
-
- def test_scrub_image(self):
- mlist = self._mlist
- msg = email.message_from_string("""\
-MIME-Version: 1.0
-Content-Type: multipart/mixed; boundary="BOUNDARY"
-
---BOUNDARY
-Content-type: text/plain; charset=us-ascii
-
-This is a message.
---BOUNDARY
-Content-Type: image/gif; name="xtest.gif"
-Content-Transfer-Encoding: base64
-Content-Disposition: attachment; filename="xtest.gif"
-
-R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw==
---BOUNDARY--
-""")
- Scrubber.process(mlist, msg, {})
- # saved file
- img = open(self._saved_file(msg.get_payload())).read()
- self.assertEqual(img.startswith('GIF87a'), True)
- self.assertEqual(len(img), 34)
- # scrubbed message
- s = '\n'.join([l for l in msg.get_payload().splitlines()
- if not l.startswith('URL: ')])
- self.assertEqual(s, """\
-This is a message.
--------------- next part --------------
-A non-text attachment was scrubbed...
-Name: xtest.gif
-Type: image/gif
-Size: 34 bytes
-Desc: not available""")
-
- def test_scrub_text(self):
- mlist = self._mlist
- msg = email.message_from_string("""\
-MIME-Version: 1.0
-Content-Type: multipart/mixed; boundary="BOUNDARY"
-
---BOUNDARY
-Content-type: text/plain; charset=us-ascii; format=flowed; delsp=no
-
-This is a message.
---BOUNDARY
-Content-type: text/plain; name="xtext.txt"
-Content-Disposition: attachment; filename="xtext.txt"
-
-This is a text attachment.
---BOUNDARY--
-""")
- Scrubber.process(mlist, msg, {})
- self.assertEqual(msg.get_param('format'), 'flowed')
- self.assertEqual(msg.get_param('delsp'), 'no')
- txt = open(self._saved_file(msg.get_payload())).read()
- self.assertEqual(txt, 'This is a text attachment.')
- s = '\n'.join([l for l in msg.get_payload().splitlines()
- if not l.startswith('URL: ')])
- self.assertEqual(s, """\
-This is a message.
--------------- next part --------------
-An embedded and charset-unspecified text was scrubbed...
-Name: xtext.txt""")
-
-
-
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestApprove))
- suite.addTest(unittest.makeSuite(TestScrubber))
return suite
--
(no title)
https://code.launchpad.net/~mailman-coders/mailman/3.0
You are receiving this branch notification because you are subscribed to it.
To unsubscribe from this branch go to
https://code.launchpad.net/~mailman-coders/mailman/3.0/+subscription/mailman-checkins.
_______________________________________________
Mailman-checkins mailing list
Mailman-checkins@python.org
Unsubscribe:
http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
