[Mailman-checkins] [Branch ~mailman-coders/mailman/3.0] (no title)
revno: 6528
committer: Barry Warsaw [EMAIL PROTECTED]
branch nick: 3.0
timestamp: Wed 2007-07-11 07:06:34 -0400
message:
Convert ToArchive tests to doctests and do a minimal amount of handler module
cleanup (really, not much was necessary).
added:
Mailman/docs/archives.txt
modified:
Mailman/Handlers/ToArchive.py
Mailman/testing/test_handlers.py
=== added file 'Mailman/docs/archives.txt'
--- a/Mailman/docs/archives.txt 1970-01-01 00:00:00 +
+++ b/Mailman/docs/archives.txt 2007-07-11 11:06:34 +
@@ -0,0 +1,141 @@
+Archives
+
+
+Updating the archives with posted messages is handled by a separate queue,
+which allows for better memory management and prevents blocking the main
+delivery processes while messages are archived. This also allows external
+archivers to work in a separate process from the main Mailman delivery
+processes.
+
+ from Mailman.Handlers.ToArchive import process
+ from Mailman.Message import Message
+ from Mailman.Queue.Switchboard import Switchboard
+ from Mailman.configuration import config
+ from Mailman.database import flush
+ from email import message_from_string
+ mlist = config.list_manager.create('[EMAIL PROTECTED]')
+ mlist.preferred_language = 'en'
+ flush()
+ switchboard = Switchboard(config.ARCHQUEUE_DIR)
+
+A helper function.
+
+ def clear():
+... for filebase in switchboard.files:
+... msg, msgdata = switchboard.dequeue(filebase)
+... switchboard.finish(filebase)
+
+
+The purpose of the ToArchive handler is to make a simple decision as to
+whether the message should get archived and if so, to drop the message in the
+archiving queue. Really the most important things are to determine when a
+message should /not/ get archived.
+
+For example, no digests should ever get archived.
+
+ mlist.archive = True
+ flush()
+ msg = message_from_string(\
+... Subject: A sample message
+...
+... A message of great import.
+... , Message)
+ process(mlist, msg, dict(isdigest=True))
+ switchboard.files
+[]
+
+If the mailing list is not configured to archive, then even regular deliveries
+won't be archived.
+
+ mlist.archive = False
+ flush()
+ process(mlist, msg, {})
+ switchboard.files
+[]
+
+There are two de-facto standards for a message to indicate that it does not
+want to be archived. We've seen both in the wild so both are supported. The
+X-No-Archive: header can be used to indicate that the message should not be
+archived. Confusingly, this header's value is actually ignored.
+
+ mlist.archive = True
+ flush()
+ msg = message_from_string(\
+... Subject: A sample message
+... X-No-Archive: YES
+...
+... A message of great import.
+... , Message)
+ process(mlist, msg, dict(isdigest=True))
+ switchboard.files
+[]
+
+Even a 'no' value will stop the archiving of the message.
+
+ msg = message_from_string(\
+... Subject: A sample message
+... X-No-Archive: No
+...
+... A message of great import.
+... , Message)
+ process(mlist, msg, dict(isdigest=True))
+ switchboard.files
+[]
+
+Another header that's been observed is the X-Archive: header. Here, the
+header's case folded value must be 'no' in order to prevent archiving.
+
+ msg = message_from_string(\
+... Subject: A sample message
+... X-Archive: No
+...
+... A message of great import.
+... , Message)
+ process(mlist, msg, dict(isdigest=True))
+ switchboard.files
+[]
+
+But if the value is 'yes', then the message will be archived.
+
+ msg = message_from_string(\
+... Subject: A sample message
+... X-Archive: Yes
+...
+... A message of great import.
+... , Message)
+ process(mlist, msg, {})
+ len(switchboard.files)
+1
+ filebase = switchboard.files[0]
+ qmsg, qdata = switchboard.dequeue(filebase)
+ switchboard.finish(filebase)
+ print qmsg.as_string()
+Subject: A sample message
+X-Archive: Yes
+BLANKLINE
+A message of great import.
+BLANKLINE
+ sorted(qdata.items())
+[('_parsemsg', False), ('received_time', ...), ('version', 3)]
+
+Without either archiving header, and all other things being the same, the
+message will get archived.
+
+ msg = message_from_string(\
+... Subject: A sample message
+...
+... A message of great import.
+... , Message)
+ process(mlist, msg, {})
+ len(switchboard.files)
+1
+ filebase = switchboard.files[0]
+ qmsg, qdata = switchboard.dequeue(filebase)
+ switchboard.finish(filebase)
+ print qmsg.as_string()
+Subject: A sample message
+BLANKLINE
+A message of great import.
+BLANKLINE
+ sorted(qdata.items())
+[('_parsemsg', False), ('received_time', ...), ('version', 3)]
=== modified file 'Mailman/Handlers/ToArchive.py'
---
[Mailman-checkins] [Branch ~mailman-coders/mailman/3.0] (no title)
revno: 6529
committer: Barry Warsaw [EMAIL PROTECTED]
branch nick: 3.0
timestamp: Thu 2007-07-12 00:12:45 -0400
message:
Convert the Scrubber test to a doctest, and fix Scrubber.py, but otherwise
don't modernize the Scrubber handler.
The is the last of the handler test conversions until we figure out what to do
with the Approve handler. In a unified user database the semantics of this
are unclear.
added:
Mailman/docs/scrubber.txt
modified:
Mailman/Handlers/Scrubber.py
Mailman/testing/test_handlers.py
=== added file 'Mailman/docs/scrubber.txt'
--- a/Mailman/docs/scrubber.txt 1970-01-01 00:00:00 +
+++ b/Mailman/docs/scrubber.txt 2007-07-12 04:12:45 +
@@ -0,0 +1,217 @@
+The scrubber
+
+
+The scrubber is an integral part of Mailman, both in the normal delivery of
+messages and in components such as the archiver. Its primary purpose is to
+scrub attachments from messages so that binary goop doesn't end up in an
+archive message.
+
+ from Mailman.Handlers.Scrubber import process, save_attachment
+ from Mailman.Message import Message
+ from Mailman.configuration import config
+ from Mailman.database import flush
+ from email import message_from_string
+ mlist = config.list_manager.create('[EMAIL PROTECTED]')
+ mlist.preferred_language = 'en'
+ flush()
+
+Helper functions for getting the attachment data.
+
+ import os, re
+ def read_attachment(filename, remove=True):
+... path = os.path.join(mlist.archive_dir(), filename)
+... fp = open(path)
+... try:
+... data = fp.read()
+... finally:
+... fp.close()
+... if remove:
+... os.unlink(path)
+... return data
+
+ from urlparse import urlparse
+ def read_url_from_message(msg):
+... url = None
+... for line in msg.get_payload().splitlines():
+... mo = re.match('URL: (?Purl[^]+)', line)
+... if mo:
+... url = mo.group('url')
+... break
+... path = '/'.join(urlparse(url).path.split('/')[3:])
+... return read_attachment(path)
+
+
+Saving attachments
+--
+
+The Scrubber handler exposes a function called save_attachments() which can be
+used to strip various types of attachments and store them in the archive
+directory. This is a public interface used by components outside the normal
+processing pipeline.
+
+Site administrators can decide whether the scrubber should use the attachment
+filename suggested in the message's Content-Disposition: header or not. If
+enabled, the filename will be used when this header attribute is present (yes,
+this is an unfortunate double negative).
+
+ config.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME = False
+ msg = message_from_string(\
+... Content-Type: image/gif; name=xtest.gif
+... Content-Transfer-Encoding: base64
+... Content-Disposition: attachment; filename=xtest.gif
+...
+... R0lGODdhAQABAIAAACwAAQABAAACAQUAOw==
+... , Message)
+ save_attachment(mlist, msg, '')
+'http://www.example.com/pipermail/[EMAIL PROTECTED]//xtest.gif'
+ data = read_attachment('xtest.gif')
+ data[:6]
+'GIF87a'
+ len(data)
+34
+
+Saving the attachment does not alter the original message.
+
+ print msg.as_string()
+Content-Type: image/gif; name=xtest.gif
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename=xtest.gif
+BLANKLINE
+R0lGODdhAQABAIAAACwAAQABAAACAQUAOw==
+
+The site administrator can also configure Mailman to ignore the
+Content-Disposition: filename. This is the default for reasons described in
+the Defaults.py.in file.
+
+ config.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME = True
+ msg = message_from_string(\
+... Content-Type: image/gif; name=xtest.gif
+... Content-Transfer-Encoding: base64
+... Content-Disposition: attachment; filename=xtest.gif
+...
+... R0lGODdhAQABAIAAACwAAQABAAACAQUAOw==
+... , Message)
+ save_attachment(mlist, msg, '')
+'http://www.example.com/pipermail/[EMAIL PROTECTED]/.../attachment.gif'
+ data = read_attachment('xtest.gif')
+Traceback (most recent call last):
+IOError: [Errno ...] No such file or directory:
+'.../archives/private/[EMAIL PROTECTED]/xtest.gif'
+ data = read_attachment('attachment.gif')
+ data[:6]
+'GIF87a'
+ len(data)
+34
+
+
+Scrubbing image attachments
+---
+
+When scrubbing image attachments, the original message is modified to include
+a reference to the attachment file as available through the on-line archive.
+
+ msg = message_from_string(\
+... MIME-Version: 1.0
+... Content-Type: multipart/mixed; boundary=BOUNDARY
+...
+... --BOUNDARY
+... Content-type: text/plain; charset=us-ascii
+...
+... This
[Mailman-checkins] [Branch ~mailman-coders/mailman/3.0] (no title)
revno: 6530
committer: Barry Warsaw [EMAIL PROTECTED]
branch nick: 3.0
timestamp: Thu 2007-07-12 00:22:33 -0400
message:
Last bit of existing test suite cleanup for now. I've removed the inmemory.py
module since we're not using that any more. I've also disabled the remaining
failing tests in test_handlers and test_security_mgr. Because of the unified
user database, both of these modules will change significantly, but I don't
want to remove them just yet.
removed:
Mailman/testing/inmemory.py
modified:
Mailman/testing/test_handlers.py
Mailman/testing/test_security_mgr.py
=== removed file 'Mailman/testing/inmemory.py'
--- a/Mailman/testing/inmemory.py 2007-05-28 20:21:41 +
+++ b/Mailman/testing/inmemory.py 1970-01-01 00:00:00 +
@@ -1,396 +0,0 @@
-# Copyright (C) 2007 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-
-In-memory implementations of Mailman interfaces, for testing purposes.
-
-import datetime
-import urlparse
-
-from Mailman import Utils
-from Mailman import passwords
-from Mailman.interfaces import *
-
-from zope.interface import implements
-
-
-
-class UserManager(object):
-implements(IUserManager)
-
-def __init__(self):
-self._users = set()
-self._next_id = 1
-
-@property
-def users(self):
-for user in self._users:
-yield user
-
-def create_user(self):
-user = User(self._next_id, self)
-self._next_id += 1
-self._users.add(user)
-return user
-
-def remove(self, user):
-self._users.discard(user)
-
-def get(self, address):
-# Yes, this is slow and icky, but it's only for testing purposes
-for user in self._users:
-if user.controls(address):
-return user
-return None
-
-
-
-class User(object):
-implements(IUser)
-
-def __init__(self, user_id, user_mgr):
-self._user_id = user_id
-self._user_mgr = user_mgr
-self._addresses = set()
-self.real_name = u''
-self.password = passwords.NoPasswordScheme.make_secret('ignore')
-self.default_profile = None
-
-def __eq__(self, other):
-return (IUser.implementedBy(other) and
-self.user_id == other.user_id and
-self.user_manager is other.user_manager)
-
-def __ne__(self, other):
-return not self.__eq__(other)
-
-@property
-def user_id(self):
-return self._user_id
-
-@property
-def user_manager(self):
-return self._user_mgr
-
-@property
-def addresses(self):
-for address in self._addresses:
-yield address
-
-def add_address(self, address):
-if self.controls(address):
-return
-user_address = Address(address, self)
-self._addresses.add(user_address)
-
-def remove_address(self, address):
-if not self.controls(address):
-return
-user_address = Address(address, self)
-self._addresses.discard(user_address)
-
-def controls(self, address):
-for user_address in self.addresses:
-if user_address == address:
-return True
-return False
-
-
-
-class Address(object):
-implements(IAddress)
-
-def __init__(self, email_address, user, profile=None):
-self._address = email_address
-self._user = user
-self.profile= profile or Profile()
-self.validated_on = None
-
-def __eq__(self, other):
-return (IAddress.implementedBy(other) and
-self.address == other.address and
-self.user == other.user)
-
-@property
-def address(self):
-return self._address
-
-@property
-def user(self):
-return self._user
-
-
-
-class OkayToPost(object):
-implements(IPostingPermission)
-
-# XXX
-okay_to_post = True
-
-
-
-class Roster(object):
-implements(IRoster)
-
-def __init__(self, name):
-self._name = name
-self._members = set()
-
-def __eq__(self, other):
-return (IRoster.implementedBy(other) and
-self.name == other.name)
-
-def
