------------------------------------------------------------ revno: 6514 committer: Barry Warsaw <[EMAIL PROTECTED]> branch nick: 3.0 timestamp: Thu 2007-06-28 01:11:50 -0400 message: Convert the rest of test_runners.py to doctests; even though incomplete, they test everything the old unit tests tested. There are XXX's left in the doctests as reminders to flesh them out. Change the NNTP_REWRITE_DUPLICATE_HEADERS to use proper capitalization. Revert a change I made in the conversion of the Switchboard class: Switchboard.files is no longer a generator. The Runner implementation is cleaner if this returns a concrete list, so that's what it does now. Update the tests to reflect that. The Runner simplifies now too because it no longer needs _open_files() or the _listcache WeakValueDictionary. The standard list manager handles all this now, so just use it directly. Also change the way the Runner sets the language context in _onefile(). It still tries to set it to the preferred language of the sender, if the sender is a member of the list. Otherwise it sets it to the list's preferred language, not the system's preferred language. Removed a conditional that can't possibly happen. removed: Mailman/Queue/tests/test_runners.py added: Mailman/docs/news-runner.txt Mailman/docs/runner.txt modified: Mailman/Defaults.py.in Mailman/Queue/Runner.py Mailman/Queue/Switchboard.py Mailman/docs/acknowledge.txt Mailman/docs/replybot.txt Mailman/docs/switchboard.txt
=== removed file 'Mailman/Queue/tests/test_runners.py' --- a/Mailman/Queue/tests/test_runners.py 2007-06-27 21:33:42 +0000 +++ b/Mailman/Queue/tests/test_runners.py 1970-01-01 00:00:00 +0000 @@ -1,163 +0,0 @@ -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Unit tests for the various Mailman qrunner modules.""" - -import os -import email -import shutil -import tempfile -import unittest - -from Mailman.Message import Message -from Mailman.Queue.NewsRunner import prepare_message -from Mailman.Queue.Runner import Runner -from Mailman.Queue.Switchboard import Switchboard -from Mailman.testing.base import TestBase - - - -class TestPrepMessage(TestBase): - def test_remove_unacceptables(self): - eq = self.assertEqual - msg = email.message_from_string("""\ -From: [EMAIL PROTECTED] -To: [EMAIL PROTECTED] -NNTP-Posting-Host: news.dom.ain -NNTP-Posting-Date: today -X-Trace: blah blah -X-Complaints-To: [EMAIL PROTECTED] -Xref: blah blah -Xref: blah blah -Date-Received: yesterday -Posted: tomorrow -Posting-Version: 99.99 -Relay-Version: 88.88 -Received: blah blah - -A message -""") - msgdata = {} - prepare_message(self._mlist, msg, msgdata) - eq(msgdata.get('prepped'), 1) - eq(msg['from'], '[EMAIL PROTECTED]') - eq(msg['to'], '[EMAIL PROTECTED]') - eq(msg['nntp-posting-host'], None) - eq(msg['nntp-posting-date'], None) - eq(msg['x-trace'], None) - eq(msg['x-complaints-to'], None) - eq(msg['xref'], None) - eq(msg['date-received'], None) - eq(msg['posted'], None) - eq(msg['posting-version'], None) - eq(msg['relay-version'], None) - eq(msg['received'], None) - - def test_munge_duplicates_no_duplicates(self): - eq = self.assertEqual - msg = email.message_from_string("""\ -From: [EMAIL PROTECTED] -To: [EMAIL PROTECTED] -Cc: [EMAIL PROTECTED] -Content-Transfer-Encoding: yes - -A message -""") - msgdata = {} - prepare_message(self._mlist, msg, msgdata) - eq(msgdata.get('prepped'), 1) - eq(msg['from'], '[EMAIL PROTECTED]') - eq(msg['to'], '[EMAIL PROTECTED]') - eq(msg['cc'], '[EMAIL PROTECTED]') - eq(msg['content-transfer-encoding'], 'yes') - - def test_munge_duplicates(self): - eq = self.assertEqual - msg = email.message_from_string("""\ -From: [EMAIL PROTECTED] -To: [EMAIL PROTECTED] -To: [EMAIL PROTECTED] -Cc: [EMAIL PROTECTED] -Cc: [EMAIL PROTECTED] -Cc: [EMAIL PROTECTED] -Content-Transfer-Encoding: yes -Content-Transfer-Encoding: no -Content-Transfer-Encoding: maybe - -A message -""") - msgdata = {} - prepare_message(self._mlist, msg, msgdata) - eq(msgdata.get('prepped'), 1) - eq(msg.get_all('from'), ['[EMAIL PROTECTED]']) - eq(msg.get_all('to'), ['[EMAIL PROTECTED]']) - eq(msg.get_all('cc'), ['[EMAIL PROTECTED]']) - eq(msg.get_all('content-transfer-encoding'), ['yes']) - eq(msg.get_all('x-original-to'), ['[EMAIL PROTECTED]']) - eq(msg.get_all('x-original-cc'), ['[EMAIL PROTECTED]', '[EMAIL PROTECTED]']) - eq(msg.get_all('x-original-content-transfer-encoding'), - ['no', 'maybe']) - - - -class TestableRunner(Runner): - def _dispose(self, mlist, msg, msgdata): - self.msg = msg - self.data = msgdata - return False - - def _doperiodic(self): - self.stop() - - def _snooze(self, filecnt): - return - - -class TestRunner(TestBase): - def setUp(self): - TestBase.setUp(self) - self._tmpdir = tempfile.mkdtemp() - self._msg = email.message_from_string("""\ -From: [EMAIL PROTECTED] -To: [EMAIL PROTECTED] - -A test message. -""", Message) - class MyRunner(TestableRunner): - QDIR = self._tmpdir - self._runner = MyRunner() - - def tearDown(self): - shutil.rmtree(self._tmpdir, True) - TestBase.tearDown(self) - - def test_run_loop(self): - eq = self.assertEqual - sb = Switchboard(self._tmpdir) - sb.enqueue(self._msg, listname='[EMAIL PROTECTED]', foo='yes') - self._runner.run() - eq(self._runner.msg['from'], self._msg['from']) - eq(self._runner.msg['to'], self._msg['to']) - eq(self._runner.data['foo'], 'yes') - - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestPrepMessage)) - suite.addTest(unittest.makeSuite(TestRunner)) - return suite === added file 'Mailman/docs/news-runner.txt' --- a/Mailman/docs/news-runner.txt 1970-01-01 00:00:00 +0000 +++ b/Mailman/docs/news-runner.txt 2007-06-28 05:11:50 +0000 @@ -0,0 +1,119 @@ +The news runner +=============== + +The news runner is the queue runner that gateways mailing list messages to an +NNTP newsgroup. One of the most important things this runner does is prepare +the message for Usenet (yes, I know that NNTP is not Usenet, but this runner +was originally written to gate to Usenet, which has its own rules). + + >>> from email import message_from_string + >>> from Mailman.Message import Message + >>> from Mailman.configuration import config + >>> from Mailman.database import flush + >>> from Mailman.Queue.NewsRunner import prepare_message + >>> mlist = config.list_manager.create('[EMAIL PROTECTED]') + >>> mlist.linked_newsgroup = 'comp.lang.python' + >>> flush() + +Some NNTP servers such as INN reject messages containing a set of prohibited +headers, so one of the things that the news runner does is remove these +prohibited headers. + + >>> msg = message_from_string("""\ + ... From: [EMAIL PROTECTED] + ... To: [EMAIL PROTECTED] + ... NNTP-Posting-Host: news.example.com + ... NNTP-Posting-Date: today + ... X-Trace: blah blah + ... X-Complaints-To: [EMAIL PROTECTED] + ... Xref: blah blah + ... Xref: blah blah + ... Date-Received: yesterday + ... Posted: tomorrow + ... Posting-Version: 99.99 + ... Relay-Version: 88.88 + ... Received: blah blah + ... + ... A message + ... """, Message) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: [EMAIL PROTECTED] + To: [EMAIL PROTECTED] + Newsgroups: comp.lang.python + Message-ID: <[EMAIL PROTECTED]> + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + +Some NNTP servers will reject messages where certain headers are duplicated, +so the news runner must collapse or move these duplicate headers to an +X-Original-* header that the news server doesn't care about. + + >>> msg = message_from_string("""\ + ... From: [EMAIL PROTECTED] + ... To: [EMAIL PROTECTED] + ... To: [EMAIL PROTECTED] + ... Cc: [EMAIL PROTECTED] + ... Cc: [EMAIL PROTECTED] + ... Cc: [EMAIL PROTECTED] + ... Content-Transfer-Encoding: yes + ... Content-Transfer-Encoding: no + ... Content-Transfer-Encoding: maybe + ... + ... A message + ... """, Message) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: [EMAIL PROTECTED] + Newsgroups: comp.lang.python + Message-ID: <[EMAIL PROTECTED]> + Lines: 1 + To: [EMAIL PROTECTED] + X-Original-To: [EMAIL PROTECTED] + CC: [EMAIL PROTECTED] + X-Original-CC: [EMAIL PROTECTED] + X-Original-CC: [EMAIL PROTECTED] + Content-Transfer-Encoding: yes + X-Original-Content-Transfer-Encoding: no + X-Original-Content-Transfer-Encoding: maybe + <BLANKLINE> + A message + <BLANKLINE> + +But if no headers are duplicated, then the news runner doesn't need to modify +the message. + + >>> msg = message_from_string("""\ + ... From: [EMAIL PROTECTED] + ... To: [EMAIL PROTECTED] + ... Cc: [EMAIL PROTECTED] + ... Content-Transfer-Encoding: yes + ... + ... A message + ... """, Message) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: [EMAIL PROTECTED] + To: [EMAIL PROTECTED] + Cc: [EMAIL PROTECTED] + Content-Transfer-Encoding: yes + Newsgroups: comp.lang.python + Message-ID: <[EMAIL PROTECTED]> + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + + +XXX More of the NewsRunner should be tested. === added file 'Mailman/docs/runner.txt' --- a/Mailman/docs/runner.txt 1970-01-01 00:00:00 +0000 +++ b/Mailman/docs/runner.txt 2007-06-28 05:11:50 +0000 @@ -0,0 +1,76 @@ +Queue runners +============= + +The queue runners (qrunner) are the processes that move messages around the +Mailman system. Each qrunner is responsible for a slice of the hash space in +a queue directory. It processes all the files in its slice, sleeps a little +while, then wakes up and runs through its queue files again. + + +Basic architecture +------------------ + +The basic architecture of qrunner is implemented in the base class that all +runners inherit from. This base class implements a .run() method that runs +continuously in a loop until the .stop() method is called. + + >>> import os + >>> from email import message_from_string + >>> from Mailman.Message import Message + >>> from Mailman.Queue.Runner import Runner + >>> from Mailman.Queue.Switchboard import Switchboard + >>> from Mailman.configuration import config + >>> from Mailman.database import flush + >>> mlist = config.list_manager.create('[EMAIL PROTECTED]') + >>> mlist.preferred_language = 'en' + >>> flush() + +Here is a very simple derived qrunner class. The class attribute QDIR tells +the qrunner which queue directory it is responsible for. Derived classes +should also implement various methods to provide the special functionality. +This is about as simple as a qrunner can be. + + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> class TestableRunner(Runner): + ... QDIR = queue_directory + ... + ... def _dispose(self, mlist, msg, msgdata): + ... self.msg = msg + ... self.msgdata = msgdata + ... return False + ... + ... def _doperiodic(self): + ... self.stop() + ... + ... def _snooze(self, filecnt): + ... return + + >>> runner = TestableRunner() + >>> switchboard = Switchboard(queue_directory) + +This qrunner doesn't do much except run once, storing the message and metadata +on instance variables. + + >>> msg = message_from_string("""\ + ... From: [EMAIL PROTECTED] + ... To: [EMAIL PROTECTED] + ... + ... A test message. + ... """, Message) + >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, + ... foo='yes', bar='no') + >>> runner.run() + >>> print runner.msg.as_string() + From: [EMAIL PROTECTED] + To: [EMAIL PROTECTED] + <BLANKLINE> + A test message. + <BLANKLINE> + >>> sorted(runner.msgdata.items()) + [('_parsemsg', False), + ('bar', 'no'), ('foo', 'yes'), + ('lang', 'en'), ('listname', '[EMAIL PROTECTED]'), + ('received_time', ...), ('version', 3)] + + +XXX More of the Runner API should be tested. === modified file 'Mailman/Defaults.py.in' --- a/Mailman/Defaults.py.in 2007-06-21 14:23:40 +0000 +++ b/Mailman/Defaults.py.in 2007-06-28 05:11:50 +0000 @@ -469,10 +469,10 @@ # original message. Any second and subsequent headers are rewritten to the # second named header (case preserved). NNTP_REWRITE_DUPLICATE_HEADERS = [ - ('to', 'X-Original-To'), - ('cc', 'X-Original-Cc'), - ('content-transfer-encoding', 'X-Original-Content-Transfer-Encoding'), - ('mime-version', 'X-MIME-Version'), + ('To', 'X-Original-To'), + ('CC', 'X-Original-CC'), + ('Content-Transfer-Encoding', 'X-Original-Content-Transfer-Encoding'), + ('MIME-Version', 'X-MIME-Version'), ] # Some list posts and mail to the -owner address may contain DomainKey or === modified file 'Mailman/Queue/Runner.py' --- a/Mailman/Queue/Runner.py 2007-06-27 21:33:42 +0000 +++ b/Mailman/Queue/Runner.py 2007-06-28 05:11:50 +0000 @@ -86,7 +86,7 @@ # Switchboard.files() is guaranteed to hand us the files in FIFO # order. Return an integer count of the number of files that were # available for this qrunner to process. - files = self._switchboard.files() + files = self._switchboard.files for filebase in files: try: # Ask the switchboard for the message and metadata objects @@ -147,7 +147,7 @@ # # Find out which mailing list this message is destined for. listname = msgdata.get('listname') - mlist = self._open_list(listname) + mlist = config.list_manager.get(listname) if not mlist: log.error('Dequeuing message destined for missing list: %s', listname) @@ -164,10 +164,11 @@ # approach, but I can't think of anything better right now. otranslation = i18n.get_translation() sender = msg.get_sender() - if mlist: - lang = mlist.getMemberLanguage(sender) + member = mlist.members.get_member(sender) + if member: + lang = member.preferred_language else: - lang = config.DEFAULT_SERVER_LANGUAGE + lang = mlist.preferred_language i18n.set_language(lang) msgdata['lang'] = lang try: @@ -181,24 +182,6 @@ if keepqueued: self._switchboard.enqueue(msg, msgdata) - # Mapping of listnames to MailList instances as a weak value dictionary. - _listcache = weakref.WeakValueDictionary() - - def _open_list(self, listname): - # Cache the open list so that any use of the list within this process - # uses the same object. We use a WeakValueDictionary so that when the - # list is no longer necessary, its memory is freed. - mlist = self._listcache.get(listname) - if not mlist: - try: - mlist = MailList.MailList(listname, lock=False) - except Errors.MMListError, e: - log.error('error opening list: %s\n%s', listname, e) - return None - else: - self._listcache[listname] = mlist - return mlist - def _log(self, exc): log.error('Uncaught runner exception: %s', exc) s = StringIO() === modified file 'Mailman/Queue/Switchboard.py' --- a/Mailman/Queue/Switchboard.py 2007-06-27 21:33:42 +0000 +++ b/Mailman/Queue/Switchboard.py 2007-06-28 05:11:50 +0000 @@ -178,8 +178,7 @@ key += DELTA times[key] = filebase # FIFO sort - for key in sorted(times): - yield times[key] + return [times[key] for key in sorted(times)] def recover_backup_files(self): # Move all .bak files in our slice to .pck. It's impossible for both === modified file 'Mailman/docs/acknowledge.txt' --- a/Mailman/docs/acknowledge.txt 2007-06-28 03:01:56 +0000 +++ b/Mailman/docs/acknowledge.txt 2007-06-28 05:11:50 +0000 @@ -21,7 +21,7 @@ >>> # for new auto-response messages. >>> from Mailman.Queue.sbcache import get_switchboard >>> virginq = get_switchboard(config.VIRGINQUEUE_DIR) - >>> list(virginq.files) + >>> virginq.files [] Subscribe a user to the mailing list. @@ -44,7 +44,7 @@ ... ... """, Message) >>> process(mlist, msg, {}) - >>> list(virginq.files) + >>> virginq.files [] We can also specify the original sender in the message's metadata. If that @@ -55,7 +55,7 @@ ... ... """, Message) >>> process(mlist, msg, dict(original_sender='[EMAIL PROTECTED]')) - >>> list(virginq.files) + >>> virginq.files [] @@ -69,7 +69,7 @@ ... ... """, Message) >>> process(mlist, msg, {}) - >>> list(virginq.files) + >>> virginq.files [] Similarly if the original sender is specified in the message metadata, and @@ -83,7 +83,7 @@ >>> flush() >>> process(mlist, msg, dict(original_sender='[EMAIL PROTECTED]')) - >>> list(virginq.files) + >>> virginq.files [] @@ -104,11 +104,10 @@ ... ... """, Message) >>> process(mlist, msg, {}) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) - >>> list(virginq.files) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) + >>> virginq.files [] >>> # Print only some of the meta data. The rest is uninteresting. >>> qdata['listname'] @@ -143,11 +142,10 @@ ... ... """, Message) >>> process(mlist, msg, {}) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) - >>> list(virginq.files) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) + >>> virginq.files [] >>> # Print only some of the meta data. The rest is uninteresting. >>> qdata['listname'] === modified file 'Mailman/docs/replybot.txt' --- a/Mailman/docs/replybot.txt 2007-06-28 03:01:56 +0000 +++ b/Mailman/docs/replybot.txt 2007-06-28 05:11:50 +0000 @@ -19,7 +19,7 @@ >>> # for new auto-response messages. >>> from Mailman.Queue.sbcache import get_switchboard >>> virginq = get_switchboard(config.VIRGINQUEUE_DIR) - >>> list(virginq.files) + >>> virginq.files [] @@ -43,10 +43,9 @@ ... help ... """, Message) >>> process(mlist, msg, dict(toowner=True)) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> # Print only some of the meta data. The rest is uninteresting. >>> qdata['listname'] '[EMAIL PROTECTED]' @@ -67,7 +66,7 @@ Precedence: bulk <BLANKLINE> admin autoresponse text - >>> list(virginq.files) + >>> virginq.files [] @@ -85,7 +84,7 @@ ... help me ... """, Message) >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] Mailman itself can suppress autoresponses for certain types of internally @@ -97,7 +96,7 @@ ... help for you ... """, Message) >>> process(mlist, msg, dict(noack=True, toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] If there is a Precedence: header with any of the values 'bulk', 'junk', or @@ -110,16 +109,16 @@ ... hey! ... """, Message) >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] >>> msg.replace_header('precedence', 'junk') >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] >>> msg.replace_header('precedence', 'list') >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] Unless the X-Ack: header has a value of "yes", in which case, the Precedence @@ -127,10 +126,9 @@ >>> msg['X-Ack'] = 'yes' >>> process(mlist, msg, dict(toowner=True)) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> del qmsg['message-id'] >>> del qmsg['date'] >>> print qmsg.as_string() @@ -164,10 +162,9 @@ ... help me ... """, Message) >>> process(mlist, msg, dict(torequest=True)) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> del qmsg['message-id'] >>> del qmsg['date'] >>> print qmsg.as_string() @@ -195,10 +192,9 @@ ... help me ... """, Message) >>> process(mlist, msg, {}) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> del qmsg['message-id'] >>> del qmsg['date'] >>> print qmsg.as_string() === modified file 'Mailman/docs/switchboard.txt' --- a/Mailman/docs/switchboard.txt 2007-06-27 21:33:42 +0000 +++ b/Mailman/docs/switchboard.txt 2007-06-28 05:11:50 +0000 @@ -72,8 +72,8 @@ >>> switchboard.finish(filebase) >>> sorted(msgdata.items()) [('_parsemsg', False), - ('bar', 2), ('foo', 1), - ('received_time', ...), ('version', 3)] + ('bar', 2), ('foo', 1), + ('received_time', ...), ('version', 3)] Keyword arguments override keys from the metadata dictionary. @@ -82,8 +82,8 @@ >>> switchboard.finish(filebase) >>> sorted(msgdata.items()) [('_parsemsg', False), - ('foo', 2), - ('received_time', ...), ('version', 3)] + ('foo', 2), + ('received_time', ...), ('version', 3)] Iterating over files -- (no title) https://code.launchpad.net/~mailman-coders/mailman/3.0 You are receiving this branch notification because you are subscribed to it. To unsubscribe from this branch go to https://code.launchpad.net/~mailman-coders/mailman/3.0/+subscription/mailman-checkins. _______________________________________________ Mailman-checkins mailing list Mailman-checkins@python.org Unsubscribe: http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org